Add node.codepage option for string conversion between JS and server

README.md

@@ -32,3 +32,44 @@ cmake .. -A Win32 -T ClangCL
 ```
 
 Open Visual Studio and build the solution.
+
+## Configuration
+
+### `node.codepage`
+
+The SA-MP client and the server core use single-byte Windows codepages, while JavaScript strings are Unicode.
+Set `node.codepage` to make omp-node convert strings between them, otherwise non-ASCII text
+(Cyrillic, Polish, Turkish, ...) is displayed incorrectly, and text typed by players reaches your script broken.
+
+The conversion is applied to every string that is passed to the server or received from it (chat, dialogs, commands, player names, events, ...)
+and to the output of `console.log`, so the server log keeps a single encoding.
+
+| Value | Language group |
+|---|---|
+| `utf-8` (default) | no conversion |
+| `windows-1250` | Central European (Polish, Czech, ...) |
+| `windows-1251` | Cyrillic (Russian, Ukrainian, ...) |
+| `windows-1252` | Western European |
+| `windows-1253` | Greek |
+| `windows-1254` | Turkish |
+| `windows-1255` | Hebrew |
+| `windows-1256` | Arabic |
+| `windows-1257` | Baltic |
+| `windows-1258` | Vietnamese |
+
+`cp1251` and `1251` are accepted as aliases. Characters that do not exist in the selected codepage (emoji, for example) are sent as `?`.
+
+Example `config.json`:
+
+```json
+{
+    "node": {
+        "resources": ["resources"],
+        "codepage": "windows-1251"
+    }
+}
+```
+
+The "language for non-Unicode programs" setting on the player's Windows must match the codepage, otherwise the game client will not display the text correctly.
+
+The codepage tables are generated by `tools/generate_codepages.js`.

src/helpers/codepage.hpp

@@ -0,0 +1,261 @@
+#pragma once
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "codepage_tables.hpp"
+
+namespace helpers
+{
+class Codepage
+{
+public:
+	static Codepage& Instance()
+	{
+		static Codepage instance;
+		return instance;
+	}
+
+	bool Set(const char* name, size_t len)
+	{
+		std::string normalized = Normalize(name, len);
+		if (normalized.empty() || normalized == "utf8")
+		{
+			table_ = nullptr;
+			reverse_.clear();
+			name_ = "utf-8";
+			return true;
+		}
+
+		normalized = StripPrefix(normalized);
+
+		for (const auto& definition : codepage_tables::ALL)
+		{
+			if (normalized == std::to_string(definition.number))
+			{
+				table_ = definition.table;
+				name_ = "windows-" + normalized;
+
+				reverse_.clear();
+				reverse_.reserve(128);
+				for (int i = 0; i < 128; i++)
+				{
+					if (table_[i] != 0xFFFD)
+					{
+						reverse_.emplace_back(char32_t(table_[i]), static_cast<uint8_t>(0x80 + i));
+					}
+				}
+				std::sort(reverse_.begin(), reverse_.end());
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	const std::string& Name() const
+	{
+		return name_;
+	}
+
+	bool IsIdentity() const
+	{
+		return table_ == nullptr;
+	}
+
+	std::string ToUtf8(const char* data, size_t len) const
+	{
+		if (!data || len == 0)
+		{
+			return std::string();
+		}
+
+		if (IsIdentity() || IsAscii(data, len))
+		{
+			return std::string(data, len);
+		}
+
+		std::string out;
+		out.reserve(len * 2);
+		for (size_t i = 0; i < len; i++)
+		{
+			unsigned char c = static_cast<unsigned char>(data[i]);
+			AppendUtf8(out, c < 0x80 ? char32_t(c) : char32_t(table_[c - 0x80]));
+		}
+		return out;
+	}
+
+	std::string FromUtf8(const char* data, size_t len) const
+	{
+		if (!data || len == 0)
+		{
+			return std::string();
+		}
+
+		if (IsIdentity() || IsAscii(data, len))
+		{
+			return std::string(data, len);
+		}
+
+		std::string out;
+		out.reserve(len);
+
+		size_t i = 0;
+		while (i < len)
+		{
+			char32_t cp = DecodeUtf8(reinterpret_cast<const unsigned char*>(data), len, i);
+			out += char(Encode(cp));
+		}
+		return out;
+	}
+
+private:
+	Codepage() = default;
+
+	static std::string Normalize(const char* name, size_t len)
+	{
+		std::string result;
+		for (size_t i = 0; name && i < len; i++)
+		{
+			char c = name[i];
+			if (c == '-' || c == '_' || c == ' ')
+			{
+				continue;
+			}
+			result += (c >= 'A' && c <= 'Z') ? char(c - 'A' + 'a') : c;
+		}
+		return result;
+	}
+
+	static std::string StripPrefix(std::string name)
+	{
+		for (const char* prefix : { "windows", "win", "cp" })
+		{
+			std::string p(prefix);
+			if (name.compare(0, p.size(), p) == 0)
+			{
+				name.erase(0, p.size());
+				break;
+			}
+		}
+		return name;
+	}
+
+	static bool IsAscii(const char* data, size_t len)
+	{
+		for (size_t i = 0; i < len; i++)
+		{
+			if (static_cast<unsigned char>(data[i]) >= 0x80)
+			{
+				return false;
+			}
+		}
+		return true;
+	}
+
+	static void AppendUtf8(std::string& out, char32_t cp)
+	{
+		if (cp < 0x80)
+		{
+			out += char(cp);
+		}
+		else if (cp < 0x800)
+		{
+			out += char(0xC0 | (cp >> 6));
+			out += char(0x80 | (cp & 0x3F));
+		}
+		else
+		{
+			out += char(0xE0 | (cp >> 12));
+			out += char(0x80 | ((cp >> 6) & 0x3F));
+			out += char(0x80 | (cp & 0x3F));
+		}
+	}
+
+	static char32_t DecodeUtf8(const unsigned char* s, size_t len, size_t& i)
+	{
+		unsigned char c = s[i];
+		if (c < 0x80)
+		{
+			i++;
+			return c;
+		}
+
+		size_t extra;
+		char32_t cp;
+		char32_t min;
+		if ((c & 0xE0) == 0xC0)
+		{
+			extra = 1;
+			cp = c & 0x1F;
+			min = 0x80;
+		}
+		else if ((c & 0xF0) == 0xE0)
+		{
+			extra = 2;
+			cp = c & 0x0F;
+			min = 0x800;
+		}
+		else if ((c & 0xF8) == 0xF0)
+		{
+			extra = 3;
+			cp = c & 0x07;
+			min = 0x10000;
+		}
+		else
+		{
+			i++;
+			return 0xFFFD;
+		}
+
+		if (i + extra >= len)
+		{
+			i++;
+			return 0xFFFD;
+		}
+
+		for (size_t k = 1; k <= extra; k++)
+		{
+			unsigned char next = s[i + k];
+			if ((next & 0xC0) != 0x80)
+			{
+				i++;
+				return 0xFFFD;
+			}
+			cp = (cp << 6) | (next & 0x3F);
+		}
+
+		if (cp < min)
+		{
+			i++;
+			return 0xFFFD;
+		}
+
+		i += extra + 1;
+		return cp;
+	}
+
+	unsigned char Encode(char32_t cp) const
+	{
+		if (cp < 0x80)
+		{
+			return static_cast<unsigned char>(cp);
+		}
+
+		auto it = std::lower_bound(reverse_.begin(), reverse_.end(), std::make_pair(cp, uint8_t(0)));
+		if (it != reverse_.end() && it->first == cp)
+		{
+			return it->second;
+		}
+		return '?';
+	}
+
+	const char16_t* table_ = nullptr;
+	std::vector<std::pair<char32_t, uint8_t>> reverse_;
+	std::string name_ = "utf-8";
+};
+}