From 25504387409c7622d23fafcbdbb9d3916e25410f Mon Sep 17 00:00:00 2001 From: noexabwuy Date: Mon, 21 Sep 2026 00:35:52 +0300 Subject: [PATCH 1/2] add table driven codepage conversion helper --- src/helpers/codepage.hpp | 261 ++++++++++++++++++++++++++++++++ src/helpers/codepage_tables.hpp | 126 +++++++++++++++ tools/generate_codepages.js | 60 ++++++++ 3 files changed, 447 insertions(+) create mode 100644 src/helpers/codepage.hpp create mode 100644 src/helpers/codepage_tables.hpp create mode 100644 tools/generate_codepages.js diff --git a/src/helpers/codepage.hpp b/src/helpers/codepage.hpp new file mode 100644 index 0000000..2c2a725 --- /dev/null +++ b/src/helpers/codepage.hpp @@ -0,0 +1,261 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "codepage_tables.hpp" + +namespace helpers +{ +class Codepage +{ +public: + static Codepage& Instance() + { + static Codepage instance; + return instance; + } + + bool Set(const char* name, size_t len) + { + std::string normalized = Normalize(name, len); + if (normalized.empty() || normalized == "utf8") + { + table_ = nullptr; + reverse_.clear(); + name_ = "utf-8"; + return true; + } + + normalized = StripPrefix(normalized); + + for (const auto& definition : codepage_tables::ALL) + { + if (normalized == std::to_string(definition.number)) + { + table_ = definition.table; + name_ = "windows-" + normalized; + + reverse_.clear(); + reverse_.reserve(128); + for (int i = 0; i < 128; i++) + { + if (table_[i] != 0xFFFD) + { + reverse_.emplace_back(char32_t(table_[i]), static_cast(0x80 + i)); + } + } + std::sort(reverse_.begin(), reverse_.end()); + return true; + } + } + + return false; + } + + const std::string& Name() const + { + return name_; + } + + bool IsIdentity() const + { + return table_ == nullptr; + } + + std::string ToUtf8(const char* data, size_t len) const + { + if (!data || len == 0) + { + return std::string(); + } + + if (IsIdentity() || IsAscii(data, len)) + { + return std::string(data, len); + } + + std::string out; + out.reserve(len * 2); + for (size_t i = 0; i < len; i++) + { + unsigned char c = static_cast(data[i]); + AppendUtf8(out, c < 0x80 ? char32_t(c) : char32_t(table_[c - 0x80])); + } + return out; + } + + std::string FromUtf8(const char* data, size_t len) const + { + if (!data || len == 0) + { + return std::string(); + } + + if (IsIdentity() || IsAscii(data, len)) + { + return std::string(data, len); + } + + std::string out; + out.reserve(len); + + size_t i = 0; + while (i < len) + { + char32_t cp = DecodeUtf8(reinterpret_cast(data), len, i); + out += char(Encode(cp)); + } + return out; + } + +private: + Codepage() = default; + + static std::string Normalize(const char* name, size_t len) + { + std::string result; + for (size_t i = 0; name && i < len; i++) + { + char c = name[i]; + if (c == '-' || c == '_' || c == ' ') + { + continue; + } + result += (c >= 'A' && c <= 'Z') ? char(c - 'A' + 'a') : c; + } + return result; + } + + static std::string StripPrefix(std::string name) + { + for (const char* prefix : { "windows", "win", "cp" }) + { + std::string p(prefix); + if (name.compare(0, p.size(), p) == 0) + { + name.erase(0, p.size()); + break; + } + } + return name; + } + + static bool IsAscii(const char* data, size_t len) + { + for (size_t i = 0; i < len; i++) + { + if (static_cast(data[i]) >= 0x80) + { + return false; + } + } + return true; + } + + static void AppendUtf8(std::string& out, char32_t cp) + { + if (cp < 0x80) + { + out += char(cp); + } + else if (cp < 0x800) + { + out += char(0xC0 | (cp >> 6)); + out += char(0x80 | (cp & 0x3F)); + } + else + { + out += char(0xE0 | (cp >> 12)); + out += char(0x80 | ((cp >> 6) & 0x3F)); + out += char(0x80 | (cp & 0x3F)); + } + } + + static char32_t DecodeUtf8(const unsigned char* s, size_t len, size_t& i) + { + unsigned char c = s[i]; + if (c < 0x80) + { + i++; + return c; + } + + size_t extra; + char32_t cp; + char32_t min; + if ((c & 0xE0) == 0xC0) + { + extra = 1; + cp = c & 0x1F; + min = 0x80; + } + else if ((c & 0xF0) == 0xE0) + { + extra = 2; + cp = c & 0x0F; + min = 0x800; + } + else if ((c & 0xF8) == 0xF0) + { + extra = 3; + cp = c & 0x07; + min = 0x10000; + } + else + { + i++; + return 0xFFFD; + } + + if (i + extra >= len) + { + i++; + return 0xFFFD; + } + + for (size_t k = 1; k <= extra; k++) + { + unsigned char next = s[i + k]; + if ((next & 0xC0) != 0x80) + { + i++; + return 0xFFFD; + } + cp = (cp << 6) | (next & 0x3F); + } + + if (cp < min) + { + i++; + return 0xFFFD; + } + + i += extra + 1; + return cp; + } + + unsigned char Encode(char32_t cp) const + { + if (cp < 0x80) + { + return static_cast(cp); + } + + auto it = std::lower_bound(reverse_.begin(), reverse_.end(), std::make_pair(cp, uint8_t(0))); + if (it != reverse_.end() && it->first == cp) + { + return it->second; + } + return '?'; + } + + const char16_t* table_ = nullptr; + std::vector> reverse_; + std::string name_ = "utf-8"; +}; +} diff --git a/src/helpers/codepage_tables.hpp b/src/helpers/codepage_tables.hpp new file mode 100644 index 0000000..1e6aa2e --- /dev/null +++ b/src/helpers/codepage_tables.hpp @@ -0,0 +1,126 @@ +// This file is generated by tools/generate_codepages.js, do not edit it manually. +// clang-format off +#pragma once + +namespace helpers +{ +namespace codepage_tables +{ +struct Definition +{ + int number; + const char16_t* table; +}; + +inline const char16_t CP1250[128] = { + 0x20AC, 0x0081, 0x201A, 0x0083, 0x201E, 0x2026, 0x2020, 0x2021, 0x0088, 0x2030, 0x0160, 0x2039, 0x015A, 0x0164, 0x017D, 0x0179, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x0161, 0x203A, 0x015B, 0x0165, 0x017E, 0x017A, + 0x00A0, 0x02C7, 0x02D8, 0x0141, 0x00A4, 0x0104, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x015E, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x017B, + 0x00B0, 0x00B1, 0x02DB, 0x0142, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x0105, 0x015F, 0x00BB, 0x013D, 0x02DD, 0x013E, 0x017C, + 0x0154, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0139, 0x0106, 0x00C7, 0x010C, 0x00C9, 0x0118, 0x00CB, 0x011A, 0x00CD, 0x00CE, 0x010E, + 0x0110, 0x0143, 0x0147, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x00D7, 0x0158, 0x016E, 0x00DA, 0x0170, 0x00DC, 0x00DD, 0x0162, 0x00DF, + 0x0155, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x013A, 0x0107, 0x00E7, 0x010D, 0x00E9, 0x0119, 0x00EB, 0x011B, 0x00ED, 0x00EE, 0x010F, + 0x0111, 0x0144, 0x0148, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x00F7, 0x0159, 0x016F, 0x00FA, 0x0171, 0x00FC, 0x00FD, 0x0163, 0x02D9, +}; + +inline const char16_t CP1251[128] = { + 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, + 0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, + 0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7, 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, + 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, 0x00B6, 0x00B7, 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, + 0x0410, 0x0411, 0x0412, 0x0413, 0x0414, 0x0415, 0x0416, 0x0417, 0x0418, 0x0419, 0x041A, 0x041B, 0x041C, 0x041D, 0x041E, 0x041F, + 0x0420, 0x0421, 0x0422, 0x0423, 0x0424, 0x0425, 0x0426, 0x0427, 0x0428, 0x0429, 0x042A, 0x042B, 0x042C, 0x042D, 0x042E, 0x042F, + 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0439, 0x043A, 0x043B, 0x043C, 0x043D, 0x043E, 0x043F, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044A, 0x044B, 0x044C, 0x044D, 0x044E, 0x044F, +}; + +inline const char16_t CP1252[128] = { + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, + 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, + 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, + 0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, + 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, + 0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF, +}; + +inline const char16_t CP1253[128] = { + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x0088, 0x2030, 0x008A, 0x2039, 0x008C, 0x008D, 0x008E, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x009A, 0x203A, 0x009C, 0x009D, 0x009E, 0x009F, + 0x00A0, 0x0385, 0x0386, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x2015, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x0384, 0x00B5, 0x00B6, 0x00B7, 0x0388, 0x0389, 0x038A, 0x00BB, 0x038C, 0x00BD, 0x038E, 0x038F, + 0x0390, 0x0391, 0x0392, 0x0393, 0x0394, 0x0395, 0x0396, 0x0397, 0x0398, 0x0399, 0x039A, 0x039B, 0x039C, 0x039D, 0x039E, 0x039F, + 0x03A0, 0x03A1, 0xFFFD, 0x03A3, 0x03A4, 0x03A5, 0x03A6, 0x03A7, 0x03A8, 0x03A9, 0x03AA, 0x03AB, 0x03AC, 0x03AD, 0x03AE, 0x03AF, + 0x03B0, 0x03B1, 0x03B2, 0x03B3, 0x03B4, 0x03B5, 0x03B6, 0x03B7, 0x03B8, 0x03B9, 0x03BA, 0x03BB, 0x03BC, 0x03BD, 0x03BE, 0x03BF, + 0x03C0, 0x03C1, 0x03C2, 0x03C3, 0x03C4, 0x03C5, 0x03C6, 0x03C7, 0x03C8, 0x03C9, 0x03CA, 0x03CB, 0x03CC, 0x03CD, 0x03CE, 0xFFFD, +}; + +inline const char16_t CP1254[128] = { + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x008E, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x009E, 0x0178, + 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, + 0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, + 0x011E, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0130, 0x015E, 0x00DF, + 0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, + 0x011F, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0131, 0x015F, 0x00FF, +}; + +inline const char16_t CP1255[128] = { + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x008A, 0x2039, 0x008C, 0x008D, 0x008E, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x009A, 0x203A, 0x009C, 0x009D, 0x009E, 0x009F, + 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x20AA, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00D7, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00F7, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, + 0x05B0, 0x05B1, 0x05B2, 0x05B3, 0x05B4, 0x05B5, 0x05B6, 0x05B7, 0x05B8, 0x05B9, 0xFFFD, 0x05BB, 0x05BC, 0x05BD, 0x05BE, 0x05BF, + 0x05C0, 0x05C1, 0x05C2, 0x05C3, 0x05F0, 0x05F1, 0x05F2, 0x05F3, 0x05F4, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, 0xFFFD, + 0x05D0, 0x05D1, 0x05D2, 0x05D3, 0x05D4, 0x05D5, 0x05D6, 0x05D7, 0x05D8, 0x05D9, 0x05DA, 0x05DB, 0x05DC, 0x05DD, 0x05DE, 0x05DF, + 0x05E0, 0x05E1, 0x05E2, 0x05E3, 0x05E4, 0x05E5, 0x05E6, 0x05E7, 0x05E8, 0x05E9, 0x05EA, 0xFFFD, 0xFFFD, 0x200E, 0x200F, 0xFFFD, +}; + +inline const char16_t CP1256[128] = { + 0x20AC, 0x067E, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x0679, 0x2039, 0x0152, 0x0686, 0x0698, 0x0688, + 0x06AF, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x06A9, 0x2122, 0x0691, 0x203A, 0x0153, 0x200C, 0x200D, 0x06BA, + 0x00A0, 0x060C, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x06BE, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x061B, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x061F, + 0x06C1, 0x0621, 0x0622, 0x0623, 0x0624, 0x0625, 0x0626, 0x0627, 0x0628, 0x0629, 0x062A, 0x062B, 0x062C, 0x062D, 0x062E, 0x062F, + 0x0630, 0x0631, 0x0632, 0x0633, 0x0634, 0x0635, 0x0636, 0x00D7, 0x0637, 0x0638, 0x0639, 0x063A, 0x0640, 0x0641, 0x0642, 0x0643, + 0x00E0, 0x0644, 0x00E2, 0x0645, 0x0646, 0x0647, 0x0648, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0649, 0x064A, 0x00EE, 0x00EF, + 0x064B, 0x064C, 0x064D, 0x064E, 0x00F4, 0x064F, 0x0650, 0x00F7, 0x0651, 0x00F9, 0x0652, 0x00FB, 0x00FC, 0x200E, 0x200F, 0x06D2, +}; + +inline const char16_t CP1257[128] = { + 0x20AC, 0x0081, 0x201A, 0x0083, 0x201E, 0x2026, 0x2020, 0x2021, 0x0088, 0x2030, 0x008A, 0x2039, 0x008C, 0x00A8, 0x02C7, 0x00B8, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x0098, 0x2122, 0x009A, 0x203A, 0x009C, 0x00AF, 0x02DB, 0x009F, + 0x00A0, 0xFFFD, 0x00A2, 0x00A3, 0x00A4, 0xFFFD, 0x00A6, 0x00A7, 0x00D8, 0x00A9, 0x0156, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00C6, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00F8, 0x00B9, 0x0157, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00E6, + 0x0104, 0x012E, 0x0100, 0x0106, 0x00C4, 0x00C5, 0x0118, 0x0112, 0x010C, 0x00C9, 0x0179, 0x0116, 0x0122, 0x0136, 0x012A, 0x013B, + 0x0160, 0x0143, 0x0145, 0x00D3, 0x014C, 0x00D5, 0x00D6, 0x00D7, 0x0172, 0x0141, 0x015A, 0x016A, 0x00DC, 0x017B, 0x017D, 0x00DF, + 0x0105, 0x012F, 0x0101, 0x0107, 0x00E4, 0x00E5, 0x0119, 0x0113, 0x010D, 0x00E9, 0x017A, 0x0117, 0x0123, 0x0137, 0x012B, 0x013C, + 0x0161, 0x0144, 0x0146, 0x00F3, 0x014D, 0x00F5, 0x00F6, 0x00F7, 0x0173, 0x0142, 0x015B, 0x016B, 0x00FC, 0x017C, 0x017E, 0x02D9, +}; + +inline const char16_t CP1258[128] = { + 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 0x02C6, 0x2030, 0x008A, 0x2039, 0x0152, 0x008D, 0x008E, 0x008F, + 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 0x02DC, 0x2122, 0x009A, 0x203A, 0x0153, 0x009D, 0x009E, 0x0178, + 0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, 0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, + 0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, 0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, + 0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x00C5, 0x00C6, 0x00C7, 0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x0300, 0x00CD, 0x00CE, 0x00CF, + 0x0110, 0x00D1, 0x0309, 0x00D3, 0x00D4, 0x01A0, 0x00D6, 0x00D7, 0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x01AF, 0x0303, 0x00DF, + 0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x00E5, 0x00E6, 0x00E7, 0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x0301, 0x00ED, 0x00EE, 0x00EF, + 0x0111, 0x00F1, 0x0323, 0x00F3, 0x00F4, 0x01A1, 0x00F6, 0x00F7, 0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x01B0, 0x20AB, 0x00FF, +}; + +inline const Definition ALL[] = { + { 1250, CP1250 }, + { 1251, CP1251 }, + { 1252, CP1252 }, + { 1253, CP1253 }, + { 1254, CP1254 }, + { 1255, CP1255 }, + { 1256, CP1256 }, + { 1257, CP1257 }, + { 1258, CP1258 }, +}; +} +} diff --git a/tools/generate_codepages.js b/tools/generate_codepages.js new file mode 100644 index 0000000..663f22d --- /dev/null +++ b/tools/generate_codepages.js @@ -0,0 +1,60 @@ +const { writeFileSync } = require("fs"); +const path = require("path"); + +const CODEPAGES = [1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258]; + +const hex = (n) => "0x" + n.toString(16).toUpperCase().padStart(4, "0"); + +let out = `// This file is generated by tools/generate_codepages.js, do not edit it manually. +// clang-format off +#pragma once + +namespace helpers +{ +namespace codepage_tables +{ +struct Definition +{ + int number; + const char16_t* table; // 128 entries, bytes 0x80..0xFF. 0xFFFD = undefined byte +}; + +`; + +for (const cp of CODEPAGES) { + const decoder = new TextDecoder(`windows-${cp}`); + const units = []; + + for (let byte = 0x80; byte <= 0xff; byte++) { + const decoded = decoder.decode(Uint8Array.of(byte)); + if (decoded.length !== 1) { + throw new Error(`windows-${cp}: byte ${byte.toString(16)} decoded to ${decoded.length} code units`); + } + units.push(decoded.charCodeAt(0)); + } + + const seen = new Map(); + units.forEach((unit, i) => { + if (unit === 0xfffd) return; + if (seen.has(unit)) { + throw new Error(`windows-${cp}: U+${unit.toString(16)} is mapped by both ${seen.get(unit)} and ${i + 0x80}`); + } + seen.set(unit, i + 0x80); + }); + + out += `inline const char16_t CP${cp}[128] = {\n`; + for (let row = 0; row < 8; row++) { + out += "\t" + units.slice(row * 16, row * 16 + 16).map(hex).join(", ") + ",\n"; + } + out += `};\n\n`; +} + +out += `inline const Definition ALL[] = {\n`; +for (const cp of CODEPAGES) { + out += `\t{ ${cp}, CP${cp} },\n`; +} +out += `};\n}\n}\n`; + +const target = path.join(__dirname, "../src/helpers/codepage_tables.hpp"); +writeFileSync(target, out); +console.log(`Written ${target}`); From bcccd08af6bee3286261e8e599647429c8fce96d Mon Sep 17 00:00:00 2001 From: noexabwuy Date: Mon, 21 Sep 2026 00:36:17 +0300 Subject: [PATCH 2/2] add node.codepage option to convert strings between JS and the server --- README.md | 41 +++++++++++++++++++++++++++++++++++++++++ src/helpers/v8.hpp | 42 +++++++++++++++++++++++++++++++++++++----- src/main.cpp | 6 ++++++ src/resource.cpp | 22 ++++++++++++++++++++-- src/runtime.cpp | 10 ++++++++++ 5 files changed, 114 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b0335c7..5ab3eb2 100644 --- a/README.md +++ b/README.md @@ -32,3 +32,44 @@ cmake .. -A Win32 -T ClangCL ``` Open Visual Studio and build the solution. + +## Configuration + +### `node.codepage` + +The SA-MP client and the server core use single-byte Windows codepages, while JavaScript strings are Unicode. +Set `node.codepage` to make omp-node convert strings between them, otherwise non-ASCII text +(Cyrillic, Polish, Turkish, ...) is displayed incorrectly, and text typed by players reaches your script broken. + +The conversion is applied to every string that is passed to the server or received from it (chat, dialogs, commands, player names, events, ...) +and to the output of `console.log`, so the server log keeps a single encoding. + +| Value | Language group | +|---|---| +| `utf-8` (default) | no conversion | +| `windows-1250` | Central European (Polish, Czech, ...) | +| `windows-1251` | Cyrillic (Russian, Ukrainian, ...) | +| `windows-1252` | Western European | +| `windows-1253` | Greek | +| `windows-1254` | Turkish | +| `windows-1255` | Hebrew | +| `windows-1256` | Arabic | +| `windows-1257` | Baltic | +| `windows-1258` | Vietnamese | + +`cp1251` and `1251` are accepted as aliases. Characters that do not exist in the selected codepage (emoji, for example) are sent as `?`. + +Example `config.json`: + +```json +{ + "node": { + "resources": ["resources"], + "codepage": "windows-1251" + } +} +``` + +The "language for non-Unicode programs" setting on the player's Windows must match the codepage, otherwise the game client will not display the text correctly. + +The codepage tables are generated by `tools/generate_codepages.js`. diff --git a/src/helpers/v8.hpp b/src/helpers/v8.hpp index 40ae7ad..38ae851 100644 --- a/src/helpers/v8.hpp +++ b/src/helpers/v8.hpp @@ -3,6 +3,7 @@ #include "v8.h" #include "json.hpp" #include "ompcapi.h" +#include "codepage.hpp" #pragma warning(disable : 4312) // Parts of this file is taken from altv-js-module, thanks to altv team @@ -104,10 +105,40 @@ inline bool SafeToString(v8::Local val, v8::Isolate* isolate, v8::Loc v8::MaybeLocal maybeVal = val->ToString(ctx); if (maybeVal.IsEmpty()) return false; - out = *v8::String::Utf8Value(isolate, maybeVal.ToLocalChecked()); + + v8::String::Utf8Value utf8(isolate, maybeVal.ToLocalChecked()); + + const auto& codepage = Codepage::Instance(); + if (codepage.IsIdentity()) + { + out = *utf8; + } + else + { + auto converted = codepage.FromUtf8(*utf8, std::char_traits::length(*utf8)); + out.assign(converted.data(), converted.size()); + } return true; } +inline v8::Local ServerStringToJS(v8::Isolate* isolate, const char* data, size_t len) +{ + if (!data) + { + data = ""; + len = 0; + } + + const auto& codepage = Codepage::Instance(); + if (codepage.IsIdentity()) + { + return v8::String::NewFromUtf8(isolate, data, v8::NewStringType::kNormal, (int)len).ToLocalChecked(); + } + + auto converted = codepage.ToUtf8(data, len); + return v8::String::NewFromUtf8(isolate, converted.data(), v8::NewStringType::kNormal, (int)converted.size()).ToLocalChecked(); +} + inline v8::Local JsonToV8(v8::Isolate* isolate, const nlohmann::json& j) { if (j.is_null()) @@ -124,7 +155,8 @@ inline v8::Local JsonToV8(v8::Isolate* isolate, const nlohmann::json& } else if (j.is_string()) { - return v8::String::NewFromUtf8(isolate, j.get().c_str(), v8::NewStringType::kNormal).ToLocalChecked(); + auto str = j.get(); + return ServerStringToJS(isolate, str.c_str(), std::char_traits::length(str.c_str())); } else if (j.is_array()) { @@ -155,17 +187,17 @@ inline v8::Local JsonToV8(v8::Isolate* isolate, const nlohmann::json& inline v8::Local JSValue(v8::Isolate* isolate, const CAPIStringView& val) { - return v8::String::NewFromUtf8(isolate, val.data, v8::NewStringType::kNormal, val.len).ToLocalChecked(); + return ServerStringToJS(isolate, val.data, val.len); } inline v8::Local JSValue(v8::Isolate* isolate, const char* val) { - return v8::String::NewFromUtf8(isolate, val).ToLocalChecked(); + return ServerStringToJS(isolate, val, val ? std::char_traits::length(val) : 0); } inline v8::Local JSValue(v8::Isolate* isolate, const Impl::String& val) { - return v8::String::NewFromUtf8(isolate, val.c_str(), v8::NewStringType::kNormal, (int)val.size()).ToLocalChecked(); + return ServerStringToJS(isolate, val.c_str(), val.size()); } inline v8::Local JSValue(v8::Isolate* isolate, bool val) diff --git a/src/main.cpp b/src/main.cpp index fe0f55b..0de39c8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -49,6 +49,7 @@ class OmpNodeComponent final : public IOmpNodeComponent, public CoreEventHandler config.setInt("node.inspector.port", 9229); config.setString("node.inspector.host", "127.0.0.1"); config.setBool("node.inspector.wait", false); + config.setString("node.codepage", "utf-8"); } else { @@ -81,6 +82,11 @@ class OmpNodeComponent final : public IOmpNodeComponent, public CoreEventHandler { config.setBool("node.inspector.wait", false); } + + if (config.getType("node.codepage") == ConfigOptionType_None) + { + config.setString("node.codepage", "utf-8"); + } } } diff --git a/src/resource.cpp b/src/resource.cpp index 6795dca..d691e26 100644 --- a/src/resource.cpp +++ b/src/resource.cpp @@ -29,6 +29,24 @@ static void ResourceLoaded(const v8::FunctionCallbackInfo& info) } } +static Impl::String LogArg(v8::Isolate* isolate, v8::Local value) +{ + v8::String::Utf8Value utf8(isolate, value); + if (!*utf8) + { + return Impl::String(); + } + + const auto& codepage = helpers::Codepage::Instance(); + if (codepage.IsIdentity()) + { + return Impl::String(*utf8); + } + + auto converted = codepage.FromUtf8(*utf8, std::char_traits::length(*utf8)); + return Impl::String(converted.data(), converted.size()); +} + static void OmpLogBridge(const v8::FunctionCallbackInfo& info) { v8::Isolate* isolate = info.GetIsolate(); @@ -48,7 +66,7 @@ static void OmpLogBridge(const v8::FunctionCallbackInfo& info) for (size_t i = 0; i < info.Length(); i++) { - Impl::String arg = *v8::String::Utf8Value(isolate, info[i]); + Impl::String arg = LogArg(isolate, info[i]); stream << arg; if (i != info.Length() - 1) stream << " "; @@ -68,7 +86,7 @@ static void OmpLogBridge(const v8::FunctionCallbackInfo& info) for (size_t i = 0; i < info.Length(); i++) { - Impl::String arg = *v8::String::Utf8Value(isolate, info[i]); + Impl::String arg = LogArg(isolate, info[i]); stream << arg; if (i != info.Length() - 1) stream << " "; diff --git a/src/runtime.cpp b/src/runtime.cpp index eec1513..e19a0aa 100644 --- a/src/runtime.cpp +++ b/src/runtime.cpp @@ -199,6 +199,16 @@ void Runtime::StoreExtraNodeConfig() inspectorConfig.host = core->getConfig().getString("node.inspector.host").to_string(); inspectorConfig.port = *core->getConfig().getInt("node.inspector.port"); inspectorConfig.wait = *core->getConfig().getBool("node.inspector.wait"); + + auto codepage = core->getConfig().getString("node.codepage"); + if (!helpers::Codepage::Instance().Set(codepage.data(), codepage.size())) + { + core->logLn(LogLevel::Warning, "Unknown node.codepage value \"%s\", strings will not be converted (sry)", codepage.to_string().c_str()); + } + else if (!helpers::Codepage::Instance().IsIdentity()) + { + core->logLn(LogLevel::Message, "Strings are converted using codepage %s", helpers::Codepage::Instance().Name().c_str()); + } } std::vector Runtime::GetResourcePathsFromconfig()