diff --git a/.gitattributes b/.gitattributes index 227cc2cb6b..aba05cbea2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,3 +10,4 @@ /lib/cfg/** working-tree-encoding=KOI8-R eol=lf /bin.win/** working-tree-encoding=windows-1251 eol=crlf *.dll -diff -working-tree-encoding +/lib/text/greeting.utf8 working-tree-encoding=UTF-8 eol=lf diff --git a/Dockerfile b/Dockerfile index c3b23fb801..2030fda148 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,15 @@ ARG WITH_OTEL=false ARG WITH_ADMIN_API=false ARG WITH_YAML=true ARG WITH_SQLITE=false +# Кодировка рантайма (issue #3681). utf8 = флип: исходники перечекаучиваются в UTF-8 +# (в git они и так UTF-8, KOI8-R даёт лишь working-tree-encoding), рантайм -- символьный. +ARG INTERNAL_ENCODING=koi8r +# Ревизия и счётчик коммитов: внутри контейнера git бесполезен (контекст может быть worktree, +# где .git -- файл-ссылка наружу, или распакованный архив), поэтому пробрасываем их снаружи: +# --build-arg GIT_REV=$(git rev-parse --short HEAD) \ +# --build-arg GIT_COUNT=$(git rev-list --count HEAD) +ARG GIT_REV= +ARG GIT_COUNT= RUN apk add --no-cache \ build-base make meson ninja git cmake samurai \ @@ -61,6 +70,19 @@ RUN if [ "$WITH_SQLITE" = "true" ]; then apk add --no-cache sqlite-dev; fi WORKDIR /mud/mud COPY . /mud/mud +# Флип кодировки исходников. Перекодируем дерево на месте, а не через git checkout: контекст +# сборки может быть worktree (там .git -- файл-ссылка) или вообще экспортированным архивом. +# Конвертируем только C++ -- .py в tests/ уже в UTF-8 (см. .gitattributes). +RUN if [ "$INTERNAL_ENCODING" = "utf8" ]; then \ + find src tests -type f \( -name '*.cpp' -o -name '*.h' -o -name '*.hpp' \) \ + ! -path 'src/third_party_libs/*' \ + -exec sh -c 'iconv -f KOI8-R -t UTF-8 "$1" > "$1.u8" && mv "$1.u8" "$1"' _ {} \; && \ + file src/utils/utils.cpp | grep -q UTF-8 ; \ + fi + +ENV BYLINS_GIT_REV=${GIT_REV} \ + BYLINS_COMMIT_COUNT=${GIT_COUNT} + RUN OTEL_OPT=$([ "$WITH_OTEL" = "true" ] && echo system || echo disabled); \ ADMIN_OPT=$([ "$WITH_ADMIN_API" = "true" ] && echo true || echo false); \ YAML_OPT=$([ "$WITH_YAML" = "true" ] && echo builtin || echo disabled); \ @@ -69,7 +91,8 @@ RUN OTEL_OPT=$([ "$WITH_OTEL" = "true" ] && echo system || echo disabled); \ -Dbuild_tests=false -Dbuild_profile=${BUILD_TYPE} \ --unity=on -Dunity_size=45 \ -Dadmin_api=${ADMIN_OPT} -Dotel=${OTEL_OPT} \ - -Dyaml=${YAML_OPT} -Dsqlite=${SQLITE_OPT} && \ + -Dyaml=${YAML_OPT} -Dsqlite=${SQLITE_OPT} \ + -Dinternal_encoding=${INTERNAL_ENCODING} && \ meson compile -C build circle:executable # ───────────────────────── Этап 2: рантайм ───────────────────────── diff --git a/lib/text/greeting.utf8 b/lib/text/greeting.utf8 new file mode 100644 index 0000000000..e4183ed6b5 --- /dev/null +++ b/lib/text/greeting.utf8 @@ -0,0 +1,22 @@ + +╭──────────────────────────────────────────────────────────────────────────╮ +│ Based on CircleMUD v3.0 — created by Jeremy Elson │ +│ DikuMUD Gamma 0.0 — Sebastian Hammer, Michael Seifert, │ +│ Hans Henrik Staerfeldt, Tom Madsen, Katja Nyboe │ +╰──────────────────────────────────────────────────────────────────────────╯ + + ███████ + ██ + ██ ████ ████ ████ ████ ████ ████ ████ ████ ████ + ███████ ██████ ██ ██ ██ ██ ████ ██ ██ ██████ ██ + ██ ██ ██ ██ ██ ██ ██ ████ ██ ████████ ██ ██ ██ + ██ ██ ██ ██ ██ ██ ██ ███ ██ ██ ██ ██ ██ ██ + ███████ ███████ ████ ███ ███ ████ ████ ████ ████ ███████ ████ + + ✦ ПО ПРЕДАНИЯМ РУССКИХ НАРОДНЫХ СКАЗОК И БЫЛИН ✦ + + По административным вопросам — Стрибог ‹stribog@bylins.su› + Сайт: www.bylins.su + Вики: https://bylinsmud.fandom.com/ru/wiki/Bylinsmud_вики + +Введите имя персонажа (или «новый», чтобы создать нового): \ No newline at end of file diff --git a/meson.build b/meson.build index 2da3288ec6..5cd4c32311 100644 --- a/meson.build +++ b/meson.build @@ -648,6 +648,7 @@ main_sources = files( 'src/utils/utf8.cpp', 'src/utils/native_text.cpp', 'src/utils/utils_encoding.cpp', + 'src/utils/translit_koi8.cpp', 'src/gameplay/mechanics/weather.cpp', 'src/engine/olc/zedit.cpp', 'src/utils/utils_string.cpp', diff --git a/src/administration/names.cpp b/src/administration/names.cpp index 7e276ef779..0cf0a4f49a 100644 --- a/src/administration/names.cpp +++ b/src/administration/names.cpp @@ -9,6 +9,7 @@ * $Revision$ * ************************************************************************ */ +#include "utils/native_text.h" #include "names.h" #include "utils/grammar/gender.h" @@ -169,9 +170,14 @@ static void NewNames::save() { return; } + // Граница записи: список уходит на диск в кодировке мира (сейчас KOI8-R), зеркально чтению + // (issue #3681). + std::ostringstream out; for (NewNameListType::const_iterator it = NewNameList.begin(); it != NewNameList.end(); ++it) - file << it->first << "\n"; + out << it->first << "\n"; + const std::string on_disk = native_text::to_disk(out.str()); + file.write(on_disk.data(), static_cast(on_disk.size())); file.close(); } diff --git a/src/engine/boot/boot_constants.h b/src/engine/boot/boot_constants.h index 2eb1cedcef..76e3c5a86a 100644 --- a/src/engine/boot/boot_constants.h +++ b/src/engine/boot/boot_constants.h @@ -115,6 +115,9 @@ enum SetStuffMode { #define PLAYER_Z_PREFIX LIB_PLRS "" LIB_Z #define HELP_PAGE_FILE LIB_TEXT_HELP "screen" // for HELP +// Rich UTF-8 login screen, sent verbatim to UTF-8 clients (issue #3681). Optional: when the +// file is absent the plain greeting from system_msg.xml is used instead. +#define GREETING_UTF8_FILE LIB_TEXT "greeting.utf8" #define PROXY_FILE LIB_MISC "proxy" // register proxy list #define XNAME_FILE LIB_MISC "xnames" // invalid name substrings diff --git a/src/engine/boot/boot_data_files.cpp b/src/engine/boot/boot_data_files.cpp index 02b3d24630..6740ace9a7 100644 --- a/src/engine/boot/boot_data_files.cpp +++ b/src/engine/boot/boot_data_files.cpp @@ -25,7 +25,7 @@ class DataFile : public BaseDataFile { void close() override; [[nodiscard]] const std::string &file_name() const { return m_file_name; } [[nodiscard]] const auto &file() const { return m_file; } - void get_one_line(char *buf); + void get_one_line(char *buf, std::size_t buf_size); private: FILE *m_file; @@ -45,14 +45,26 @@ void DataFile::close() { fclose(m_file); } -void DataFile::get_one_line(char *buf) { - if (fgets(buf, READ_SIZE, file()) == nullptr) { +void DataFile::get_one_line(char *buf, std::size_t buf_size) { + char raw[READ_SIZE]; + if (fgets(raw, sizeof(raw), file()) == nullptr) { mudlog("SYSERR: error reading help file: not terminated with $?", DEF, kLvlImmortal, SYSLOG, true); buf[0] = '$'; buf[1] = 0; return; } - buf[strlen(buf) - 1] = '\0'; // take off the trailing \n + size_t len = strlen(raw); + while (len > 0 && (raw[len - 1] == '\n' || raw[len - 1] == '\r')) { + raw[--len] = '\0'; // take off the trailing \r\n + } + + // Граница кодировки (issue #3681): файлы справки лежат на диске в KOI8-R, а движок под + // UTF-8 ждёт нативный текст -- без этого ключи разделов не совпадали бы с тем, что ввёл + // игрок, а тело раздела уезжало бы клиенту байтами чужой кодировки. Строка может при этом + // вырасти (KOI8-R байт -> до трёх байт UTF-8), поэтому размер буфера передаётся явно. + const std::string native = native_text::from_disk_line(raw); + strncpy(buf, native.c_str(), buf_size - 1); + buf[buf_size - 1] = '\0'; } class DiscreteFile : public DataFile { @@ -1730,20 +1742,23 @@ class HelpFile : public DataFile { }; bool HelpFile::load_help() { + // В нативной кодировке строка может быть втрое длиннее прочитанной с диска (KOI8-R байт + // разворачивается в UTF-8 максимум в три), поэтому буферы с запасом (issue #3681). + constexpr size_t kLineBufSize = 3 * READ_SIZE + 1; #if defined(CIRCLE_MACINTOSH) - static char key[READ_SIZE + 1], next_key[READ_SIZE + 1], entry[32384]; // ? + static char key[kLineBufSize], next_key[kLineBufSize], entry[32384]; // ? #else - char key[READ_SIZE + 1], next_key[READ_SIZE + 1], entry[32384]; + char key[kLineBufSize], next_key[kLineBufSize], entry[32384]; #endif - char line[READ_SIZE + 1]; + char line[kLineBufSize]; const char *scan; // get the first keyword line - get_one_line(key); + get_one_line(key, sizeof(key)); while (*key != '$') // read in the corresponding help entry { snprintf(entry, sizeof(entry), "%s\r\n", key); - get_one_line(line); + get_one_line(line, sizeof(line)); while (*line != '#') { // если вдруг файл внезапно закончился и '#' так и не встретился // логаем ошибку и заканчиваем парсинг во избежание зацикливания @@ -1755,7 +1770,7 @@ bool HelpFile::load_help() { } size_t entry_len = strlen(entry); snprintf(entry + entry_len, sizeof(entry) - entry_len, "%s\r\n", line); - get_one_line(line); + get_one_line(line, sizeof(line)); } // Assign read level int min_level = 0; @@ -1773,7 +1788,7 @@ bool HelpFile::load_help() { } // get next keyword line (or $) - get_one_line(key); + get_one_line(key, sizeof(key)); } return true; diff --git a/src/engine/core/iosystem.cpp b/src/engine/core/iosystem.cpp index e998805265..84f096c01e 100644 --- a/src/engine/core/iosystem.cpp +++ b/src/engine/core/iosystem.cpp @@ -7,6 +7,7 @@ */ #include "engine/core/iosystem.h" +#include "utils/native_text.h" #include #include #include "gameplay/core/experience.h" @@ -431,7 +432,11 @@ int process_input(DescriptorData *t) { // Буква задана СТРОКОВЫМ литералом: он байт-прозрачен, поэтому один и тот // же код верен и для KOI8-R (1 байт), и для UTF-8 (2 байта) -- в отличие от // символьного литерала, который под UTF-8 не помещается в char (issue #3681). - static const std::string_view kYaLetter = "я"; + // В этой точке буфер содержит KOI8-R (таблицы легаси-кодировок выше отдают + // именно его), поэтому букву тоже берём в KOI8-R. Под KOI8-R-рантаймом + // to_koi8 - тождество, под UTF-8 - перекодировка литерала. + static const std::string kYaStorage = native_text::to_koi8("я"); + const std::string_view kYaLetter = kYaStorage; if (*(write_point - 1) == 'z' && space_left + 1 >= kYaLetter.size()) { --write_point; ++space_left; @@ -446,20 +451,32 @@ int process_input(DescriptorData *t) { *write_point = '\0'; + // Приводим собранную строку к нативной кодировке движка (issue #3681). После разбора + // выше в tmp лежит либо UTF-8 (клиент UTF-8), либо KOI8-R (все остальные кодировки + // клиентов - их таблицы отдают именно KOI8-R). if (t->keytable == kCodePageUTF8) { - int i; - char utf8_tmp[kMaxSockBuf * 2 * 3]; - size_t len_i, len_o; + if (!native_text::native_is_utf8()) { + int i; + char utf8_tmp[kMaxSockBuf * 2 * 3]; + size_t len_i, len_o; - len_i = strlen(tmp); + len_i = strlen(tmp); - for (i = 0; i < kMaxSockBuf * 2 * 3; i++) { - utf8_tmp[i] = 0; + for (i = 0; i < kMaxSockBuf * 2 * 3; i++) { + utf8_tmp[i] = 0; + } + codepages::utf8_to_koi(tmp, utf8_tmp); + len_o = strlen(utf8_tmp); + strncpy(tmp, utf8_tmp, kMaxInputLength - 1); + space_left = space_left + len_i - len_o; } - codepages::utf8_to_koi(tmp, utf8_tmp); - len_o = strlen(utf8_tmp); - strncpy(tmp, utf8_tmp, kMaxInputLength - 1); - space_left = space_left + len_i - len_o; + // иначе клиент прислал уже нативную кодировку - трогать нечего + } else if (native_text::native_is_utf8()) { + const size_t len_i = strlen(tmp); + const std::string native = native_text::from_koi8(tmp); + strncpy(tmp, native.c_str(), kMaxInputLength - 1); + tmp[kMaxInputLength - 1] = '\0'; + space_left = space_left + len_i - strlen(tmp); } if ((space_left <= 0) && (ptr < nl_pos)) { diff --git a/src/engine/db/db.cpp b/src/engine/db/db.cpp index 4d0538e6dd..2511088d26 100644 --- a/src/engine/db/db.cpp +++ b/src/engine/db/db.cpp @@ -1020,7 +1020,10 @@ void ZoneTrafficSave() { } void zone_traffic_load() { pugi::xml_document doc; - pugi::xml_parse_result result = doc.load_file(ZONE_TRAFFIC_FILE); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_db = native_text::read_data_file(ZONE_TRAFFIC_FILE); + pugi::xml_parse_result result = doc.load_buffer(xml_db.data(), xml_db.size()); if (!result) { snprintf(buf, kMaxStringLength, "...%s", result.description()); mudlog(buf, CMP, kLvlImmortal, SYSLOG, true); diff --git a/src/engine/db/help.cpp b/src/engine/db/help.cpp index b3e833cdb4..3e2f55e68e 100644 --- a/src/engine/db/help.cpp +++ b/src/engine/db/help.cpp @@ -586,12 +586,12 @@ std::string OutRecipiesHelp(ECharClass ch_class) { utils::SortKoiString(skills_list); for (auto it : skills_list) { tmpstr = !(++columns % 2) ? "\r\n" : "\t"; - out << "\t" << std::left << std::setw(30) << it << tmpstr; + out << "\t" << fmt::format("{:<30}", it) << tmpstr; } utils::SortKoiString(skills_list2); for (auto it : skills_list2) { tmpstr = !(++columns2 % 2) ? "\r\n" : "\t"; - out2 << "\t" << "&C" << std::left << std::setw(30) << it << "&n" << tmpstr; + out2 << "\t" << "&C" << fmt::format("{:<30}", it) << "&n" << tmpstr; } if (out.str().back() == '\t') out << "\r\n"; @@ -711,20 +711,20 @@ std::string OutSkillsHelp(ECharClass ch_class) { } } if (num == 1) { - skills_list2.push_back(utils::SubstKtoW(skill.GetName())); + skills_list2.push_back(skill.GetName()); continue; } - skills_list.push_back(utils::SubstKtoW(skill.GetName())); + skills_list.push_back(skill.GetName()); } std::sort(skills_list.begin(), skills_list.end()); for (auto it : skills_list) { tmpstr = !(++columns % 2) ? "\r\n" : "\t"; - out << "\t" << std::left << std::setw(30) << utils::SubstWtoK(it) << tmpstr; + out << "\t" << fmt::format("{:<30}", it) << tmpstr; } std::sort(skills_list2.begin(), skills_list2.end()); for (auto it : skills_list2) { tmpstr = !(++columns2 % 2) ? "\r\n" : "\t"; - out2 << "\t" << "&C" << std::left << std::setw(30) << utils::SubstWtoK(it) << "&n" << tmpstr; + out2 << "\t" << "&C" << fmt::format("{:<30}", it) << "&n" << tmpstr; } if (out.str().back() == '\t') out << "\r\n"; @@ -970,17 +970,17 @@ std::string OutFeatureHelp(ECharClass ch_class) { utils::SortKoiString(feat_list); for (auto it : feat_list) { tmpstr = !(++columns % 2) ? "\r\n" : "\t"; - out << "\t" << std::left << std::setw(30) << it << tmpstr; + out << "\t" << fmt::format("{:<30}", it) << tmpstr; } utils::SortKoiString(feat_list2); for (auto it : feat_list2) { tmpstr = !(++columns2 % 2) ? "\r\n" : "\t"; - out2 << "\t" << "&C" << std::left << std::setw(30) << it << "&n" << tmpstr; + out2 << "\t" << "&C" << fmt::format("{:<30}", it) << "&n" << tmpstr; } utils::SortKoiString(feat_list3); for (auto it : feat_list3) { tmpstr = !(++columns3 % 2) ? "\r\n" : "\t"; - out3 << "\t" << "&C" << std::left << std::setw(30) << it << "&n" << tmpstr; + out3 << "\t" << "&C" << fmt::format("{:<30}", it) << "&n" << tmpstr; } if (out.str().back() == '\t') out << "\r\n"; @@ -1285,8 +1285,12 @@ void SetsHelp() { str_out = "все"; else if (count_class == 0) str_out = "никто"; - if (str_out.back() == '\n') - str_out.back() = '\0'; + // Именно pop_back(), а не запись '\0': присваивание оставляло NUL ВНУТРИ строки, + // не меняя её длины. Байтовая таблица это терпела, а libfort в режиме utf8_table + // на встроенном нуле уводит расчёт ширины в переполнение и роняет процесс. + if (str_out.back() == '\n') { + str_out.pop_back(); + } table << str_out << table_wrapper::kSeparator << table_wrapper::kEndRow; } table.SetColumnAlign(0, table_wrapper::align::kRight); diff --git a/src/engine/db/obj_save.cpp b/src/engine/db/obj_save.cpp index 2f911620d5..0f2e7d0f48 100644 --- a/src/engine/db/obj_save.cpp +++ b/src/engine/db/obj_save.cpp @@ -2093,7 +2093,10 @@ int save_char_objects(CharData *ch, int savetype, int rentcost) { Crash_delete_files(iplayer); return false; } - file.write(obj_content.data(), static_cast(obj_content.size())); + // Граница записи: рента уходит на диск в кодировке мира (сейчас KOI8-R), зеркально + // чтению -- иначе первое сохранение переводит файл в UTF-8 (issue #3681). + const std::string on_disk = native_text::to_disk(obj_content); + file.write(on_disk.data(), static_cast(on_disk.size())); file.close(); #ifndef _WIN32 if (chmod(fname, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP) < 0) { @@ -2104,7 +2107,7 @@ int save_char_objects(CharData *ch, int savetype, int rentcost) { #endif utils::CExecutionTimer crc_timer; FileCRC::update_from_content(ch->get_uid(), FileCRC::kTextObjs, - obj_content.data(), obj_content.size()); + on_disk.data(), on_disk.size()); crc_sec = crc_timer.delta().count(); g_obj_crash_save_hash[ch->get_uid()] = content_hash; } diff --git a/src/engine/db/player_index.cpp b/src/engine/db/player_index.cpp index 349ba6d866..43c1b1d227 100644 --- a/src/engine/db/player_index.cpp +++ b/src/engine/db/player_index.cpp @@ -7,6 +7,7 @@ */ #include "player_index.h" +#include "utils/native_text.h" #include "administration/accounts.h" #include "global_objects.h" @@ -103,24 +104,24 @@ std::size_t PlayersIndex::hasher::operator()(const std::string &value) const { // FNV-1a implementation using p = fnv_params; std::size_t result = p::offset_basis; - for (unsigned char c : value) { - result ^= static_cast(LOWER(c)); + // Регистр сворачивается ПО СИМВОЛУ, а не по байту (issue #3681): побайтная свёртка + // оставляет многобайтные буквы нетронутыми, из-за чего "Дрегвий" и "дрегвий" дают разные + // хэши и точный поиск персонажа промахивается. Компаратор ниже обязан согласоваться. + std::string folded = value; + native_text::to_lower(folded); + for (unsigned char c : folded) { + result ^= static_cast(c); result *= p::prime; } return result; } bool PlayersIndex::equal_to::operator()(const std::string &left, const std::string &right) const { - if (left.size() != right.size()) { + // Посимвольное сравнение без учёта регистра -- в паре к hasher выше. + if (native_text::compare_ci(left, right) != 0) { return false; } - for (std::size_t i = 0; i < left.size(); ++i) { - if (LOWER(left[i]) != LOWER(right[i])) { - return false; - } - } - return true; } @@ -258,7 +259,11 @@ void ActualizePlayersIndex(char *name) { int deleted; char filename[kMaxStringLength]; - for (int i = 0; (name[i] = LOWER(name[i])); i++); + // По символу, а не по байту (issue #3681). Побайтная свёртка бьёт многобайтные буквы: + // у "г" второй байт UTF-8 равен 0xB3, а в KOI8-R это "Ё", и таблица опускала его в 0xA3 -- + // буква превращалась в "У". Имя файла получалось неверным, персонаж не загружался и просто + // не попадал в индекс (в боевом мире так терялось 1278 из 9117 записей). + native_text::to_lower(name); if (get_filename(name, filename, kPlayersFile)) { Player t_short_ch; Player *short_ch = &t_short_ch; @@ -346,6 +351,13 @@ void BuildPlayerIndexNew() { if (sscanf(name, "%s ", playername) == 0) continue; + // players.lst читается обычным fopen, минуя FBFILE, поэтому границу кодировки надо + // пройти здесь: иначе индекс останется в KOI8-R, а поиск пойдёт по нативной кодировке + // и ни одного существующего персонажа не найдёт (issue #3681). + const std::string native_name = native_text::from_disk_line(playername); + strncpy(playername, native_name.c_str(), sizeof(playername) - 1); + playername[sizeof(playername) - 1] = '\0'; + if (!player_table.IsPlayerExists(playername)) { ActualizePlayersIndex(playername); } @@ -372,8 +384,10 @@ void FlushPlayerIndex() { } ++saved; + // Имя уходит на диск в кодировке мира (сейчас KOI8-R) -- зеркало from_disk_line, + // которым индекс читается обратно (issue #3681). sprintf(name, "%s %ld %d %d\n", - i.name().c_str(), + native_text::to_disk(i.name()).c_str(), i.uid(), i.level, i.last_logon); fputs(name, players); } diff --git a/src/engine/entities/char_player.cpp b/src/engine/entities/char_player.cpp index 4a5d4b8192..b2da5cc54d 100644 --- a/src/engine/entities/char_player.cpp +++ b/src/engine/entities/char_player.cpp @@ -2,6 +2,7 @@ // Copyright (c) 2008 Krodo // Part of Bylins http://www.mud.ru +#include "utils/native_text.h" #include "char_player.h" #include "gameplay/core/experience.h" #include "administration/privilege.h" @@ -819,7 +820,9 @@ void Player::save_char() { // Накопленный буфер пишем на диск в бинарном режиме (байты файла == байты // буфера на всех платформах) и из него же считаем CRC -- без перечитывания // только что записанного файла. - const std::string &pfile = saved.str(); + // Граница записи: сейв уходит на диск в той же кодировке, в какой лежит остальной мир + // (сейчас KOI8-R), а не в нативной -- зеркало от from_disk_line на чтении (issue #3681). + const std::string pfile = native_text::to_disk(saved.str()); utils::CExecutionTimer wt; FILE *pf = fopen(filename, "wb"); if (!pf) { @@ -1853,7 +1856,7 @@ int Player::load_char_ascii(const char *name, const int load_flags) { // иначе в таблице crc будут пустые имена, т.к. сама плеер-таблица еще не сформирована // и в любом случае при ребуте это все пересчитывать не нужно if (!(load_flags & ELoadCharFlags::kNoCrcCheck)) { - FileCRC::verify_from_content(this->get_uid(), FileCRC::kPlayer, fl->buf, fl->size); + FileCRC::verify_from_content(this->get_uid(), FileCRC::kPlayer, fl->raw, fl->raw_size); } fbclose(fl); diff --git a/src/engine/network/descriptor_data.cpp b/src/engine/network/descriptor_data.cpp index ab69b11b61..bb385928f9 100644 --- a/src/engine/network/descriptor_data.cpp +++ b/src/engine/network/descriptor_data.cpp @@ -6,6 +6,9 @@ */ #include "descriptor_data.h" +#include +#include +#include "utils/native_text.h" #include "utils/utils_encoding.h" #include "engine/entities/char_player.h" @@ -164,6 +167,16 @@ void DescriptorData::msdp_report_changed_vars() { } void DescriptorData::string_to_client_encoding(const char *in_str, char *out_str) const { + // Легаси-кодировки клиентов заданы таблицами "байт KOI8-R -> байт целевой кодировки", + // поэтому перед ними текст надо привести к KOI8-R. Под KOI8-R-рантаймом это тождество, + // под UTF-8 - настоящая перекодировка (issue #3681). Для UTF-8-клиента ничего приводить + // не нужно: см. case kCodePageUTF8 ниже. + std::string koi8_text; + if (keytable != kCodePageUTF8) { + koi8_text = native_text::to_koi8(in_str); + in_str = koi8_text.c_str(); + } + switch (keytable) { case kCodePageAlt: for (; *in_str; *out_str = codepages::KtoA(*in_str), in_str++, out_str++); @@ -204,7 +217,13 @@ void DescriptorData::string_to_client_encoding(const char *in_str, char *out_str // Anton Gorev (2016-04-25): we have to be careful. String in UTF-8 encoding may // contain character with code 0xff which telnet interprets as IAC. // II: FE and FF were never defined for any purpose in UTF-8, we are safe - codepages::koi_to_utf8(const_cast(in_str), out_str); + if (native_text::native_is_utf8()) { + // Рантайм уже в UTF-8 - отдаём как есть. Повторная перекодировка испортила бы + // текст (именно так и выглядела первая флип-сборка). + strcpy(out_str, in_str); + } else { + codepages::koi_to_utf8(const_cast(in_str), out_str); + } break; default: diff --git a/src/engine/ui/cmd/do_score.cpp b/src/engine/ui/cmd/do_score.cpp index bd877500ba..17426e2620 100644 --- a/src/engine/ui/cmd/do_score.cpp +++ b/src/engine/ui/cmd/do_score.cpp @@ -6,6 +6,7 @@ слабовидящих. */ +#include "utils/native_text.h" #include "engine/ui/color.h" #include "gameplay/affects/affect_messages.h" #include "utils/utils_string.h" @@ -118,9 +119,9 @@ void PrintBonusStateInfo(CharData *ch, std::ostringstream &out); // \todo Переписать на вывод в поток с использованием общих со "счет все" функций void PrintScoreList(CharData *ch) { sprintf(buf, "%s", MUD::RaceMessages().GetMessage(GET_RACE(ch), ch->get_sex()).c_str()); - buf[0] = LOWER(buf[0]); + native_text::copy_lower_char(buf, buf); sprintf(buf1, "%s", religion_name[GET_RELIGION(ch)][static_cast(ch->get_sex())]); - buf1[0] = LOWER(buf1[0]); + native_text::copy_lower_char(buf1, buf1); SendMsgToChar(ch, "Вы %s, %s, %s, %s, уровень %d, перевоплощений %d.\r\n", ch->get_name().c_str(), buf, MUD::Class(ch->GetClass()).GetCName(), diff --git a/src/engine/ui/cmd/do_spells.cpp b/src/engine/ui/cmd/do_spells.cpp index 1ebb0dd5fe..55175fd269 100644 --- a/src/engine/ui/cmd/do_spells.cpp +++ b/src/engine/ui/cmd/do_spells.cpp @@ -11,6 +11,7 @@ #include "engine/db/global_objects.h" #include "gameplay/mechanics/weather.h" #include "gameplay/core/remort.h" +#include "utils/native_text.h" #include @@ -48,11 +49,17 @@ void DisplaySpells(CharData *ch, CharData *vict, bool all) { char names[kMaxMemoryCircle][kMaxStringLength]; std::string time_str; int slots[kMaxMemoryCircle], i, max_slot = 0, slot_num, gcount = 0; + // Разбивка на колонки считается в символах, а не в байтах: под UTF-8 байт на символ уже не + // один, и прежняя арифметика по длине строки разносила колонки (issue #3681). slots[] + // остаётся смещением в буфере -- оно по природе байтовое, -- а chars[] хранит, сколько + // символов в этот круг уже набрано. + int chars[kMaxMemoryCircle]; auto have_spells{false}; max_slot = 0; for (i = 0; i < kMaxMemoryCircle; i++) { *names[i] = '\0'; slots[i] = 0; + chars[i] = 0; } for (const auto &spl_info : MUD::Spells()) { @@ -89,18 +96,20 @@ void DisplaySpells(CharData *ch, CharData *vict, bool all) { continue; if (CheckRecipeItems(ch, spell_id, ESpellType::kRunes, false)) { const auto line = fmt::format("{}|<...{:4}.> {}{:<38}&n|", - slots[slot_num] % 114 < 10 ? "\r\n" : " ", + chars[slot_num] % 114 < 10 ? "\r\n" : " ", CalcSpellManacost(ch, spell_id), GetSpellColor(spell_id), MUD::Spell(spell_id).GetCName()); strcpy(names[slot_num] + slots[slot_num], line.c_str()); slots[slot_num] += static_cast(line.size()); + chars[slot_num] += static_cast(native_text::char_count(line)); } else { if (all) { const auto line = fmt::format("{}|+--------+ {}{:<38}&n|", - slots[slot_num] % 114 < 10 ? "\r\n" : " ", GetSpellColor(spell_id), + chars[slot_num] % 114 < 10 ? "\r\n" : " ", GetSpellColor(spell_id), MUD::Spell(spell_id).GetCName()); strcpy(names[slot_num] + slots[slot_num], line.c_str()); slots[slot_num] += static_cast(line.size()); + chars[slot_num] += static_cast(native_text::char_count(line)); } } } else { @@ -118,9 +127,10 @@ void DisplaySpells(CharData *ch, CharData *vict, bool all) { else { sprintf(buf1, "%s", "K"); } - slots[slot_num] += sprintf(names[slot_num] + slots[slot_num], - "%s|<%s%c%c%c%c%c%c%c>%s%s%-30s %-7s&n|", - slots[slot_num] % 116 < 10 ? "\r\n" : " ", + // fmt, а не sprintf: ширину поля с названием заклинания надо мерить в символах. + // printf считает %-30s в байтах, и под UTF-8 колонка разъезжалась (issue #3681). + const auto line = fmt::format("{}|<{}{}{}{}{}{}{}{}>{}{}{:<30} {:<7}&n|", + chars[slot_num] % 116 < 10 ? "\r\n" : " ", IS_SET(GET_SPELL_TYPE(ch, spell_id), ESpellType::kKnow) ? buf1 : ".", IS_SET(GET_SPELL_TYPE(ch, spell_id), ESpellType::kTemp) ? 'T' : '.', IS_SET(GET_SPELL_TYPE(ch, spell_id), ESpellType::kPotionCast) ? 'P' : '.', @@ -128,11 +138,14 @@ void DisplaySpells(CharData *ch, CharData *vict, bool all) { IS_SET(GET_SPELL_TYPE(ch, spell_id), ESpellType::kScrollCast) ? 'S' : '.', IS_SET(GET_SPELL_TYPE(ch, spell_id), ESpellType::kItemCast) ? 'I' : '.', IS_SET(GET_SPELL_TYPE(ch, spell_id), ESpellType::kRunes) ? 'R' : '.', - '.', + '.', (CalcMinSpellLvl(ch, spell_id) - GetRealLevel(ch) < 10) ? " " : " ", GetSpellColor(spell_id), MUD::Spell(spell_id).GetCName(), - time_str.c_str()); + time_str); + strcpy(names[slot_num] + slots[slot_num], line.c_str()); + slots[slot_num] += static_cast(line.size()); + chars[slot_num] += static_cast(native_text::char_count(line)); } have_spells = true; }; diff --git a/src/engine/ui/login.cpp b/src/engine/ui/login.cpp index f5f21776e1..cbbd775d09 100644 --- a/src/engine/ui/login.cpp +++ b/src/engine/ui/login.cpp @@ -6,6 +6,11 @@ #include "interpreter.h" #include "utils/russian_keys.h" #include "utils/native_text.h" +#include "utils/utf8.h" +#include "engine/boot/boot_constants.h" + +#include +#include #include "engine/ui/system_messages.h" #include "engine/core/config.h" #include "gameplay/mechanics/condition.h" @@ -182,18 +187,28 @@ std::map new_loc_codes; // имя чара на код, отправленный на почту для подтверждения мыла при создании std::map new_char_codes; +// Посимвольно, а не побайтно (issue #3681): у многобайтной буквы хвостовой байт не проходит +// проверку "это буква", и раньше здесь отвергалось любое русское имя. Условие "*argument > 0" +// означало "символ не ASCII" (у старших байтов знаковый char отрицателен) - теперь это прямая +// проверка кодовой точки. Регистр тоже сворачивается по символу: первая буква заглавная, +// остальные строчные; длина при этом не меняется, поэтому буфер name не переполняется. int _parse_name(char *argument, char *name) { - int i; + int i = 0; - // skip whitespaces - for (i = 0; (*name = (i ? LOWER(*argument) : UPPER(*argument))); argument++, i++, name++) { - if (native_text::first_char_code(argument) == rus::kYo - || native_text::first_char_code(argument) == rus::kYoUpper - || !a_isalpha(*argument) - || *argument > 0) { + while (*argument) { + const char32_t code = native_text::first_char_code(argument); + if (code == rus::kYo || code == rus::kYoUpper + || !native_text::is_alpha_char(argument) + || code < 0x80) { return (1); } + const size_t bytes = i ? native_text::copy_lower_char(argument, name) + : native_text::copy_upper_char(argument, name); + argument += bytes; + name += bytes; + ++i; } + *name = '\0'; if (!i) { return (1); @@ -207,12 +222,19 @@ int _parse_name(char *argument, char *name) { * чтобы их в игру вообще пускало, а новых с Ё/ё соответственно брило. */ int parse_exist_name(char *argument, char *name) { - int i; + int i = 0; - // skip whitespaces - for (i = 0; (*name = (i ? LOWER(*argument) : UPPER(*argument))); argument++, i++, name++) - if (!a_isalpha(*argument) || *argument > 0) + while (*argument) { + if (!native_text::is_alpha_char(argument) || native_text::first_char_code(argument) < 0x80) { return (1); + } + const size_t bytes = i ? native_text::copy_lower_char(argument, name) + : native_text::copy_upper_char(argument, name); + argument += bytes; + name += bytes; + ++i; + } + *name = '\0'; if (!i) return (1); @@ -1514,6 +1536,50 @@ static void HandleInit(DescriptorData *d, char * /*argument*/) { return; } +// Экран приветствия. Файл lib/text/greeting.utf8 всегда в UTF-8 и написан полной палитрой: +// скруглённая рамка, типографское тире, кавычки-лапки. Он один на все кодировки -- клиенту в +// alt/win/koi8 (и всему KOI8-R-сборке целиком) то же самое доезжает уже приведённым к KOI8-R, +// где скруглённые углы становятся обычными, тире -- дефисом, лапки -- палочками (issue #3681). +// +// Файл необязателен: нет его или в нём битый UTF-8 -- берётся прежняя шапка из system_msg.xml. +static const std::string &GetGreeting() { + static std::string greeting; + static bool loaded = false; + if (!loaded) { + loaded = true; + std::ifstream in(GREETING_UTF8_FILE, std::ios::binary); + if (in) { + std::ostringstream body; + body << in.rdbuf(); + const std::string raw = body.str(); + if (utf8::is_valid(raw)) { + // В файле переводы строк обычные, а выводу нужен CRLF -- ровно та же + // нормализация, что делает загрузчик system_msg.xml. + std::string crlf; + crlf.reserve(raw.size() + raw.size() / 32); + for (const char c : raw) { + if (c == '\r') { + continue; + } + if (c == '\n') { + crlf += "\r\n"; + } else { + crlf += c; + } + } + greeting = native_text::from_utf8(crlf); + } else { + log("SYSERR: %s is not valid UTF-8, falling back to the plain greeting", + GREETING_UTF8_FILE); + } + } + } + if (!greeting.empty()) { + return greeting; + } + return system_messages::GetText(system_messages::ESystemMsg::kGreetings); +} + static void HandleGetKeytable(DescriptorData *d, char *argument) { if (strlen(argument) > 0) argument[0] = argument[strlen(argument) - 1]; @@ -1527,7 +1593,7 @@ static void HandleGetKeytable(DescriptorData *d, char *argument) { } d->keytable = (ubyte) *argument - (ubyte) '0'; ip_log(d->host); - iosystem::write_to_output(system_messages::GetText(system_messages::ESystemMsg::kGreetings).c_str(), d); + iosystem::write_to_output(GetGreeting().c_str(), d); d->state = EConState::kGetName; return; } diff --git a/src/engine/ui/mapsystem.cpp b/src/engine/ui/mapsystem.cpp index dc2b4744ca..9852aa6860 100644 --- a/src/engine/ui/mapsystem.cpp +++ b/src/engine/ui/mapsystem.cpp @@ -615,11 +615,16 @@ void print_map(CharData *ch, CharData *imm) { } } - if (found && start_line < 0) { - start_line = i; - } else if (!found && start_line > 0) { - end_line = i; - break; + // Ищем именно первую и последнюю непустую строку, а не первый разрыв: в строке + // посередине карты может не оказаться ни одной клетки (комнаты редко стоят плотным + // прямоугольником), и прежний код обрубал карту прямо там. Чем больше глубина, тем + // вероятнее разрыв, поэтому "карта богов" с глубиной 25 выглядела не крупнее обычной. + // Ровно та же правка уже сделана для правой границы по столбцам. + if (found) { + if (start_line < 0) { + start_line = i; + } + end_line = static_cast(i) + 1; } } diff --git a/src/gameplay/clans/house.cpp b/src/gameplay/clans/house.cpp index 0a7df152a0..045d3523ec 100644 --- a/src/gameplay/clans/house.cpp +++ b/src/gameplay/clans/house.cpp @@ -4,6 +4,7 @@ * (c) 2005 Krodo * ******************************************************************************/ +#include "utils/utils_string.h" #include "house.h" #include "utils/russian_keys.h" #include "utils/native_text.h" @@ -441,7 +442,7 @@ void Clan::ClanLoadSingle(const std::string &index) { log("Owner %ld is no longer exist (%s).", unique, filename.c_str()); break; } - tempMember->name[0] = UPPER(tempMember->name[0]); + native_text::capitalize_first(tempMember->name); tempMember->rank_num = 0; tempMember->money = money; tempMember->exp = exp; @@ -477,7 +478,7 @@ void Clan::ClanLoadSingle(const std::string &index) { log("Member %ld is no longer exist (%s).", unique, filename.c_str()); continue; } - tempMember->name[0] = UPPER(tempMember->name[0]); + native_text::capitalize_first(tempMember->name); tempMember->rank_num = rank; tempMember->money = money; tempMember->exp = exp; @@ -2432,9 +2433,11 @@ void Clan::save_chest() { log("Save obj: %s", this->abbrev.c_str()); ObjSaveSync::check(this->GetRent(), ObjSaveSync::CLAN_SAVE); + // Имя каталога дружины строится из байтов KOI8-R, как и до миграции: под UTF-8 байтовый + // AtoL резал русскую букву пополам, и добро дружины уезжало в каталог с другим именем + // (issue #3681). std::string buffer = this->abbrev; - for (unsigned i = 0; i != buffer.length(); ++i) - buffer[i] = LOWER(codepages::AtoL(buffer[i])); + CreateFileName(buffer); std::string filename = LIB_HOUSE + buffer + "/" + buffer + ".obj"; for (auto chest : world[GetRoomRnum(this->chest_room)]->contents) { if (Clan::is_clan_chest(chest)) { @@ -2494,9 +2497,7 @@ void Clan::ChestLoad() { for (ClanListType::const_iterator clan = Clan::ClanList.begin(); clan != Clan::ClanList.end(); ++clan) { buffer = (*clan)->abbrev; - for (unsigned i = 0; i != buffer.length(); ++i) { - buffer[i] = LOWER(codepages::AtoL(buffer[i])); - } + CreateFileName(buffer); std::string filename = LIB_HOUSE + buffer + "/" + buffer + ".obj"; //лоадим сундук. в зонах его лоадить не нужно. @@ -2597,9 +2598,7 @@ void Clan::ChestUpdate() { // * Запись сообщения дружины в файл и поле клана. void Clan::write_mod(const std::string &arg) { std::string abbrev = this->get_abbrev(); - for (unsigned i = 0; i != abbrev.length(); ++i) { - abbrev[i] = LOWER(codepages::AtoL(abbrev[i])); - } + CreateFileName(abbrev); std::string filename = LIB_HOUSE + abbrev + "/" + abbrev + ".mod"; std::ofstream file(filename.c_str()); @@ -4226,9 +4225,7 @@ std::string Clan::get_remember(unsigned int num, int flag) const { std::string Clan::get_file_abbrev() const { std::string text = this->get_abbrev(); - for (unsigned i = 0; i != text.length(); ++i) { - text[i] = LOWER(codepages::AtoL(text[i])); - } + CreateFileName(text); return text; } diff --git a/src/gameplay/communication/boards/boards_changelog_loaders.cpp b/src/gameplay/communication/boards/boards_changelog_loaders.cpp index f8f363a5b0..2a3b4cdfc3 100644 --- a/src/gameplay/communication/boards/boards_changelog_loaders.cpp +++ b/src/gameplay/communication/boards/boards_changelog_loaders.cpp @@ -1,4 +1,5 @@ #include "boards_changelog_loaders.h" +#include "utils/native_text.h" #include "utils/logger.h" #include "boards_constants.h" @@ -203,6 +204,9 @@ bool GitChangeLogLoader::GitCommitReader::next_line(std::string &buffer, std::is if (!std::getline(is, buffer)) { return false; } + // Changelog лежит на диске в KOI8-R и читается потоком; границу кодировки проходим здесь + // (issue #3681). Под KOI8-R это тождество. + buffer = native_text::from_disk_line(buffer.c_str()); ++line; return true; @@ -221,6 +225,7 @@ bool GitChangeLogLoader::load(std::istream &is) { int line = 0; std::string buffer; while (std::getline(is, buffer)) { + buffer = native_text::from_disk_line(buffer.c_str()); ++line; switch (parser_state) { case REVISION: diff --git a/src/gameplay/communication/boards/boards_types.cpp b/src/gameplay/communication/boards/boards_types.cpp index 0b542e21ab..5cbcd0fc35 100644 --- a/src/gameplay/communication/boards/boards_types.cpp +++ b/src/gameplay/communication/boards/boards_types.cpp @@ -1,4 +1,5 @@ #include "boards_types.h" +#include "utils/native_text.h" #include "boards_constants.h" #include "utils/logger.h" @@ -58,6 +59,13 @@ void Board::Load() { ReadEndString(file); std::getline(file, message->text, '~'); + // Доски лежат на диске в KOI8-R и читаются потоком, минуя FBFILE, поэтому границу + // кодировки проходим здесь -- по текстовым полям (issue #3681). Под KOI8-R это + // тождество. + message->author = native_text::from_disk_line(message->author.c_str()); + message->subject = native_text::from_disk_line(message->subject.c_str()); + message->text = native_text::from_disk_line(message->text.c_str()); + // не помешает глянуть че мы там залоадили if (message->author.empty() || !message->unique @@ -142,20 +150,26 @@ void Board::Save() { log("Error open file: %s! (%s %s %d)", file_.c_str(), __FILE__, __func__, __LINE__); return; } - file << "Type: " << type_ << " " - << "Clan: " << clan_rent_ << " " - << "PersUID: " << pers_unique_ << " " - << "PersName: " << (pers_name_.empty() ? "none" : pers_name_) << "\n"; + // Собираем всё в строку и приводим к кодировке диска одним куском: граница записи -- зеркало + // границы чтения (доски читаются через from_disk_line), иначе первое же сохранение доски + // переводит её файл в UTF-8, а мир вокруг остаётся в KOI8-R (issue #3681). + std::ostringstream out; + out << "Type: " << type_ << " " + << "Clan: " << clan_rent_ << " " + << "PersUID: " << pers_unique_ << " " + << "PersName: " << (pers_name_.empty() ? "none" : pers_name_) << "\n"; for (MessageListType::const_reverse_iterator message = messages.rbegin(); message != messages.rend(); ++message) { - file << "Message: " << (*message)->num << "\n" - << (*message)->author << " " - << (*message)->unique << " " - << (*message)->level << " " - << (*message)->date << " " - << (*message)->rank << "\n" - << (*message)->subject << "~\n" - << (*message)->text << "~\n"; + out << "Message: " << (*message)->num << "\n" + << (*message)->author << " " + << (*message)->unique << " " + << (*message)->level << " " + << (*message)->date << " " + << (*message)->rank << "\n" + << (*message)->subject << "~\n" + << (*message)->text << "~\n"; } + const std::string on_disk = native_text::to_disk(out.str()); + file.write(on_disk.data(), static_cast(on_disk.size())); file.close(); } diff --git a/src/gameplay/communication/mail.cpp b/src/gameplay/communication/mail.cpp index 19e9d0e0c4..a46ecbb2e2 100644 --- a/src/gameplay/communication/mail.cpp +++ b/src/gameplay/communication/mail.cpp @@ -9,6 +9,7 @@ ************************************************************************ */ #include "mail.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/global_objects.h" #include "gameplay/economics/currencies.h" @@ -594,7 +595,10 @@ void save() { void load() { pugi::xml_document doc; - pugi::xml_parse_result result = doc.load_file(MAIL_XML_FILE); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_mail = native_text::read_data_file(MAIL_XML_FILE); + pugi::xml_parse_result result = doc.load_buffer(xml_mail.data(), xml_mail.size()); if (!result) { snprintf(buf, kMaxStringLength, "...%s", result.description()); mudlog(buf, CMP, kLvlImmortal, SYSLOG, true); diff --git a/src/gameplay/crafting/craft.cpp b/src/gameplay/crafting/craft.cpp index 621b97b2ed..37b7772162 100644 --- a/src/gameplay/crafting/craft.cpp +++ b/src/gameplay/crafting/craft.cpp @@ -5,6 +5,7 @@ */ #include "craft.h" +#include "utils/native_text.h" #include "gameplay/mechanics/magic_item.h" #include "engine/db/obj_prototypes.h" @@ -1074,7 +1075,10 @@ bool CCraftModel::load() { Logger::CPrefix prefix(logger, BODY_PREFIX); pugi::xml_document doc; - const auto result = doc.load_file(FILE_NAME.c_str()); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_craft = native_text::read_data_file(FILE_NAME.c_str()); + const auto result = doc.load_buffer(xml_craft.data(), xml_craft.size()); if (!result) { logger("Craft load error: '%s' at offset %zu\n", diff --git a/src/gameplay/economics/shop_ext.cpp b/src/gameplay/economics/shop_ext.cpp index f1bd6748b8..c6a25a63ff 100644 --- a/src/gameplay/economics/shop_ext.cpp +++ b/src/gameplay/economics/shop_ext.cpp @@ -3,6 +3,7 @@ // Part of Bylins http://www.mud.ru #include "shop_ext.h" +#include "utils/native_text.h" #include "engine/olc/vedun/enum_registry.h" // vedun::RegisterEditorEnums (refresh ShopItemSetId) #include #include "gameplay/economics/currencies.h" @@ -148,7 +149,10 @@ void log_shop_load() { void load_item_desc() { pugi::xml_document doc; - pugi::xml_parse_result result = doc.load_file(LIB_PLRSTUFF"/shop/item_desc.xml"); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_shop_ext = native_text::read_data_file(LIB_PLRSTUFF"/shop/item_desc.xml"); + pugi::xml_parse_result result = doc.load_buffer(xml_shop_ext.data(), xml_shop_ext.size()); if (!result) { snprintf(buf, kMaxStringLength, "...%s", result.description()); mudlog(buf, CMP, kLvlImmortal, SYSLOG, true); diff --git a/src/gameplay/mechanics/glory_const.cpp b/src/gameplay/mechanics/glory_const.cpp index 1f9de1b0ae..7a9c3080e8 100644 --- a/src/gameplay/mechanics/glory_const.cpp +++ b/src/gameplay/mechanics/glory_const.cpp @@ -900,7 +900,10 @@ void save() { void load() { int ver = 0; pugi::xml_document doc; - pugi::xml_parse_result result = doc.load_file(LIB_PLRSTUFF"glory_const.xml"); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_glory_const = native_text::read_data_file(LIB_PLRSTUFF"glory_const.xml"); + pugi::xml_parse_result result = doc.load_buffer(xml_glory_const.data(), xml_glory_const.size()); if (!result) { snprintf(buf, kMaxStringLength, "WARNING: glory_const.xml not found or unreadable (%s), skipping (non-fatal)", result.description()); perror(buf); diff --git a/src/gameplay/mechanics/named_stuff.cpp b/src/gameplay/mechanics/named_stuff.cpp index 9abda42e83..1f1d58ee8d 100644 --- a/src/gameplay/mechanics/named_stuff.cpp +++ b/src/gameplay/mechanics/named_stuff.cpp @@ -553,7 +553,10 @@ void load() { stuff_list.clear(); pugi::xml_document doc; - doc.load_file(LIB_PLRSTUFF"named_stuff_list.xml"); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_named_stuff = native_text::read_data_file(LIB_PLRSTUFF"named_stuff_list.xml"); + doc.load_buffer(xml_named_stuff.data(), xml_named_stuff.size()); pugi::xml_node obj_list = doc.child("named_stuff_list"); for (pugi::xml_node node = obj_list.child("obj"); node; node = node.next_sibling("obj")) { diff --git a/src/gameplay/mechanics/sets_drop.cpp b/src/gameplay/mechanics/sets_drop.cpp index 866cea8d70..bdc5f0b6ed 100644 --- a/src/gameplay/mechanics/sets_drop.cpp +++ b/src/gameplay/mechanics/sets_drop.cpp @@ -2,6 +2,7 @@ // Part of Bylins http://www.mud.ru #include "sets_drop.h" +#include "utils/native_text.h" #include #include @@ -217,7 +218,10 @@ void create_clone_miniset(int vnum) { // * Инициализация списка сетов на лоад. void init_obj_list() { pugi::xml_document doc; - pugi::xml_parse_result result = doc.load_file(CONFIG_FILE); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_sets_drop = native_text::read_data_file(CONFIG_FILE); + pugi::xml_parse_result result = doc.load_buffer(xml_sets_drop.data(), xml_sets_drop.size()); if (!result) { snprintf(buf, kMaxStringLength, "...%s", result.description()); mudlog(buf, CMP, kLvlImmortal, SYSLOG, true); diff --git a/src/gameplay/mechanics/sight.cpp b/src/gameplay/mechanics/sight.cpp index d76284a088..61e7bdd8e5 100644 --- a/src/gameplay/mechanics/sight.cpp +++ b/src/gameplay/mechanics/sight.cpp @@ -912,12 +912,18 @@ void do_auto_exits(CharData *ch) { // Наконец-то добавлена отрисовка в автовыходах закрытых дверей if (EXIT(ch, door) && EXIT(ch, door)->to_room() != kNowhere) { if (EXIT_FLAGGED(EXIT(ch, door), EExitFlag::kClosed)) { - slen += sprintf(buf + slen, "(%c) ", LOWER(*dirs[door])); + // Печатаем первую БУКВУ направления: под UTF-8 у русской буквы два байта, и %c + // выводил половину (issue #3681). + slen += sprintf(buf + slen, "("); + slen += static_cast(native_text::copy_lower_char(dirs[door], buf + slen)); + slen += sprintf(buf + slen, ") "); } else if (!EXIT_FLAGGED(EXIT(ch, door), EExitFlag::kHidden)) { if (world[EXIT(ch, door)->to_room()]->zone_rn == world[ch->in_room]->zone_rn) { - slen += sprintf(buf + slen, "%c ", LOWER(*dirs[door])); + slen += static_cast(native_text::copy_lower_char(dirs[door], buf + slen)); + slen += sprintf(buf + slen, " "); } else { - slen += sprintf(buf + slen, "%c ", UPPER(*dirs[door])); + slen += static_cast(native_text::copy_upper_char(dirs[door], buf + slen)); + slen += sprintf(buf + slen, " "); } } } diff --git a/src/gameplay/mechanics/title.cpp b/src/gameplay/mechanics/title.cpp index d71ee91d20..4da8e38d12 100644 --- a/src/gameplay/mechanics/title.cpp +++ b/src/gameplay/mechanics/title.cpp @@ -2,6 +2,7 @@ // Copyright (c) 2006 Krodo // Part of Bylins http://www.bylins.su +#include "utils/native_text.h" #include "title.h" #include "gameplay/economics/currencies.h" #include "engine/db/player_index.h" @@ -134,8 +135,8 @@ void TitleSystem::do_title(CharData *ch, char *argument, int/* cmd*/, int/* subc utils::Trim(title); utils::Trim(pre_title); if (!pre_title.empty()) { - sprintf(buf2, "%c%s", UPPER(pre_title.substr(0, 1).c_str()[0]), pre_title.substr(1).c_str()); - pre_title = buf2; + // Поднимаем первую букву целиком, а не первый байт (issue #3681). + native_text::capitalize_first(pre_title); } if (!pre_title.empty() && !check_pre_title(pre_title, ch)) return; if (!title.empty() && !check_title(title, ch)) return; diff --git a/src/gameplay/statistics/mob_stat.cpp b/src/gameplay/statistics/mob_stat.cpp index 61ffa68482..408907e467 100644 --- a/src/gameplay/statistics/mob_stat.cpp +++ b/src/gameplay/statistics/mob_stat.cpp @@ -2,6 +2,7 @@ // Part of Bylins http://www.mud.ru #include "mob_stat.h" +#include "utils/native_text.h" #include "third_party_libs/pugixml/pugixml.h" #include "utils/utils_parse.h" @@ -275,7 +276,10 @@ static void LoadXmlLegacy() { char buf_[kMaxInputLength]; pugi::xml_document doc; - pugi::xml_parse_result result = doc.load_file(kMobStatFileNew); + // Файл лежит на диске в KOI8-R; читаем через границу кодировки, а разбираем уже + // буфер в нативной кодировке движка (issue #3681). Под KOI8-R это тождество. + const std::string xml_mob_stat = native_text::read_data_file(kMobStatFileNew); + pugi::xml_parse_result result = doc.load_buffer(xml_mob_stat.data(), xml_mob_stat.size()); if (!result) { snprintf(buf_, sizeof(buf_), "...%s", result.description()); mudlog(buf_, CMP, kLvlImmortal, SYSLOG, true); diff --git a/src/utils/diskio.cpp b/src/utils/diskio.cpp index f60cd12e44..a4af5eb956 100644 --- a/src/utils/diskio.cpp +++ b/src/utils/diskio.cpp @@ -10,6 +10,9 @@ **************************************************************************/ #include +#include +#include +#include "utils/native_text.h" #include "engine/core/sysdep.h" #include "utils/utils.h" #include "diskio.h" @@ -146,6 +149,24 @@ FBFILE *fbopen_for_read(char *fname) { strcpy(fbfl->name, fname); auto dummy = fread(fbfl->buf, sizeof(char), fbfl->size, fl); + fbfl->raw = fbfl->buf; + fbfl->raw_size = fbfl->size; + + // Данные на диске лежат в KOI8-R. Поднимаем весь буфер до нативной кодировки движка здесь, + // где размер под нашим контролем -- построчная конверсия рисковала бы переполнить буфер + // вызывающего, ведь кириллица при перекодировке удлиняется. Исходные байты остаются в raw + // для CRC. Под KOI8-R ветка не компилируется вовсе, buf и raw -- один и тот же указатель. + if (native_text::native_is_utf8()) { + const std::string native = native_text::from_disk_line(fbfl->buf); + if (native.size() != static_cast(fbfl->size)) { + char *converted = nullptr; + CREATE(converted, native.size() + 1); + memcpy(converted, native.c_str(), native.size() + 1); + fbfl->buf = converted; + fbfl->size = static_cast(native.size()); + fbfl->ptr = fbfl->buf; + } + } UNUSED_ARG(dummy); fclose(fl); @@ -193,6 +214,10 @@ size_t fbclose_for_read(FBFILE *fbfl) { return 0; } + // buf мог быть перекодирован в отдельный буфер; raw при этом указывает на исходный. + if (fbfl->raw && fbfl->raw != fbfl->buf) { + free(fbfl->raw); + } if (fbfl->buf) { free(fbfl->buf); } diff --git a/src/utils/diskio.h b/src/utils/diskio.h index 8d1bb08ad6..dfc563586c 100644 --- a/src/utils/diskio.h +++ b/src/utils/diskio.h @@ -38,6 +38,11 @@ typedef struct { int size; // size in bytes of buffer // int flags; // read/write/append, future expansion // char *name; // filename (for delayed writing) // + // Исходные байты файла, как они лежат на диске. buf может быть перекодирован в нативную + // кодировку движка (issue #3681), а CRC файлов игроков считается по ИСХОДНОМУ содержимому, + // иначе сумма разъедется. Под KOI8-R это тот же указатель, что и buf. + char *raw; // original on-disk bytes (for CRC) // + int raw_size; // size of raw in bytes // } FBFILE; void ExtractTagFromArgument(char *argument, char *tag); diff --git a/src/utils/logger.cpp b/src/utils/logger.cpp index ad3a0bd2ce..7f5634fe4b 100644 --- a/src/utils/logger.cpp +++ b/src/utils/logger.cpp @@ -41,12 +41,12 @@ void pers_log(CharData *ch, const char *format, ...) { } if (!ch->desc->pers_log) { - char filename[128], name[64], *ptr; - strcpy(name, GET_NAME(ch)); - for (ptr = name; *ptr; ptr++) { - *ptr = LOWER(codepages::AtoL(*ptr)); - } - sprintf(filename, "%s/perslog/%s.log", runtime_config.log_dir().c_str(), name); + char filename[128]; + // Имя файла персонального лога строится из байтов KOI8-R, как и до миграции + // (issue #3681). + std::string name = GET_NAME(ch); + CreateFileName(name); + sprintf(filename, "%s/perslog/%s.log", runtime_config.log_dir().c_str(), name.c_str()); ch->desc->pers_log = fopen(filename, "a"); if (!ch->desc->pers_log) { log("SYSERR: error open %s (%s %s %d)", filename, __FILE__, __func__, __LINE__); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 9ebddeb3b7..8a4538ca17 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -8,7 +8,9 @@ through them changes nothing until the encoding flip. */ #include "native_text.h" +#include "utf8.h" #include "utils_encoding.h" +#include "translit_koi8.h" #ifdef INTERNAL_ENCODING_UTF8 #include "utf8.h" @@ -26,6 +28,8 @@ constexpr unsigned char kYoLowerByte = 0xA3; constexpr unsigned char kYoUpperByte = 0xB3; #endif +#include +#include #include #include @@ -387,15 +391,46 @@ void to_upper(char *s) { fold_range_utf8(s, s + std::char_traits::length(s std::string from_koi8(const std::string &text) { - if (text.empty()) { + // This runs per attribute/field while the world and the configs load, and the overwhelming + // majority of those are pure ASCII (keys, aliases, numbers), which KOI8-R and UTF-8 spell + // identically. Detect that first and hand the text back untouched instead of transcoding. + bool has_high_byte = false; + for (const char c : text) { + if (static_cast(c) >= 0x80) { + has_high_byte = true; + break; + } + } + if (!has_high_byte) { return text; } - // koi_to_utf8() can grow the text; utils_encoding sizes its own buffers at 6x, so match that. - std::vector out(text.size() * 6 + 1, '\0'); + // Every KOI8-R character lives below U+FFFF, so three bytes per input byte is a hard bound. + std::vector out(text.size() * 3 + 1, '\0'); codepages::koi_to_utf8(const_cast(text.c_str()), out.data()); return std::string(out.data()); } +std::string to_koi8(const std::string &text) { + if (text.empty()) { + return text; + } + bool has_high_byte = false; + for (const char c : text) { + if (static_cast(c) >= 0x80) { + has_high_byte = true; + break; + } + } + if (!has_high_byte) { + return text; // pure ASCII is spelled identically in both encodings + } + return codepages::Utf8ToKoi8(text); +} + +std::string from_utf8(const std::string &text) { + return text; // the native encoding already is UTF-8 +} + std::string translit_to_filename(std::string_view name) { // Code point -> the very same Latin character the KOI8-R byte table yields, so a player's // file name is identical before and after the flip. Upper and lower case collapse together @@ -599,6 +634,16 @@ std::string from_koi8(const std::string &text) { return text; // the native encoding already is KOI8-R } +std::string to_koi8(const std::string &text) { + return text; // the native encoding already is KOI8-R +} + +std::string from_utf8(const std::string &text) { + // The native encoding is KOI8-R, so this is the real conversion: reduce what KOI8-R lacks + // (see translit_koi8.h), then transcode. + return codepages::Utf8ToKoi8(text); +} + std::string translit_to_filename(std::string_view name) { // Byte-wise, exactly as get_filename() did before this was extracted: transliterate through // the Latin table, then lowercase. Yo is not in that table and had its own special case. @@ -701,6 +746,55 @@ bool list_contains_char(std::string_view list, std::string_view ch) { return false; } +std::string from_disk_line(const char *line) { +#ifdef INTERNAL_ENCODING_UTF8 + if (line == nullptr || *line == '\0') { + return {}; + } + const std::string_view view(line); + return utf8::is_valid(view) ? std::string(view) : from_koi8(std::string(view)); +#else + return line == nullptr ? std::string() : std::string(line); +#endif +} + +std::string from_disk_text(const std::string &text) { +#ifdef INTERNAL_ENCODING_UTF8 + // Same discriminator as from_disk_line, applied to the whole file: well-formed UTF-8 is taken + // as already native, anything else as KOI8-R. Cyrillic in KOI8-R is almost never valid UTF-8, + // which makes validity a reliable test, and it is what keeps a load/save cycle idempotent. + return utf8::is_valid(text) ? text : from_koi8(text); +#else + return text; +#endif +} + +std::string to_disk(const std::string &text) { +#ifdef INTERNAL_ENCODING_UTF8 + return to_koi8(text); +#else + return text; +#endif +} + +std::string read_data_file(const std::string &path) { + std::ifstream in(path, std::ios::binary); + if (!in) { + return {}; + } + std::string raw((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + return from_disk_text(raw); +} + + +std::string sort_key(const std::string &text) { + std::string key = to_koi8(text); + for (char &c : key) { + c = codepages::KtoW(c); + } + return key; +} + } // namespace native_text // vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/native_text.h b/src/utils/native_text.h index f7715296e2..0d1f276c2d 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -150,6 +150,60 @@ void to_upper(char *s); // free of #ifdefs and gives one place to revisit when the data files themselves move. std::string from_koi8(const std::string &text); +// The inverse: take native text down to KOI8-R. Needed wherever something downstream is defined +// in terms of KOI8-R bytes -- the legacy client code pages are KOI8-R -> target byte tables, and +// the on-disk formats are KOI8-R. Identity under KOI8-R. A character KOI8-R does not have is +// first reduced to the closest one it does (see translit_koi8.h): a typographic dash to "-", a +// rounded frame corner to a square one, an accented letter to its base. Only what has no +// equivalent at all becomes the converter's placeholder. +std::string to_koi8(const std::string &text); + +// Bring text that is UTF-8 on disk into the native encoding. The counterpart of from_koi8 for +// the files that are deliberately kept in UTF-8 rather than KOI8-R (the login screen). Identity +// under UTF-8; under KOI8-R it goes through the same reduction as to_koi8, so a file written +// with the full Unicode repertoire still renders sensibly on a KOI8-R build. +std::string from_utf8(const std::string &text); + +// Collation key for sorting Russian text. The Russian letters are not in alphabetical order in +// KOI8-R, so sorting has always gone through Windows-1251 bytes, where they are. The key +// reproduces exactly that order and does not depend on the native encoding: compare two keys +// instead of transcoding the strings themselves (which corrupted them under UTF-8, since the +// KOI8-R -> Windows-1251 byte table means nothing for a multibyte string). +std::string sort_key(const std::string &text); + +// Read a data file (all of which are stored in KOI8-R) and hand back its contents in the +// engine's native encoding. The counterpart of from_koi8 for whole files: parsers that take a +// buffer should go through this instead of reading the path themselves, so the boundary stays +// in one place. Returns an empty string if the file cannot be read -- callers report that the +// same way they did when the parser failed to open it. +std::string read_data_file(const std::string &path); + +// Bring one line read from a data file into the native encoding, tolerating a file that has +// already been converted. Under KOI8-R this is the identity. Under UTF-8 the text is taken as +// already-native when it is well-formed UTF-8 and transcoded otherwise -- Cyrillic in KOI8-R is +// almost never valid UTF-8, which makes validity a reliable discriminator and lets old and new +// player files coexist without a version field. +std::string from_disk_line(const char *line); + +// The same for a whole file's contents. Use this, not from_koi8, for anything the engine also +// WRITES back: from_koi8 transcodes unconditionally, so a file the engine already saved in the +// native encoding would be transcoded a second time -- and since the save then writes that back, +// every Cyrillic byte doubles on each load/save cycle and the file grows exponentially (that is +// exactly what happened to cfg/mechanics/obj_sets.xml, issue #3681). +std::string from_disk_text(const std::string &text); + +// The write side of the same boundary, and the exact mirror of from_disk_text: whatever the +// engine puts on disk goes out in the encoding the disk format is in, which during the migration +// is still KOI8-R. Identity under KOI8-R; under UTF-8 the text is reduced and transcoded exactly +// as it is for a legacy client (see to_koi8). +// +// Read and write MUST stay symmetric. If the engine writes the native encoding while the rest of +// the world is KOI8-R, then the first save quietly converts every file it touches, rolling back +// to a KOI8-R build stops being possible, and the world is no longer the world we started with. +// The files that are deliberately UTF-8 (lib/text/greeting.utf8) are read raw and never go +// through here (issue #3681). +std::string to_disk(const std::string &text); + // Transliterate `name` into the ASCII form used for save-file names: Russian letters become // Latin ones, ASCII is lowercased. The mapping is fixed by what the byte-wise implementation // produced before the migration and MUST NOT drift -- the result is the on-disk file name of a diff --git a/src/utils/parser_wrapper.cpp b/src/utils/parser_wrapper.cpp index 972e85d16c..1aaf3661ee 100644 --- a/src/utils/parser_wrapper.cpp +++ b/src/utils/parser_wrapper.cpp @@ -1,4 +1,6 @@ #include "parser_wrapper.h" +#include +#include "utils/native_text.h" #include "utils/logger.h" @@ -20,7 +22,15 @@ DataNode::DataNode() : DataNode::DataNode(const std::filesystem::path &file_name) : DataNode() { - if (auto result = impl_->xml_doc->load_file(file_name.c_str()); !result) { + // Содержимое приводится к нативной кодировке движка ОДИН раз на документ, а не на каждое + // поле: тогда всё, что читается через DataNode, приходит уже в нужной кодировке. Под KOI8-R + // это тождество (issue #3681). + // + // read_data_file, а не самодельное чтение с from_koi8: конфиг движок не только читает, но и + // пишет обратно, а безусловная перекодировка уже сохранённого в UTF-8 файла удваивала бы + // каждую кириллическую букву на каждом цикле загрузки-сохранения. + const std::string converted = native_text::read_data_file(file_name.string()); + if (auto result = impl_->xml_doc->load_buffer(converted.data(), converted.size()); !result) { std::ostringstream buffer; buffer << "..." << result.description() << "\r\n" << " (file: " << file_name << ")" << "\r\n"; err_log("%s", buffer.str().c_str()); @@ -112,7 +122,18 @@ bool DataNode::Save(const std::filesystem::path &file) const { decl.append_attribute("encoding"); } decl.attribute("encoding").set_value("koi8-r"); - return doc.save_file(file.string().c_str()); + // Пишем в той же кодировке, в какой файл лежит на диске (сейчас KOI8-R), а не в нативной: + // граница записи обязана быть зеркалом границы чтения, иначе первое же сохранение молча + // переводит файл в UTF-8 и откат на KOI8-R-сборку становится невозможен (issue #3681). + std::ostringstream xml; + doc.save(xml, "\t", pugi::format_default, pugi::encoding_utf8); + const std::string on_disk = native_text::to_disk(xml.str()); + std::ofstream out(file, std::ios::binary); + if (!out) { + return false; + } + out.write(on_disk.data(), static_cast(on_disk.size())); + return out.good(); } std::string DataNode::ToXmlString() const { diff --git a/src/utils/translit_koi8.cpp b/src/utils/translit_koi8.cpp new file mode 100644 index 0000000000..439ee717f5 --- /dev/null +++ b/src/utils/translit_koi8.cpp @@ -0,0 +1,550 @@ +/** +\file translit_koi8.cpp - a part of the Bylins engine. +\brief Reducing text to what KOI8-R can actually represent (issue #3681). + +Legacy clients speak KOI8-R, CP866 or Windows-1251, and all of those are reached through +KOI8-R. When the engine runs on UTF-8 the text may contain characters none of them has -- a +typographic dash, curly quotes, an accented Latin letter, an emoji. Dropping such a character +to a single placeholder loses information that a reader could still have used, so this table +maps each one to the closest thing KOI8-R does have ("--", '"', "e", "..."), and only what has +no sensible equivalent falls back to the placeholder. + +The table is generated, not written by hand: every entry was checked against the KOI8-R codec +to be genuinely absent from it, and the Latin replacements come from Unicode NFKD with the +combining marks stripped. Replacements are UTF-8, because the substitution happens before the +conversion, and are spelled as \xNN escapes so they do not depend on this file's encoding. +*/ + +#include "translit_koi8.h" + +#include "utf8.h" +#include "utils_encoding.h" + +#include +#include + +namespace codepages { + +namespace { + +struct TranslitEntry { + char32_t code_point; + const char *replacement; // UTF-8 +}; + +// Sorted by code point: looked up with a binary search. +constexpr TranslitEntry kTable[] = { + {0x00A2, "\x63"}, // CENT SIGN + {0x00A3, "\x4C"}, // POUND SIGN + {0x00A5, "\x59"}, // YEN SIGN + {0x00AB, "\x22"}, // LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00AD, ""}, // SOFT HYPHEN + {0x00AE, "\x52"}, // REGISTERED SIGN + {0x00BB, "\x22"}, // RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + {0x00C0, "\x41"}, // LATIN CAPITAL LETTER A WITH GRAVE + {0x00C1, "\x41"}, // LATIN CAPITAL LETTER A WITH ACUTE + {0x00C2, "\x41"}, // LATIN CAPITAL LETTER A WITH CIRCUMFLEX + {0x00C3, "\x41"}, // LATIN CAPITAL LETTER A WITH TILDE + {0x00C4, "\x41"}, // LATIN CAPITAL LETTER A WITH DIAERESIS + {0x00C5, "\x41"}, // LATIN CAPITAL LETTER A WITH RING ABOVE + {0x00C6, "\x41\x45"}, // LATIN CAPITAL LETTER AE + {0x00C7, "\x43"}, // LATIN CAPITAL LETTER C WITH CEDILLA + {0x00C8, "\x45"}, // LATIN CAPITAL LETTER E WITH GRAVE + {0x00C9, "\x45"}, // LATIN CAPITAL LETTER E WITH ACUTE + {0x00CA, "\x45"}, // LATIN CAPITAL LETTER E WITH CIRCUMFLEX + {0x00CB, "\x45"}, // LATIN CAPITAL LETTER E WITH DIAERESIS + {0x00CC, "\x49"}, // LATIN CAPITAL LETTER I WITH GRAVE + {0x00CD, "\x49"}, // LATIN CAPITAL LETTER I WITH ACUTE + {0x00CE, "\x49"}, // LATIN CAPITAL LETTER I WITH CIRCUMFLEX + {0x00CF, "\x49"}, // LATIN CAPITAL LETTER I WITH DIAERESIS + {0x00D0, "\x44"}, // LATIN CAPITAL LETTER ETH + {0x00D1, "\x4E"}, // LATIN CAPITAL LETTER N WITH TILDE + {0x00D2, "\x4F"}, // LATIN CAPITAL LETTER O WITH GRAVE + {0x00D3, "\x4F"}, // LATIN CAPITAL LETTER O WITH ACUTE + {0x00D4, "\x4F"}, // LATIN CAPITAL LETTER O WITH CIRCUMFLEX + {0x00D5, "\x4F"}, // LATIN CAPITAL LETTER O WITH TILDE + {0x00D6, "\x4F"}, // LATIN CAPITAL LETTER O WITH DIAERESIS + {0x00D7, "\x78"}, // MULTIPLICATION SIGN + {0x00D8, "\x4F"}, // LATIN CAPITAL LETTER O WITH STROKE + {0x00D9, "\x55"}, // LATIN CAPITAL LETTER U WITH GRAVE + {0x00DA, "\x55"}, // LATIN CAPITAL LETTER U WITH ACUTE + {0x00DB, "\x55"}, // LATIN CAPITAL LETTER U WITH CIRCUMFLEX + {0x00DC, "\x55"}, // LATIN CAPITAL LETTER U WITH DIAERESIS + {0x00DD, "\x59"}, // LATIN CAPITAL LETTER Y WITH ACUTE + {0x00DE, "\x54\x68"}, // LATIN CAPITAL LETTER THORN + {0x00DF, "\x73\x73"}, // LATIN SMALL LETTER SHARP S + {0x00E0, "\x61"}, // LATIN SMALL LETTER A WITH GRAVE + {0x00E1, "\x61"}, // LATIN SMALL LETTER A WITH ACUTE + {0x00E2, "\x61"}, // LATIN SMALL LETTER A WITH CIRCUMFLEX + {0x00E3, "\x61"}, // LATIN SMALL LETTER A WITH TILDE + {0x00E4, "\x61"}, // LATIN SMALL LETTER A WITH DIAERESIS + {0x00E5, "\x61"}, // LATIN SMALL LETTER A WITH RING ABOVE + {0x00E6, "\x61\x65"}, // LATIN SMALL LETTER AE + {0x00E7, "\x63"}, // LATIN SMALL LETTER C WITH CEDILLA + {0x00E8, "\x65"}, // LATIN SMALL LETTER E WITH GRAVE + {0x00E9, "\x65"}, // LATIN SMALL LETTER E WITH ACUTE + {0x00EA, "\x65"}, // LATIN SMALL LETTER E WITH CIRCUMFLEX + {0x00EB, "\x65"}, // LATIN SMALL LETTER E WITH DIAERESIS + {0x00EC, "\x69"}, // LATIN SMALL LETTER I WITH GRAVE + {0x00ED, "\x69"}, // LATIN SMALL LETTER I WITH ACUTE + {0x00EE, "\x69"}, // LATIN SMALL LETTER I WITH CIRCUMFLEX + {0x00EF, "\x69"}, // LATIN SMALL LETTER I WITH DIAERESIS + {0x00F0, "\x64"}, // LATIN SMALL LETTER ETH + {0x00F1, "\x6E"}, // LATIN SMALL LETTER N WITH TILDE + {0x00F2, "\x6F"}, // LATIN SMALL LETTER O WITH GRAVE + {0x00F3, "\x6F"}, // LATIN SMALL LETTER O WITH ACUTE + {0x00F4, "\x6F"}, // LATIN SMALL LETTER O WITH CIRCUMFLEX + {0x00F5, "\x6F"}, // LATIN SMALL LETTER O WITH TILDE + {0x00F6, "\x6F"}, // LATIN SMALL LETTER O WITH DIAERESIS + {0x00F8, "\x6F"}, // LATIN SMALL LETTER O WITH STROKE + {0x00F9, "\x75"}, // LATIN SMALL LETTER U WITH GRAVE + {0x00FA, "\x75"}, // LATIN SMALL LETTER U WITH ACUTE + {0x00FB, "\x75"}, // LATIN SMALL LETTER U WITH CIRCUMFLEX + {0x00FC, "\x75"}, // LATIN SMALL LETTER U WITH DIAERESIS + {0x00FD, "\x79"}, // LATIN SMALL LETTER Y WITH ACUTE + {0x00FE, "\x74\x68"}, // LATIN SMALL LETTER THORN + {0x00FF, "\x79"}, // LATIN SMALL LETTER Y WITH DIAERESIS + {0x0100, "\x41"}, // LATIN CAPITAL LETTER A WITH MACRON + {0x0101, "\x61"}, // LATIN SMALL LETTER A WITH MACRON + {0x0102, "\x41"}, // LATIN CAPITAL LETTER A WITH BREVE + {0x0103, "\x61"}, // LATIN SMALL LETTER A WITH BREVE + {0x0104, "\x41"}, // LATIN CAPITAL LETTER A WITH OGONEK + {0x0105, "\x61"}, // LATIN SMALL LETTER A WITH OGONEK + {0x0106, "\x43"}, // LATIN CAPITAL LETTER C WITH ACUTE + {0x0107, "\x63"}, // LATIN SMALL LETTER C WITH ACUTE + {0x0108, "\x43"}, // LATIN CAPITAL LETTER C WITH CIRCUMFLEX + {0x0109, "\x63"}, // LATIN SMALL LETTER C WITH CIRCUMFLEX + {0x010A, "\x43"}, // LATIN CAPITAL LETTER C WITH DOT ABOVE + {0x010B, "\x63"}, // LATIN SMALL LETTER C WITH DOT ABOVE + {0x010C, "\x43"}, // LATIN CAPITAL LETTER C WITH CARON + {0x010D, "\x63"}, // LATIN SMALL LETTER C WITH CARON + {0x010E, "\x44"}, // LATIN CAPITAL LETTER D WITH CARON + {0x010F, "\x64"}, // LATIN SMALL LETTER D WITH CARON + {0x0112, "\x45"}, // LATIN CAPITAL LETTER E WITH MACRON + {0x0113, "\x65"}, // LATIN SMALL LETTER E WITH MACRON + {0x0114, "\x45"}, // LATIN CAPITAL LETTER E WITH BREVE + {0x0115, "\x65"}, // LATIN SMALL LETTER E WITH BREVE + {0x0116, "\x45"}, // LATIN CAPITAL LETTER E WITH DOT ABOVE + {0x0117, "\x65"}, // LATIN SMALL LETTER E WITH DOT ABOVE + {0x0118, "\x45"}, // LATIN CAPITAL LETTER E WITH OGONEK + {0x0119, "\x65"}, // LATIN SMALL LETTER E WITH OGONEK + {0x011A, "\x45"}, // LATIN CAPITAL LETTER E WITH CARON + {0x011B, "\x65"}, // LATIN SMALL LETTER E WITH CARON + {0x011C, "\x47"}, // LATIN CAPITAL LETTER G WITH CIRCUMFLEX + {0x011D, "\x67"}, // LATIN SMALL LETTER G WITH CIRCUMFLEX + {0x011E, "\x47"}, // LATIN CAPITAL LETTER G WITH BREVE + {0x011F, "\x67"}, // LATIN SMALL LETTER G WITH BREVE + {0x0120, "\x47"}, // LATIN CAPITAL LETTER G WITH DOT ABOVE + {0x0121, "\x67"}, // LATIN SMALL LETTER G WITH DOT ABOVE + {0x0122, "\x47"}, // LATIN CAPITAL LETTER G WITH CEDILLA + {0x0123, "\x67"}, // LATIN SMALL LETTER G WITH CEDILLA + {0x0124, "\x48"}, // LATIN CAPITAL LETTER H WITH CIRCUMFLEX + {0x0125, "\x68"}, // LATIN SMALL LETTER H WITH CIRCUMFLEX + {0x0128, "\x49"}, // LATIN CAPITAL LETTER I WITH TILDE + {0x0129, "\x69"}, // LATIN SMALL LETTER I WITH TILDE + {0x012A, "\x49"}, // LATIN CAPITAL LETTER I WITH MACRON + {0x012B, "\x69"}, // LATIN SMALL LETTER I WITH MACRON + {0x012C, "\x49"}, // LATIN CAPITAL LETTER I WITH BREVE + {0x012D, "\x69"}, // LATIN SMALL LETTER I WITH BREVE + {0x012E, "\x49"}, // LATIN CAPITAL LETTER I WITH OGONEK + {0x012F, "\x69"}, // LATIN SMALL LETTER I WITH OGONEK + {0x0130, "\x49"}, // LATIN CAPITAL LETTER I WITH DOT ABOVE + {0x0132, "\x49\x4A"}, // LATIN CAPITAL LIGATURE IJ + {0x0133, "\x69\x6A"}, // LATIN SMALL LIGATURE IJ + {0x0134, "\x4A"}, // LATIN CAPITAL LETTER J WITH CIRCUMFLEX + {0x0135, "\x6A"}, // LATIN SMALL LETTER J WITH CIRCUMFLEX + {0x0136, "\x4B"}, // LATIN CAPITAL LETTER K WITH CEDILLA + {0x0137, "\x6B"}, // LATIN SMALL LETTER K WITH CEDILLA + {0x0139, "\x4C"}, // LATIN CAPITAL LETTER L WITH ACUTE + {0x013A, "\x6C"}, // LATIN SMALL LETTER L WITH ACUTE + {0x013B, "\x4C"}, // LATIN CAPITAL LETTER L WITH CEDILLA + {0x013C, "\x6C"}, // LATIN SMALL LETTER L WITH CEDILLA + {0x013D, "\x4C"}, // LATIN CAPITAL LETTER L WITH CARON + {0x013E, "\x6C"}, // LATIN SMALL LETTER L WITH CARON + {0x013F, "\x4C"}, // LATIN CAPITAL LETTER L WITH MIDDLE DOT + {0x0140, "\x6C"}, // LATIN SMALL LETTER L WITH MIDDLE DOT + {0x0141, "\x4C"}, // LATIN CAPITAL LETTER L WITH STROKE + {0x0142, "\x6C"}, // LATIN SMALL LETTER L WITH STROKE + {0x0143, "\x4E"}, // LATIN CAPITAL LETTER N WITH ACUTE + {0x0144, "\x6E"}, // LATIN SMALL LETTER N WITH ACUTE + {0x0145, "\x4E"}, // LATIN CAPITAL LETTER N WITH CEDILLA + {0x0146, "\x6E"}, // LATIN SMALL LETTER N WITH CEDILLA + {0x0147, "\x4E"}, // LATIN CAPITAL LETTER N WITH CARON + {0x0148, "\x6E"}, // LATIN SMALL LETTER N WITH CARON + {0x0149, "\x6E"}, // LATIN SMALL LETTER N PRECEDED BY APOSTROPHE + {0x014C, "\x4F"}, // LATIN CAPITAL LETTER O WITH MACRON + {0x014D, "\x6F"}, // LATIN SMALL LETTER O WITH MACRON + {0x014E, "\x4F"}, // LATIN CAPITAL LETTER O WITH BREVE + {0x014F, "\x6F"}, // LATIN SMALL LETTER O WITH BREVE + {0x0150, "\x4F"}, // LATIN CAPITAL LETTER O WITH DOUBLE ACUTE + {0x0151, "\x6F"}, // LATIN SMALL LETTER O WITH DOUBLE ACUTE + {0x0154, "\x52"}, // LATIN CAPITAL LETTER R WITH ACUTE + {0x0155, "\x72"}, // LATIN SMALL LETTER R WITH ACUTE + {0x0156, "\x52"}, // LATIN CAPITAL LETTER R WITH CEDILLA + {0x0157, "\x72"}, // LATIN SMALL LETTER R WITH CEDILLA + {0x0158, "\x52"}, // LATIN CAPITAL LETTER R WITH CARON + {0x0159, "\x72"}, // LATIN SMALL LETTER R WITH CARON + {0x015A, "\x53"}, // LATIN CAPITAL LETTER S WITH ACUTE + {0x015B, "\x73"}, // LATIN SMALL LETTER S WITH ACUTE + {0x015C, "\x53"}, // LATIN CAPITAL LETTER S WITH CIRCUMFLEX + {0x015D, "\x73"}, // LATIN SMALL LETTER S WITH CIRCUMFLEX + {0x015E, "\x53"}, // LATIN CAPITAL LETTER S WITH CEDILLA + {0x015F, "\x73"}, // LATIN SMALL LETTER S WITH CEDILLA + {0x0160, "\x53"}, // LATIN CAPITAL LETTER S WITH CARON + {0x0161, "\x73"}, // LATIN SMALL LETTER S WITH CARON + {0x0162, "\x54"}, // LATIN CAPITAL LETTER T WITH CEDILLA + {0x0163, "\x74"}, // LATIN SMALL LETTER T WITH CEDILLA + {0x0164, "\x54"}, // LATIN CAPITAL LETTER T WITH CARON + {0x0165, "\x74"}, // LATIN SMALL LETTER T WITH CARON + {0x0168, "\x55"}, // LATIN CAPITAL LETTER U WITH TILDE + {0x0169, "\x75"}, // LATIN SMALL LETTER U WITH TILDE + {0x016A, "\x55"}, // LATIN CAPITAL LETTER U WITH MACRON + {0x016B, "\x75"}, // LATIN SMALL LETTER U WITH MACRON + {0x016C, "\x55"}, // LATIN CAPITAL LETTER U WITH BREVE + {0x016D, "\x75"}, // LATIN SMALL LETTER U WITH BREVE + {0x016E, "\x55"}, // LATIN CAPITAL LETTER U WITH RING ABOVE + {0x016F, "\x75"}, // LATIN SMALL LETTER U WITH RING ABOVE + {0x0170, "\x55"}, // LATIN CAPITAL LETTER U WITH DOUBLE ACUTE + {0x0171, "\x75"}, // LATIN SMALL LETTER U WITH DOUBLE ACUTE + {0x0172, "\x55"}, // LATIN CAPITAL LETTER U WITH OGONEK + {0x0173, "\x75"}, // LATIN SMALL LETTER U WITH OGONEK + {0x0174, "\x57"}, // LATIN CAPITAL LETTER W WITH CIRCUMFLEX + {0x0175, "\x77"}, // LATIN SMALL LETTER W WITH CIRCUMFLEX + {0x0176, "\x59"}, // LATIN CAPITAL LETTER Y WITH CIRCUMFLEX + {0x0177, "\x79"}, // LATIN SMALL LETTER Y WITH CIRCUMFLEX + {0x0178, "\x59"}, // LATIN CAPITAL LETTER Y WITH DIAERESIS + {0x0179, "\x5A"}, // LATIN CAPITAL LETTER Z WITH ACUTE + {0x017A, "\x7A"}, // LATIN SMALL LETTER Z WITH ACUTE + {0x017B, "\x5A"}, // LATIN CAPITAL LETTER Z WITH DOT ABOVE + {0x017C, "\x7A"}, // LATIN SMALL LETTER Z WITH DOT ABOVE + {0x017D, "\x5A"}, // LATIN CAPITAL LETTER Z WITH CARON + {0x017E, "\x7A"}, // LATIN SMALL LETTER Z WITH CARON + {0x017F, "\x73"}, // LATIN SMALL LETTER LONG S + {0x01A0, "\x4F"}, // LATIN CAPITAL LETTER O WITH HORN + {0x01A1, "\x6F"}, // LATIN SMALL LETTER O WITH HORN + {0x01AF, "\x55"}, // LATIN CAPITAL LETTER U WITH HORN + {0x01B0, "\x75"}, // LATIN SMALL LETTER U WITH HORN + {0x01C4, "\x44\x5A"}, // LATIN CAPITAL LETTER DZ WITH CARON + {0x01C5, "\x44\x7A"}, // LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON + {0x01C6, "\x64\x7A"}, // LATIN SMALL LETTER DZ WITH CARON + {0x01C7, "\x4C\x4A"}, // LATIN CAPITAL LETTER LJ + {0x01C8, "\x4C\x6A"}, // LATIN CAPITAL LETTER L WITH SMALL LETTER J + {0x01C9, "\x6C\x6A"}, // LATIN SMALL LETTER LJ + {0x01CA, "\x4E\x4A"}, // LATIN CAPITAL LETTER NJ + {0x01CB, "\x4E\x6A"}, // LATIN CAPITAL LETTER N WITH SMALL LETTER J + {0x01CC, "\x6E\x6A"}, // LATIN SMALL LETTER NJ + {0x01CD, "\x41"}, // LATIN CAPITAL LETTER A WITH CARON + {0x01CE, "\x61"}, // LATIN SMALL LETTER A WITH CARON + {0x01CF, "\x49"}, // LATIN CAPITAL LETTER I WITH CARON + {0x01D0, "\x69"}, // LATIN SMALL LETTER I WITH CARON + {0x01D1, "\x4F"}, // LATIN CAPITAL LETTER O WITH CARON + {0x01D2, "\x6F"}, // LATIN SMALL LETTER O WITH CARON + {0x01D3, "\x55"}, // LATIN CAPITAL LETTER U WITH CARON + {0x01D4, "\x75"}, // LATIN SMALL LETTER U WITH CARON + {0x01D5, "\x55"}, // LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON + {0x01D6, "\x75"}, // LATIN SMALL LETTER U WITH DIAERESIS AND MACRON + {0x01D7, "\x55"}, // LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE + {0x01D8, "\x75"}, // LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE + {0x01D9, "\x55"}, // LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON + {0x01DA, "\x75"}, // LATIN SMALL LETTER U WITH DIAERESIS AND CARON + {0x01DB, "\x55"}, // LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE + {0x01DC, "\x75"}, // LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE + {0x01DE, "\x41"}, // LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON + {0x01DF, "\x61"}, // LATIN SMALL LETTER A WITH DIAERESIS AND MACRON + {0x01E0, "\x41"}, // LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON + {0x01E1, "\x61"}, // LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON + {0x01E6, "\x47"}, // LATIN CAPITAL LETTER G WITH CARON + {0x01E7, "\x67"}, // LATIN SMALL LETTER G WITH CARON + {0x01E8, "\x4B"}, // LATIN CAPITAL LETTER K WITH CARON + {0x01E9, "\x6B"}, // LATIN SMALL LETTER K WITH CARON + {0x01EA, "\x4F"}, // LATIN CAPITAL LETTER O WITH OGONEK + {0x01EB, "\x6F"}, // LATIN SMALL LETTER O WITH OGONEK + {0x01EC, "\x4F"}, // LATIN CAPITAL LETTER O WITH OGONEK AND MACRON + {0x01ED, "\x6F"}, // LATIN SMALL LETTER O WITH OGONEK AND MACRON + {0x01F0, "\x6A"}, // LATIN SMALL LETTER J WITH CARON + {0x01F1, "\x44\x5A"}, // LATIN CAPITAL LETTER DZ + {0x01F2, "\x44\x7A"}, // LATIN CAPITAL LETTER D WITH SMALL LETTER Z + {0x01F3, "\x64\x7A"}, // LATIN SMALL LETTER DZ + {0x01F4, "\x47"}, // LATIN CAPITAL LETTER G WITH ACUTE + {0x01F5, "\x67"}, // LATIN SMALL LETTER G WITH ACUTE + {0x01F8, "\x4E"}, // LATIN CAPITAL LETTER N WITH GRAVE + {0x01F9, "\x6E"}, // LATIN SMALL LETTER N WITH GRAVE + {0x01FA, "\x41"}, // LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE + {0x01FB, "\x61"}, // LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE + {0x0200, "\x41"}, // LATIN CAPITAL LETTER A WITH DOUBLE GRAVE + {0x0201, "\x61"}, // LATIN SMALL LETTER A WITH DOUBLE GRAVE + {0x0202, "\x41"}, // LATIN CAPITAL LETTER A WITH INVERTED BREVE + {0x0203, "\x61"}, // LATIN SMALL LETTER A WITH INVERTED BREVE + {0x0204, "\x45"}, // LATIN CAPITAL LETTER E WITH DOUBLE GRAVE + {0x0205, "\x65"}, // LATIN SMALL LETTER E WITH DOUBLE GRAVE + {0x0206, "\x45"}, // LATIN CAPITAL LETTER E WITH INVERTED BREVE + {0x0207, "\x65"}, // LATIN SMALL LETTER E WITH INVERTED BREVE + {0x0208, "\x49"}, // LATIN CAPITAL LETTER I WITH DOUBLE GRAVE + {0x0209, "\x69"}, // LATIN SMALL LETTER I WITH DOUBLE GRAVE + {0x020A, "\x49"}, // LATIN CAPITAL LETTER I WITH INVERTED BREVE + {0x020B, "\x69"}, // LATIN SMALL LETTER I WITH INVERTED BREVE + {0x020C, "\x4F"}, // LATIN CAPITAL LETTER O WITH DOUBLE GRAVE + {0x020D, "\x6F"}, // LATIN SMALL LETTER O WITH DOUBLE GRAVE + {0x020E, "\x4F"}, // LATIN CAPITAL LETTER O WITH INVERTED BREVE + {0x020F, "\x6F"}, // LATIN SMALL LETTER O WITH INVERTED BREVE + {0x0210, "\x52"}, // LATIN CAPITAL LETTER R WITH DOUBLE GRAVE + {0x0211, "\x72"}, // LATIN SMALL LETTER R WITH DOUBLE GRAVE + {0x0212, "\x52"}, // LATIN CAPITAL LETTER R WITH INVERTED BREVE + {0x0213, "\x72"}, // LATIN SMALL LETTER R WITH INVERTED BREVE + {0x0214, "\x55"}, // LATIN CAPITAL LETTER U WITH DOUBLE GRAVE + {0x0215, "\x75"}, // LATIN SMALL LETTER U WITH DOUBLE GRAVE + {0x0216, "\x55"}, // LATIN CAPITAL LETTER U WITH INVERTED BREVE + {0x0217, "\x75"}, // LATIN SMALL LETTER U WITH INVERTED BREVE + {0x0218, "\x53"}, // LATIN CAPITAL LETTER S WITH COMMA BELOW + {0x0219, "\x73"}, // LATIN SMALL LETTER S WITH COMMA BELOW + {0x021A, "\x54"}, // LATIN CAPITAL LETTER T WITH COMMA BELOW + {0x021B, "\x74"}, // LATIN SMALL LETTER T WITH COMMA BELOW + {0x021E, "\x48"}, // LATIN CAPITAL LETTER H WITH CARON + {0x021F, "\x68"}, // LATIN SMALL LETTER H WITH CARON + {0x0226, "\x41"}, // LATIN CAPITAL LETTER A WITH DOT ABOVE + {0x0227, "\x61"}, // LATIN SMALL LETTER A WITH DOT ABOVE + {0x0228, "\x45"}, // LATIN CAPITAL LETTER E WITH CEDILLA + {0x0229, "\x65"}, // LATIN SMALL LETTER E WITH CEDILLA + {0x022A, "\x4F"}, // LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + {0x022B, "\x6F"}, // LATIN SMALL LETTER O WITH DIAERESIS AND MACRON + {0x022C, "\x4F"}, // LATIN CAPITAL LETTER O WITH TILDE AND MACRON + {0x022D, "\x6F"}, // LATIN SMALL LETTER O WITH TILDE AND MACRON + {0x022E, "\x4F"}, // LATIN CAPITAL LETTER O WITH DOT ABOVE + {0x022F, "\x6F"}, // LATIN SMALL LETTER O WITH DOT ABOVE + {0x0230, "\x4F"}, // LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + {0x0231, "\x6F"}, // LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON + {0x0232, "\x59"}, // LATIN CAPITAL LETTER Y WITH MACRON + {0x0233, "\x79"}, // LATIN SMALL LETTER Y WITH MACRON + {0x0404, "\xD0\xAD"}, // CYRILLIC CAPITAL LETTER UKRAINIAN IE + {0x0405, "\xD0\xA1"}, // CYRILLIC CAPITAL LETTER DZE + {0x0406, "\xD0\x98"}, // CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I + {0x0407, "\xD0\x98"}, // CYRILLIC CAPITAL LETTER YI + {0x0408, "\x4A"}, // CYRILLIC CAPITAL LETTER JE + {0x040E, "\xD0\xA3"}, // CYRILLIC CAPITAL LETTER SHORT U + {0x0454, "\xD1\x8D"}, // CYRILLIC SMALL LETTER UKRAINIAN IE + {0x0455, "\xD1\x81"}, // CYRILLIC SMALL LETTER DZE + {0x0456, "\xD0\xB8"}, // CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I + {0x0457, "\xD0\xB8"}, // CYRILLIC SMALL LETTER YI + {0x0458, "\x6A"}, // CYRILLIC SMALL LETTER JE + {0x045E, "\xD1\x83"}, // CYRILLIC SMALL LETTER SHORT U + {0x0490, "\xD0\x93"}, // CYRILLIC CAPITAL LETTER GHE WITH UPTURN + {0x0491, "\xD0\xB3"}, // CYRILLIC SMALL LETTER GHE WITH UPTURN + {0x2007, "\x20"}, // FIGURE SPACE + {0x2009, "\x20"}, // THIN SPACE + {0x200B, ""}, // ZERO WIDTH SPACE + {0x2010, "\x2D"}, // HYPHEN + {0x2011, "\x2D"}, // NON-BREAKING HYPHEN + {0x2012, "\x2D"}, // FIGURE DASH + {0x2013, "\x2D"}, // EN DASH + {0x2014, "\x2D"}, // EM DASH + {0x2015, "\x2D"}, // HORIZONTAL BAR + {0x2018, "\x27"}, // LEFT SINGLE QUOTATION MARK + {0x2019, "\x27"}, // RIGHT SINGLE QUOTATION MARK + {0x201A, "\x2C"}, // SINGLE LOW-9 QUOTATION MARK + {0x201B, "\x27"}, // SINGLE HIGH-REVERSED-9 QUOTATION MARK + {0x201C, "\x22"}, // LEFT DOUBLE QUOTATION MARK + {0x201D, "\x22"}, // RIGHT DOUBLE QUOTATION MARK + {0x201E, "\x22"}, // DOUBLE LOW-9 QUOTATION MARK + {0x201F, "\x22"}, // DOUBLE HIGH-REVERSED-9 QUOTATION MARK + {0x2022, "\x2A"}, // BULLET + {0x2023, "\x3E"}, // TRIANGULAR BULLET + {0x2026, "\x2E\x2E\x2E"}, // HORIZONTAL ELLIPSIS + {0x2027, "\x2A"}, // HYPHENATION POINT + {0x2028, "\x20"}, // LINE SEPARATOR + {0x2029, "\x20"}, // PARAGRAPH SEPARATOR + {0x202F, "\x20"}, // NARROW NO-BREAK SPACE + {0x2032, "\x27"}, // PRIME + {0x2033, "\x22"}, // DOUBLE PRIME + {0x2039, "\x3C"}, // SINGLE LEFT-POINTING ANGLE QUOTATION MARK + {0x203A, "\x3E"}, // SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + {0x2043, "\x2D"}, // HYPHEN BULLET + {0x20AC, "\x45\x55\x52"}, // EURO SIGN + {0x2122, "\x74\x6D"}, // TRADE MARK SIGN + {0x2190, "\x3C\x2D"}, // LEFTWARDS ARROW + {0x2191, "\x5E"}, // UPWARDS ARROW + {0x2192, "\x2D\x3E"}, // RIGHTWARDS ARROW + {0x2193, "\x76"}, // DOWNWARDS ARROW + {0x21D0, "\x3C\x3D"}, // LEFTWARDS DOUBLE ARROW + {0x21D2, "\x3D\x3E"}, // RIGHTWARDS DOUBLE ARROW + {0x2212, "\x2D"}, // MINUS SIGN + {0x221E, "\x69\x6E\x66"}, // INFINITY + {0x2260, "\x21\x3D"}, // NOT EQUAL TO + {0x2501, "\xE2\x94\x80"}, // BOX DRAWINGS HEAVY HORIZONTAL + {0x2503, "\xE2\x94\x82"}, // BOX DRAWINGS HEAVY VERTICAL + {0x2504, "\xE2\x94\x80"}, // BOX DRAWINGS LIGHT TRIPLE DASH HORIZONTAL + {0x2505, "\xE2\x94\x80"}, // BOX DRAWINGS HEAVY TRIPLE DASH HORIZONTAL + {0x2506, "\xE2\x94\x82"}, // BOX DRAWINGS LIGHT TRIPLE DASH VERTICAL + {0x2507, "\xE2\x94\x82"}, // BOX DRAWINGS HEAVY TRIPLE DASH VERTICAL + {0x2508, "\xE2\x94\x80"}, // BOX DRAWINGS LIGHT QUADRUPLE DASH HORIZONTAL + {0x2509, "\xE2\x94\x80"}, // BOX DRAWINGS HEAVY QUADRUPLE DASH HORIZONTAL + {0x250A, "\xE2\x94\x82"}, // BOX DRAWINGS LIGHT QUADRUPLE DASH VERTICAL + {0x250B, "\xE2\x94\x82"}, // BOX DRAWINGS HEAVY QUADRUPLE DASH VERTICAL + {0x250D, "\x2B"}, // BOX DRAWINGS DOWN LIGHT AND RIGHT HEAVY + {0x250E, "\x2B"}, // BOX DRAWINGS DOWN HEAVY AND RIGHT LIGHT + {0x250F, "\xE2\x94\x8C"}, // BOX DRAWINGS HEAVY DOWN AND RIGHT + {0x2511, "\x2B"}, // BOX DRAWINGS DOWN LIGHT AND LEFT HEAVY + {0x2512, "\x2B"}, // BOX DRAWINGS DOWN HEAVY AND LEFT LIGHT + {0x2513, "\xE2\x94\x90"}, // BOX DRAWINGS HEAVY DOWN AND LEFT + {0x2515, "\x2B"}, // BOX DRAWINGS UP LIGHT AND RIGHT HEAVY + {0x2516, "\x2B"}, // BOX DRAWINGS UP HEAVY AND RIGHT LIGHT + {0x2517, "\xE2\x94\x94"}, // BOX DRAWINGS HEAVY UP AND RIGHT + {0x2519, "\x2B"}, // BOX DRAWINGS UP LIGHT AND LEFT HEAVY + {0x251A, "\x2B"}, // BOX DRAWINGS UP HEAVY AND LEFT LIGHT + {0x251B, "\xE2\x94\x98"}, // BOX DRAWINGS HEAVY UP AND LEFT + {0x251D, "\x7C"}, // BOX DRAWINGS VERTICAL LIGHT AND RIGHT HEAVY + {0x251E, "\x2B"}, // BOX DRAWINGS UP HEAVY AND RIGHT DOWN LIGHT + {0x251F, "\x2B"}, // BOX DRAWINGS DOWN HEAVY AND RIGHT UP LIGHT + {0x2520, "\x7C"}, // BOX DRAWINGS VERTICAL HEAVY AND RIGHT LIGHT + {0x2521, "\x2B"}, // BOX DRAWINGS DOWN LIGHT AND RIGHT UP HEAVY + {0x2522, "\x2B"}, // BOX DRAWINGS UP LIGHT AND RIGHT DOWN HEAVY + {0x2523, "\xE2\x94\x9C"}, // BOX DRAWINGS HEAVY VERTICAL AND RIGHT + {0x2525, "\x7C"}, // BOX DRAWINGS VERTICAL LIGHT AND LEFT HEAVY + {0x2526, "\x2B"}, // BOX DRAWINGS UP HEAVY AND LEFT DOWN LIGHT + {0x2527, "\x2B"}, // BOX DRAWINGS DOWN HEAVY AND LEFT UP LIGHT + {0x2528, "\x7C"}, // BOX DRAWINGS VERTICAL HEAVY AND LEFT LIGHT + {0x2529, "\x2B"}, // BOX DRAWINGS DOWN LIGHT AND LEFT UP HEAVY + {0x252A, "\x2B"}, // BOX DRAWINGS UP LIGHT AND LEFT DOWN HEAVY + {0x252B, "\xE2\x94\xA4"}, // BOX DRAWINGS HEAVY VERTICAL AND LEFT + {0x252D, "\x2B"}, // BOX DRAWINGS LEFT HEAVY AND RIGHT DOWN LIGHT + {0x252E, "\x2B"}, // BOX DRAWINGS RIGHT HEAVY AND LEFT DOWN LIGHT + {0x252F, "\x2D"}, // BOX DRAWINGS DOWN LIGHT AND HORIZONTAL HEAVY + {0x2530, "\x2D"}, // BOX DRAWINGS DOWN HEAVY AND HORIZONTAL LIGHT + {0x2531, "\x2B"}, // BOX DRAWINGS RIGHT LIGHT AND LEFT DOWN HEAVY + {0x2532, "\x2B"}, // BOX DRAWINGS LEFT LIGHT AND RIGHT DOWN HEAVY + {0x2533, "\xE2\x94\xAC"}, // BOX DRAWINGS HEAVY DOWN AND HORIZONTAL + {0x2535, "\x2B"}, // BOX DRAWINGS LEFT HEAVY AND RIGHT UP LIGHT + {0x2536, "\x2B"}, // BOX DRAWINGS RIGHT HEAVY AND LEFT UP LIGHT + {0x2537, "\x2D"}, // BOX DRAWINGS UP LIGHT AND HORIZONTAL HEAVY + {0x2538, "\x2D"}, // BOX DRAWINGS UP HEAVY AND HORIZONTAL LIGHT + {0x2539, "\x2B"}, // BOX DRAWINGS RIGHT LIGHT AND LEFT UP HEAVY + {0x253A, "\x2B"}, // BOX DRAWINGS LEFT LIGHT AND RIGHT UP HEAVY + {0x253B, "\xE2\x94\xB4"}, // BOX DRAWINGS HEAVY UP AND HORIZONTAL + {0x253D, "\x7C"}, // BOX DRAWINGS LEFT HEAVY AND RIGHT VERTICAL LIGHT + {0x253E, "\x7C"}, // BOX DRAWINGS RIGHT HEAVY AND LEFT VERTICAL LIGHT + {0x253F, "\x7C"}, // BOX DRAWINGS VERTICAL LIGHT AND HORIZONTAL HEAVY + {0x2540, "\x2D"}, // BOX DRAWINGS UP HEAVY AND DOWN HORIZONTAL LIGHT + {0x2541, "\x2D"}, // BOX DRAWINGS DOWN HEAVY AND UP HORIZONTAL LIGHT + {0x2542, "\x7C"}, // BOX DRAWINGS VERTICAL HEAVY AND HORIZONTAL LIGHT + {0x2543, "\x2B"}, // BOX DRAWINGS LEFT UP HEAVY AND RIGHT DOWN LIGHT + {0x2544, "\x2B"}, // BOX DRAWINGS RIGHT UP HEAVY AND LEFT DOWN LIGHT + {0x2545, "\x2B"}, // BOX DRAWINGS LEFT DOWN HEAVY AND RIGHT UP LIGHT + {0x2546, "\x2B"}, // BOX DRAWINGS RIGHT DOWN HEAVY AND LEFT UP LIGHT + {0x2547, "\x2D"}, // BOX DRAWINGS DOWN LIGHT AND UP HORIZONTAL HEAVY + {0x2548, "\x2D"}, // BOX DRAWINGS UP LIGHT AND DOWN HORIZONTAL HEAVY + {0x2549, "\x7C"}, // BOX DRAWINGS RIGHT LIGHT AND LEFT VERTICAL HEAVY + {0x254A, "\x7C"}, // BOX DRAWINGS LEFT LIGHT AND RIGHT VERTICAL HEAVY + {0x254B, "\xE2\x94\xBC"}, // BOX DRAWINGS HEAVY VERTICAL AND HORIZONTAL + {0x254C, "\xE2\x94\x80"}, // BOX DRAWINGS LIGHT DOUBLE DASH HORIZONTAL + {0x254D, "\xE2\x94\x80"}, // BOX DRAWINGS HEAVY DOUBLE DASH HORIZONTAL + {0x254E, "\xE2\x94\x82"}, // BOX DRAWINGS LIGHT DOUBLE DASH VERTICAL + {0x254F, "\xE2\x94\x82"}, // BOX DRAWINGS HEAVY DOUBLE DASH VERTICAL + {0x256D, "\xE2\x94\x8C"}, // BOX DRAWINGS LIGHT ARC DOWN AND RIGHT + {0x256E, "\xE2\x94\x90"}, // BOX DRAWINGS LIGHT ARC DOWN AND LEFT + {0x256F, "\xE2\x94\x98"}, // BOX DRAWINGS LIGHT ARC UP AND LEFT + {0x2570, "\xE2\x94\x94"}, // BOX DRAWINGS LIGHT ARC UP AND RIGHT + {0x2571, "\x2B"}, // BOX DRAWINGS LIGHT DIAGONAL UPPER RIGHT TO LOWER LEFT + {0x2572, "\x2B"}, // BOX DRAWINGS LIGHT DIAGONAL UPPER LEFT TO LOWER RIGHT + {0x2573, "\x2B"}, // BOX DRAWINGS LIGHT DIAGONAL CROSS + {0x2574, "\x2B"}, // BOX DRAWINGS LIGHT LEFT + {0x2575, "\x2B"}, // BOX DRAWINGS LIGHT UP + {0x2576, "\x2B"}, // BOX DRAWINGS LIGHT RIGHT + {0x2577, "\x2B"}, // BOX DRAWINGS LIGHT DOWN + {0x2578, "\x2B"}, // BOX DRAWINGS HEAVY LEFT + {0x2579, "\x2B"}, // BOX DRAWINGS HEAVY UP + {0x257A, "\x2B"}, // BOX DRAWINGS HEAVY RIGHT + {0x257B, "\x2B"}, // BOX DRAWINGS HEAVY DOWN + {0x257C, "\x2B"}, // BOX DRAWINGS LIGHT LEFT AND HEAVY RIGHT + {0x257D, "\x2B"}, // BOX DRAWINGS LIGHT UP AND HEAVY DOWN + {0x257E, "\x2B"}, // BOX DRAWINGS HEAVY LEFT AND LIGHT RIGHT + {0x257F, "\x2B"}, // BOX DRAWINGS HEAVY UP AND LIGHT DOWN + {0x2581, "\xE2\x96\x84"}, // LOWER ONE EIGHTH BLOCK + {0x2582, "\xE2\x96\x84"}, // LOWER ONE QUARTER BLOCK + {0x2583, "\xE2\x96\x84"}, // LOWER THREE EIGHTHS BLOCK + {0x2585, "\xE2\x96\x84"}, // LOWER FIVE EIGHTHS BLOCK + {0x2586, "\xE2\x96\x84"}, // LOWER THREE QUARTERS BLOCK + {0x2587, "\xE2\x96\x84"}, // LOWER SEVEN EIGHTHS BLOCK + {0x2589, "\xE2\x96\x88"}, // LEFT SEVEN EIGHTHS BLOCK + {0x258A, "\xE2\x96\x88"}, // LEFT THREE QUARTERS BLOCK + {0x258B, "\xE2\x96\x88"}, // LEFT FIVE EIGHTHS BLOCK + {0x258D, "\xE2\x96\x8C"}, // LEFT THREE EIGHTHS BLOCK + {0x258E, "\xE2\x96\x8C"}, // LEFT ONE QUARTER BLOCK + {0x258F, "\xE2\x96\x8C"}, // LEFT ONE EIGHTH BLOCK + {0x2594, "\xE2\x96\x80"}, // UPPER ONE EIGHTH BLOCK + {0x2595, "\xE2\x96\x90"}, // RIGHT ONE EIGHTH BLOCK + {0x2596, "\xE2\x96\x84"}, // QUADRANT LOWER LEFT + {0x2597, "\xE2\x96\x84"}, // QUADRANT LOWER RIGHT + {0x2598, "\xE2\x96\x80"}, // QUADRANT UPPER LEFT + {0x2599, "\xE2\x96\x88"}, // QUADRANT UPPER LEFT AND LOWER LEFT AND LOWER RIGHT + {0x259A, "\xE2\x96\x92"}, // QUADRANT UPPER LEFT AND LOWER RIGHT + {0x259B, "\xE2\x96\x88"}, // QUADRANT UPPER LEFT AND UPPER RIGHT AND LOWER LEFT + {0x259C, "\xE2\x96\x88"}, // QUADRANT UPPER LEFT AND UPPER RIGHT AND LOWER RIGHT + {0x259D, "\xE2\x96\x80"}, // QUADRANT UPPER RIGHT + {0x259E, "\xE2\x96\x92"}, // QUADRANT UPPER RIGHT AND LOWER LEFT + {0x259F, "\xE2\x96\x88"}, // QUADRANT UPPER RIGHT AND LOWER LEFT AND LOWER RIGHT + {0x25A1, "\x23"}, // WHITE SQUARE + {0x25AA, "\x2A"}, // BLACK SMALL SQUARE + {0x25AB, "\x2A"}, // WHITE SMALL SQUARE + {0x25B2, "\x5E"}, // BLACK UP-POINTING TRIANGLE + {0x25B6, "\x3E"}, // BLACK RIGHT-POINTING TRIANGLE + {0x25BC, "\x76"}, // BLACK DOWN-POINTING TRIANGLE + {0x25C0, "\x3C"}, // BLACK LEFT-POINTING TRIANGLE + {0x25CB, "\x6F"}, // WHITE CIRCLE + {0x25CF, "\x2A"}, // BLACK CIRCLE + {0x2605, "\x2A"}, // BLACK STAR + {0x2606, "\x2A"}, // WHITE STAR + {0x2620, "\x21"}, // SKULL AND CROSSBONES + {0x2660, "\x53"}, // BLACK SPADE SUIT + {0x2663, "\x43"}, // BLACK CLUB SUIT + {0x2665, "\x48"}, // BLACK HEART SUIT + {0x2666, "\x44"}, // BLACK DIAMOND SUIT + {0x2694, "\x78"}, // CROSSED SWORDS + {0x2713, "\x2B"}, // CHECK MARK + {0x2714, "\x2B"}, // HEAVY CHECK MARK + {0x2717, "\x78"}, // BALLOT X + {0x2718, "\x78"}, // HEAVY BALLOT X + {0x2726, "\x2A"}, // BLACK FOUR POINTED STAR + {0x2727, "\x2A"}, // WHITE FOUR POINTED STAR + {0x272A, "\x2A"}, // CIRCLED WHITE STAR + {0x2734, "\x2A"}, // EIGHT POINTED BLACK STAR + {0x2736, "\x2A"}, // SIX POINTED BLACK STAR +}; + +} // namespace + +const char *TranslitToKoi8(char32_t code_point) { + const auto *end = kTable + std::size(kTable); + const auto *it = std::lower_bound(kTable, end, code_point, + [](const TranslitEntry &entry, char32_t value) { return entry.code_point < value; }); + return (it != end && it->code_point == code_point) ? it->replacement : nullptr; +} + +std::string Utf8ToKoi8(const std::string &text) { + if (text.empty()) { + return text; + } + std::string reduced; + reduced.reserve(text.size()); + for (std::size_t pos = 0; pos < text.size();) { + char32_t cp = 0; + const std::size_t len = utf8::decode(text, pos, cp); + if (len == 0) { + break; + } + const char *const replacement = TranslitToKoi8(cp); + if (replacement != nullptr) { + reduced.append(replacement); + } else { + reduced.append(text, pos, len); + } + pos += len; + } + + // KOI8-R is never longer than the UTF-8 it came from, so the input size is a safe bound. + std::vector out(reduced.size() + 1, '\0'); + utf8_to_koi(const_cast(reduced.c_str()), out.data()); + return std::string(out.data()); +} + +} // namespace codepages + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/translit_koi8.h b/src/utils/translit_koi8.h new file mode 100644 index 0000000000..34cc77dd30 --- /dev/null +++ b/src/utils/translit_koi8.h @@ -0,0 +1,28 @@ +/** +\file translit_koi8.h - a part of the Bylins engine. +\brief Closest KOI8-R equivalent for characters KOI8-R does not have (issue #3681). +*/ + +#ifndef BYLINS_SRC_UTILS_TRANSLIT_KOI8_H_ +#define BYLINS_SRC_UTILS_TRANSLIT_KOI8_H_ + +#include + +namespace codepages { + +// UTF-8 replacement for a code point KOI8-R cannot represent, or nullptr when there is no +// sensible one (the caller then falls back to its placeholder). Replacements may be longer +// than one character: "..." for an ellipsis, "(tm)" for a trademark sign. +const char *TranslitToKoi8(char32_t code_point); + +// UTF-8 text -> KOI8-R, running the table above as a pre-pass. Doing the substitution before the +// conversion is what allows a replacement to be longer than one character: the converter itself +// writes exactly one output byte per code point. No replacement ever grows the text (pinned by a +// test), so the result is never longer than the input. +std::string Utf8ToKoi8(const std::string &text); + +} // namespace codepages + +#endif // BYLINS_SRC_UTILS_TRANSLIT_KOI8_H_ + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/utils_encoding.cpp b/src/utils/utils_encoding.cpp index 5d25112cb5..e842d685fc 100644 --- a/src/utils/utils_encoding.cpp +++ b/src/utils/utils_encoding.cpp @@ -16,23 +16,70 @@ namespace codepages { +// Tables of raw bytes, spelled as \xNN escapes on purpose. They used to be written as string +// literals of high-byte characters, which made their contents depend on the encoding of this +// file: with UTF-8 sources every Cyrillic/pseudographic character became two or three bytes and +// the tables silently grew past 128 entries, corrupting every legacy client's output. Escapes +// are plain ASCII and mean the same thing in any source encoding (issue #3681). char AltToKoi[] = { - "АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмноп░▒▓│┤╡+++╣║╗╝+╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨++╙╘╒++╪┘┌█▄▌▐▀рстуфхцчшщъыьэюяЁё╫╜╢╓╤╕╥╖·√??■ " + "\xE1\xE2\xF7\xE7\xE4\xE5\xF6\xFA\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0" + "\xF2\xF3\xF4\xF5\xE6\xE8\xE3\xFE\xFB\xFD\xFF\xF9\xF8\xFC\xE0\xF1" + "\xC1\xC2\xD7\xC7\xC4\xC5\xD6\xDA\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0" + "\x90\x91\x92\x81\x87\xB2\x2B\x2B\x2B\xB5\xA1\xA8\xAE\x2B\xAC\x83" + "\x84\x89\x88\x86\x80\x8A\xAF\xB0\xAB\xA5\xBB\xB8\xB1\xA0\xBE\xB9" + "\xBA\x2B\x2B\xAA\xA9\xA2\x2B\x2B\xBC\x85\x82\x8D\x8C\x8E\x8F\x8B" + "\xD2\xD3\xD4\xD5\xC6\xC8\xC3\xDE\xDB\xDD\xDF\xD9\xD8\xDC\xC0\xD1" + "\xB3\xA3\xBD\xAD\xB4\xA4\xB6\xA6\xB7\xA7\x9E\x96\x3F\x3F\x94\x9A" }; char KoiToAlt[] = { - "дЁз©юыц╢баеъэшщч╟╠╡+Ч+Ш+++Ъ+++З+м╨уЯУиВЫ╩тсх╬С╪фгл╣ПТ╧ЖЬкопйьРн+Н═║Ф╓╔ДёЕ╗╘╙╚╛╜╝╞ОЮАБЦ╕╒ЛК╖ХМИГЙ·─│√└┘■┐∙┬┴┼▀▄█▌▐÷░▒▓⌠├┌°⌡┤≤²≥≈ " + "\xC4\xB3\xDA\xBF\xC0\xD9\xC3\xB4\xC2\xC1\xC5\xDF\xDC\xDB\xDD\xDE" + "\xB0\xB1\xB2\x2B\xFE\x2B\xFB\x2B\x2B\x2B\xFF\x2B\x2B\x2B\xFA\x2B" + "\xCD\xBA\xD5\xF1\xF5\xC9\xF7\xF9\xBB\xD4\xD3\xC8\xBE\xF3\xBC\xC6" + "\xC7\xCC\xB5\xF0\xF4\xB9\xF6\xF8\xCB\xCF\xD0\xCA\xD8\xF2\xCE\x2B" + "\xEE\xA0\xA1\xE6\xA4\xA5\xE4\xA3\xE5\xA8\xA9\xAA\xAB\xAC\xAD\xAE" + "\xAF\xEF\xE0\xE1\xE2\xE3\xA6\xA2\xEC\xEB\xA7\xE8\xED\xE9\xE7\xEA" + "\x9E\x80\x81\x96\x84\x85\x94\x83\x95\x88\x89\x8A\x8B\x8C\x8D\x8E" + "\x8F\x9F\x90\x91\x92\x93\x86\x82\x9C\x9B\x87\x98\x9D\x99\x97\x9A" }; char WinToKoi[] = { - "++++++++++++++++++++++++++++++++ ++++╫++Ё©╢++++╥°+╤╕╜++·ё+╓++++╖АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя" + "\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B" + "\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B" + "\x9A\x2B\x2B\x2B\x2B\xBD\x2B\x2B\xB3\xBF\xB4\x2B\x2B\x2B\x2B\xB7" + "\x9C\x2B\xB6\xA6\xAD\x2B\x2B\x9E\xA3\x2B\xA4\x2B\x2B\x2B\x2B\xA7" + "\xE1\xE2\xF7\xE7\xE4\xE5\xF6\xFA\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0" + "\xF2\xF3\xF4\xF5\xE6\xE8\xE3\xFE\xFB\xFD\xFF\xF9\xF8\xFC\xE0\xF1" + "\xC1\xC2\xD7\xC7\xC4\xC5\xD6\xDA\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0" + "\xD2\xD3\xD4\xD5\xC6\xC8\xC3\xDE\xDB\xDD\xDF\xD9\xD8\xDC\xC0\xD1" }; char KoiToWin[] = { - "++++++++++++++++++++++++++═+╟+╥++++╦╨+Ё©+++++╢+++++╗╙+╡╞+++++╔+╘ЧЮАЖДЕТЦУХИЙКЛМНОЪПЯРСФБЭШГЬЩЫВЗчюаждетцухийклмноъпярсфбэшгьщывз" + "\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B" + "\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\xA0\x2B\xB0\x2B\xB7\x2B" + "\x2B\x2B\x2B\xB8\xBA\x2B\xB3\xBF\x2B\x2B\x2B\x2B\x2B\xB4\x2B\x2B" + "\x2B\x2B\x2B\xA8\xAA\x2B\xB2\xAF\x2B\x2B\x2B\x2B\x2B\xA5\x2B\xA9" + "\xFE\xE0\xE1\xF6\xE4\xE5\xF4\xE3\xF5\xE8\xE9\xEA\xEB\xEC\xED\xEE" + "\xEF\xFF\xF0\xF1\xF2\xF3\xE6\xE2\xFC\xFB\xE7\xF8\xFD\xF9\xF7\xFA" + "\xDE\xC0\xC1\xD6\xC4\xC5\xD4\xC3\xD5\xC8\xC9\xCA\xCB\xCC\xCD\xCE" + "\xCF\xDF\xD0\xD1\xD2\xD3\xC6\xC2\xDC\xDB\xC7\xD8\xDD\xD9\xD7\xDA" }; char KoiToWin2[] = { - "++++++++++++++++++++++++++═+╟+╥++++╦╨+Ё©+++++╢+++++╗╙+╡╞+++++╔+╘ЧЮАЖДЕТЦУХИЙКЛМНОzПЯРСФБЭШГЬЩЫВЗчюаждетцухийклмноъпярсфбэшгьщывз" + "\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B" + "\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\x2B\xA0\x2B\xB0\x2B\xB7\x2B" + "\x2B\x2B\x2B\xB8\xBA\x2B\xB3\xBF\x2B\x2B\x2B\x2B\x2B\xB4\x2B\x2B" + "\x2B\x2B\x2B\xA8\xAA\x2B\xB2\xAF\x2B\x2B\x2B\x2B\x2B\xA5\x2B\xA9" + "\xFE\xE0\xE1\xF6\xE4\xE5\xF4\xE3\xF5\xE8\xE9\xEA\xEB\xEC\xED\xEE" + "\xEF\x7A\xF0\xF1\xF2\xF3\xE6\xE2\xFC\xFB\xE7\xF8\xFD\xF9\xF7\xFA" + "\xDE\xC0\xC1\xD6\xC4\xC5\xD4\xC3\xD5\xC8\xC9\xCA\xCB\xCC\xCD\xCE" + "\xCF\xDF\xD0\xD1\xD2\xD3\xC6\xC2\xDC\xDB\xC7\xD8\xDD\xD9\xD7\xDA" }; char AltToLat[] = { - "─│┌┐└┘├┤┬┴┼▀▄█▌▐░▒▓⌠■∙√≈≤≥ ⌡°²·÷═║╒ё╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡Ё╢╣╤╥╦╧╨╩╪╫╬©0abcdefghijklmnopqrstY1v23z456780ABCDEFGHIJKLMNOPQRSTY1V23Z45678" + "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F" + "\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9A\x9B\x9C\x9D\x9E\x9F" + "\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF" + "\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF" + "\x30\x61\x62\x63\x64\x65\x66\x67\x68\x69\x6A\x6B\x6C\x6D\x6E\x6F" + "\x70\x71\x72\x73\x74\x59\x31\x76\x32\x33\x7A\x34\x35\x36\x37\x38" + "\x30\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4A\x4B\x4C\x4D\x4E\x4F" + "\x50\x51\x52\x53\x54\x59\x31\x56\x32\x33\x5A\x34\x35\x36\x37\x38" }; void koi_to_win(char *str, int size) { @@ -103,7 +150,10 @@ void utf8_to_koi(char *str_i, char *str_o) #else // HAVE_ICONV -#define KOI8_UNKNOWN_CHAR '+' // char to use when cannot represent unicode codepoint in KOI8-R +// Placeholder for a code point KOI8-R cannot represent AND for which translit_koi8 has no +// equivalent. '?' rather than the former '+': a plus is ordinary text in this game ("+5 to +// damage"), so it read as content rather than as a lost character (issue #3681). +#define KOI8_UNKNOWN_CHAR '?' // Simple implementation of UTF-8/KOI8-R converter, supports all codes available in KOI8-R void utf8_to_koi(char *str_i, char *str_o) { diff --git a/src/utils/utils_parse.cpp b/src/utils/utils_parse.cpp index 622ddd0fd6..39a6fe34ff 100644 --- a/src/utils/utils_parse.cpp +++ b/src/utils/utils_parse.cpp @@ -352,6 +352,11 @@ int AttrInt(const parser_wrapper::DataNode &node, const char *key, int def) { std::string AttrStr(const parser_wrapper::DataNode &node, const char *key, const char *def) { const char *v = node.GetValue(key); + // Перекодировки здесь НЕТ и быть не должно: DataNode приводит содержимое файла к нативной + // кодировке один раз на документ, поэтому значение уже нативное. Пока перекодировка стояла + // ещё и здесь, каждое поле переводилось дважды, а так как конфиги движок ещё и пишет + // обратно, файл рос на каждой загрузке -- cfg/mechanics/obj_sets.xml так дорос со 120 КБ + // до 6,7 ГБ и убивал загрузку по памяти (issue #3681). return (v && *v) ? std::string(v) : std::string(def); } diff --git a/src/utils/utils_string.cpp b/src/utils/utils_string.cpp index e4adbbe044..74015f3933 100644 --- a/src/utils/utils_string.cpp +++ b/src/utils/utils_string.cpp @@ -365,25 +365,21 @@ std::string FixDot(std::string s) { return s; } +// Сортировка идёт по ключу, а не по перекодированной на месте строке. Прежний трюк -- перегнать +// список в Windows-1251 (там русские буквы стоят по алфавиту, в отличие от KOI8-R), отсортировать +// и перегнать обратно -- под UTF-8 портил текст: побайтовая таблица KOI8-R -> Windows-1251 для +// многобайтовой строки не значит ничего, и обратное преобразование уже не восстанавливало исходник +// (issue #3681). native_text::sort_key даёт ровно тот же порядок, ничего не меняя в самих строках. void SortKoiString(std::vector &str) { - for (auto &it : str) { - ConvertKtoW(it); - } - std::sort(str.begin(), str.end(), std::less()); - for (auto &it : str) { - ConvertWtoK(it); - } + std::sort(str.begin(), str.end(), [](const std::string &a, const std::string &b) { + return native_text::sort_key(a) < native_text::sort_key(b); + }); } - void SortKoiStringReverse(std::vector &str) { - for (auto &it : str) { - ConvertKtoW(it); - } - std::sort(str.begin(), str.end(), std::greater()); - for (auto &it : str) { - ConvertWtoK(it); - } + std::sort(str.begin(), str.end(), [](const std::string &a, const std::string &b) { + return native_text::sort_key(a) > native_text::sort_key(b); + }); } void ReplaceFirst(std::string &s, const std::string &toSearch, const std::string &replacer) { @@ -1165,6 +1161,14 @@ std::string sprintGender(int gender_value) { // замена в name русских символов на англ в нижнем регистре (для файлов) void CreateFileName(std::string &name) { + // Имя файла собирается по байтам KOI8-R -- ровно так, как собиралось до миграции. Иначе у + // уже существующего персонажа доска получит другое имя файла, и старая просто потеряется: + // под UTF-8 байтовый AtoL резал русскую букву пополам и оставлял её в имени как есть, отчего + // рядом со 105 старыми досками завелось 70 новых, с кириллицей в имени (issue #3681). + // + // Приводим строку к KOI8-R и дальше работаем прежним байтовым кодом: под KOI8-R to_koi8 -- + // тождество, так что поведение там не меняется ни на байт. + name = native_text::to_koi8(name); for (unsigned i = 0; i != name.length(); ++i) name[i] = LOWER(codepages::AtoL(name[i])); } @@ -1190,7 +1194,10 @@ std::string ExpFormat(long long exp) { void name_convert(std::string &text) { if (!text.empty()) { utils::ConvertToLow(text); - *text.begin() = UPPER(*text.begin()); + // Первую букву поднимаем целиком, а не первый байт: под UTF-8 у русской буквы их два, + // и байтовый UPPER превращал имя в мусор -- "имя щщщщщ одобрить" не находило + // персонажа (issue #3681). + native_text::capitalize_first(text); } } diff --git a/src/version.cpp.in b/src/version.cpp.in index 0136dacf98..97a4b82372 100644 --- a/src/version.cpp.in +++ b/src/version.cpp.in @@ -9,6 +9,7 @@ #include "engine/entities/char_data.h" #include "engine/structs/structs.h" #include "utils/logger.h" +#include "utils/native_text.h" const char* REVISION = "$ Build revision: ${REVISION} $"; const char* revision = "${REVISION}"; @@ -21,17 +22,25 @@ const char* build_datetime = "${BUILD_DATETIME}"; const char* build_compiler = "${BUILD_COMPILER}"; const char* build_features = "${BUILD_FEATURES}"; +// Кодировка берётся из рантайма, а не из строки сборки: так она не может разойтись с тем, +// как движок на самом деле работает с текстом (issue #3681). +static const char *InternalEncodingName() { + return native_text::native_is_utf8() ? "UTF-8" : "KOI8-R"; +} + void ShowBuildInfo(CharData *ch) { SendMsgToChar(ch, "%s %s, build from %s, revision %s\n", engine_name, engine_version, build_datetime, revision); SendMsgToChar(ch, "Based on CircleMUD, version 3.00 beta patchlevel 16\n"); if (privilege::IsImmortal(ch)) { - SendMsgToChar(ch, "Compiler: %s\nEnabled features: %s\n", build_compiler, build_features); + SendMsgToChar(ch, "Compiler: %s; internal encoding: %s\nEnabled features: %s\n", + build_compiler, InternalEncodingName(), build_features); } } void LogBuildInfo() { - log("%s %s, build from %s, revision: %s\r\nCompiler: %s\r\nEnabled features: %s", - engine_name, engine_version, build_datetime, revision, build_compiler, build_features); + log("%s %s, build from %s, revision: %s\r\nCompiler: %s; internal encoding: %s\r\nEnabled features: %s", + engine_name, engine_version, build_datetime, revision, + build_compiler, InternalEncodingName(), build_features); } // vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/boards.encoding.cpp b/tests/boards.encoding.cpp new file mode 100644 index 0000000000..66921192da --- /dev/null +++ b/tests/boards.encoding.cpp @@ -0,0 +1,73 @@ +// Загрузка доски с русским текстом: проверяет, что сообщения приходят в НАТИВНОЙ кодировке +// движка. Файл доски лежит на диске в KOI8-R и читается потоком, минуя FBFILE, поэтому это +// отдельная граница кодировки (issue #3681). Тест осмыслен в обеих сборках: литералы ниже +// компилируются в той же кодировке, в какой работает движок. + +#include "gameplay/communication/boards/boards_types.h" +#include "utils/native_text.h" +#include "utils/utf8.h" + +#include + +#include +#include + +namespace { + +const char *const kNewsSample = "data/boards/news.sample"; + +Boards::Board::shared_ptr LoadSampleBoard() { + auto board = std::make_shared(Boards::NEWS_BOARD); + board->set_file_name(kNewsSample); + board->Load(); + return board; +} + +} // namespace + +TEST(BoardsEncoding, MessagesLoadInTheNativeEncoding) { + std::ifstream probe(kNewsSample); + ASSERT_TRUE(probe.is_open()) << "нет тестовых данных " << kNewsSample; + probe.close(); + + const auto board = LoadSampleBoard(); + ASSERT_FALSE(board->empty()) << "доска не загрузилась"; + + bool seen_cyrillic = false; + for (std::size_t i = 0; i < board->messages_count(); ++i) { + const auto message = board->get_message(i); + for (const std::string *field : {&message->author, &message->subject, &message->text}) { + if (field->empty()) { + continue; + } + const bool has_high_byte = std::any_of(field->begin(), field->end(), + [](char c) { return static_cast(c) >= 0x80; }); + if (!has_high_byte) { + continue; + } + seen_cyrillic = true; + // В UTF-8-сборке текст обязан быть корректным UTF-8; в KOI8-R-сборке -- наоборот, + // это KOI8-R, который валидным UTF-8 не является. + EXPECT_EQ(utf8::is_valid(*field), native_text::native_is_utf8()) + << "поле не в нативной кодировке: " << *field; + } + } + EXPECT_TRUE(seen_cyrillic) << "в тестовых данных не оказалось русского текста -- проверять нечего"; +} + +TEST(BoardsEncoding, AuthorNameSurvivesTheBoundary) { + const auto board = LoadSampleBoard(); + ASSERT_FALSE(board->empty()); + + // Автор одного из сообщений в news.sample -- "Стрибог". + bool found = false; + for (std::size_t i = 0; i < board->messages_count(); ++i) { + if (board->get_message(i)->author == "Стрибог") { + found = true; + break; + } + } + EXPECT_TRUE(found) << "имя автора не совпало -- текст доски пришёл не в той кодировке"; +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/meson.build b/tests/meson.build index 2493ba0373..b6861a5f65 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -24,6 +24,7 @@ test_sources = files( 'blocking.queue.cpp', 'boards.changelog.cpp', 'boards.news.cpp', + 'boards.encoding.cpp', 'msdp.parser.cpp', 'msdp.builder.cpp', 'char.affects.cpp', @@ -39,6 +40,8 @@ test_sources = files( 'utils.editor.cpp', 'utils.string.cpp', 'utils.encoding.cpp', + 'translit_koi8.cpp', + 'parser_wrapper.encoding.cpp', 'utf8.cpp', 'native_text.cpp', 'text_semantics.cpp', diff --git a/tests/native_text.cpp b/tests/native_text.cpp index c351efa161..8c6fd90c88 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -5,6 +5,7 @@ // flag the library was built with), so this one test file is correct under either build. #include "utils/native_text.h" +#include "utils/utf8.h" #include "utils/russian_keys.h" #include @@ -433,4 +434,104 @@ TEST(NativeText, TransliterationIsStableAcrossTheFlip) { EXPECT_EQ(native_text::translit_to_filename(name), "vasq"); } +TEST(NativeText, FromKoi8BringsDiskTextIntoTheNativeEncoding) { + // Data files (world, configs, help, boards, saves) are stored in KOI8-R. from_koi8() is the + // boundary that brings them into whatever the engine runs on: a no-op today, a transcode + // after the flip. Both directions are asserted here so the wiring can be trusted before the + // sources that need it are converted. + const char *const koi8_privet = "\xF0\xD2\xC9\xD7\xC5\xD4"; // "Privet", KOI8-R + const char *const utf8_privet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; // the same, UTF-8 + + EXPECT_EQ(native_text::from_koi8(""), ""); + EXPECT_EQ(native_text::from_koi8("plain ascii 123"), "plain ascii 123"); // ASCII never changes + + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::from_koi8(koi8_privet), utf8_privet); + // The result must be well-formed UTF-8 -- libfort aborts the process on anything else. + EXPECT_TRUE(utf8::is_valid(native_text::from_koi8(koi8_privet))); + EXPECT_EQ(native_text::char_count(native_text::from_koi8(koi8_privet)), 6u); + } else { + EXPECT_EQ(native_text::from_koi8(koi8_privet), koi8_privet); // identity under KOI8-R + } +} + +TEST(NativeText, ToKoi8IsTheInverseBoundary) { + // The counterpart of from_koi8: used where something downstream speaks KOI8-R -- the legacy + // client code pages (their tables are indexed by KOI8-R bytes) and the on-disk formats. + const char *const koi8_privet = "\xF0\xD2\xC9\xD7\xC5\xD4"; + const char *const utf8_privet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; + + EXPECT_EQ(native_text::to_koi8(""), ""); + EXPECT_EQ(native_text::to_koi8("plain ascii 123"), "plain ascii 123"); + + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::to_koi8(utf8_privet), koi8_privet); + // Round trip through the boundary must return the original text. + EXPECT_EQ(native_text::from_koi8(native_text::to_koi8(utf8_privet)), utf8_privet); + } else { + EXPECT_EQ(native_text::to_koi8(koi8_privet), koi8_privet); + } +} + +// KOI8-R spellings, so the fixture is built through from_koi8 and comes out native in either +// build. Words chosen to expose the KOI8-R byte order: there the Russian letters are NOT in +// alphabetical order, so a plain byte comparison would sort these wrong. +TEST(NativeText, SortKeyOrdersRussianAlphabetically) { + const auto native = [](const char *koi8) { return native_text::from_koi8(koi8); }; + // "arbuz", "banan", "vishnya", "yabloko" -- a, b, v, ya. + const std::string a = native("\xC1\xD2\xC2\xD5\xDA"); + const std::string b = native("\xC2\xC1\xCE\xC1\xCE"); + const std::string v = native("\xD7\xC9\xDB\xCE\xD1"); + const std::string ya = native("\xD1\xC2\xCC\xCF\xCB\xCF"); + + EXPECT_LT(native_text::sort_key(a), native_text::sort_key(b)); + EXPECT_LT(native_text::sort_key(b), native_text::sort_key(v)); + EXPECT_LT(native_text::sort_key(v), native_text::sort_key(ya)); + // "zhaba" and "ivan": alphabetically zh comes before i, but in KOI8-R its byte (0xD6) is + // above i's (0xC9) -- which is the whole reason the key exists. + const std::string zh = native("\xD6\xC1\xC2\xC1"); + const std::string i = native("\xC9\xD7\xC1\xCE"); + EXPECT_LT(native_text::sort_key(zh), native_text::sort_key(i)); + if (!native_text::native_is_utf8()) { + EXPECT_GT(zh, i); + } + // ASCII keeps its own order and stays below the Cyrillic. + EXPECT_LT(native_text::sort_key("abc"), native_text::sort_key("abd")); + EXPECT_LT(native_text::sort_key("zzz"), native_text::sort_key(a)); +} + +TEST(NativeText, SortKeyIsStableAcrossEncodings) { + // The key is a byte string in a fixed encoding, so the very same words must produce the very + // same key no matter which encoding the engine runs in. Pinned literally. + EXPECT_EQ(native_text::sort_key(native_text::from_koi8("\xC1\xD2\xC2\xD5\xDA")), + "\xE0\xF0\xE1\xF3\xE7"); // "arbuz" in Windows-1251 + EXPECT_EQ(native_text::sort_key("plain ascii"), "plain ascii"); +} + +TEST(NativeText, FromDiskTextIsIdempotent) { + // The bug this guards against: a config the engine both reads and writes was transcoded + // unconditionally on load, so text already saved in the native encoding got transcoded a + // second time -- and since the save wrote that back, every Cyrillic byte doubled on each + // load/save cycle. cfg/mechanics/obj_sets.xml grew from 120 KB to 6.7 GB that way. + const std::string koi8 = "\xD0\xD2\xC9\xD7\xC5\xD4"; // "privet" in KOI8-R + const std::string once = native_text::from_disk_text(koi8); + const std::string twice = native_text::from_disk_text(once); + EXPECT_EQ(once, twice) << "reading back what we wrote must not change it"; + // However many times it goes through, the size must not grow. + std::string acc = koi8; + for (int i = 0; i < 10; ++i) { + acc = native_text::from_disk_text(acc); + } + EXPECT_EQ(acc, once); + + if (native_text::native_is_utf8()) { + EXPECT_EQ(once, "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"); + } else { + EXPECT_EQ(once, koi8); // no conversion at all under KOI8-R + } + // ASCII is spelled the same in both encodings and must pass through untouched. + EXPECT_EQ(native_text::from_disk_text(""), ""); + EXPECT_EQ(native_text::from_disk_text(""), ""); +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/parser_wrapper.encoding.cpp b/tests/parser_wrapper.encoding.cpp new file mode 100644 index 0000000000..12a7c30336 --- /dev/null +++ b/tests/parser_wrapper.encoding.cpp @@ -0,0 +1,105 @@ +// The load/save cycle of an XML config must not change what it holds (issue #3681). +// +// This pins a bug that cost a world: the document was brought into the native encoding once in +// DataNode, and then AGAIN per field in parse::AttrStr. Under KOI8-R both conversions are the +// identity, so nothing showed. Under UTF-8 every field was converted twice -- and because the +// engine writes configs back (ObjSetsLoader::Load normalises the file through save()), the damage +// accumulated: cfg/mechanics/obj_sets.xml doubled on every boot until, at 6.7 GB, reading it into +// memory killed the process. +// +// Pure ASCII: the Cyrillic fixture is spelled as UTF-8 byte escapes and brought into the native +// encoding through native_text, so the same test is correct under either build. + +#include "utils/parser_wrapper.h" +#include "utils/utils_parse.h" +#include "utils/native_text.h" +#include "utils/translit_koi8.h" + +#include + +#include +#include +#include +#include +#include + +namespace { + +// "privet, mir" in UTF-8, plus an ampersand: colour codes in real configs are written "&W", and +// the ampersand is the character XML escaping is most likely to mangle on a re-save. +const char *const kCyrillicUtf8 = + "&W" "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82" ", " "\xD0\xBC\xD0\xB8\xD1\x80" "&n"; + +std::string ReadWhole(const std::filesystem::path &path) { + std::ifstream in(path, std::ios::binary); + std::ostringstream buffer; + buffer << in.rdbuf(); + return buffer.str(); +} + +// A scratch file removed on destruction, so a failing assertion cannot leave litter behind. +class TempFile { + public: + explicit TempFile(const char *name) : path_(std::filesystem::temp_directory_path() / name) {} + ~TempFile() { std::error_code ec; std::filesystem::remove(path_, ec); } + const std::filesystem::path &path() const { return path_; } + + private: + std::filesystem::path path_; +}; + +} // namespace + +TEST(ParserWrapperEncoding, SaveLoadSaveIsByteIdentical) { + const std::string text = native_text::from_koi8(codepages::Utf8ToKoi8(kCyrillicUtf8)); + TempFile file("bylins_cfg_roundtrip.xml"); + + { + auto doc = parser_wrapper::DataNode::NewDocument(); + auto root = doc.AddChild("obj_sets"); + auto set = root.AddChild("set"); + set.SetValue("name", text); + ASSERT_TRUE(doc.Save(file.path())); + } + const std::string first = ReadWhole(file.path()); + ASSERT_FALSE(first.empty()); + + // Read it back the way the loaders do and write it out again from what we read. + { + parser_wrapper::DataNode loaded(file.path()); + ASSERT_TRUE(loaded.IsNotEmpty()); + ASSERT_TRUE(loaded.GoToChild("set")); + // The value must come back exactly as it went in -- this is where the double conversion + // showed up first. + EXPECT_EQ(parse::AttrStr(loaded, "name"), text); + + auto doc = parser_wrapper::DataNode::NewDocument(); + auto root = doc.AddChild("obj_sets"); + auto set = root.AddChild("set"); + set.SetValue("name", parse::AttrStr(loaded, "name")); + ASSERT_TRUE(doc.Save(file.path())); + } + const std::string second = ReadWhole(file.path()); + + EXPECT_EQ(first.size(), second.size()) << "the file grew on a load/save cycle"; + EXPECT_EQ(first, second) << "a load/save cycle must reproduce the file byte for byte"; +} + +TEST(ParserWrapperEncoding, ReadsAConfigStoredInKoi8) { + // Configs on disk are still KOI8-R, so reading one must transcode; and the result, once + // written back, must then stay put (covered by the test above). + TempFile file("bylins_cfg_koi8.xml"); + { + std::ofstream out(file.path(), std::ios::binary); + out << "\n\n"; + } + + parser_wrapper::DataNode loaded(file.path()); + ASSERT_TRUE(loaded.IsNotEmpty()); + ASSERT_TRUE(loaded.GoToChild("set")); + EXPECT_EQ(parse::AttrStr(loaded, "name"), + native_text::from_koi8(codepages::Utf8ToKoi8(kCyrillicUtf8))); +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/translit_koi8.cpp b/tests/translit_koi8.cpp new file mode 100644 index 0000000000..43b9bce9df --- /dev/null +++ b/tests/translit_koi8.cpp @@ -0,0 +1,145 @@ +// Unit tests for the KOI8-R transliteration table (src/utils/translit_koi8.*, issue #3681). +// +// Pure ASCII: every non-ASCII fixture is spelled as UTF-8 byte escapes, so the file means the +// same thing under either source encoding. The end-to-end expectations go through +// native_text::to_koi8, which only converts in the UTF-8 build, so they branch on +// native_text::native_is_utf8(). + +#include "utils/translit_koi8.h" +#include "utils/native_text.h" +#include "utils/utf8.h" + +#include + +#include +#include + +using codepages::TranslitToKoi8; + +namespace { + +// Convenience: the replacement as a std::string, or "" so a failure prints readably. +std::string Tr(char32_t cp) { + const char *const r = TranslitToKoi8(cp); + return r == nullptr ? std::string("") : std::string(r); +} + +} // namespace + +TEST(TranslitKoi8, TypographyBecomesPlainAscii) { + EXPECT_EQ(Tr(0x2014), "-"); // EM DASH + EXPECT_EQ(Tr(0x2013), "-"); // EN DASH + EXPECT_EQ(Tr(0x2026), "..."); // HORIZONTAL ELLIPSIS + EXPECT_EQ(Tr(0x00AB), "\""); // LEFT-POINTING DOUBLE ANGLE QUOTATION MARK + EXPECT_EQ(Tr(0x00BB), "\""); // RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK + EXPECT_EQ(Tr(0x201C), "\""); // LEFT DOUBLE QUOTATION MARK + EXPECT_EQ(Tr(0x2019), "'"); // RIGHT SINGLE QUOTATION MARK + EXPECT_EQ(Tr(0x2122), "tm"); // TRADE MARK SIGN + EXPECT_EQ(Tr(0x00AD), ""); // SOFT HYPHEN: drops out entirely + // A no-break space needs no replacement: KOI8-R has one of its own (0x9A). + EXPECT_EQ(TranslitToKoi8(0x00A0), nullptr); +} + +TEST(TranslitKoi8, AccentedLatinLosesItsAccent) { + EXPECT_EQ(Tr(0x00E9), "e"); // e with acute + EXPECT_EQ(Tr(0x00FC), "u"); // u with diaeresis + EXPECT_EQ(Tr(0x00C0), "A"); // A with grave: case is kept + EXPECT_EQ(Tr(0x0141), "L"); // L with stroke (no NFKD decomposition; entered by hand) +} + +TEST(TranslitKoi8, NonRussianCyrillicMapsToItsRussianNeighbour) { + EXPECT_EQ(Tr(0x0404), "\xD0\xAD"); // Ukrainian Ye -> Russian E + EXPECT_EQ(Tr(0x0456), "\xD0\xB8"); // Ukrainian i -> Russian i + EXPECT_EQ(Tr(0x0490), "\xD0\x93"); // Ukrainian Ghe with upturn -> Russian Ghe +} + +TEST(TranslitKoi8, WhatKoi8AlreadyHasIsNotTouched) { + // ASCII and the whole Russian alphabet are representable, so there is nothing to replace + // and the table must stay out of the way. + for (char32_t cp = 0x20; cp < 0x7F; ++cp) { + EXPECT_EQ(TranslitToKoi8(cp), nullptr) << "ASCII " << static_cast(cp); + } + for (char32_t cp = 0x0410; cp <= 0x044F; ++cp) { + EXPECT_EQ(TranslitToKoi8(cp), nullptr) << "Cyrillic U+" << static_cast(cp); + } + EXPECT_EQ(TranslitToKoi8(0x0401), nullptr); // Yo + EXPECT_EQ(TranslitToKoi8(0x0451), nullptr); // yo + EXPECT_EQ(TranslitToKoi8(0x2500), nullptr); // box drawing: KOI8-R has these +} + +TEST(TranslitKoi8, NoEquivalentYieldsNullptr) { + EXPECT_EQ(TranslitToKoi8(0x1F600), nullptr); // grinning face + EXPECT_EQ(TranslitToKoi8(0x4E00), nullptr); // CJK ideograph + EXPECT_EQ(TranslitToKoi8(0x05D0), nullptr); // Hebrew alef +} + +TEST(TranslitKoi8, TableIsSortedAndUnique) { + // The lookup is a binary search; a table out of order would silently miss entries. + char32_t previous = 0; + unsigned found = 0; + for (char32_t cp = 1; cp <= 0x2FFF; ++cp) { + if (TranslitToKoi8(cp) != nullptr) { + EXPECT_LT(previous, cp); + previous = cp; + ++found; + } + } + EXPECT_GT(found, 200u) << "the generated table looks truncated"; +} + +TEST(TranslitKoi8, ToKoi8AppliesTheTable) { + if (!native_text::native_is_utf8()) { + GTEST_SKIP() << "to_koi8 is a no-op when the native encoding already is KOI8-R"; + } + // "-- privet ..." with an em dash and an ellipsis around Cyrillic text. + const std::string source = + "\xE2\x80\x94" " \xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82" "\xE2\x80\xA6"; + const std::string koi = native_text::to_koi8(source); + EXPECT_EQ(koi.front(), '-'); + EXPECT_EQ(koi.substr(koi.size() - 3), "..."); + // The Cyrillic in the middle survived as KOI8-R "privet". + EXPECT_EQ(koi, "- \xD0\xD2\xC9\xD7\xC5\xD4..."); +} + +TEST(TranslitKoi8, ToKoi8FallsBackToThePlaceholder) { + if (!native_text::native_is_utf8()) { + GTEST_SKIP() << "to_koi8 is a no-op when the native encoding already is KOI8-R"; + } + // An emoji has no KOI8-R equivalent at all, so it becomes the single placeholder character. + EXPECT_EQ(native_text::to_koi8("a\xF0\x9F\x98\x80" "b"), "a?b"); +} + +TEST(TranslitKoi8, GraphicsDegradeToWhatKoi8CanDraw) { + // Frames: the rounded and heavy corners reduce to the plain ones KOI8-R does have. + EXPECT_EQ(Tr(0x256D), "\xE2\x94\x8C"); // arc down and right -> U+250C + EXPECT_EQ(Tr(0x256E), "\xE2\x94\x90"); // arc down and left -> U+2510 + EXPECT_EQ(Tr(0x256F), "\xE2\x94\x98"); // arc up and left -> U+2518 + EXPECT_EQ(Tr(0x2570), "\xE2\x94\x94"); // arc up and right -> U+2514 + EXPECT_EQ(Tr(0x2501), "\xE2\x94\x80"); // heavy horizontal -> U+2500 + EXPECT_EQ(Tr(0x2503), "\xE2\x94\x82"); // heavy vertical -> U+2502 + // Partial blocks and stars reduce to something that still draws. + EXPECT_EQ(Tr(0x2589), "\xE2\x96\x88"); // 7/8 block -> full block + EXPECT_EQ(Tr(0x2726), "*"); // black four pointed star + EXPECT_EQ(Tr(0x2605), "*"); // black star +} + +TEST(TranslitKoi8, ReplacementNeverGrowsTheText) { + // The substitution is a pre-pass over the UTF-8, and callers rely on the conversion to + // KOI8-R never making the string longer. So a replacement must fit in the bytes the + // character itself occupied. + for (char32_t cp = 1; cp <= 0x10FFFF; ++cp) { + if (cp >= 0xD800 && cp <= 0xDFFF) { + continue; // surrogates are not encodable + } + const char *const r = TranslitToKoi8(cp); + if (r == nullptr) { + continue; + } + std::string source; + ASSERT_GT(utf8::encode(cp, source), 0u) << "U+" << static_cast(cp); + EXPECT_LE(std::strlen(r), source.size()) + << "replacement for U+" << static_cast(cp) << " is longer than the character"; + } +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/utils.encoding.cpp b/tests/utils.encoding.cpp index 8716438155..84e28dc7cb 100644 --- a/tests/utils.encoding.cpp +++ b/tests/utils.encoding.cpp @@ -119,11 +119,13 @@ TEST(Utils_Encoding, Utf8ToKoi_YoLowercase) TEST(Utils_Encoding, Utf8ToKoi_UnknownCharReplacement) { - // Characters not in KOI8-R should be replaced with KOI8_UNKNOWN_CHAR (+) + // Characters not in KOI8-R should be replaced with KOI8_UNKNOWN_CHAR (?) // For example, Euro sign: U+20AC = UTF-8 E2 82 AC + // (This is the raw converter. native_text::to_koi8 runs the transliteration table first, + // so through that path the euro sign becomes "EUR" and never reaches the placeholder.) std::string utf8_euro = "\xE2\x82\xAC"; std::string koi_euro = Utf8ToKoi(utf8_euro.c_str()); - EXPECT_EQ("+", koi_euro); + EXPECT_EQ("?", koi_euro); } TEST(Utils_Encoding, Utf8ToKoi_BoxDrawingChars) diff --git a/tools/meson/generate_version.py b/tools/meson/generate_version.py index 851fef3164..7bc5b61358 100755 --- a/tools/meson/generate_version.py +++ b/tools/meson/generate_version.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import os import sys import subprocess from datetime import datetime @@ -20,6 +21,11 @@ except Exception: pass +# Сборка может идти там, где git недоступен или бесполезен: контейнер, worktree (у него .git -- +# файл-ссылка наружу контекста), распакованный архив. Тогда значения пробрасываются окружением. +if not git_rev or git_rev == 'unknown': + git_rev = os.environ.get('BYLINS_GIT_REV', '') or git_rev or 'unknown' + # issue.versioning: the 4th version part ("release") = total commit count -- monotonic, # +1 per commit. Computed at build time like the revision hash. If git is unavailable # (e.g. build from an unpacked archive) it falls back to 0, mirroring git_rev above. @@ -34,6 +40,9 @@ except Exception: pass +if commit_count == "0": + commit_count = os.environ.get('BYLINS_COMMIT_COUNT', '') or "0" + # major.minor.patch: explicit, from the repo-root VERSION file (single source of truth). try: with open(f'{source_root}/VERSION.txt', encoding='ascii') as vf: