diff --git a/.github/workflows/_linux.yml b/.github/workflows/_linux.yml index 8dc3ca035f..06db4c4cc9 100644 --- a/.github/workflows/_linux.yml +++ b/.github/workflows/_linux.yml @@ -90,6 +90,39 @@ jobs: - name: Run tests run: meson test -C build + utf8-flip: + name: GCC / UTF-8 flip rehearsal + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install base dependencies + uses: ./.github/actions/install-linux-deps + with: + add-universe: 'true' + + # Репетиция флипа кодировки (issue #3681). Исходники и рантайм переключаются ВМЕСТЕ: + # блобы в git и так лежат в UTF-8, поэтому снятие working-tree-encoding + повторный + # checkout даёт UTF-8-исходники, а -Dinternal_encoding=utf8 переводит на символьную + # семантику рантайм. Задача держит UTF-8-ветку компилируемой и протестированной до + # самого флипа -- иначе она не собирается вообще ничем. + - name: Check out sources as UTF-8 + run: | + sed -i -E 's|^(/(src\|tests)/\*\* +)working-tree-encoding=KOI8-R |\1|' .gitattributes + rm -rf src tests + git checkout -- src tests + file src/utils/utils.cpp | grep -q UTF-8 || { echo "sources did not become UTF-8"; exit 1; } + + - name: Configure meson + run: meson setup build --wipe -Dbuild_profile=release -Dinternal_encoding=utf8 -Dunity_size=45 + + - name: Build + run: meson compile -C build tests + + - name: Run tests + run: meson test -C build + gcc15: name: GCC 15 / Base runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0bba9fd8cf..7029a85f69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ При написании кода вы должны следовать правилу "одна команда - одна строка". Желательно это же правило распространять и на объявления переменных. -Былины используют стандарт C++17. +Былины используют стандарт C++20 (задан в meson.build: cpp_std=c++20). Тела конструкций if, if ... else for, while, do ... while всегда должны быть заключены в символы '{', '}'. diff --git a/VERSION.txt b/VERSION.txt index db7a480479..4e44204c6f 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -0.1.31 +0.1.32 \ No newline at end of file diff --git a/meson.build b/meson.build index 69a1ab0cbe..2da3288ec6 100644 --- a/meson.build +++ b/meson.build @@ -120,6 +120,11 @@ if host_system == 'windows' endif endif +# Native runtime string encoding (issue #3681). Default koi8r keeps byte semantics unchanged. +if get_option('internal_encoding') == 'utf8' + project_args += '-DINTERNAL_ENCODING_UTF8' +endif + linker_choice = get_option('linker') if linker_choice != '' add_project_link_arguments('-fuse-ld=' + linker_choice, language: 'cpp') @@ -640,6 +645,8 @@ main_sources = files( 'src/gameplay/mechanics/title.cpp', 'src/gameplay/statistics/top.cpp', 'src/utils/utils.cpp', + 'src/utils/utf8.cpp', + 'src/utils/native_text.cpp', 'src/utils/utils_encoding.cpp', 'src/gameplay/mechanics/weather.cpp', 'src/engine/olc/zedit.cpp', diff --git a/meson_options.txt b/meson_options.txt index bf8598bb5d..f870dc1133 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -10,6 +10,10 @@ option('lua_formatter', type: 'boolean', value: true, description: 'Build the em option('with_asan', type: 'boolean', value: false, description: 'Build with Address Sanitizer') option('use_pch', type: 'boolean', value: true, description: 'Use precompiled headers for circle.library') +# KOI8-R -> UTF-8 migration (issue #3681). Selects the engine's native runtime string encoding. +# koi8r = current byte semantics (default); utf8 = character semantics via the utf8:: helpers. +option('internal_encoding', type: 'combo', choices: ['koi8r', 'utf8'], value: 'koi8r', description: 'Engine-native runtime string encoding: koi8r (byte semantics, current) or utf8 (character semantics)') + # Options for Admin API and Web features option('admin_api', type: 'boolean', value: false, description: 'Enable Admin API and JSON-related features') diff --git a/src/administration/accounts.cpp b/src/administration/accounts.cpp index 5fcf86a8e7..87b4523378 100644 --- a/src/administration/accounts.cpp +++ b/src/administration/accounts.cpp @@ -3,6 +3,7 @@ * 2018 (c) bodrich */ #include "accounts.h" +#include "utils/native_text.h" #include "password.h" #include "engine/entities/zone.h" #include @@ -85,7 +86,7 @@ void Account::show_players(CharData *ch) { ss << "Данные аккаунта: " << this->email << "\r\n"; for (auto &x : this->players_list) { std::string name = GetNameByUnique(x); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); ss << count << ") " << name << "\r\n"; count++; } @@ -101,7 +102,7 @@ void Account::list_players(DescriptorData *d) { for (auto &x : this->players_list) { std::string name = GetNameByUnique(x); iosystem::write_to_output((std::to_string(count) + ") ").c_str(), d); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); iosystem::write_to_output(name.c_str(), d); iosystem::write_to_output("\r\n", d); count++; diff --git a/src/engine/boot/boot_data_files.cpp b/src/engine/boot/boot_data_files.cpp index a04ee6c62f..02b3d24630 100644 --- a/src/engine/boot/boot_data_files.cpp +++ b/src/engine/boot/boot_data_files.cpp @@ -1,4 +1,5 @@ #include "boot_data_files.h" +#include "utils/native_text.h" #include "engine/db/obj_prototypes.h" #include "engine/scripting/dg_olc.h" @@ -397,7 +398,7 @@ void WorldFile::parse_room(int virtual_nr) { world[room_realnum]->zone_rn = zone; world[room_realnum]->vnum = virtual_nr; std::string tmpstr = fread_string(); - tmpstr[0] = UPPER(tmpstr[0]); + native_text::capitalize_first(tmpstr); world[room_realnum]->set_name(tmpstr); // if (zone_table[zone].RnumRoomsLocation.first == -1) { // zone_table[zone].RnumRoomsLocation.first = room_realnum; diff --git a/src/engine/core/iosystem.cpp b/src/engine/core/iosystem.cpp index 1a75220b79..e998805265 100644 --- a/src/engine/core/iosystem.cpp +++ b/src/engine/core/iosystem.cpp @@ -7,6 +7,8 @@ */ #include "engine/core/iosystem.h" +#include +#include #include "gameplay/core/experience.h" #include "administration/privilege.h" #include "utils/utils_encoding.h" @@ -426,8 +428,16 @@ int process_input(DescriptorData *t) { // Увы, это кое-что ломает, напр. wizhelp, или "г я использую zMUD" if (t->state == EConState::kPlaying || (t->state == EConState::kExdesc)) { if (t->keytable == kCodePageWinzZ || t->keytable == kCodePageWinzOld) { - if (*(write_point - 1) == 'z') { - *(write_point - 1) = 'я'; + // Буква задана СТРОКОВЫМ литералом: он байт-прозрачен, поэтому один и тот + // же код верен и для KOI8-R (1 байт), и для UTF-8 (2 байта) -- в отличие от + // символьного литерала, который под UTF-8 не помещается в char (issue #3681). + static const std::string_view kYaLetter = "я"; + if (*(write_point - 1) == 'z' && space_left + 1 >= kYaLetter.size()) { + --write_point; + ++space_left; + std::memcpy(write_point, kYaLetter.data(), kYaLetter.size()); + write_point += kYaLetter.size(); + space_left -= kYaLetter.size(); } } } diff --git a/src/engine/db/db.cpp b/src/engine/db/db.cpp index 912ac9efba..4d0538e6dd 100644 --- a/src/engine/db/db.cpp +++ b/src/engine/db/db.cpp @@ -1,4 +1,5 @@ #include +#include "utils/native_text.h" #include "gameplay/affects/affect_messages.h" #include "gameplay/abilities/feats.h" // issue.perk-action-patching: BuildTalentPatchIndex #include "utils/utils_encoding.h" @@ -3595,7 +3596,7 @@ Rooms::~Rooms() { int get_filename(const char *orig_name, char *filename, int mode) { const char *prefix, *middle, *suffix; - char name[64], *ptr; + char name[64]; if (orig_name == nullptr || *orig_name == '\0' || filename == nullptr) { log("SYSERR: NULL pointer or empty string passed to get_filename(), %p or %p.", orig_name, filename); @@ -3630,13 +3631,9 @@ int get_filename(const char *orig_name, char *filename, int mode) { default: return (0); } - strcpy(name, orig_name); - for (ptr = name; *ptr; ptr++) { - if (*ptr == 'Ё' || *ptr == 'ё') - *ptr = '9'; - else - *ptr = LOWER(codepages::AtoL(*ptr)); - } + // Транслитерация вынесена в native_text: имя файла игрока обязано совпадать до и после + // смены кодировки, иначе сохранёнки перестанут находиться (issue #3681). + strcpy(name, native_text::translit_to_filename(orig_name).c_str()); switch (LOWER(*name)) { case 'a': diff --git a/src/engine/db/obj_save.cpp b/src/engine/db/obj_save.cpp index 7443a64f6a..2f911620d5 100644 --- a/src/engine/db/obj_save.cpp +++ b/src/engine/db/obj_save.cpp @@ -10,6 +10,7 @@ // * AutoEQ by Burkhard Knopf #include "engine/core/char_handler.h" +#include "utils/native_text.h" #include "gameplay/mechanics/equipment.h" #include "obj_save.h" #include "gameplay/mechanics/groups.h" @@ -1628,7 +1629,7 @@ int Crash_load(CharData *ch) { } std::string cap = obj->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); // Предмет разваливается от старости if (obj->get_timer() <= 0) { diff --git a/src/engine/db/sqlite_world_data_source.cpp b/src/engine/db/sqlite_world_data_source.cpp index 2688bc2b3a..d2adfa9964 100644 --- a/src/engine/db/sqlite_world_data_source.cpp +++ b/src/engine/db/sqlite_world_data_source.cpp @@ -4,6 +4,7 @@ #ifdef HAVE_SQLITE #include "sqlite_world_data_source.h" +#include "utils/native_text.h" #include "utils/utils_encoding.h" #include "db.h" #include "obj_prototypes.h" @@ -1256,7 +1257,7 @@ std::vector SqliteWorldDataSource::LoadRooms(const std::vector auto room = new RoomData; room->vnum = vnum; // Apply UPPER to first character (same as Legacy loader) - if (!name.empty()) { name[0] = UPPER(name[0]); } + if (!name.empty()) { native_text::capitalize_first(name); } room->set_name(name); if (!description.empty()) diff --git a/src/engine/db/yaml_world_data_source.cpp b/src/engine/db/yaml_world_data_source.cpp index 22fb053769..142442fdd4 100644 --- a/src/engine/db/yaml_world_data_source.cpp +++ b/src/engine/db/yaml_world_data_source.cpp @@ -4,6 +4,7 @@ #ifdef HAVE_YAML #include "yaml_world_data_source.h" +#include "utils/native_text.h" #include "utils/utils_encoding.h" #include "dictionary_loader.h" #include "db.h" @@ -700,8 +701,9 @@ std::string YamlWorldDataSource::GetText(const YAML::Node &node, const std::stri { if (node[key]) { - // YAML files are already in KOI8-R, no conversion needed - std::string text = node[key].as(); + // Файлы мира лежат на диске в KOI8-R; переводим в нативную кодировку движка + // (под KOI8-R это тождество, под UTF-8 - перекодировка). Issue #3681. + std::string text = native_text::from_koi8(node[key].as()); // Convert line endings if configured for DOS format if (m_convert_lf_to_crlf) { @@ -1425,7 +1427,7 @@ RoomData* YamlWorldDataSource::ParseRoomNode(const YAML::Node &root, int vnum, i room->zone_rn = zone_rnum; std::string name = GetText(root, "name", "Untitled Room"); - if (!name.empty()) { name[0] = UPPER(name[0]); } + if (!name.empty()) { native_text::capitalize_first(name); } room->set_name(name); std::string description = GetText(root, "description", ""); diff --git a/src/engine/olc/medit.cpp b/src/engine/olc/medit.cpp index 0298cdc4f4..5d78a4cd9d 100644 --- a/src/engine/olc/medit.cpp +++ b/src/engine/olc/medit.cpp @@ -8,6 +8,8 @@ ***************************************************************************/ #include "engine/db/world_characters.h" +#include "utils/native_text.h" +#include "utils/russian_keys.h" #include "gameplay/affects/affect_messages.h" #include "gameplay/fight/fight_messages.h" #include "engine/entities/obj_data.h" @@ -1296,11 +1298,11 @@ void medit_parse(DescriptorData *d, char *arg) { case MEDIT_CONFIRM_SAVESTRING: // * Ensure mob has MOB_ISNPC set or things will go pair shaped. OLC_MOB(d)->SetFlag(EMobFlag::kNpc); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': + case rus::kDe: + case rus::kDeUpper: // * Save the mob in memory and to disk. // SendMsgToChar("Saving mobile to memory a.\r\n", d->character.get()); medit_save_internally(d); @@ -1314,8 +1316,8 @@ void medit_parse(DescriptorData *d, char *arg) { case 'n': case 'N': - case 'н': - case 'Н': cleanup_olc(d, CLEANUP_ALL); + case rus::kEn: + case rus::kEnUpper: cleanup_olc(d, CLEANUP_ALL); break; default: SendMsgToChar("Неверный выбор!\r\n", d->character.get()); @@ -1327,7 +1329,7 @@ void medit_parse(DescriptorData *d, char *arg) { //------------------------------------------------------------------- case MEDIT_MAIN_MENU: i = 0; olc_log("%s command %c", GET_NAME(d->character), *arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'q': case 'Q': if (OLC_VAL(d)) // Anything been changed? @@ -1543,118 +1545,118 @@ void medit_parse(DescriptorData *d, char *arg) { medit_disp_helpers(d); return; - case 'а': - case 'А': OLC_MODE(d) = MEDIT_SKILLS; + case rus::kA: + case rus::kAUpper: OLC_MODE(d) = MEDIT_SKILLS; medit_disp_skills(d); return; - case 'б': - case 'Б': OLC_MODE(d) = MEDIT_SPELLS; + case rus::kBe: + case rus::kBeUpper: OLC_MODE(d) = MEDIT_SPELLS; medit_disp_spells(d); return; - case 'в': - case 'В': OLC_MODE(d) = MEDIT_STR; + case rus::kVe: + case rus::kVeUpper: OLC_MODE(d) = MEDIT_STR; i++; break; - case 'г': - case 'Г': OLC_MODE(d) = MEDIT_DEX; + case rus::kGe: + case rus::kGeUpper: OLC_MODE(d) = MEDIT_DEX; i++; break; - case 'д': - case 'Д': OLC_MODE(d) = MEDIT_CON; + case rus::kDe: + case rus::kDeUpper: OLC_MODE(d) = MEDIT_CON; i++; break; - case 'е': - case 'Е': OLC_MODE(d) = MEDIT_WIS; + case rus::kIe: + case rus::kIeUpper: OLC_MODE(d) = MEDIT_WIS; i++; break; - case 'ж': - case 'Ж': OLC_MODE(d) = MEDIT_INT; + case rus::kZhe: + case rus::kZheUpper: OLC_MODE(d) = MEDIT_INT; i++; break; - case 'з': - case 'З': OLC_MODE(d) = MEDIT_CHA; + case rus::kZe: + case rus::kZeUpper: OLC_MODE(d) = MEDIT_CHA; i++; break; - case 'и': - case 'И': OLC_MODE(d) = MEDIT_HEIGHT; + case rus::kI: + case rus::kIUpper: OLC_MODE(d) = MEDIT_HEIGHT; i++; break; - case 'к': - case 'К': OLC_MODE(d) = MEDIT_WEIGHT; + case rus::kKa: + case rus::kKaUpper: OLC_MODE(d) = MEDIT_WEIGHT; i++; break; - case 'л': - case 'Л': OLC_MODE(d) = MEDIT_SIZE; + case rus::kEl: + case rus::kElUpper: OLC_MODE(d) = MEDIT_SIZE; i++; break; - case 'м': - case 'М': OLC_MODE(d) = MEDIT_EXTRA; + case rus::kEm: + case rus::kEmUpper: OLC_MODE(d) = MEDIT_EXTRA; i++; break; - case 'х': - case 'Х': OLC_MODE(d) = MEDIT_REMORT; + case rus::kHa: + case rus::kHaUpper: OLC_MODE(d) = MEDIT_REMORT; i++; break; - case 'Ю': - case 'ю': OLC_MODE(d) = MEDIT_MAXFACTOR; + case rus::kYuUpper: + case rus::kYu: OLC_MODE(d) = MEDIT_MAXFACTOR; i++; break; - case 'н': - case 'Н': SendMsgToChar(d->character.get(), "\r\nВведите новое значение от 0 до 100%% :"); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar(d->character.get(), "\r\nВведите новое значение от 0 до 100%% :"); OLC_MODE(d) = MEDIT_LIKE; return; - case 'п': - case 'П': OLC_MODE(d) = MEDIT_DLIST_MENU; + case rus::kPe: + case rus::kPeUpper: OLC_MODE(d) = MEDIT_DLIST_MENU; disp_dl_list(d); return; - case 'р': - case 'Р': OLC_MODE(d) = MEDIT_ROLE; + case rus::kEr: + case rus::kErUpper: OLC_MODE(d) = MEDIT_ROLE; medit_disp_role(d); return; - case 'с': - case 'С': OLC_MODE(d) = MEDIT_RESISTANCES; + case rus::kEs: + case rus::kEsUpper: OLC_MODE(d) = MEDIT_RESISTANCES; medit_disp_resistances(d); return; - case 'т': - case 'Т': OLC_MODE(d) = MEDIT_SAVES; + case rus::kTe: + case rus::kTeUpper: OLC_MODE(d) = MEDIT_SAVES; medit_disp_saves(d); return; - case 'у': - case 'У': OLC_MODE(d) = MEDIT_ADD_PARAMETERS; + case rus::kU: + case rus::kUUpper: OLC_MODE(d) = MEDIT_ADD_PARAMETERS; medit_disp_add_parameters(d); return; - case 'ф': - case 'Ф': OLC_MODE(d) = MEDIT_FEATURES; + case rus::kEf: + case rus::kEfUpper: OLC_MODE(d) = MEDIT_FEATURES; medit_disp_features(d); return; - case 'ц': - case 'Ц': OLC_MODE(d) = MEDIT_RACE; + case rus::kTse: + case rus::kTseUpper: OLC_MODE(d) = MEDIT_RACE; medit_disp_race(d); return; - case 'ч': - case 'Ч': OLC_MODE(d) = MEDIT_CLONE; + case rus::kChe: + case rus::kCheUpper: OLC_MODE(d) = MEDIT_CLONE; medit_disp_clone_menu(d); return; @@ -2172,9 +2174,9 @@ void medit_parse(DescriptorData *d, char *arg) { case MEDIT_DLIST_MENU: if (*arg) { // Обрабатываем комнады добавить удалить и.т.п - switch (*arg) { - case 'а': - case 'А': + switch (native_text::first_char_code(arg)) { + case rus::kA: + case rus::kAUpper: // Добавляем запись. OLC_MODE(d) = MEDIT_DLIST_ADD; SendMsgToChar("\r\nVNUM - виртуальный номер прототипа\r\n" @@ -2193,8 +2195,8 @@ void medit_parse(DescriptorData *d, char *arg) { return; - case 'б': - case 'Б': + case rus::kBe: + case rus::kBeUpper: // Удаляем запись. OLC_MODE(d) = MEDIT_DLIST_DEL; SendMsgToChar("\r\nВведите номер удаляемой записи:", d->character.get()); diff --git a/src/engine/olc/oedit.cpp b/src/engine/olc/oedit.cpp index dedc6aa740..31b4c605f3 100644 --- a/src/engine/olc/oedit.cpp +++ b/src/engine/olc/oedit.cpp @@ -9,6 +9,8 @@ ************************************************************************/ #include "engine/db/world_objects.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "gameplay/fight/fight_messages.h" #include "engine/db/obj_prototypes.h" #include "engine/core/conf.h" @@ -1486,11 +1488,11 @@ void oedit_parse(DescriptorData *d, char *arg) { switch (OLC_MODE(d)) { case OEDIT_CONFIRM_SAVESTRING: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': SendMsgToChar("Объект сохранен.\r\n", d->character.get()); + case rus::kDe: + case rus::kDeUpper: SendMsgToChar("Объект сохранен.\r\n", d->character.get()); OLC_OBJ(d)->remove_incorrect_values_keys(OLC_OBJ(d)->get_type()); oedit_save_internally(d); snprintf(buf, sizeof(buf), "OLC: %s edits obj %d", GET_NAME(d->character), OLC_NUM(d)); @@ -1501,8 +1503,8 @@ void oedit_parse(DescriptorData *d, char *arg) { case 'n': case 'N': - case 'н': - case 'Н': cleanup_olc(d, CLEANUP_ALL); + case rus::kEn: + case rus::kEnUpper: cleanup_olc(d, CLEANUP_ALL); break; default: SendMsgToChar("Неверный выбор!\r\n", d->character.get()); diff --git a/src/engine/olc/redit.cpp b/src/engine/olc/redit.cpp index 78b8008b41..f88147a984 100644 --- a/src/engine/olc/redit.cpp +++ b/src/engine/olc/redit.cpp @@ -9,6 +9,8 @@ ************************************************************************/ #include "engine/entities/obj_data.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/core/comm.h" #include "engine/db/db.h" #include "engine/db/world_data_source_manager.h" @@ -564,11 +566,11 @@ void redit_parse(DescriptorData *d, char *arg) { switch (OLC_MODE(d)) { case REDIT_CONFIRM_SAVESTRING: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': redit_save_internally(d); + case rus::kDe: + case rus::kDeUpper: redit_save_internally(d); snprintf(buf, sizeof(buf), "OLC: %s edits room %d.", GET_NAME(d->character), OLC_NUM(d)); olc_log("%s edit room %d", GET_NAME(d->character), OLC_NUM(d)); mudlog(buf, NRM, std::max(kLvlBuilder, GET_INVIS_LEV(d->character)), SYSLOG, true); @@ -579,8 +581,8 @@ void redit_parse(DescriptorData *d, char *arg) { case 'n': case 'N': - case 'н': - case 'Н': + case rus::kEn: + case rus::kEnUpper: // * Free everything up, including strings, etc. cleanup_olc(d, CLEANUP_ALL); break; diff --git a/src/engine/olc/zedit.cpp b/src/engine/olc/zedit.cpp index c69ca37498..c41407d961 100644 --- a/src/engine/olc/zedit.cpp +++ b/src/engine/olc/zedit.cpp @@ -5,6 +5,8 @@ ************************************************************************/ #include "engine/db/obj_prototypes.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/entities/obj_data.h" #include "engine/core/comm.h" #include "engine/db/db.h" @@ -1370,11 +1372,11 @@ void zedit_parse(DescriptorData *d, char *arg) { switch (OLC_MODE(d)) { case ZEDIT_CONFIRM_SAVESTRING: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': + case rus::kDe: + case rus::kDeUpper: // * Save the zone in memory, hiding invisible people. SendMsgToChar("Зона сохранена.\r\n", d->character.get()); zedit_save_internally(d); @@ -1384,8 +1386,8 @@ void zedit_parse(DescriptorData *d, char *arg) { // FALL THROUGH case 'n': case 'N': - case 'н': - case 'Н': cleanup_olc(d, CLEANUP_ALL); + case rus::kEn: + case rus::kEnUpper: cleanup_olc(d, CLEANUP_ALL); break; default: SendMsgToChar("Неверный выбор!\r\n", d->character.get()); SendMsgToChar("Вы желаете сохранить зону? : ", d->character.get()); diff --git a/src/engine/scripting/dg_scripts.cpp b/src/engine/scripting/dg_scripts.cpp index 017c25a373..e2aa3c4be7 100644 --- a/src/engine/scripting/dg_scripts.cpp +++ b/src/engine/scripting/dg_scripts.cpp @@ -31,6 +31,7 @@ #include "gameplay/mechanics/illumination.h" #include "gameplay/mechanics/inventory.h" #include "utils/utils_parse.h" +#include "utils/native_text.h" #include "dg_event.h" #include "engine/ui/color.h" #include "gameplay/clans/house.h" @@ -4312,8 +4313,13 @@ int eval_lhs_op_rhs(const char *expr, char *result, size_t result_size, void *go p = matching_paren(p) + 1; else if (*p == '"') p = matching_quote(p) + 1; - else if (a_isalnum(*p)) - for (p++; *p && (a_isalnum(*p) || isspace(*p)); p++); + // Step over whole characters (issue #3681): a byte-wise scan ends a token in the middle + // of a multibyte letter. isspace() takes an unsigned value -- a raw char is negative for + // any non-ASCII byte, which is undefined behaviour. + else if (native_text::is_alnum_char(p)) + for (p += native_text::char_bytes(p); + *p && (native_text::is_alnum_char(p) || isspace(static_cast(*p))); + p += native_text::char_bytes(p)); else p++; } @@ -6251,8 +6257,8 @@ void do_tlist(CharData *ch, char *argument, int cmd, int/* subcmd*/) { char trgtypes[256]; for (; nr < top_of_trigt && (trig_index[nr]->vnum <= last); nr++) { std::string out = ""; - snprintf(buf, sizeof(buf), "%2d) [%5d] %-50s ", ++found, - trig_index[nr]->vnum, trig_index[nr]->proto->get_name().c_str()); + strcpy(buf, fmt::format("{:2}) [{:5}] {:<50} ", ++found, + trig_index[nr]->vnum, trig_index[nr]->proto->get_name()).c_str()); out += buf; if (trig_index[nr]->proto->get_attach_type() == MOB_TRIGGER) { sprintbit(trig_index[nr]->proto->get_trigger_type(), trig_types, trgtypes, sizeof(trgtypes)); diff --git a/src/engine/ui/cmd/do_affects.cpp b/src/engine/ui/cmd/do_affects.cpp index db94732efa..9100bc4a11 100644 --- a/src/engine/ui/cmd/do_affects.cpp +++ b/src/engine/ui/cmd/do_affects.cpp @@ -3,6 +3,7 @@ // #include "engine/entities/char_data.h" +#include #include "gameplay/affects/affect_messages.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -65,9 +66,12 @@ void do_affects(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { (mod + 1) / kSecsPerMudHour + 1, grammar::GetDeclensionInNumber((mod + 1) / kSecsPerMudHour + 1, grammar::EWhat::kHour)) : sprintf(buf2, "(менее часа)"); - snprintf(buf, kMaxStringLength, "%s%s%-21s %-12s%s ", + // Ширина колонок - в символах, а не в байтах (issue #3681). + strcpy(buf, fmt::format("{}{}{:<21} {:<12}{} ", *sp_name == '!' ? "Состояние : " : "Заклинание : ", - kColorBoldCyn, sp_name, buf2, kColorNrm); + kColorBoldCyn, + sp_name, + buf2, kColorNrm).c_str()); *buf2 = '\0'; if (!privilege::IsImmortal(ch)) { auto next_affect_i = affect_i; diff --git a/src/engine/ui/cmd/do_alias.cpp b/src/engine/ui/cmd/do_alias.cpp index de102d0e2f..c7421ee0e1 100644 --- a/src/engine/ui/cmd/do_alias.cpp +++ b/src/engine/ui/cmd/do_alias.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include #include "engine/ui/alias.h" void do_alias(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { @@ -24,8 +25,7 @@ void do_alias(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar(" Нет алиасов.\r\n", ch); else { while (a != nullptr) { - sprintf(buf, "%-15s %s\r\n", a->alias, a->replacement); - SendMsgToChar(buf, ch); + SendMsgToChar(fmt::format("{:<15} {}\r\n", a->alias, a->replacement), ch); a = a->next; } } diff --git a/src/engine/ui/cmd/do_commands.cpp b/src/engine/ui/cmd/do_commands.cpp index 85c460ba04..3520f446bc 100644 --- a/src/engine/ui/cmd/do_commands.cpp +++ b/src/engine/ui/cmd/do_commands.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include #include "administration/privilege.h" #include "gameplay/communication/social.h" #include "engine/db/global_objects.h" @@ -37,7 +38,7 @@ void do_commands(CharData *ch, char *argument, int/* cmd*/, int subcmd) { continue; } for (const auto &kw : soc.GetKeywords()) { - sprintf(buf + strlen(buf), "%-19s", kw.c_str()); + strcat(buf, fmt::format("{:<19}", kw).c_str()); if (!(no % 4)) strcat(buf, "\r\n"); no++; @@ -49,13 +50,13 @@ void do_commands(CharData *ch, char *argument, int/* cmd*/, int subcmd) { i = cmd_sort_info[cmd_num].sort_pos; if (wizhelp) { if (privilege::HasPrivilege(vict, std::string(cmd_info[i].command), 0, 0, 0)) { - sprintf(buf + strlen(buf), "%-15s", cmd_info[i].command); + strcat(buf, fmt::format("{:<15}", cmd_info[i].command).c_str()); if (!(no % 5)) strcat(buf, "\r\n"); no++; } } else if (cmd_info[i].minimum_level >= 0 && (static_cast(socials) == cmd_sort_info[i].is_social)) { - sprintf(buf + strlen(buf), "%-15s", cmd_info[i].command); + strcat(buf, fmt::format("{:<15}", cmd_info[i].command).c_str()); if (!(no % 5)) strcat(buf, "\r\n"); no++; diff --git a/src/engine/ui/cmd/do_display.cpp b/src/engine/ui/cmd/do_display.cpp index ef7ba374cd..6cc71f950e 100644 --- a/src/engine/ui/cmd/do_display.cpp +++ b/src/engine/ui/cmd/do_display.cpp @@ -7,6 +7,8 @@ */ #include "engine/entities/char_data.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "administration/privilege.h" const char *DISPLAY_HELP = "Формат: статус { { Ж | Э | З | В | Д | У | О | Б | П | К } | все | нет }\r\n"; @@ -38,35 +40,35 @@ void do_display(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { const size_t len = strlen(argument); for (size_t i = 0; i < len; i++) { - switch (LOWER(argument[i])) { + switch (native_text::first_char_code_lower(argument + i)) { case 'h': - case 'ж': ch->SetFlag(EPrf::kDispHp); + case rus::kZhe: ch->SetFlag(EPrf::kDispHp); break; case 'w': - case 'з': ch->SetFlag(EPrf::kDispMana); + case rus::kZe: ch->SetFlag(EPrf::kDispMana); break; case 'm': - case 'э': ch->SetFlag(EPrf::kDispMove); + case rus::kE: ch->SetFlag(EPrf::kDispMove); break; case 'e': - case 'в': ch->SetFlag(EPrf::kDispExits); + case rus::kVe: ch->SetFlag(EPrf::kDispExits); break; case 'g': - case 'д': ch->SetFlag(EPrf::kDispMoney); + case rus::kDe: ch->SetFlag(EPrf::kDispMoney); break; case 'l': - case 'у': ch->SetFlag(EPrf::kDispLvl); + case rus::kU: ch->SetFlag(EPrf::kDispLvl); break; case 'x': - case 'о': ch->SetFlag(EPrf::kDispExp); + case rus::kO: ch->SetFlag(EPrf::kDispExp); break; - case 'б': + case rus::kBe: case 'f': ch->SetFlag(EPrf::kDispFight); break; - case 'п': + case rus::kPe: case 't': ch->SetFlag(EPrf::kDispTimed); break; - case 'к': + case rus::kKa: case 'c': ch->SetFlag(EPrf::kDispCooldowns); break; case ' ': break; diff --git a/src/engine/ui/cmd/do_exits.cpp b/src/engine/ui/cmd/do_exits.cpp index e5549e9edf..16ebec4a32 100644 --- a/src/engine/ui/cmd/do_exits.cpp +++ b/src/engine/ui/cmd/do_exits.cpp @@ -6,6 +6,7 @@ */ #include "engine/entities/char_data.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/sight.h" #include "gameplay/mechanics/illumination.h" @@ -29,10 +30,10 @@ void DoExits(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/) { for (door = 0; door < EDirection::kMaxDirNum; door++) if (EXIT(ch, door) && EXIT(ch, door)->to_room() != kNowhere && !EXIT_FLAGGED(EXIT(ch, door), EExitFlag::kClosed)) { if (privilege::IsGod(ch)) - sprintf(buf2, "%-6s - [%5d] %s\r\n", dirs_rus[door], - GET_ROOM_VNUM(EXIT(ch, door)->to_room()), world[EXIT(ch, door)->to_room()]->name); + strcpy(buf2, fmt::format("{:<6} - [{:5}] {}\r\n", dirs_rus[door], + GET_ROOM_VNUM(EXIT(ch, door)->to_room()), world[EXIT(ch, door)->to_room()]->name).c_str()); else { - sprintf(buf2, "%-6s - ", dirs_rus[door]); + strcpy(buf2, fmt::format("{:<6} - ", dirs_rus[door]).c_str()); if (is_dark(EXIT(ch, door)->to_room()) && !sight::CanSeeInDark(ch)) strcat(buf2, "слишком темно\r\n"); else { diff --git a/src/engine/ui/cmd/do_features.cpp b/src/engine/ui/cmd/do_features.cpp index 406081057e..01f03bce12 100644 --- a/src/engine/ui/cmd/do_features.cpp +++ b/src/engine/ui/cmd/do_features.cpp @@ -1,4 +1,5 @@ #include "engine/ui/color.h" +#include #include "gameplay/core/remort.h" #include "engine/entities/char_data.h" #include "gameplay/abilities/timed_abilities.h" @@ -81,17 +82,17 @@ void DisplayFeats(CharData *ch, CharData *vict, bool all_feats) { continue; } if (!ch->IsFlagged(EPrf::kBlindMode)) { - sprintf(buf, " %s%s %-30s%s\r\n", + strcpy(buf, fmt::format(" {}{} {:<30}{}\r\n", ch->HaveFeat(feat.GetId()) ? kColorGrn : CanGetFeat(ch, feat.GetId()) ? kColorNrm : kColorRed, ch->HaveFeat(feat.GetId()) ? "[И]" : CanGetFeat(ch, feat.GetId()) ? "[Д]" : "[Н]", - MUD::Feat(feat.GetId()).GetCName(), kColorNrm); + MUD::Feat(feat.GetId()).GetCName(), kColorNrm).c_str()); } else { - sprintf(buf, " %s %-30s\r\n", + strcpy(buf, fmt::format(" {} {:<30}\r\n", ch->HaveFeat(feat.GetId()) ? "[И]" : CanGetFeat(ch, feat.GetId()) ? "[Д]" : "[Н]", - MUD::Feat(feat.GetId()).GetCName()); + MUD::Feat(feat.GetId()).GetCName()).c_str()); } if (feat.IsInborn() || diff --git a/src/engine/ui/cmd/do_gen_comm.cpp b/src/engine/ui/cmd/do_gen_comm.cpp index 83c1138185..30c2dc3f15 100644 --- a/src/engine/ui/cmd/do_gen_comm.cpp +++ b/src/engine/ui/cmd/do_gen_comm.cpp @@ -8,6 +8,7 @@ #include "do_gen_comm.h" #include "administration/privilege.h" +#include "utils/native_text.h" #include "utils/grammar/gender.h" #include "gameplay/mechanics/sight.h" @@ -156,17 +157,25 @@ void do_gen_comm(CharData *ch, char *argument, int/* cmd*/, int subcmd) { int bad_simb_cnt = 0, bad_seq_cnt = 0; // фильтруем верхний регистр - for (int k = 0; argument[k] != '\0'; k++) { - if (a_isupper(argument[k])) { + // Counted and folded per character (issue #3681): a byte-wise scan cannot see an + // uppercase Cyrillic letter at all, so the filter silently stopped working for Russian. + // The denominator is the character count for the same reason -- with byte lengths the + // percentage would be halved for Russian text. + const size_t total_chars = native_text::char_count(argument); + for (auto letter : native_text::chars(argument)) { + if (native_text::is_upper_char(letter.data())) { bad_simb_cnt++; bad_seq_cnt++; } else bad_seq_cnt = 0; if ((bad_seq_cnt > 1) && - (((bad_simb_cnt * 100 / strlen(argument)) > bad_smb_procent) || - (bad_seq_cnt > MAX_UPPERS_SEQ_CHAR))) - argument[k] = a_lcc(argument[k]); + (((bad_simb_cnt * 100 / total_chars) > bad_smb_procent) || + (bad_seq_cnt > MAX_UPPERS_SEQ_CHAR))) { + // letter указывает внутрь argument; свёртка регистра не меняет длину. + char *at = argument + (letter.data() - argument); + native_text::copy_lower_char(at, at); + } } // фильтруем одинаковые сообщения в эфире if (!str_cmp(ch->get_last_tell().c_str(), argument)) { @@ -293,7 +302,7 @@ std::string format_gossip_name(CharData *ch, CharData *vict) { return ""; } std::string name = privilege::IsImmortal(ch) ? GET_NAME(ch) : sight::PersonName(ch, vict, 0); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); return name; } diff --git a/src/engine/ui/cmd/do_ignore.cpp b/src/engine/ui/cmd/do_ignore.cpp index 8aa55b0470..2ba9d612f3 100644 --- a/src/engine/ui/cmd/do_ignore.cpp +++ b/src/engine/ui/cmd/do_ignore.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "utils/utils_string.h" #include "utils/utils.h" #include "engine/core/comm.h" @@ -43,7 +44,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { strcpy(name, "Все"); } else { strcpy(name, ign_find_name(ignore->id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); } sprintf(buf, " %s: ", name); SendMsgToChar(buf, ch); @@ -146,7 +147,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar("Вы и так не игнорируете всех сразу.\r\n", ch); } else { strcpy(name, ign_find_name(vict_id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); sprintf(buf, "Вы и так не игнорируете " "персонажа %s%s%s.\r\n", kColorWht, name, kColorNrm); @@ -173,7 +174,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar(buf, ch); } else { strcpy(name, ign_find_name(ignore->id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); sprintf(buf, "Для персонажа %s%s%s вы игнорируете:%s.\r\n", kColorWht, name, kColorNrm, text_ignore_modes(ignore->mode, buf1)); SendMsgToChar(buf, ch); @@ -183,7 +184,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar("Вы больше не игнорируете всех сразу.\r\n", ch); } else { strcpy(name, ign_find_name(vict_id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); sprintf(buf, "Вы больше не игнорируете персонажа %s%s%s.\r\n", kColorWht, name, kColorNrm); SendMsgToChar(buf, ch); diff --git a/src/engine/ui/cmd/do_score.cpp b/src/engine/ui/cmd/do_score.cpp index 53e91692ce..bd877500ba 100644 --- a/src/engine/ui/cmd/do_score.cpp +++ b/src/engine/ui/cmd/do_score.cpp @@ -232,12 +232,12 @@ void PrintScoreList(CharData *ch) { } else if (NAME_BAD(ch)) { SendMsgToChar(ch, "ВНИМАНИЕ! ваше имя запрещено богами. Очень скоро вы прекратите получать опыт.\r\n"); } - SendMsgToChar(ch, "Вы можете вступить в группу с максимальной разницей в %2d %-75s\r\n", + SendMsgToChar(fmt::format("Вы можете вступить в группу с максимальной разницей в {:2} {:<75.76}\r\n", grouping[ch->GetClass()][static_cast(remort::GetRealRemort(ch))], - (std::string( + grammar::GetDeclensionInNumber(grouping[ch->GetClass()][static_cast(remort::GetRealRemort( ch))], grammar::EWhat::kLvl) - + std::string(" без потерь для опыта.")).substr(0, 76).c_str())); + + std::string(" без потерь для опыта.")), ch); SendMsgToChar(ch, "Вы можете принять в группу максимум %d соратников.\r\n", group::max_group_size(ch)); std::ostringstream out; diff --git a/src/engine/ui/cmd/do_skills.cpp b/src/engine/ui/cmd/do_skills.cpp index a3ff7c1d74..fbdccf49e3 100644 --- a/src/engine/ui/cmd/do_skills.cpp +++ b/src/engine/ui/cmd/do_skills.cpp @@ -1,4 +1,5 @@ #include "do_skills.h" +#include #include "engine/ui/color.h" #include "engine/entities/char_data.h" @@ -77,12 +78,13 @@ void DisplaySkills(CharData *ch, CharData *vict, const char *filter/* = nullptr* default: sprintf(buf, " "); } - sprintf(buf + strlen(buf), "%-23s %s (%d)%s \r\n", + // Ширина колонки - в символах, а не в байтах (issue #3681). + strcat(buf, fmt::format("{:<23} {} ({}){} \r\n", skill.GetName(), how_good(GetSkill(ch, skill_id), CalcSkillHardCap(ch, skill_id)), GetTrainedSkill(ch, skill_id) == 0 ? GetEquippedSkill(ch, skill_id) : std::min(CalcSkillMinCap(ch, skill_id) + GetEquippedSkill(ch, skill_id), MUD::Skill(skill_id).cap), - kColorNrm); + kColorNrm).c_str()); skills_names.emplace_back(buf); i++; } diff --git a/src/engine/ui/cmd/do_spells.cpp b/src/engine/ui/cmd/do_spells.cpp index 690bfbb6a2..1ebb0dd5fe 100644 --- a/src/engine/ui/cmd/do_spells.cpp +++ b/src/engine/ui/cmd/do_spells.cpp @@ -1,4 +1,5 @@ #include "do_spells.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/magic_item.h" @@ -87,16 +88,19 @@ void DisplaySpells(CharData *ch, CharData *vict, bool all) { if (CalcSpellManacost(ch, spell_id) > Mana(GetRealWis(ch))) continue; if (CheckRecipeItems(ch, spell_id, ESpellType::kRunes, false)) { - slots[slot_num] += sprintf(names[slot_num] + slots[slot_num], - "%s|<...%4d.> %s%-38s&n|", - slots[slot_num] % 114 < - 10 ? "\r\n" : " ", - CalcSpellManacost(ch, spell_id), GetSpellColor(spell_id), MUD::Spell(spell_id).GetCName()); + const auto line = fmt::format("{}|<...{:4}.> {}{:<38}&n|", + slots[slot_num] % 114 < 10 ? "\r\n" : " ", + CalcSpellManacost(ch, spell_id), GetSpellColor(spell_id), + MUD::Spell(spell_id).GetCName()); + strcpy(names[slot_num] + slots[slot_num], line.c_str()); + slots[slot_num] += static_cast(line.size()); } else { if (all) { - slots[slot_num] += sprintf(names[slot_num] + slots[slot_num], - "%s|+--------+ %s%-38s&n|", slots[slot_num] % 114 < 10 ? "\r\n" - : " ", GetSpellColor(spell_id), MUD::Spell(spell_id).GetCName()); + const auto line = fmt::format("{}|+--------+ {}{:<38}&n|", + slots[slot_num] % 114 < 10 ? "\r\n" : " ", GetSpellColor(spell_id), + MUD::Spell(spell_id).GetCName()); + strcpy(names[slot_num] + slots[slot_num], line.c_str()); + slots[slot_num] += static_cast(line.size()); } } } else { diff --git a/src/engine/ui/cmd/do_where.cpp b/src/engine/ui/cmd/do_where.cpp index 73ad5cb5e5..2dd5fc28e6 100644 --- a/src/engine/ui/cmd/do_where.cpp +++ b/src/engine/ui/cmd/do_where.cpp @@ -3,6 +3,7 @@ // #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/world_objects.h" #include "gameplay/economics/exchange.h" @@ -145,8 +146,8 @@ void PerformMortalWhere(CharData *ch, char *arg) { continue; } - sprintf(buf, "%-20s - %s\r\n", GET_NAME(i), world[i->in_room]->name); - SendMsgToChar(buf, ch); + // Ширина колонки - в символах, а не в байтах (issue #3681). + SendMsgToChar(fmt::format("{:<20} - {}\r\n", GET_NAME(i), world[i->in_room]->name), ch); } } else // print only FIRST char, not all. { @@ -165,8 +166,7 @@ void PerformMortalWhere(CharData *ch, char *arg) { continue; } - sprintf(buf, "%-25s - %s\r\n", GET_NAME(i), world[i->in_room]->name); - SendMsgToChar(buf, ch); + SendMsgToChar(fmt::format("{:<25} - {}\r\n", GET_NAME(i), world[i->in_room]->name), ch); return; } SendMsgToChar("Никого похожего с этим именем нет.\r\n", ch); @@ -196,9 +196,11 @@ std::string where_format::FormatWhere(const std::vector &rows) { for (const auto &row : rows) { const std::string prefix = fmt::format("{:>{}}. {:<5} [{:>7}] {:<25} - ", row.num, num_width, RowKindLabel(row.kind), row.vnum, row.name); - // Отступ строк-продолжений = длине префикса, чтобы разделитель " - " - // встал ровно под разделителем первой строки. - const std::string cont = fmt::format("{:>{}}", " - ", static_cast(prefix.size())); + // Отступ строк-продолжений = ШИРИНЕ префикса в символах, чтобы разделитель " - " + // встал ровно под разделителем первой строки. Именно в символах, а не в байтах: + // prefix.size() под UTF-8 больше числа колонок, и отступ уезжал (issue #3681). + const std::string cont = + fmt::format("{:>{}}", " - ", static_cast(native_text::char_count(prefix))); out += prefix; if (!row.location_lines.empty()) { diff --git a/src/engine/ui/cmd/do_who.cpp b/src/engine/ui/cmd/do_who.cpp index 6dfbbb12fb..198d7b2955 100644 --- a/src/engine/ui/cmd/do_who.cpp +++ b/src/engine/ui/cmd/do_who.cpp @@ -3,6 +3,9 @@ // #include "engine/ui/cmd/do_who.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "utils/grammar/gender.h" #include "gameplay/mechanics/sight.h" @@ -51,10 +54,10 @@ void DoWho(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { sscanf(arg, "%d-%d", &low, &high); strcpy(buf, buf1); } else if (*arg == '-') { - const char mode = *(arg + 1); // just in case; we destroy arg in the switch + const char32_t mode = native_text::first_char_code(arg + 1); // just in case; we destroy arg in the switch switch (mode) { case 'b': - case 'и': + case rus::kI: if (privilege::IsImmortal(ch) || GET_GOD_FLAG(ch, EGf::kDemigod) || ch->IsFlagged(EPrf::kCoderinfo)) showname = true; strcpy(buf, buf1); @@ -164,15 +167,19 @@ void DoWho(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (short_list) { char tmp[kMaxInputLength]; snprintf(tmp, sizeof(tmp), "%s%s%s", GetPkNameColor(tch), GET_NAME(tch), kColorNrm); + // Ширина колонки - в символах, а не в байтах (issue #3681): fmt "{:<30}" считает + // байты, из-за чего колонка с русским именем под UTF-8 выходит вдвое уже. if (privilege::IsImpl(ch) || ch->IsFlagged(EPrf::kCoderinfo)) { - sprintf(buf, "%s[%2d %s] %-30s%s", + strcpy(buf, fmt::format("{}[{:2} {}] {:<30}{}", privilege::IsGod(tch.get()) ? kColorWht : "", GetRealLevel(tch), MUD::Class(tch->GetClass()).GetCName(), - tmp, privilege::IsGod(tch.get()) ? kColorNrm : ""); + tmp, + privilege::IsGod(tch.get()) ? kColorNrm : "").c_str()); } else { - sprintf(buf, "%s%-30s%s", + strcpy(buf, fmt::format("{}{:<30}{}", privilege::IsImmortal(tch.get()) ? kColorWht : "", - tmp, privilege::IsImmortal(tch.get()) ? kColorNrm : ""); + tmp, + privilege::IsImmortal(tch.get()) ? kColorNrm : "").c_str()); } } else { if (privilege::IsImpl(ch) diff --git a/src/engine/ui/cmd/do_who_am_i.cpp b/src/engine/ui/cmd/do_who_am_i.cpp index c9189e44b7..31998bcc34 100644 --- a/src/engine/ui/cmd/do_who_am_i.cpp +++ b/src/engine/ui/cmd/do_who_am_i.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "gameplay/clans/house.h" #include "engine/db/player_index.h" #include "gameplay/core/remort.h" @@ -29,7 +30,7 @@ void DoWhoAmI(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/) { } else { const int god_level = (ch)->player_specials->saved.NameGod > 1000 ? (ch)->player_specials->saved.NameGod - 1000 : (ch)->player_specials->saved.NameGod; sprintf(buf1, "%s", GetNameById((ch)->player_specials->saved.NameIDGod).c_str()); - *buf1 = UPPER(*buf1); + native_text::capitalize_first(buf1); static const char *by_rank_god = "Богом"; static const char *by_rank_privileged = "привилегированным игроком"; diff --git a/src/engine/ui/cmd_god/do_last.cpp b/src/engine/ui/cmd_god/do_last.cpp index b1286ab210..5c040dcdb5 100644 --- a/src/engine/ui/cmd_god/do_last.cpp +++ b/src/engine/ui/cmd_god/do_last.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include #include "administration/privilege.h" #include "engine/entities/char_player.h" #include "engine/db/global_objects.h" @@ -28,11 +29,13 @@ void DoPageLastLogins(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) SendMsgToChar("Вы не столь уж и божественны для этого.\r\n", ch); } else { time_t tmp_time = chdata->get_last_logon(); - sprintf(buf, "[%5ld] [%2d %s] %-12s : %-18s : %-20s\r\n", + SendMsgToChar(fmt::format("[{:5}] [{:2} {}] {:<12} : {:<18} : {:<20}\r\n", chdata->get_uid(), GetRealLevel(chdata), - MUD::Class(chdata->GetClass()).GetAbbr().c_str(), GET_NAME(chdata), - chdata->player_specials->saved.LastIP[0] ? chdata->player_specials->saved.LastIP : "Unknown", ctime(&tmp_time)); - SendMsgToChar(buf, ch); + MUD::Class(chdata->GetClass()).GetAbbr(), + GET_NAME(chdata), + chdata->player_specials->saved.LastIP[0] + ? chdata->player_specials->saved.LastIP : "Unknown", + ctime(&tmp_time)), ch); } } diff --git a/src/engine/ui/cmd_god/do_liblist.cpp b/src/engine/ui/cmd_god/do_liblist.cpp index 66abc672df..f7a5da3b3d 100644 --- a/src/engine/ui/cmd_god/do_liblist.cpp +++ b/src/engine/ui/cmd_god/do_liblist.cpp @@ -263,8 +263,8 @@ void Print(CharData *ch, int first, int last, const std::string &options) { int cnt = 0; for (int i = 0; i <= top_of_mobt; ++i) { if (mob_index[i].vnum >= first && mob_index[i].vnum <= last) { - fmt::format_to(std::back_inserter(out), "{:5}. {:<45} [{:<6}] [{:<2}]{}", - ++cnt, mob_proto[i].get_name_str().substr(0, 45), + fmt::format_to(std::back_inserter(out), "{:5}. {:<45.45} [{:<6}] [{:<2}]{}", + ++cnt, mob_proto[i].get_name_str(), mob_index[i].vnum, mob_proto[i].GetLevel(), PrintFlag(mob_proto + i, options)); if (!mob_proto[i].proto_script->empty()) { diff --git a/src/engine/ui/cmd_god/do_print_armor.cpp b/src/engine/ui/cmd_god/do_print_armor.cpp index 7507a96f0a..8ecdeb8a3c 100644 --- a/src/engine/ui/cmd_god/do_print_armor.cpp +++ b/src/engine/ui/cmd_god/do_print_armor.cpp @@ -7,6 +7,8 @@ */ #include "engine/entities/char_data.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/obj_prototypes.h" #include "engine/db/global_objects.h" @@ -34,8 +36,8 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { char tmpbuf[kMaxInputLength]; bool find_param = false; while (*argument) { - switch (*argument) { - case 'М': argument = one_argument(++argument, tmpbuf); + switch (native_text::first_char_code(argument)) { + case rus::kEmUpper: argument = one_argument(++argument, tmpbuf); if (utils::IsAbbr(tmpbuf, "булат")) { filter.material = EObjMaterial::kBulat; } else if (utils::IsAbbr(tmpbuf, "бронза")) { @@ -78,7 +80,7 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } find_param = true; break; - case 'Т': argument = one_argument(++argument, tmpbuf); + case rus::kTeUpper: argument = one_argument(++argument, tmpbuf); if (utils::IsAbbr(tmpbuf, "броня") || utils::IsAbbr(tmpbuf, "armor")) { filter.type = EObjType::kArmor; } else if (utils::IsAbbr(tmpbuf, "легкие") || utils::IsAbbr(tmpbuf, "легкая")) { @@ -93,7 +95,7 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } find_param = true; break; - case 'О': argument = one_argument(++argument, tmpbuf); + case rus::kOUpper: argument = one_argument(++argument, tmpbuf); if (utils::IsAbbr(tmpbuf, "тело")) { filter.wear = EWearFlag::kBody; filter.wear_message = 3; @@ -118,7 +120,7 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } find_param = true; break; - case 'А': { + case rus::kAUpper: { bool tmp_find = false; argument = one_argument(++argument, tmpbuf); if (!strlen(tmpbuf)) { diff --git a/src/engine/ui/cmd_god/do_set_all.cpp b/src/engine/ui/cmd_god/do_set_all.cpp index e832949bc2..b254ce4721 100644 --- a/src/engine/ui/cmd_god/do_set_all.cpp +++ b/src/engine/ui/cmd_god/do_set_all.cpp @@ -7,6 +7,7 @@ */ #include "engine/ui/cmd_god/do_set_all.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/karma.h" @@ -167,7 +168,7 @@ void setall_inspect() { } Password::set_password(vict, std::string(it->second->pwd)); std::string str = player_table[it->second->pos].name(); - str[0] = UPPER(str[0]); + native_text::capitalize_first(str); sprintf(buf2, "У персонажа %s изменен пароль (setall).", player_table[it->second->pos].name().c_str()); it->second->out += buf2; sprintf(buf1, "\r\n"); diff --git a/src/engine/ui/cmd_god/do_show.cpp b/src/engine/ui/cmd_god/do_show.cpp index 1bc4d0adff..c7a16a52cf 100644 --- a/src/engine/ui/cmd_god/do_show.cpp +++ b/src/engine/ui/cmd_god/do_show.cpp @@ -3,6 +3,7 @@ // #include "administration/accounts.h" +#include "utils/native_text.h" #include "administration/ban.h" #include "administration/privilege.h" #include "engine/ui/cmd/do_features.h" @@ -252,12 +253,12 @@ void print_mob_bosses(CharData *ch, bool lvl_sort) { const auto mob = mob_proto + mob_rnum; const auto vnum = GET_MOB_VNUM(mob); - out += fmt::format("{:<3} {:<31}s [{:<2}][{:<6}] {:<31}s\r\n", + out += fmt::format("{:<3} {:<31.31} [{:<2}][{:<6}] {:<31.31}\r\n", ++cnt, - mob->get_name_str().substr(0, 31), + mob->get_name_str(), zone_table[mob_index[mob_rnum].zone].mob_level, vnum, - zone_name_str.substr(0, 31)); + zone_name_str); } page_string(ch->desc, out); } @@ -460,9 +461,9 @@ void ListSpellCreate(CharData *ch) { if (r > 0) runes_str += '|'; runes_str += std::to_string(info.runes[r]); } - SendMsgToChar(ch, "%3d) Rune spell [%3d] &W%-30s&n runes: %s level %d\r\n", + SendMsgToChar(fmt::format("{:3}) Rune spell [{:3}] &W{:<30}&n runes: {} level {}\r\n", ++i, to_underlying(spell_id), MUD::Spell(spell_id).GetCName(), - runes_str.c_str(), info.min_caster_level); + runes_str, info.min_caster_level), ch); } } @@ -594,11 +595,11 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { sprintf(buf + strlen(buf), "Имя никем не одобрено!\r\n"); } else if ((vict)->player_specials->saved.NameGod < 1000) { sprintf(buf1, "%s", GetNameById((vict)->player_specials->saved.NameIDGod).c_str()); - *buf1 = UPPER(*buf1); + native_text::capitalize_first(buf1); snprintf(buf + strlen(buf), kMaxStringLength, "Имя запрещено богом %s\r\n", buf1); } else { sprintf(buf1, "%s", GetNameById((vict)->player_specials->saved.NameIDGod).c_str()); - *buf1 = UPPER(*buf1); + native_text::capitalize_first(buf1); snprintf(buf + strlen(buf), kMaxStringLength, "Имя одобрено богом %s\r\n", buf1); } if (remort::GetRealRemort(vict) < 4) @@ -717,18 +718,19 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { && d->character->in_room != kNowhere && ((sight::CanSee(ch, d->character) && GetRealLevel(ch) >= GetRealLevel(d->character)) || ch->IsFlagged(EPrf::kCoderinfo))) { - sprintf(buf + strlen(buf), - "%-10s - подслушивается %s (map %s).\r\n", + strcat(buf, fmt::format( + "{:<10} - подслушивается {} (map {}).\r\n", GET_NAME(d->snooping->character), GET_PAD(d->character, 4), - d->snoop_with_map ? "on" : "off"); + d->snoop_with_map ? "on" : "off").c_str()); } } SendMsgToChar(*buf ? buf : "Никто не подслушивается.\r\n", ch); break; // snoop case 9: // show linkdrop SendMsgToChar(" Список игроков в состоянии 'link drop'\r\n", ch); - sprintf(buf, "%-50s%-16s %s\r\n", " Имя", "Комната", "Бездействие (тики)"); + strcpy(buf, fmt::format("{:<50}{:<16} {}\r\n", " Имя", + "Комната", "Бездействие (тики)").c_str()); SendMsgToChar(buf, ch); i = 0; for (const auto &character : character_list) { @@ -737,9 +739,9 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { continue; } ++i; - sprintf(buf, "%-50s[%6d][%6d] %d\r\n", - character->GetNameWithTitleOrRace().c_str(), GET_ROOM_VNUM(character->in_room), - GET_ROOM_VNUM(character->get_was_in_room()), character->char_specials.timer); + strcpy(buf, fmt::format("{:<50}[{:6}][{:6}] {}\r\n", + character->GetNameWithTitleOrRace(), GET_ROOM_VNUM(character->in_room), + GET_ROOM_VNUM(character->get_was_in_room()), character->char_specials.timer).c_str()); SendMsgToChar(buf, ch); } sprintf(buf, "Всего - %d\r\n", i); diff --git a/src/engine/ui/cmd_god/do_stat.cpp b/src/engine/ui/cmd_god/do_stat.cpp index a3e5359b00..8a06898a17 100644 --- a/src/engine/ui/cmd_god/do_stat.cpp +++ b/src/engine/ui/cmd_god/do_stat.cpp @@ -1,4 +1,5 @@ #include "gameplay/mechanics/equipment.h" +#include "utils/native_text.h" #include "gameplay/affects/affect_messages.h" #include "do_stat.h" #include "utils/utils_string.h" @@ -844,7 +845,7 @@ void do_stat_object(CharData *ch, ObjData *j, const int virt = 0) { } } if (!str.empty()) { - str[0] = UPPER(str[0]); + native_text::capitalize_first(str); SendMsgToChar(ch, "&C%s&n", str.c_str()); } else { auto room = get_room_where_obj(j); diff --git a/src/engine/ui/cmd_god/do_tabulate.cpp b/src/engine/ui/cmd_god/do_tabulate.cpp index 7afa9b019d..7a60eb611b 100644 --- a/src/engine/ui/cmd_god/do_tabulate.cpp +++ b/src/engine/ui/cmd_god/do_tabulate.cpp @@ -96,10 +96,10 @@ int TabulateObjsByFilter(char *argument, CharData *ch) { for (const auto &i : obj_proto) { // ch не передаём: у прототипов нет наносимых меток (custom label). if (filter.check(i.get(), nullptr)) { - snprintf(line, sizeof(line), "%3d. [%7d] %-50s %s\r\n", + strcpy(line, fmt::format("{:3}. [{:7}] {:<50} {}\r\n", ++found, i->get_vnum(), - utils::RemoveColors(i->get_short_description()).c_str(), - filter.show_obj_aff(i.get()).c_str()); + utils::RemoveColors(i->get_short_description()), + filter.show_obj_aff(i.get())).c_str()); out += line; } } @@ -117,8 +117,9 @@ int TabulateMobsByName(char *searchname, CharData *ch) { for (nr = 0; nr <= top_of_mobt; nr++) { if (isname(searchname, mob_proto[nr].GetCharAliases())) { - sprintf(buf, "%3d. [%5d] %-30s (%s)\r\n", ++found, mob_index[nr].vnum, mob_proto[nr].get_npc_name().c_str(), - npc_race_types[mob_proto[nr].player_data.Race - ENpcRace::kBasic]); + strcpy(buf, fmt::format("{:3}. [{:5}] {:<30} ({})\r\n", ++found, mob_index[nr].vnum, + mob_proto[nr].get_npc_name(), + npc_race_types[mob_proto[nr].player_data.Race - ENpcRace::kBasic]).c_str()); SendMsgToChar(buf, ch); } } diff --git a/src/engine/ui/cmd_god/do_users.cpp b/src/engine/ui/cmd_god/do_users.cpp index a89f377f20..7398ea12dd 100644 --- a/src/engine/ui/cmd_god/do_users.cpp +++ b/src/engine/ui/cmd_god/do_users.cpp @@ -3,6 +3,7 @@ // #include "engine/ui/color.h" +#include #include "administration/privilege.h" #include "gameplay/classes/pc_classes.h" #include "engine/entities/char_data.h" @@ -105,7 +106,9 @@ void do_users(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } } // end while (parser) - const char *format = "%3d %-7s %-20s %-17s %-3s %-8s "; + // Ширина колонок - в символах, а не в байтах (issue #3681): поля ниже паддятся +// через native_text, поэтому формат содержит голые "%s". + const char *format = "{:3} {:<7} {:<20} {:<17} {:<3} {:<8} "; if (showemail) { strcpy(line, "Ном Професс Имя Состояние Idl Логин Сайт E-mail\r\n"); } else { @@ -260,26 +263,17 @@ void do_users(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (d->character && d->character->GetCharAliases().c_str()) { if (d->original) { - sprintf(line, - format, - d->desc_num, - classname, - d->original->GetCharAliases().c_str(), - state, - idletime, - timeptr); + strcpy(line, fmt::format(fmt::runtime(format), + d->desc_num, classname, d->original->GetCharAliases().c_str(), + state, idletime, timeptr).c_str()); } else { - sprintf(line, - format, - d->desc_num, - classname, - d->character->GetCharAliases().c_str(), - state, - idletime, - timeptr); + strcpy(line, fmt::format(fmt::runtime(format), + d->desc_num, classname, d->character->GetCharAliases().c_str(), + state, idletime, timeptr).c_str()); } } else { - sprintf(line, format, d->desc_num, " - ", "UNDEFINED", state, idletime, timeptr); + strcpy(line, fmt::format(fmt::runtime(format), d->desc_num, " - ", + "UNDEFINED", state, idletime, timeptr).c_str()); } if (d && *d->host) { diff --git a/src/engine/ui/login.cpp b/src/engine/ui/login.cpp index 5d75a7a582..f5f21776e1 100644 --- a/src/engine/ui/login.cpp +++ b/src/engine/ui/login.cpp @@ -4,6 +4,8 @@ extracted from interpreter.cpp. Entry point ProcessLoginInput (was nanny). */ #include "interpreter.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/ui/system_messages.h" #include "engine/core/config.h" #include "gameplay/mechanics/condition.h" @@ -185,8 +187,8 @@ int _parse_name(char *argument, char *name) { // skip whitespaces for (i = 0; (*name = (i ? LOWER(*argument) : UPPER(*argument))); argument++, i++, name++) { - if (*argument == 'ё' - || *argument == 'Ё' + if (native_text::first_char_code(argument) == rus::kYo + || native_text::first_char_code(argument) == rus::kYoUpper || !a_isalpha(*argument) || *argument > 0) { return (1); @@ -1532,7 +1534,8 @@ static void HandleGetKeytable(DescriptorData *d, char *argument) { static void HandleNameConfirm(DescriptorData *d, char *argument) { char buffer[kMaxStringLength]; - if (UPPER(*argument) == 'Y' || UPPER(*argument) == 'Д') { + if (native_text::first_char_code_upper(argument) == 'Y' + || native_text::first_char_code_upper(argument) == rus::kDeUpper) { if (ban->IsBanned(d->host) >= BanList::BAN_NEW) { sprintf(buffer, "Попытка создания персонажа %s отклонена для [%s] (siteban)", GET_PC_NAME(d->character), d->host); @@ -1570,7 +1573,8 @@ static void HandleNameConfirm(DescriptorData *d, char *argument) { d->state = EConState::kQsex; return; - } else if (UPPER(*argument) == 'N' || UPPER(*argument) == 'Н') { + } else if (native_text::first_char_code_upper(argument) == 'N' + || native_text::first_char_code_upper(argument) == rus::kEnUpper) { iosystem::write_to_output("Итак, чего изволите? Учтите, бананов нет :)\r\n" "Имя : ", d); d->character->SetCharAliases(nullptr); d->state = EConState::kGetName; @@ -1677,12 +1681,12 @@ static void HandleQuerySex(DescriptorData *d, char *argument) { return; } - switch (UPPER(*argument)) { - case 'М': + switch (native_text::first_char_code_upper(argument)) { + case rus::kEmUpper: case 'M': d->character->set_sex(EGender::kMale); break; - case 'Ж': + case rus::kZheUpper: case 'F': d->character->set_sex(EGender::kFemale); break; @@ -1706,9 +1710,9 @@ static void HandleQueryReligion(DescriptorData *d, char *argument) { return; } - switch (UPPER(*argument)) { - case 'Я': - case 'З': + switch (native_text::first_char_code_upper(argument)) { + case rus::kYaUpper: + case rus::kZeUpper: case 'P': if (class_religion[to_underlying(d->character->GetClass())] == kReligionMono) { iosystem::write_to_output("Персонаж выбранной вами профессии не желает быть язычником!\r\n" @@ -1718,7 +1722,7 @@ static void HandleQueryReligion(DescriptorData *d, char *argument) { GET_RELIGION(d->character) = kReligionPoly; break; - case 'Х': + case rus::kHaUpper: case 'C': if (class_religion[to_underlying(d->character->GetClass())] == kReligionPoly) { iosystem::write_to_output("Персонажу выбранной вами профессии противно христианство!\r\n" @@ -2049,9 +2053,9 @@ static void HandleResetReligion(DescriptorData *d, char *argument) { return; } - switch (UPPER(*argument)) { - case 'Я': - case 'З': + switch (native_text::first_char_code_upper(argument)) { + case rus::kYaUpper: + case rus::kZeUpper: case 'P': if (class_religion[to_underlying(d->character->GetClass())] == kReligionMono) { iosystem::write_to_output("Персонаж выбранной вами профессии не желает быть язычником!\r\n" @@ -2061,7 +2065,7 @@ static void HandleResetReligion(DescriptorData *d, char *argument) { GET_RELIGION(d->character) = kReligionPoly; break; - case 'Х': + case rus::kHaUpper: case 'C': if (class_religion[to_underlying(d->character->GetClass())] == kReligionPoly) { iosystem::write_to_output("Персонажу выбранной вами профессии противно христианство!\r\n" diff --git a/src/engine/ui/modify.cpp b/src/engine/ui/modify.cpp index 11fe7132e3..8378412488 100644 --- a/src/engine/ui/modify.cpp +++ b/src/engine/ui/modify.cpp @@ -13,10 +13,12 @@ ************************************************************************ */ #include +#include "utils/russian_keys.h" #include "engine/db/player_index.h" #include #include "modify.h" +#include "utils/native_text.h" #include "engine/olc/vedun/vedun.h" #include "interpreter.h" #include "engine/core/target_resolver.h" @@ -652,11 +654,11 @@ void string_add(DescriptorData *d, char *str) { if (!d->writer->get_string()) { if (strlen(str) + 3 > d->max_str) { SendMsgToChar("Слишком длинная строка - усечена.\r\n", d->character.get()); - strcpy(&str[d->max_str - 3], "\r\n"); + strcpy(&str[native_text::truncate_offset(str, d->max_str - 3)], "\r\n"); d->writer->set_string(str); } else if (EConState::kWriteMod == d->state && strlen(str) + 3 > 80) { SendMsgToChar("Слишком длинная строка - усечена.\r\n", d->character.get()); - str[80 - 3] = '\0'; + str[native_text::truncate_offset(str, 80 - 3)] = '\0'; d->writer->set_string(str); } else { d->writer->set_string(str); @@ -664,7 +666,7 @@ void string_add(DescriptorData *d, char *str) { } else { if (EConState::kWriteMod == d->state && strlen(str) + 3 > 80) { SendMsgToChar("Слишком длинная строка - усечена.\r\n", d->character.get()); - str[80 - 3] = '\0'; + str[native_text::truncate_offset(str, 80 - 3)] = '\0'; } if (strlen(str) + d->writer->length() + 3 > d->max_str) // \r\n\0 // @@ -1188,9 +1190,14 @@ char *next_page(char *str, CharData *ch) { // * We need to check here and see if we are over the page width, // * and if so, compensate by going to the begining of the next line. - else if ((ch)->player_specials->saved.stringLength && ++col > (ch)->player_specials->saved.stringLength) { - col = 1; - line++; + // * A multibyte character counts as one column; skip its trailing bytes so + // * they are not counted again (native_text::char_bytes == 1 under KOI8-R). + else if ((ch)->player_specials->saved.stringLength) { + if (++col > (ch)->player_specials->saved.stringLength) { + col = 1; + line++; + } + str += native_text::char_bytes(str) - 1; } } } @@ -1263,7 +1270,8 @@ void show_string(DescriptorData *d, char *input) { any_one_arg(input, buf); //* Q is for quit. :) - if (LOWER(*buf) == 'q' || LOWER(*buf) == 'к') { + if (native_text::first_char_code_lower(buf) == 'q' + || native_text::first_char_code_lower(buf) == rus::kKa) { free(d->showstr_vector); d->showstr_count = 0; if (d->showstr_head) { @@ -1275,12 +1283,14 @@ void show_string(DescriptorData *d, char *input) { } // R is for refresh, so back up one page internally so we can display // it again. - else if (LOWER(*buf) == 'r' || LOWER(*buf) == 'п') { + else if (native_text::first_char_code_lower(buf) == 'r' + || native_text::first_char_code_lower(buf) == rus::kPe) { d->showstr_page = MAX(0, d->showstr_page - 1); } // B is for back, so back up two pages internally so we can display the // correct page here. - else if (LOWER(*buf) == 'b' || LOWER(*buf) == 'н') { + else if (native_text::first_char_code_lower(buf) == 'b' + || native_text::first_char_code_lower(buf) == rus::kEn) { d->showstr_page = MAX(0, d->showstr_page - 2); } // Feature to 'goto' a page. Just type the number of the page and you diff --git a/src/engine/ui/objects_filter.cpp b/src/engine/ui/objects_filter.cpp index ef969c4334..f612622f4f 100644 --- a/src/engine/ui/objects_filter.cpp +++ b/src/engine/ui/objects_filter.cpp @@ -6,6 +6,8 @@ */ #include "objects_filter.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "gameplay/mechanics/sight.h" #include "gameplay/economics/exchange.h" @@ -825,45 +827,45 @@ bool ParseFilter::parse_filter(const CharData *ch, ParseFilter &filter, const ch return false; } while (*argument) { - switch (*argument) { - case 'И': argument = one_argument(++argument, buf_tmp); + switch (native_text::first_char_code(argument)) { + case rus::kIUpper: argument = one_argument(++argument, buf_tmp); if (strlen(buf_tmp) == 0) { SendMsgToChar("Укажите имя предмета.\r\n", ch); return false; } filter.name = buf_tmp; break; - case 'Т': argument = one_argument(++argument, buf_tmp); + case rus::kTeUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_type(buf_tmp)) { SendMsgToChar("Неверный тип предмета.\r\n", ch); return false; } break; - case 'С': argument = one_argument(++argument, buf_tmp); + case rus::kEsUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_state(buf_tmp)) { SendMsgToChar("Неверное состояние предмета.\r\n", ch); return false; } break; - case 'О': argument = one_argument(++argument, buf_tmp); + case rus::kOUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_wear(buf_tmp)) { SendMsgToChar("Неверное место одевания предмета.\r\n", ch); return false; } break; - case 'Ц': argument = one_argument(++argument, buf_tmp); + case rus::kTseUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_cost(buf_tmp)) { SendMsgToChar("Неверный формат в фильтре: Ц<цена><+->.\r\n", ch); return false; } break; - case 'К': argument = one_argument(++argument, buf_tmp); + case rus::kKaUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_weap_class(buf_tmp)) { SendMsgToChar("Неверный класс оружия.\r\n", ch); return false; } break; - case 'А': { + case rus::kAUpper: { argument = one_argument(++argument, buf_tmp); size_t len = strlen(buf_tmp); if (len == 0) { @@ -878,29 +880,29 @@ bool ParseFilter::parse_filter(const CharData *ch, ParseFilter &filter, const ch return false; } break; - } // case 'А' - case 'Р':// стоимость ренты + } // case rus::kAUpper + case rus::kErUpper:// стоимость ренты argument = one_argument(++argument, buf_tmp); if (!filter.init_rent(buf_tmp)) { SendMsgToChar("Неверный формат в фильтре: Р<стоимость><+->.\r\n", ch); return false; } break; - case 'М':// количество мортов + case rus::kEmUpper:// количество мортов argument = one_argument(++argument, buf_tmp); if (!filter.init_remorts(buf_tmp)) { SendMsgToChar("Неверный формат в фильтре: М<количество мортов><+->.\r\n", ch); return false; } break; - case 'У':// умения + case rus::kUUpper:// умения argument = one_argument(++argument, buf_tmp); if (!filter.init_skill(buf_tmp)) { SendMsgToChar("Неверное умение.\r\n", ch); return false; } break; - case 'В':// имя выставившего на базаре + case rus::kVeUpper:// имя выставившего на базаре argument = one_argument(++argument, buf_tmp); if (filter_type != EXCHANGE) { SendMsgToChar("Только для базара.\r\n", ch); @@ -908,7 +910,7 @@ bool ParseFilter::parse_filter(const CharData *ch, ParseFilter &filter, const ch } owner = buf_tmp; break; - case 'П':// профессия (отсечь предметы запрещенные данному классу) + case rus::kPeUpper:// профессия (отсечь предметы запрещенные данному классу) argument = one_argument(++argument, buf_tmp); if (!filter.init_profession(buf_tmp)) { SendMsgToChar("Неверное название профессии.\r\n", ch); diff --git a/src/engine/ui/table_wrapper.h b/src/engine/ui/table_wrapper.h index 94d684c471..e3a36238d5 100644 --- a/src/engine/ui/table_wrapper.h +++ b/src/engine/ui/table_wrapper.h @@ -41,9 +41,21 @@ using Color = fort::color; using TextAlign = fort::text_align; /** - * Таблица в стандартной, не unicode кодировке. + * Базовый тип таблицы выбирается под нативную кодировку движка (issue #3681). + * libfort считает ширину ячейки по-разному: char_table меряет байтами, utf8_table - кодовыми + * точками. Под KOI8-R (1 байт = 1 символ) верен первый, под UTF-8 - второй; при неверном выборе + * колонки с русским текстом съезжают вдвое. */ -class Table : public fort::char_table { +#ifdef INTERNAL_ENCODING_UTF8 +using TableBase = fort::utf8_table; +#else +using TableBase = fort::char_table; +#endif + +/** + * Таблица в нативной кодировке движка. + */ +class Table : public TableBase { public: void SetColumnAlign(std::size_t column_index, TextAlign align) { this->column(column_index).set_cell_text_align(align); diff --git a/src/gameplay/clans/house.cpp b/src/gameplay/clans/house.cpp index 4cd8b5db45..0a7df152a0 100644 --- a/src/gameplay/clans/house.cpp +++ b/src/gameplay/clans/house.cpp @@ -5,6 +5,8 @@ ******************************************************************************/ #include "house.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "gameplay/economics/currencies.h" #include "utils/utils_encoding.h" @@ -991,7 +993,7 @@ void Clan::HouseInfo(CharData *ch) { for (const auto &it : temp_list) { if (temp != ranks[it->rank_num]) { std::string rnk = ranks[it->rank_num]; - rnk[0] = UPPER(rnk[0]); + native_text::capitalize_first(rnk); if (temp == "") { buffer << rnk << ": "; @@ -1116,7 +1118,7 @@ void Clan::HouseAdd(CharData *ch, std::string &buffer) { return; } std::string name = buffer2; - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); if (unique == ch->get_uid()) { SendMsgToChar("Сам себя повысил, самому себе вынес благодарность?\r\n", ch); return; @@ -1308,7 +1310,7 @@ void Clan::remove_member(const ClanMembersList::key_type &key, char *reason) { if (d->character && CLAN(d->character) && CLAN(d->character)->GetRent() == this->GetRent()) { - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); SendMsgToChar(d->character.get(), "%s более не является членом вашей дружины.\r\n", name.c_str()); } } @@ -1426,7 +1428,7 @@ void Clan::hcon_outcast(CharData *ch, std::string &buffer) { char tmpstr[kMaxInputLength]; sprintf(tmpstr, "Богом %s", GET_NAME(ch)); clan->remove_member(member_uid, tmpstr); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); SendMsgToChar(ch, "%s исключен(a) из дружины '%s'.\r\n", name.c_str(), clan->name.c_str()); return; } @@ -2073,7 +2075,7 @@ void Clan::HcontrolBuild(CharData *ch, std::string &buffer) { tempClan->chest_room = rent; tempClan->guard = guard; // пишем воеводу - owner[0] = UPPER(owner[0]); + native_text::capitalize_first(owner); tempClan->owner = owner; const auto tempMember = std::make_shared(); tempMember->name = owner; @@ -2751,9 +2753,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { switch (d->clan_olc->mode) { case CLAN_MAIN_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // есть вариант, что за время в олц в клане изменят кол-во званий @@ -2833,9 +2835,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_PRIVILEGE_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // выход в общее меню @@ -2878,11 +2880,11 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_SAVE_MENU: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': d->clan_olc->clan->privileges.clear(); + case rus::kDe: + case rus::kDeUpper: d->clan_olc->clan->privileges.clear(); d->clan_olc->clan->privileges = d->clan_olc->privileges; d->clan_olc.reset(); // Clan::ClanSave(); @@ -2892,8 +2894,8 @@ void Clan::Manage(DescriptorData *d, const char *arg) { case 'n': case 'N': - case 'н': - case 'Н': d->clan_olc.reset(); + case rus::kEn: + case rus::kEnUpper: d->clan_olc.reset(); d->state = EConState::kPlaying; SendMsgToChar("Редактирование отменено.\r\n", d->character.get()); return; @@ -2908,9 +2910,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_ADDALL_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // выход в общее меню с изменением всех званий @@ -2962,9 +2964,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_DELALL_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // выход в общее меню с изменением всех званий @@ -3475,7 +3477,7 @@ void Clan::HouseOwner(CharData *ch, std::string &buffer) { else if (CLAN(d->character) && CLAN(ch) != CLAN(d->character)) SendMsgToChar("Вы не можете передать свои права члену другой дружины.\r\n", ch); else { - buffer2[0] = UPPER(buffer2[0]); + native_text::capitalize_first(buffer2); // воевода идет рангом ниже this->m_members.set_rank(ch->get_uid(), 1); Clan::SetClanData(ch); diff --git a/src/gameplay/communication/parcel.cpp b/src/gameplay/communication/parcel.cpp index f1463863b2..047c885cb4 100644 --- a/src/gameplay/communication/parcel.cpp +++ b/src/gameplay/communication/parcel.cpp @@ -3,6 +3,7 @@ // Part of Bylins http://www.mud.ru #include "parcel.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "gameplay/economics/currencies.h" @@ -828,13 +829,13 @@ bool print_imm_where_obj(CharData *ch, const ObjData *arg, int num) { std::string sender = GetNameByUnique(it2->first); found = true; - SendMsgToChar(ch, "%2d. [%6d] %-25s - наход%sся на почте (отправитель: %s, получатель: %s).\r\n", + SendMsgToChar(fmt::format("{:2}. [{:6}] {:<25} - наход{}ся на почте (отправитель: {}, получатель: {}).\r\n", num++, GET_OBJ_VNUM(it3->obj_.get()), - it3->obj_->get_short_description().c_str(), + it3->obj_->get_short_description(), grammar::ObjPluralVerbEnding((it3->obj_)->get_sex()), - sender.c_str(), - target.c_str()); + sender, + target), ch); } } } @@ -852,8 +853,8 @@ std::string FindParcelObj(const ObjData *obj) { std::string target = GetNameByUnique(it->first); std::string sender = GetNameByUnique(it2->first); - target[0] = UPPER(target[0]); - sender[0] = UPPER(sender[0]); + native_text::capitalize_first(target); + native_text::capitalize_first(sender); str = fmt::format("наход{}ся на почте (отправитель: {}, получатель: {}).\r\n", grammar::ObjPluralVerbEnding((it3->obj_)->get_sex()), sender.c_str(), diff --git a/src/gameplay/core/game_limits.cpp b/src/gameplay/core/game_limits.cpp index de51ea5b6d..d7809574fd 100644 --- a/src/gameplay/core/game_limits.cpp +++ b/src/gameplay/core/game_limits.cpp @@ -13,6 +13,7 @@ ************************************************************************ */ #include "gameplay/core/game_limits.h" +#include "utils/native_text.h" #include "gameplay/core/experience.h" #include "gameplay/affects/affect_data.h" // issue.mob-flag-affect-materialization: restore re-materialize #include "administration/privilege.h" @@ -1154,7 +1155,7 @@ void exchange_point_update() { if (GET_EXCHANGE_ITEM(exch_item)->get_timer() == 0) { std::string cap = GET_EXCHANGE_ITEM(exch_item)->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); sprintf(buf, "Exchange: - %s рассыпал%s от длительного использования.\r\n", cap.c_str(), grammar::ObjSexEnding((GET_EXCHANGE_ITEM(exch_item))->get_sex(), 2)); log("%s", buf); @@ -1237,7 +1238,7 @@ void charmee_obj_decay_tell(CharData *charmee, ObjData *obj, ECharmeeObjPos obj_ короче, рефакторинг приветствуется, если кто-нибудь придумает лучше. */ std::string cap = obj->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); snprintf(local_buf, kMaxStringLength, "%s сказал%s вам : '%s%s рассыпал%s %s...'", GET_NAME(charmee), grammar::SexEnding((charmee)->get_sex(), 1), diff --git a/src/gameplay/core/genchar.cpp b/src/gameplay/core/genchar.cpp index fb1a6c327f..777d24e887 100644 --- a/src/gameplay/core/genchar.cpp +++ b/src/gameplay/core/genchar.cpp @@ -13,6 +13,7 @@ ************************************************************************ */ #include "genchar.h" +#include "utils/russian_keys.h" #include "engine/core/conf.h" #include "engine/core/sysdep.h" @@ -20,6 +21,7 @@ #include "engine/core/comm.h" #include "utils/logger.h" #include "utils/utils.h" +#include "utils/native_text.h" #include "gameplay/magic/spells.h" #include "engine/entities/char_data.h" #include "engine/entities/char_player.h" @@ -94,48 +96,48 @@ void genchar_disp_menu(CharData *ch) { int genchar_parse(CharData *ch, char *arg) { const auto &ch_class = MUD::Class(ch->GetClass()); - switch (*arg) { - case 'А': - case 'а': ch->set_str(std::max(ch->GetInbornStr() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kStr))); + switch (native_text::first_char_code(arg)) { + case rus::kAUpper: + case rus::kA: ch->set_str(std::max(ch->GetInbornStr() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kStr))); break; - case 'Б': - case 'б': ch->set_dex(std::max(ch->GetInbornDex() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kDex))); + case rus::kBeUpper: + case rus::kBe: ch->set_dex(std::max(ch->GetInbornDex() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kDex))); break; - case 'Г': - case 'г': ch->set_int(std::max(ch->GetInbornInt() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kInt))); + case rus::kGeUpper: + case rus::kGe: ch->set_int(std::max(ch->GetInbornInt() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kInt))); break; - case 'Д': - case 'д': ch->set_wis(std::max(ch->GetInbornWis() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kWis))); + case rus::kDeUpper: + case rus::kDe: ch->set_wis(std::max(ch->GetInbornWis() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kWis))); break; - case 'Е': - case 'е': ch->set_con(std::max(ch->GetInbornCon() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCon))); + case rus::kIeUpper: + case rus::kIe: ch->set_con(std::max(ch->GetInbornCon() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCon))); break; - case 'Ж': - case 'ж': ch->set_cha(std::max(ch->GetInbornCha() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCha))); + case rus::kZheUpper: + case rus::kZhe: ch->set_cha(std::max(ch->GetInbornCha() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCha))); break; - case 'З': - case 'з': ch->set_str(std::min(ch->GetInbornStr() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kStr))); + case rus::kZeUpper: + case rus::kZe: ch->set_str(std::min(ch->GetInbornStr() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kStr))); break; - case 'И': - case 'и': ch->set_dex(std::min(ch->GetInbornDex() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kDex))); + case rus::kIUpper: + case rus::kI: ch->set_dex(std::min(ch->GetInbornDex() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kDex))); break; - case 'К': - case 'к': ch->set_int(std::min(ch->GetInbornInt() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kInt))); + case rus::kKaUpper: + case rus::kKa: ch->set_int(std::min(ch->GetInbornInt() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kInt))); break; - case 'Л': - case 'л': ch->set_wis(std::min(ch->GetInbornWis() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kWis))); + case rus::kElUpper: + case rus::kEl: ch->set_wis(std::min(ch->GetInbornWis() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kWis))); break; - case 'М': - case 'м': ch->set_con(std::min(ch->GetInbornCon() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCon))); + case rus::kEmUpper: + case rus::kEm: ch->set_con(std::min(ch->GetInbornCon() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCon))); break; - case 'Н': - case 'н': ch->set_cha(std::min(ch->GetInbornCha() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCha))); + case rus::kEnUpper: + case rus::kEn: ch->set_cha(std::min(ch->GetInbornCha() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCha))); break; - case 'П': - case 'п': SendMsgToChar(genchar_help, ch); + case rus::kPeUpper: + case rus::kPe: SendMsgToChar(genchar_help, ch); break; - case 'В': - case 'в': + case rus::kVeUpper: + case rus::kVe: if (CalcBasseStatsSum(ch) != kBaseStatsSum) break; // по случаю успешной генерации сохраняем стартовые статы @@ -146,8 +148,8 @@ int genchar_parse(CharData *ch, char *arg) { ch->set_start_stat(G_CON, ch->GetInbornCon()); ch->set_start_stat(G_CHA, ch->GetInbornCha()); return kGencharExit; - case 'О': - case 'о': { + case rus::kOUpper: + case rus::kO: { const auto &tmp_class = MUD::Class(ch->GetClass()); ch->set_str(tmp_class.GetBaseStatGenAuto(EBaseStat::kStr)); ch->set_dex(tmp_class.GetBaseStatGenAuto(EBaseStat::kDex)); @@ -249,10 +251,18 @@ void SetStartAbils(CharData *ch) { // 5 - предложный (о ком? о чем?) // result - результат void GetCase(std::string name, const EGender sex, int caseNum, char *data) { - size_t len = name.size(); std::string result = data; - if (strchr("цкнгшщзхфвпрлджчсмтб", name[len - 1]) != nullptr + // The declension is chosen by the last letter of the name (and sometimes the one before it). + // Those are *characters*, not bytes (issue #3681): under KOI8-R `stem`/`last`/`prev` are the + // same single bytes the old name[len - 1] / name[len - 2] / substr(0, len - 1) produced, + // under UTF-8 they are whole letters. + const size_t last_off = native_text::last_char_offset(name); + const std::string stem = name.substr(0, last_off); + const std::string last = name.substr(last_off); + const std::string prev = stem.substr(native_text::last_char_offset(stem)); + + if (native_text::list_contains_char("цкнгшщзхфвпрлджчсмтб", last) && sex == EGender::kMale) { result = name; if (caseNum == 1) @@ -265,8 +275,8 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { result += "ом"; // Иваном, Ретичем else if (caseNum == 5) result += "е"; // Иване - } else if (name[len - 1] == 'я') { - result = name.substr(0, len - 1); + } else if (last == "я") { + result = stem; if (caseNum == 1) result += "и"; // Ани, Вани else if (caseNum == 2) @@ -279,9 +289,9 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { result += "е"; // Ане, Ване else result += "я"; // Аня, Ваня - } else if (name[len - 1] == 'й' + } else if (last == "й" && sex == EGender::kMale) { - result = name.substr(0, len - 1); + result = stem; if (caseNum == 1) result += "я"; // Дрегвия else if (caseNum == 2) @@ -294,10 +304,10 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { result += "и"; // Дрегвии else result += "й"; // Дрегвий - } else if (name[len - 1] == 'а') { - result = name.substr(0, len - 1); + } else if (last == "а") { + result = stem; if (caseNum == 1) { - if (strchr("шщжч", name[len - 2]) != nullptr) + if (native_text::list_contains_char("шщжч", prev)) result += "и"; // Маши, Паши else result += "ы"; // Анны @@ -306,7 +316,7 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { else if (caseNum == 3) result += "у"; // Пашу, Анну else if (caseNum == 4) { - if (strchr("шщч", name[len - 2]) != nullptr) + if (native_text::list_contains_char("шщч", prev)) result += "ей"; // Машей, Пашей else result += "ой"; // Анной, Ханжой diff --git a/src/gameplay/crafting/im.cpp b/src/gameplay/crafting/im.cpp index 5d555bdcdc..e35dc3e955 100644 --- a/src/gameplay/crafting/im.cpp +++ b/src/gameplay/crafting/im.cpp @@ -11,8 +11,10 @@ // Реализация ингредиентной магии #include "im.h" +#include #include "utils/parser_wrapper.h" #include "utils/utils_parse.h" +#include "utils/native_text.h" #include #include #include @@ -246,7 +248,17 @@ const char *replace_alias(const char *ptr, im_memb *sample, int rnum, const char if (*ptr == VAR_CHAR) { int k; ++ptr; - for (k = 0; (*ptr) && a_isalnum(*ptr); aname[k++] = *ptr++); + // One whole character per step (issue #3681), with a bounds check: the + // previous loop had none, and multibyte text fills aname[] twice as fast. + for (k = 0; *ptr && native_text::is_alnum_char(ptr);) { + const size_t bytes = native_text::char_bytes(ptr); + if (static_cast(k) + bytes >= sizeof(aname)) { + break; + } + for (size_t i = 0; i < bytes; ++i) { + aname[k++] = *ptr++; + } + } aname[k] = 0; al = get_im_alias(sample, aname); strcpy(dst, al ? al : aname); @@ -942,14 +954,14 @@ void list_recipes(CharData *ch, bool all_recipes) { rs = im_get_char_rskill(ch, sortpos); const bool unavailable = req->level > GetRealLevel(ch) || req->remort > remort::GetRealRemort(ch); if (!ch->IsFlagged(EPrf::kBlindMode)) { - sprintf(buf, " %s%-30s%s %2d (%2d)%s\r\n", + strcpy(buf, fmt::format(" {}{:<30}{} {:2} ({:2}){}\r\n", unavailable ? kColorRed : rs ? kColorGrn : kColorNrm, imrecipes[sortpos].name, kColorCyn, - req->level, req->remort, kColorNrm); + req->level, req->remort, kColorNrm).c_str()); } else { - sprintf(buf, " %s %-30s %2d (%2d)\r\n", + strcpy(buf, fmt::format(" {} {:<30} {:2} ({:2})\r\n", unavailable ? "[Н]" : rs ? "[И]" : "[Д]", imrecipes[sortpos].name, - req->level, req->remort); + req->level, req->remort).c_str()); } strcat(buf1, buf); ++i; @@ -971,7 +983,7 @@ void list_recipes(CharData *ch, bool all_recipes) { } if (rs->perc <= 0) continue; - sprintf(buf, "%-30s %s%s\r\n", imrecipes[rs->rid].name, how_good(rs->perc, kMaxRecipeLevel), kColorBoldBlk); + strcpy(buf, fmt::format("{:<30} {}{}\r\n", imrecipes[rs->rid].name, how_good(rs->perc, kMaxRecipeLevel), kColorBoldBlk).c_str()); strcat(buf2, buf); ++i; } diff --git a/src/gameplay/crafting/item_creation.cpp b/src/gameplay/crafting/item_creation.cpp index 6103c48f3e..37cdd0e775 100644 --- a/src/gameplay/crafting/item_creation.cpp +++ b/src/gameplay/crafting/item_creation.cpp @@ -7,6 +7,7 @@ * $Revision$ * ************************************************************************ */ #include "item_creation.h" +#include #include "utils/utils_parse.h" #include "utils/parser_wrapper.h" #include "administration/privilege.h" @@ -333,9 +334,9 @@ void do_list_make(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/ } j++; } - sprintf(tmpbuf, "%3zd %-1s %-6s %-40s(%5d) :", - i + 1, (trec->locked ? "*" : " "), skill_name.c_str(), obj_name.c_str(), trec->obj_proto); - tmpstr += string(tmpbuf); + tmpstr += fmt::format("{:3} {:<1} {:<6} {:<40}({:5}) :", + i + 1, (trec->locked ? "*" : " "), skill_name, + obj_name, trec->obj_proto); for (int j = 0; j < MAX_PARTS; j++) { if (trec->parts[j].proto != 0) { obj = GetObjectPrototype(trec->parts[j].proto); @@ -344,7 +345,7 @@ void do_list_make(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/ } else { obj_name = "Нет"; } - sprintf(tmpbuf, " %-35s(%5d)", obj_name.c_str(), trec->parts[j].proto); + strcpy(tmpbuf, fmt::format(" {:<35}({:5})", obj_name, trec->parts[j].proto).c_str()); if (j > 0) { if (j % 2 == 0) { // разбиваем строчки если ингров больше 2; diff --git a/src/gameplay/economics/exchange.cpp b/src/gameplay/economics/exchange.cpp index 460528edf1..a252b4b7af 100644 --- a/src/gameplay/economics/exchange.cpp +++ b/src/gameplay/economics/exchange.cpp @@ -9,6 +9,7 @@ ************************************************************************ */ #include "exchange.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/global_objects.h" #include "gameplay/economics/currencies.h" @@ -472,7 +473,7 @@ int exchange_information(CharData *ch, char *arg) { } auto seller_name = GetNameById(GET_EXCHANGE_ITEM_SELLERID(item)); snprintf(buf2, sizeof(buf2), "%s", seller_name.empty() ? "(сожран долгоносиком)" : seller_name.c_str()); - *buf2 = UPPER(*buf2); + native_text::capitalize_first(buf2); out += fmt::sprintf("Продавец %s\n", buf2); if (GET_EXCHANGE_ITEM_COMMENT(item)) { out += fmt::sprintf("Берестовая наклейка на лоте гласит: '%s'.\n", GET_EXCHANGE_ITEM_COMMENT(item)); @@ -668,7 +669,7 @@ int exchange_offers(const CharData *ch, const char *arg) { filter += GET_NAME(ch); } else { while (*arg1) { - arg1[0] = UPPER(arg1[0]); + native_text::capitalize_first(arg1); filter += arg1; filter += ' '; arg = one_argument(arg, arg1); @@ -907,7 +908,7 @@ int LoadExchange() { // Предмет разваливается от старости if (GET_EXCHANGE_ITEM(item)->get_timer() <= 0) { std::string cap = GET_EXCHANGE_ITEM(item)->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); log("Exchange: - %s рассыпал%s от длительного использования.\r\n", cap.c_str(), grammar::ObjSexEnding((GET_EXCHANGE_ITEM(item))->get_sex(), 2)); extract_exchange_item(item); @@ -1002,7 +1003,7 @@ int exchange_database_reload(bool loadbackup) { // Предмет разваливается от старости if (GET_EXCHANGE_ITEM(item)->get_timer() <= 0) { std::string cap = GET_EXCHANGE_ITEM(item)->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); log("Exchange: - %s рассыпал%s от длительного использования.\r\n", cap.c_str(), grammar::ObjSexEnding((GET_EXCHANGE_ITEM(item))->get_sex(), 2)); extract_exchange_item(item); diff --git a/src/gameplay/fight/pk.cpp b/src/gameplay/fight/pk.cpp index d927493e2a..7b092d481d 100644 --- a/src/gameplay/fight/pk.cpp +++ b/src/gameplay/fight/pk.cpp @@ -12,6 +12,8 @@ ************************************************************************ */ #include "pk.h" +#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" #include "gameplay/mechanics/mount.h" @@ -652,7 +654,7 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { continue; } - temp[0] = UPPER(temp[0]); + native_text::capitalize_first(temp); // если нада исключаем тех, кто находится оффлайн if (bOnlineOnly) { for (const auto &tch : character_list) { @@ -663,9 +665,9 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (tch->get_uid() == uid) { found = true; if (pk.battle_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp.c_str()); + strcat(buf, fmt::format(" {:<40} <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp).c_str()); } else { - sprintf(buf + strlen(buf), " %-40s %3ld %3ld\r\n", temp.c_str(), pk.kill_num, pk.revenge_num); + strcat(buf, fmt::format(" {:<40} {:3} {:3}\r\n", temp, pk.kill_num, pk.revenge_num).c_str()); } break; } @@ -673,9 +675,9 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } else { found = true; if (pk.battle_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp.c_str()); + strcat(buf, fmt::format(" {:<40} <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp).c_str()); } else { - sprintf(buf + strlen(buf), " %-40s %3ld %3ld\r\n", temp.c_str(), pk.kill_num, pk.revenge_num); + strcat(buf, fmt::format(" {:<40} {:3} {:3}\r\n", temp, pk.kill_num, pk.revenge_num).c_str()); } } } @@ -708,12 +710,12 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { // Сначала проверка клан флага if (CLAN(ch) && pk.clan_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <ВОЙНА>\r\n", GET_NAME(tch)); + strcat(buf, fmt::format(" {:<40} <ВОЙНА>\r\n", GET_NAME(tch)).c_str()); } else if (pk.clan_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <ВРЕМЕННЫЙ ФЛАГ>\r\n", GET_NAME(tch)); + strcat(buf, fmt::format(" {:<40} <ВРЕМЕННЫЙ ФЛАГ>\r\n", GET_NAME(tch)).c_str()); } else if (pk.kill_num + pk.revenge_num > 0) { - sprintf(buf + strlen(buf), " %-40s %3ld %3ld\r\n", - GET_NAME(tch), pk.kill_num, pk.revenge_num); + strcat(buf, fmt::format(" {:<40} {:3} {:3}\r\n", + GET_NAME(tch), pk.kill_num, pk.revenge_num).c_str()); } else { continue; } diff --git a/src/gameplay/mechanics/glory.cpp b/src/gameplay/mechanics/glory.cpp index aec1f17a68..f5e66bcab9 100644 --- a/src/gameplay/mechanics/glory.cpp +++ b/src/gameplay/mechanics/glory.cpp @@ -3,6 +3,8 @@ // Part of Bylins http://www.mud.ru #include "glory.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -478,57 +480,57 @@ void parse_add_stat(CharData *ch, int stat) { // * Парс олц меню 'слава'. bool parse_spend_glory_menu(CharData *ch, const char *arg) { - switch (*arg) { - case 'А': - case 'а': + switch (native_text::first_char_code(arg)) { + case rus::kAUpper: + case rus::kA: if (ch->desc->glory->olc_add_str >= 1) { if (!parse_remove_stat(ch, G_STR)) break; ch->desc->glory->olc_str -= 1; ch->desc->glory->olc_add_str -= 1; } break; - case 'Б': - case 'б': + case rus::kBeUpper: + case rus::kBe: if (ch->desc->glory->olc_add_dex >= 1) { if (!parse_remove_stat(ch, G_DEX)) break; ch->desc->glory->olc_dex -= 1; ch->desc->glory->olc_add_dex -= 1; } break; - case 'Г': - case 'г': + case rus::kGeUpper: + case rus::kGe: if (ch->desc->glory->olc_add_int >= 1) { if (!parse_remove_stat(ch, G_INT)) break; ch->desc->glory->olc_int -= 1; ch->desc->glory->olc_add_int -= 1; } break; - case 'Д': - case 'д': + case rus::kDeUpper: + case rus::kDe: if (ch->desc->glory->olc_add_wis >= 1) { if (!parse_remove_stat(ch, G_WIS)) break; ch->desc->glory->olc_wis -= 1; ch->desc->glory->olc_add_wis -= 1; } break; - case 'Е': - case 'е': + case rus::kIeUpper: + case rus::kIe: if (ch->desc->glory->olc_add_con >= 1) { if (!parse_remove_stat(ch, G_CON)) break; ch->desc->glory->olc_con -= 1; ch->desc->glory->olc_add_con -= 1; } break; - case 'Ж': - case 'ж': + case rus::kZheUpper: + case rus::kZhe: if (ch->desc->glory->olc_add_cha >= 1) { if (!parse_remove_stat(ch, G_CHA)) break; ch->desc->glory->olc_cha -= 1; ch->desc->glory->olc_add_cha -= 1; } break; - case 'З': - case 'з': + case rus::kZeUpper: + case rus::kZe: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_STR); @@ -536,8 +538,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_str += 1; } break; - case 'И': - case 'и': + case rus::kIUpper: + case rus::kI: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_DEX); @@ -545,8 +547,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_dex += 1; } break; - case 'К': - case 'к': + case rus::kKaUpper: + case rus::kKa: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_INT); @@ -554,8 +556,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_int += 1; } break; - case 'Л': - case 'л': + case rus::kElUpper: + case rus::kEl: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_WIS); @@ -563,8 +565,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_wis += 1; } break; - case 'М': - case 'м': + case rus::kEmUpper: + case rus::kEm: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_CON); @@ -572,8 +574,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_con += 1; } break; - case 'Н': - case 'н': + case rus::kEnUpper: + case rus::kEn: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_CHA); @@ -581,8 +583,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_cha += 1; } break; - case 'В': - case 'в': { + case rus::kVeUpper: + case rus::kVe: { // проверка, чтобы не записывать зря, а только при изменения // и чтобы нельзя было из стата славу вытащить if ((ch->desc->glory->olc_str == ch->GetInbornStr() @@ -642,8 +644,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { SendMsgToChar("Ваши изменения сохранены.\r\n", ch); return true; } - case 'Х': - case 'х': ch->desc->glory.reset(); + case rus::kHaUpper: + case rus::kHa: ch->desc->glory.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование прервано.\r\n", ch); return true; diff --git a/src/gameplay/mechanics/glory_const.cpp b/src/gameplay/mechanics/glory_const.cpp index c4e4c20d7b..1f9de1b0ae 100644 --- a/src/gameplay/mechanics/glory_const.cpp +++ b/src/gameplay/mechanics/glory_const.cpp @@ -4,6 +4,8 @@ // Part of Bylins http://www.mud.ru #include "glory_const.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -226,7 +228,8 @@ void print_glory(CharData *ch, GloryListType::iterator &it) { *buf = '\0'; for (auto i = it->second->stats.begin(), iend = it->second->stats.end(); i != iend; ++i) { if ((i->first >= 0) && (i->first < (int) sizeof(olc_stat_name))) { - sprintf(buf + strlen(buf), "%-16s: +%d", olc_stat_name[i->first], i->second * stat_multi(i->first)); + strcat(buf, fmt::format("{:<16}: +{}", olc_stat_name[i->first], + i->second * stat_multi(i->first)).c_str()); if (stat_multi(i->first) > 1) sprintf(buf + strlen(buf), "(%d)", i->second); strcat(buf, "\r\n"); @@ -465,32 +468,32 @@ int olc_real_stat(CharData *ch, int stat) { } bool parse_spend_glory_menu(CharData *ch, char *arg) { - switch (LOWER(*arg)) { - case 'а': olc_del_stat(ch, GLORY_STR); + switch (native_text::first_char_code_lower(arg)) { + case rus::kA: olc_del_stat(ch, GLORY_STR); break; - case 'б': olc_del_stat(ch, GLORY_DEX); + case rus::kBe: olc_del_stat(ch, GLORY_DEX); break; - case 'г': olc_del_stat(ch, GLORY_INT); + case rus::kGe: olc_del_stat(ch, GLORY_INT); break; - case 'д': olc_del_stat(ch, GLORY_WIS); + case rus::kDe: olc_del_stat(ch, GLORY_WIS); break; - case 'е': olc_del_stat(ch, GLORY_CON); + case rus::kIe: olc_del_stat(ch, GLORY_CON); break; - case 'ж': olc_del_stat(ch, GLORY_CHA); + case rus::kZhe: olc_del_stat(ch, GLORY_CHA); break; - case 'з': olc_del_stat(ch, GLORY_HIT); + case rus::kZe: olc_del_stat(ch, GLORY_HIT); break; - case 'и': olc_del_stat(ch, GLORY_SUCCESS); + case rus::kI: olc_del_stat(ch, GLORY_SUCCESS); break; - case 'к': olc_del_stat(ch, GLORY_WILL); + case rus::kKa: olc_del_stat(ch, GLORY_WILL); break; - case 'л': olc_del_stat(ch, GLORY_STABILITY); + case rus::kEl: olc_del_stat(ch, GLORY_STABILITY); break; - case 'м': olc_del_stat(ch, GLORY_REFLEX); + case rus::kEm: olc_del_stat(ch, GLORY_REFLEX); break; - case 'н': olc_del_stat(ch, GLORY_MIND); + case rus::kEn: olc_del_stat(ch, GLORY_MIND); break; - case 'э': olc_del_stat(ch, GLORY_MANAREG); + case rus::kE: olc_del_stat(ch, GLORY_MANAREG); break; case 'x': olc_add_stat(ch, GLORY_BONUSPSYS); break; @@ -500,33 +503,33 @@ bool parse_spend_glory_menu(CharData *ch, char *arg) { break; case 'd': olc_del_stat(ch, GLORY_BONUSMAG); break; - case 'о': olc_add_stat(ch, GLORY_STR); + case rus::kO: olc_add_stat(ch, GLORY_STR); break; - case 'п': olc_add_stat(ch, GLORY_DEX); + case rus::kPe: olc_add_stat(ch, GLORY_DEX); break; - case 'р': olc_add_stat(ch, GLORY_INT); + case rus::kEr: olc_add_stat(ch, GLORY_INT); break; - case 'с': olc_add_stat(ch, GLORY_WIS); + case rus::kEs: olc_add_stat(ch, GLORY_WIS); break; - case 'т': olc_add_stat(ch, GLORY_CON); + case rus::kTe: olc_add_stat(ch, GLORY_CON); break; - case 'у': olc_add_stat(ch, GLORY_CHA); + case rus::kU: olc_add_stat(ch, GLORY_CHA); break; - case 'ф': olc_add_stat(ch, GLORY_HIT); + case rus::kEf: olc_add_stat(ch, GLORY_HIT); break; - case 'х': olc_add_stat(ch, GLORY_SUCCESS); + case rus::kHa: olc_add_stat(ch, GLORY_SUCCESS); break; - case 'ц': olc_add_stat(ch, GLORY_WILL); + case rus::kTse: olc_add_stat(ch, GLORY_WILL); break; - case 'ч': olc_add_stat(ch, GLORY_STABILITY); + case rus::kChe: olc_add_stat(ch, GLORY_STABILITY); break; - case 'ш': olc_add_stat(ch, GLORY_REFLEX); + case rus::kSha: olc_add_stat(ch, GLORY_REFLEX); break; - case 'щ': olc_add_stat(ch, GLORY_MIND); + case rus::kScha: olc_add_stat(ch, GLORY_MIND); break; - case 'ю': olc_add_stat(ch, GLORY_MANAREG); + case rus::kYu: olc_add_stat(ch, GLORY_MANAREG); break; - case 'в': { + case rus::kVe: { // получившиеся статы ch->set_str(olc_real_stat(ch, GLORY_STR)); ch->set_dex(olc_real_stat(ch, GLORY_DEX)); @@ -572,7 +575,7 @@ bool parse_spend_glory_menu(CharData *ch, char *arg) { save(); return 1; } - case 'я': ch->desc->glory_const.reset(); + case rus::kYa: ch->desc->glory_const.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование прервано.\r\n", ch); return 1; @@ -1126,7 +1129,7 @@ void PrintGloryChart(CharData *ch) { t_it != playerGloryList.end() && i < kPlayerChartSize; ++t_it, ++i) { std::string name = GetNameByUnique(t_it->get()->uid); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); if (name.length() == 0) { name = "*скрыто*"; } diff --git a/src/gameplay/mechanics/named_stuff.cpp b/src/gameplay/mechanics/named_stuff.cpp index 2451c7b8f5..9abda42e83 100644 --- a/src/gameplay/mechanics/named_stuff.cpp +++ b/src/gameplay/mechanics/named_stuff.cpp @@ -3,6 +3,9 @@ // Part of Bylins http://www.mud.ru #include "named_stuff.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" @@ -165,11 +168,15 @@ bool parse_nedit_menu(CharData *ch, char *arg) { if (!*buf1) { return false; } - if ((*buf1 < '1' || *buf1 > '8') && (LOWER(*buf1) != 'в' && LOWER(*buf1) != 'х' && LOWER(*buf1) != 'у')) { + if ((*buf1 < '1' || *buf1 > '8') && (native_text::first_char_code_lower(buf1) != rus::kVe + && native_text::first_char_code_lower(buf1) != rus::kHa + && native_text::first_char_code_lower(buf1) != rus::kU)) { SendMsgToChar(ch, "Неверный параметр %c!\r\n", *buf1); return false; } - if (!*buf2 && LOWER(*buf1) != 'в' && LOWER(*buf1) != 'х' && LOWER(*buf1) != 'у') { + if (!*buf2 && native_text::first_char_code_lower(buf1) != rus::kVe + && native_text::first_char_code_lower(buf1) != rus::kHa + && native_text::first_char_code_lower(buf1) != rus::kU) { if (*buf1 < '5' || *buf1 > '8') { SendMsgToChar("Не указан второй параметр!\r\n", ch); } else { @@ -190,7 +197,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { return false; } - switch (LOWER(*buf1)) { + switch (native_text::first_char_code_lower(buf1)) { case '1': if (a_isdigit(*buf2) && sscanf(buf2, "%d", &num)) { if (GetObjRnum(num) < 0) { @@ -262,7 +269,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { } break; - case 'у': + case rus::kU: if (!ch->desc->old_vnum) return false; stuff_list.erase(ch->desc->old_vnum); @@ -271,7 +278,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { save(); return true; - case 'в': tmp_node->uid = ch->desc->named_obj->uid; + case rus::kVe: tmp_node->uid = ch->desc->named_obj->uid; tmp_node->can_clan = ch->desc->named_obj->can_clan; tmp_node->can_alli = ch->desc->named_obj->can_alli; tmp_node->mail = ch->desc->named_obj->mail; @@ -287,7 +294,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { save(); return true; - case 'х': ch->desc->state = EConState::kPlaying; + case rus::kHa: ch->desc->state = EConState::kPlaying; SendMsgToChar(CommonMsg(ECommonMsg::kOk) + "\r\n", ch); return true; @@ -369,12 +376,12 @@ void do_named(CharData *ch, char *argument, int cmd, int subcmd) { out += buf1; } found++; - sprintf(buf2, "%6ld) &R*&n%-31s Владелец:%-16s e-mail:&S%s&s\r\n", + strcpy(buf2, fmt::format("{:6}) &R*&n{:<31} Владелец:{:<16} e-mail:&S{}&s\r\n", it->first + 1, "Несуществующий предмет", - GetNameByUnique(it->second->uid, false).c_str(), - str_dup(it->second->mail.c_str()) - ); + GetNameByUnique(it->second->uid, false), + it->second->mail + ).c_str()); out += buf2; } } else { @@ -388,9 +395,9 @@ void do_named(CharData *ch, char *argument, int cmd, int subcmd) { obj_proto[r_num]->get_vnum(), colored_name(obj_proto[r_num]->get_short_description().c_str(), -32)); if (privilege::IsGrGod(ch) || ch->IsFlagged(EPrf::kCoderinfo)) { - snprintf(buf2, kMaxStringLength, "%s Игра:%d Пост:%d Владелец:%-16s e-mail:&S%s&s\r\n", buf1, + strcpy(buf2, fmt::format("{} Игра:{} Пост:{} Владелец:{:<16} e-mail:&S{}&s\r\n", buf1, obj_proto.total_online(r_num), obj_proto.stored(r_num), - GetNameByUnique(it->second->uid, false).c_str(), it->second->mail.c_str()); + GetNameByUnique(it->second->uid, false), it->second->mail).c_str()); } else { snprintf(buf2, kMaxStringLength, "%s\r\n", buf1); } diff --git a/src/gameplay/mechanics/obj_sets_olc.cpp b/src/gameplay/mechanics/obj_sets_olc.cpp index 7195f13be8..fb411cb649 100644 --- a/src/gameplay/mechanics/obj_sets_olc.cpp +++ b/src/gameplay/mechanics/obj_sets_olc.cpp @@ -2,6 +2,8 @@ // Part of Bylins http://www.mud.ru #include "obj_sets.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "utils/grammar/declensions.h" #include @@ -525,19 +527,19 @@ void sedit::save_olc(CharData *ch) { void parse_main_exit(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': ch->desc->state = EConState::kPlaying; + case rus::kDe: + case rus::kDeUpper: ch->desc->state = EConState::kPlaying; ch->desc->sedit->save_olc(ch); ch->desc->sedit.reset(); SendMsgToChar("Изменения сохранены.\r\n", ch); break; case 'n': case 'N': - case 'н': - case 'Н': ch->desc->sedit.reset(); + case rus::kEn: + case rus::kEnUpper: ch->desc->sedit.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование отменено.\r\n", ch); break; @@ -551,11 +553,11 @@ void parse_main_exit(CharData *ch, const char *arg) { void parse_set_remove(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': { + case rus::kDe: + case rus::kDeUpper: { for (auto i = sets_list.begin(); i != sets_list.end(); ++i) { if ((*i)->uid == ch->desc->sedit->olc_set.uid) { sets_list.erase(i); @@ -571,8 +573,8 @@ void parse_set_remove(CharData *ch, const char *arg) { } case 'n': case 'N': - case 'н': - case 'Н': SendMsgToChar("Удаление отменено.\r\n", ch); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar("Удаление отменено.\r\n", ch); ch->desc->sedit->show_main(ch); break; default: @@ -585,11 +587,11 @@ void parse_set_remove(CharData *ch, const char *arg) { void sedit::parse_obj_remove(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': { + case rus::kDe: + case rus::kDeUpper: { auto i = olc_set.obj_list.find(obj_edit); if (i != olc_set.obj_list.end()) { olc_set.obj_list.erase(i); @@ -601,8 +603,8 @@ void sedit::parse_obj_remove(CharData *ch, const char *arg) { } case 'n': case 'N': - case 'н': - case 'Н': SendMsgToChar("Удаление отменено.\r\n", ch); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar("Удаление отменено.\r\n", ch); show_obj_edit(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); @@ -613,11 +615,11 @@ void sedit::parse_obj_remove(CharData *ch, const char *arg) { void sedit::parse_activ_remove(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': { + case rus::kDe: + case rus::kDeUpper: { auto i = olc_set.activ_list.find(activ_edit); if (i != olc_set.activ_list.end()) { olc_set.activ_list.erase(i); @@ -629,8 +631,8 @@ void sedit::parse_activ_remove(CharData *ch, const char *arg) { } case 'n': case 'N': - case 'н': - case 'Н': SendMsgToChar("Удаление отменено.\r\n", ch); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar("Удаление отменено.\r\n", ch); show_activ_edit(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); @@ -667,11 +669,11 @@ void sedit::parse_global_msg(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': + case rus::kVeUpper: + case rus::kVe: if (msg_edit != global_msg) { SendMsgToChar("Вы хотите сохранить изменения? Y(Д)/N(Н) : ", ch); state = STATE_GLOBAL_MSG_EXIT; @@ -721,11 +723,11 @@ void sedit::parse_global_msg(CharData *ch, const char *arg) { void parse_global_msg_exit(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': ch->desc->state = EConState::kPlaying; + case rus::kDe: + case rus::kDeUpper: ch->desc->state = EConState::kPlaying; global_msg = ch->desc->sedit->msg_edit; obj_sets::save(); ch->desc->sedit.reset(); @@ -733,8 +735,8 @@ void parse_global_msg_exit(CharData *ch, const char *arg) { break; case 'n': case 'N': - case 'н': - case 'Н': ch->desc->sedit.reset(); + case rus::kEn: + case rus::kEnUpper: ch->desc->sedit.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование отменено.\r\n", ch); break; @@ -754,11 +756,11 @@ void sedit::parse_main(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': + case rus::kVeUpper: + case rus::kVe: if (new_entry || changed()) { SendMsgToChar("Вы хотите сохранить изменения? Y(Д)/N(Н) : ", ch); state = STATE_MAIN_EXIT; @@ -1132,11 +1134,11 @@ void sedit::parse_obj_edit(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': show_main(ch); + case rus::kVeUpper: + case rus::kVe: show_main(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); show_obj_edit(ch); @@ -1337,11 +1339,11 @@ void sedit::parse_activ_edit(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': show_main(ch); + case rus::kVeUpper: + case rus::kVe: show_main(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); show_activ_edit(ch); diff --git a/src/gameplay/mechanics/sight.cpp b/src/gameplay/mechanics/sight.cpp index bd3cbcfddb..d76284a088 100644 --- a/src/gameplay/mechanics/sight.cpp +++ b/src/gameplay/mechanics/sight.cpp @@ -6,6 +6,7 @@ */ #include "engine/core/char_movement.h" +#include "utils/native_text.h" #include "engine/core/target_resolver.h" #include "sight.h" #include "gameplay/mechanics/hide.h" @@ -1357,7 +1358,7 @@ const char *show_obj_to_char(ObjData *object, CharData *ch, int mode, int show_s } } else if (mode >= 2 && how <= 1) { std::string obj_name = OBJN(object, ch, grammar::ECase::kNom); - obj_name[0] = UPPER(obj_name[0]); + native_text::capitalize_first(obj_name); if (object->get_type() == EObjType::kLightSource) { if (GET_OBJ_VAL(object, 2) == -1) { sprintf(buf2, "\r\n%s дает вечный свет.", obj_name.c_str()); diff --git a/src/utils/mud_string.cpp b/src/utils/mud_string.cpp index 5a9fcd3d98..6bd64f25be 100644 --- a/src/utils/mud_string.cpp +++ b/src/utils/mud_string.cpp @@ -1,6 +1,7 @@ #include "mud_string.h" #include "utils.h" +#include "utils/native_text.h" int search_block(const char *target_string, const char **list, int exact); @@ -45,9 +46,13 @@ T one_argument_template(T argument, char *first_arg) { do { skip_spaces(&argument); first_arg = begin; + // Lowercase one whole character at a time (issue #3681). The a_isspace() test stays + // byte-based on purpose: it only ever runs at a character boundary, and every + // whitespace character is ASCII, so no multibyte lead byte can be mistaken for one. while (*argument && !a_isspace(*argument)) { - *(first_arg++) = a_lcc(*argument); - argument++; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; } *first_arg = '\0'; } while (fill_word(begin)); @@ -64,11 +69,12 @@ T any_one_arg_template(T argument, char *first_arg) { skip_spaces(&argument); int num = 0; + // As above: one character per step, `num` still counts bytes so it remains a buffer guard. while (*argument && !a_isspace(*argument) && num < kMaxStringLength - 1) { - *first_arg = a_lcc(*argument); - ++first_arg; - ++argument; - ++num; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; + num += static_cast(n); } *first_arg = '\0'; skip_spaces(&argument); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp new file mode 100644 index 0000000000..9ebddeb3b7 --- /dev/null +++ b/src/utils/native_text.cpp @@ -0,0 +1,706 @@ +/** +\file native_text.cpp - a part of the Bylins engine. +\brief Native-encoding character helpers declared in native_text.h (issue #3681). + +Two implementations selected by the INTERNAL_ENCODING_UTF8 build macro. The KOI8-R branch is kept +byte-for-byte identical to the open-coded logic these helpers replace so that routing call sites +through them changes nothing until the encoding flip. +*/ + +#include "native_text.h" +#include "utils_encoding.h" + +#ifdef INTERNAL_ENCODING_UTF8 +#include "utf8.h" +#else +// The KOI8-R case tables (defined in utils.cpp). Declared directly instead of including utils.h, +// which drags in fmt/ and much of the engine for what is just two 256-byte lookups. +extern const char a_ucc_table[]; +extern const char a_lcc_table[]; +extern const bool a_isalnum_table[]; +extern const bool a_isalpha_table[]; +extern const bool a_isupper_table[]; + +// Yo is absent from the Latin table and always had a special case in get_filename(). +constexpr unsigned char kYoLowerByte = 0xA3; +constexpr unsigned char kYoUpperByte = 0xB3; +#endif + +#include +#include + +namespace native_text { + +#ifdef INTERNAL_ENCODING_UTF8 + +bool native_is_utf8() { + return true; +} + +std::size_t char_count(const char *begin, const char *end) { + return utf8::length(std::string_view(begin, static_cast(end - begin))); +} + +std::size_t char_count(std::string_view s) { + return utf8::length(s); +} + +void capitalize_first(char *s) { + if (s == nullptr || *s == '\0') { + return; + } + const std::string_view sv(s); + char32_t cp = 0; + const std::size_t len = utf8::decode(sv, 0, cp); + if (len == 0) { + return; + } + const char32_t upper = utf8::to_upper(cp); + if (upper == cp) { + return; + } + std::string encoded; + if (utf8::encode(upper, encoded) == len) { + for (std::size_t i = 0; i < len; ++i) { + s[i] = encoded[i]; + } + } +} + +std::size_t truncate_offset(std::string_view s, std::size_t max_bytes) { + if (max_bytes >= s.size()) { + return s.size(); + } + std::size_t pos = 0; + while (true) { + char32_t cp = 0; + const std::size_t len = utf8::decode(s, pos, cp); + if (len == 0 || pos + len > max_bytes) { + break; + } + pos += len; + } + return pos; +} + +namespace { + +// Local copy of the lead-byte length table. utf8::sequence_length lives in another translation +// unit, and this runs once per character in every scan -- a cross-module call there costs more +// than the work itself. +inline std::size_t lead_len(unsigned char c) { + if (c < 0x80) { + return 1; + } + if (c >= 0xC0 && c <= 0xDF) { + return 2; + } + if (c >= 0xE0 && c <= 0xEF) { + return 3; + } + if (c >= 0xF0 && c <= 0xF7) { + return 4; + } + return 1; +} + +} // namespace + +std::size_t char_bytes(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return 1; + } + const std::size_t want = lead_len(lead); + std::size_t n = 1; + while (n < want && (static_cast(s[n]) & 0xC0) == 0x80) { + ++n; + } + return n; +} + +namespace { + +// Shared driver for the two case-insensitive comparisons. `limit` caps how many bytes of `a` may +// be consumed (npos = unlimited): once that budget is spent the strings count as equal, which is +// what the strn_cmp callers -- who pass a prefix length in bytes -- expect. +int compare_folded(std::string_view a, std::string_view b, std::size_t limit) { + std::size_t pa = 0; + std::size_t pb = 0; + while (true) { + if (limit != std::string_view::npos && pa >= limit) { + return 0; + } + char32_t ca = 0; + char32_t cb = 0; + const std::size_t la = utf8::decode(a, pa, ca); + const std::size_t lb = utf8::decode(b, pb, cb); + if (la == 0 && lb == 0) { + return 0; + } + if (la == 0) { + return -1; + } + if (lb == 0) { + return 1; + } + const char32_t fa = utf8::to_lower(ca); + const char32_t fb = utf8::to_lower(cb); + if (fa != fb) { + return fa < fb ? -1 : 1; + } + pa += la; + pb += lb; + } +} + +} // namespace + +int compare_ci(std::string_view a, std::string_view b) { + return compare_folded(a, b, std::string_view::npos); +} + +int ncompare_ci(std::string_view a, std::string_view b, std::size_t n) { + return compare_folded(a, b, n); +} + +bool is_alnum_char(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return (lead >= '0' && lead <= '9') || (lead >= 'A' && lead <= 'Z') || (lead >= 'a' && lead <= 'z'); + } + char32_t cp = 0; + if (utf8::decode(std::string_view(s, char_bytes(s)), 0, cp) == 0) { + return false; + } + // Russian Cyrillic block, including Yo. + return (cp >= 0x0410 && cp <= 0x044F) || cp == 0x0401 || cp == 0x0451; +} + +bool is_alpha_char(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return (lead >= 'A' && lead <= 'Z') || (lead >= 'a' && lead <= 'z'); + } + char32_t cp = 0; + if (utf8::decode(std::string_view(s, char_bytes(s)), 0, cp) == 0) { + return false; + } + return (cp >= 0x0410 && cp <= 0x044F) || cp == 0x0401 || cp == 0x0451; +} + +bool is_upper_char(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return lead >= 'A' && lead <= 'Z'; + } + char32_t cp = 0; + if (utf8::decode(std::string_view(s, char_bytes(s)), 0, cp) == 0) { + return false; + } + return (cp >= 0x0410 && cp <= 0x042F) || cp == 0x0401; +} + +bool chars_equal_ci(const char *a, const char *b) { + char32_t ca = 0; + char32_t cb = 0; + if (utf8::decode(std::string_view(a, char_bytes(a)), 0, ca) == 0 + || utf8::decode(std::string_view(b, char_bytes(b)), 0, cb) == 0) { + return false; + } + return utf8::to_lower(ca) == utf8::to_lower(cb); +} + +namespace { + +// Case folding for the repertoire the engine actually carries -- ASCII and the two-byte Cyrillic +// block -- done directly on the bytes. This runs per character in hot paths (whole-string case +// conversion, argument parsing), so it must not decode, allocate, or make a cross-module call: +// +// A-Z / a-z : one byte, +-0x20 +// A-P (D0 90..D0 9F) <-> a-p (D0 B0..D0 BF) : lead stays D0, trail +-0x20 +// R-Ya(D0 A0..D0 AF) <-> r-ya(D1 80..D1 8F) : lead flips D0<->D1, trail -+0x20 +// Yo (D0 81) <-> yo (D1 91) +// +// Anything else (other scripts, malformed bytes) falls through to the general path below, which +// is correct but slower -- and effectively never taken by this codebase. +inline bool fold_fast(const char *src, char *dst, std::size_t len, bool upper) { + const unsigned char c0 = static_cast(src[0]); + if (c0 < 0x80) { + char c = src[0]; + if (upper) { + if (c >= 'a' && c <= 'z') { + c = static_cast(c - 0x20); + } + } else if (c >= 'A' && c <= 'Z') { + c = static_cast(c + 0x20); + } + dst[0] = c; + return true; + } + if (len != 2) { + return false; + } + const unsigned char c1 = static_cast(src[1]); + unsigned char o0 = c0; + unsigned char o1 = c1; + if (upper) { + if (c0 == 0xD0 && c1 >= 0xB0 && c1 <= 0xBF) { // a-p -> A-P + o1 = static_cast(c1 - 0x20); + } else if (c0 == 0xD1 && c1 >= 0x80 && c1 <= 0x8F) { // r-ya -> R-Ya + o0 = 0xD0; + o1 = static_cast(c1 + 0x20); + } else if (c0 == 0xD1 && c1 == 0x91) { // yo -> Yo + o0 = 0xD0; + o1 = 0x81; + } else if (!((c0 == 0xD0 && c1 >= 0x90 && c1 <= 0xAF) || (c0 == 0xD0 && c1 == 0x81))) { + return false; // not Cyrillic: general path + } + } else { + if (c0 == 0xD0 && c1 >= 0x90 && c1 <= 0x9F) { // A-P -> a-p + o1 = static_cast(c1 + 0x20); + } else if (c0 == 0xD0 && c1 >= 0xA0 && c1 <= 0xAF) { // R-Ya -> r-ya + o0 = 0xD1; + o1 = static_cast(c1 - 0x20); + } else if (c0 == 0xD0 && c1 == 0x81) { // Yo -> yo + o0 = 0xD1; + o1 = 0x91; + } else if (!((c0 == 0xD0 && c1 >= 0xB0) || (c0 == 0xD1 && c1 <= 0x8F) || (c0 == 0xD1 && c1 == 0x91))) { + return false; + } + } + dst[0] = static_cast(o0); + dst[1] = static_cast(o1); + return true; +} + +std::size_t copy_folded_char(const char *src, char *dst, bool upper) { + const std::size_t len = char_bytes(src); + if (fold_fast(src, dst, len, upper)) { + return len; + } + // General path: decode, fold, re-encode; only taken for characters outside ASCII+Cyrillic. + char32_t cp = 0; + if (utf8::decode(std::string_view(src, len), 0, cp) != 0) { + const char32_t folded = upper ? utf8::to_upper(cp) : utf8::to_lower(cp); + char tmp[4]; + if (folded != cp && utf8::encode(folded, tmp) == len) { + for (std::size_t i = 0; i < len; ++i) { + dst[i] = tmp[i]; + } + return len; + } + } + if (dst != src) { + for (std::size_t i = 0; i < len; ++i) { + dst[i] = src[i]; + } + } + return len; +} + +} // namespace + +std::size_t copy_lower_char(const char *src, char *dst) { + return copy_folded_char(src, dst, false); +} + +std::size_t copy_upper_char(const char *src, char *dst) { + return copy_folded_char(src, dst, true); +} + + +char32_t first_char_code(const char *s) { + if (s == nullptr || *s == '\0') { + return 0; + } + char32_t cp = 0; + utf8::decode(std::string_view(s, char_bytes(s)), 0, cp); + return cp; +} + +char32_t first_char_code_lower(const char *s) { + return utf8::to_lower(first_char_code(s)); +} + +char32_t first_char_code_upper(const char *s) { + return utf8::to_upper(first_char_code(s)); +} + +namespace { + +// Whole-buffer case conversion as one tight loop with no calls in the hot path: ASCII and the +// two-byte Cyrillic block are folded straight on the bytes. Anything else falls back to the +// general helper, which this codebase never hits in practice. Written this way deliberately -- +// a per-character dispatch measured several times slower than the byte loop it replaces. +inline void fold_range_utf8(char *p, char *const end, bool upper) { + while (p < end) { + const unsigned char c0 = static_cast(*p); + if (c0 < 0x80) { + char c = *p; + if (upper) { + if (c >= 'a' && c <= 'z') { + c = static_cast(c - 0x20); + } + } else if (c >= 'A' && c <= 'Z') { + c = static_cast(c + 0x20); + } + *p++ = c; + continue; + } + if ((c0 == 0xD0 || c0 == 0xD1) && p + 1 < end) { + const unsigned char c1 = static_cast(p[1]); + if (upper) { + if (c0 == 0xD0 && c1 >= 0xB0) { + p[1] = static_cast(c1 - 0x20); + } else if (c0 == 0xD1 && c1 <= 0x8F) { + p[0] = static_cast(0xD0); + p[1] = static_cast(c1 + 0x20); + } else if (c0 == 0xD1 && c1 == 0x91) { + p[0] = static_cast(0xD0); + p[1] = static_cast(0x81); + } + } else { + if (c0 == 0xD0 && c1 >= 0x90 && c1 <= 0x9F) { + p[1] = static_cast(c1 + 0x20); + } else if (c0 == 0xD0 && c1 >= 0xA0 && c1 <= 0xAF) { + p[0] = static_cast(0xD1); + p[1] = static_cast(c1 - 0x20); + } else if (c0 == 0xD0 && c1 == 0x81) { + p[0] = static_cast(0xD1); + p[1] = static_cast(0x91); + } + } + p += 2; + continue; + } + p += upper ? copy_upper_char(p, p) : copy_lower_char(p, p); + } +} + +} // namespace + +void to_lower(std::string &s) { fold_range_utf8(s.data(), s.data() + s.size(), false); } +void to_upper(std::string &s) { fold_range_utf8(s.data(), s.data() + s.size(), true); } +void to_lower(char *s) { fold_range_utf8(s, s + std::char_traits::length(s), false); } +void to_upper(char *s) { fold_range_utf8(s, s + std::char_traits::length(s), true); } + + +std::string from_koi8(const std::string &text) { + if (text.empty()) { + return text; + } + // koi_to_utf8() can grow the text; utils_encoding sizes its own buffers at 6x, so match that. + std::vector out(text.size() * 6 + 1, '\0'); + codepages::koi_to_utf8(const_cast(text.c_str()), out.data()); + return std::string(out.data()); +} + +std::string translit_to_filename(std::string_view name) { + // Code point -> the very same Latin character the KOI8-R byte table yields, so a player's + // file name is identical before and after the flip. Upper and lower case collapse together + // because the byte-wise original lowercased after transliterating. + static const struct { char32_t cp; char latin; } kMap[] = { + {0x0430, 'a'}, {0x0410, 'a'}, + {0x0431, 'b'}, {0x0411, 'b'}, + {0x0432, 'v'}, {0x0412, 'v'}, + {0x0433, 'g'}, {0x0413, 'g'}, + {0x0434, 'd'}, {0x0414, 'd'}, + {0x0435, 'e'}, {0x0415, 'e'}, + {0x0451, '9'}, {0x0401, '9'}, + {0x0436, '1'}, {0x0416, '1'}, + {0x0437, 'z'}, {0x0417, 'z'}, + {0x0438, 'i'}, {0x0418, 'i'}, + {0x0439, 'j'}, {0x0419, 'j'}, + {0x043A, 'k'}, {0x041A, 'k'}, + {0x043B, 'l'}, {0x041B, 'l'}, + {0x043C, 'm'}, {0x041C, 'm'}, + {0x043D, 'n'}, {0x041D, 'n'}, + {0x043E, 'o'}, {0x041E, 'o'}, + {0x043F, 'p'}, {0x041F, 'p'}, + {0x0440, 'r'}, {0x0420, 'r'}, + {0x0441, 's'}, {0x0421, 's'}, + {0x0442, 't'}, {0x0422, 't'}, + {0x0443, 'y'}, {0x0423, 'y'}, + {0x0444, 'f'}, {0x0424, 'f'}, + {0x0445, 'h'}, {0x0425, 'h'}, + {0x0446, 'c'}, {0x0426, 'c'}, + {0x0447, '7'}, {0x0427, '7'}, + {0x0448, '4'}, {0x0428, '4'}, + {0x0449, '6'}, {0x0429, '6'}, + {0x044A, '8'}, {0x042A, '8'}, + {0x044B, '3'}, {0x042B, '3'}, + {0x044C, '2'}, {0x042C, '2'}, + {0x044D, '5'}, {0x042D, '5'}, + {0x044E, '0'}, {0x042E, '0'}, + {0x044F, 'q'}, {0x042F, 'q'}, + }; + std::string out; + out.reserve(name.size()); + std::size_t pos = 0; + while (pos < name.size()) { + char32_t cp = 0; + const std::size_t len = utf8::decode(name, pos, cp); // decode() already reports the length + if (len == 0) { + break; + } + if (cp < 0x80) { + char c = static_cast(cp); + if (c >= 'A' && c <= 'Z') { + c = static_cast(c + 0x20); + } + out.push_back(c); + } else { + char mapped = '_'; + for (const auto &e : kMap) { + if (e.cp == cp) { + mapped = e.latin; + break; + } + } + out.push_back(mapped); + } + pos += len; + } + return out; +} + +#else // KOI8-R: 1 byte == 1 character + +bool native_is_utf8() { + return false; +} + +std::size_t char_count(const char *begin, const char *end) { + return static_cast(end - begin); +} + +std::size_t char_count(std::string_view s) { + return s.size(); +} + +void capitalize_first(char *s) { + if (s != nullptr && *s != '\0') { + *s = a_ucc_table[static_cast(*s)]; + } +} + +std::size_t truncate_offset(std::string_view s, std::size_t max_bytes) { + return max_bytes < s.size() ? max_bytes : s.size(); +} + +std::size_t char_bytes(const char *) { + return 1; +} + +namespace { + +// Byte-wise fold-and-subtract, identical to the open-coded `LOWER(a[i]) - LOWER(b[i])` loops in +// utils_string.cpp: the magnitude of the result (not just its sign) is preserved, since some +// callers propagate it. A string that ended compares as LOWER('\0') against the other's byte. +int compare_bytes(std::string_view a, std::string_view b, std::size_t limit) { + std::size_t i = 0; + while (true) { + if (limit != std::string_view::npos && i >= limit) { + return 0; + } + const bool a_end = i >= a.size(); + const bool b_end = i >= b.size(); + if (a_end && b_end) { + return 0; + } + const unsigned char ca = a_end ? '\0' : static_cast(a[i]); + const unsigned char cb = b_end ? '\0' : static_cast(b[i]); + const int chk = a_lcc_table[ca] - a_lcc_table[cb]; + if (chk != 0) { + return chk; + } + ++i; + } +} + +} // namespace + +int compare_ci(std::string_view a, std::string_view b) { + return compare_bytes(a, b, std::string_view::npos); +} + +int ncompare_ci(std::string_view a, std::string_view b, std::size_t n) { + return compare_bytes(a, b, n); +} + +bool is_alnum_char(const char *s) { + return a_isalnum_table[static_cast(*s)]; +} + +bool is_alpha_char(const char *s) { + return a_isalpha_table[static_cast(*s)]; +} + +bool is_upper_char(const char *s) { + return a_isupper_table[static_cast(*s)]; +} + +bool chars_equal_ci(const char *a, const char *b) { + return a_lcc_table[static_cast(*a)] == a_lcc_table[static_cast(*b)]; +} + +std::size_t copy_lower_char(const char *src, char *dst) { + *dst = a_lcc_table[static_cast(*src)]; + return 1; +} + +std::size_t copy_upper_char(const char *src, char *dst) { + *dst = a_ucc_table[static_cast(*src)]; + return 1; +} + + +char32_t first_char_code(const char *s) { + return (s == nullptr || *s == '\0') ? 0 : static_cast(*s); +} + +char32_t first_char_code_lower(const char *s) { + return (s == nullptr || *s == '\0') + ? 0 : static_cast(a_lcc_table[static_cast(*s)]); +} + +char32_t first_char_code_upper(const char *s) { + return (s == nullptr || *s == '\0') + ? 0 : static_cast(a_ucc_table[static_cast(*s)]); +} + +void to_lower(std::string &s) { + for (char &c : s) { + c = a_lcc_table[static_cast(c)]; + } +} + +void to_upper(std::string &s) { + for (char &c : s) { + c = a_ucc_table[static_cast(c)]; + } +} + +void to_lower(char *s) { + for (; *s; ++s) { + *s = a_lcc_table[static_cast(*s)]; + } +} + +void to_upper(char *s) { + for (; *s; ++s) { + *s = a_ucc_table[static_cast(*s)]; + } +} + + +std::string from_koi8(const std::string &text) { + return text; // the native encoding already is KOI8-R +} + +std::string translit_to_filename(std::string_view name) { + // Byte-wise, exactly as get_filename() did before this was extracted: transliterate through + // the Latin table, then lowercase. Yo is not in that table and had its own special case. + std::string out; + out.reserve(name.size()); + for (const char ch : name) { + const unsigned char b = static_cast(ch); + if (b == kYoLowerByte || b == kYoUpperByte) { + out.push_back('9'); + } else { + out.push_back(a_lcc_table[static_cast(codepages::AtoL(ch))]); + } + } + return out; +} + +#endif + +// --------------------------------------------------------------------------------------------- +// Encoding-independent helpers: expressed purely in terms of the primitives above, so they need +// no per-encoding branch. Unlike char_bytes() these take a bounded view, not a C string, so they +// are safe on a string_view that is not null-terminated. +// --------------------------------------------------------------------------------------------- + +namespace { + +inline std::size_t lead_len_shared(unsigned char c) { + if (c < 0x80) { + return 1; + } + if (c >= 0xC0 && c <= 0xDF) { + return 2; + } + if (c >= 0xE0 && c <= 0xEF) { + return 3; + } + if (c >= 0xF0 && c <= 0xF7) { + return 4; + } + return 1; +} + +// Byte length of the character at `pos`, clamped to the end of `s`. +std::size_t char_bytes_at(std::string_view s, std::size_t pos) { +#ifdef INTERNAL_ENCODING_UTF8 + const unsigned char lead = static_cast(s[pos]); + if (lead < 0x80) { + return 1; + } + const std::size_t want = lead_len_shared(lead); + std::size_t n = 1; + while (n < want && pos + n < s.size() && (static_cast(s[pos + n]) & 0xC0) == 0x80) { + ++n; + } + return n; +#else + (void) s; + (void) pos; + return 1; +#endif +} + +} // namespace + +void capitalize_first(std::string &s) { + if (s.empty()) { + return; + } + // The uppercase form keeps the byte length for ASCII and the whole Russian alphabet, so + // capitalising in place never resizes the string. + capitalize_first(&s[0]); +} + +std::size_t CharRange::Iterator::step(std::string_view s, std::size_t pos) { + return pos < s.size() ? char_bytes_at(s, pos) : 0; +} + +std::size_t last_char_offset(std::string_view s) { + std::size_t last = 0; + std::size_t pos = 0; + while (pos < s.size()) { + last = pos; + pos += char_bytes_at(s, pos); + } + return last; +} + +bool list_contains_char(std::string_view list, std::string_view ch) { + if (ch.empty()) { + return false; + } + std::size_t pos = 0; + while (pos < list.size()) { + const std::size_t len = char_bytes_at(list, pos); + if (len == ch.size() && list.compare(pos, len, ch) == 0) { + return true; + } + pos += len; + } + return false; +} + +} // namespace native_text + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/native_text.h b/src/utils/native_text.h new file mode 100644 index 0000000000..f7715296e2 --- /dev/null +++ b/src/utils/native_text.h @@ -0,0 +1,168 @@ +/** +\file native_text.h - a part of the Bylins engine. +\brief Character-semantic operations in the engine's *native runtime encoding* (issue #3681). + +The migration flips the engine's internal string encoding from KOI8-R (1 byte == 1 character) to +UTF-8 (multibyte). Code that must reason about characters -- counting display width, capitalising +a letter, truncating without splitting a character -- should go through this thin dispatch layer +instead of assuming bytes. The active encoding is chosen at build time by the `internal_encoding` +Meson option, which defines INTERNAL_ENCODING_UTF8 for the UTF-8 build: + + * KOI8-R (default, current behaviour): every helper is byte-for-byte identical to the open-coded + byte logic it replaces, so routing call sites through it is a no-op. + * UTF-8: helpers operate on code points via the utf8:: primitives. + +This lets the whole byte-vs-char refactor land and ship on KOI8-R (safely, as a no-op) ahead of +the encoding flip. Once the flip is permanent the KOI8-R branch and this indirection are removed. +*/ + +#ifndef BYLINS_SRC_UTILS_NATIVE_TEXT_H_ +#define BYLINS_SRC_UTILS_NATIVE_TEXT_H_ + +#include +#include +#include + +namespace native_text { + +// True iff the engine was built with UTF-8 as its native runtime encoding. Reflects the flag the +// library itself was compiled with (not the translation unit that calls this), so callers/tests +// can branch reliably regardless of their own compile flags. +bool native_is_utf8(); + +// Number of display characters in a byte range / view. KOI8-R: the byte count. UTF-8: the number +// of code points (malformed bytes counted as one each, so it never stalls on legacy data). +std::size_t char_count(const char *begin, const char *end); +std::size_t char_count(std::string_view s); + +// Uppercase the first character of the null-terminated string `s` in place (ASCII + Russian +// Cyrillic incl. Yo). No-op on an empty string, on a non-cased first character, or in the (never +// occurring for these alphabets) case where the uppercase form has a different byte length. +void capitalize_first(char *s); +void capitalize_first(std::string &s); + +// Largest byte offset <= max_bytes that lands on a character boundary, so cutting the string +// there never splits a multibyte character. KOI8-R: min(max_bytes, s.size()). +std::size_t truncate_offset(std::string_view s, std::size_t max_bytes); + +// Byte length of the character that starts at `s` (KOI8-R: 1), for stepping over a whole +// character byte-by-byte. Always >= 1; on a malformed/truncated UTF-8 lead it returns only the +// bytes actually present (never counts past a terminator or a non-continuation byte). +std::size_t char_bytes(const char *s); + +// Numeric identity of the character starting at `s`, for dispatching a switch on a letter: +// the raw byte under KOI8-R, the code point under UTF-8. Compare against the constants in +// utils/russian_keys.h (Cyrillic) or ordinary character literals (ASCII, identical in both). +// Returns 0 on an empty string. +char32_t first_char_code(const char *s); + +// The same, case-folded -- the replacements for switch (LOWER(*s)) / switch (UPPER(*s)). +char32_t first_char_code_lower(const char *s); +char32_t first_char_code_upper(const char *s); + +// Case-insensitive comparison in the native encoding: lexicographic over lowered characters, +// the shorter string orders first, returns the signed difference at the first mismatch (0 when +// equal). KOI8-R: per byte, via LOWER() -- matches str_cmp/str/str semantics. UTF-8: per code +// point, folded via utf8::to_lower (so the sign is meaningful; the magnitude is a code-point +// difference). In ncompare_ci, `n` is a byte budget -- the callers pass strlen()/length() -- and +// the comparison stops (as a match) once that many bytes of equal text have been consumed. +int compare_ci(std::string_view a, std::string_view b); +int ncompare_ci(std::string_view a, std::string_view b, std::size_t n); + +// Is the character starting at `s` alphanumeric? KOI8-R: the a_isalnum byte table. UTF-8: ASCII +// letters/digits plus the Russian Cyrillic block -- so a multibyte letter is classified as one +// alphanumeric character rather than a lead byte followed by "punctuation" trail bytes. +bool is_alnum_char(const char *s); + +// Is the character starting at `s` a letter? Same contract as is_alnum_char, minus the digits. +bool is_alpha_char(const char *s); + +// Is the character starting at `s` an uppercase letter? KOI8-R: the a_isupper byte table. +// UTF-8: ASCII A-Z plus the uppercase Cyrillic range (and Yo) as whole code points -- the byte +// table cannot see these at all, since a UTF-8 Cyrillic lead byte is not in its uppercase range. +bool is_upper_char(const char *s); + +// Do the characters starting at `a` and `b` match ignoring case? KOI8-R: LOWER(*a) == LOWER(*b). +// UTF-8: compares whole folded code points, so "P" matches "p" in Cyrillic too. +bool chars_equal_ci(const char *a, const char *b); + +// Copy the character starting at `src` to `dst`, lowercased, and return how many bytes were +// consumed (always the same number written, so a caller's buffer accounting is unaffected). +// KOI8-R: one byte through a_lcc_table. UTF-8: folds the code point; in the rare case where the +// lowercase form would not be the same byte length, the character is copied unchanged rather +// than resized. `dst` may alias `src` (the length-preserving property makes that safe). +std::size_t copy_lower_char(const char *src, char *dst); + +// Uppercase counterpart of copy_lower_char, with the same contract. +std::size_t copy_upper_char(const char *src, char *dst); + +// Byte offset at which the final character of `s` begins (0 for an empty string), so that +// s.substr(0, last_char_offset(s)) drops exactly one character and s.substr(last_char_offset(s)) +// is that character. KOI8-R: s.size() - 1. +std::size_t last_char_offset(std::string_view s); + +// Range-for over the characters of `s`: each element is a string_view covering exactly one +// character, so scanning code never does pointer arithmetic and never lands mid-character. +// +// for (auto ch : native_text::chars(name)) { ... } // ch is one character, whatever its size +// +// The view must outlive the loop (it is not copied). Malformed bytes yield one element each. +class CharRange { + public: + explicit CharRange(std::string_view s) : m_str(s) {} + + class Iterator { + public: + Iterator(std::string_view s, std::size_t pos) : m_str(s), m_pos(pos), m_len(step(s, pos)) {} + std::string_view operator*() const { return m_str.substr(m_pos, m_len); } + Iterator &operator++() { + m_pos += m_len; + m_len = step(m_str, m_pos); + return *this; + } + bool operator!=(const Iterator &other) const { return m_pos != other.m_pos; } + + private: + static std::size_t step(std::string_view s, std::size_t pos); + std::string_view m_str; + std::size_t m_pos; + std::size_t m_len; + }; + + [[nodiscard]] Iterator begin() const { return Iterator(m_str, 0); } + [[nodiscard]] Iterator end() const { return Iterator(m_str, m_str.size()); } + + private: + std::string_view m_str; +}; + +inline CharRange chars(std::string_view s) { return CharRange(s); } + +// Whole-string case conversion in place. Prefer these over hand-rolled per-character loops. +// Length-preserving for ASCII and the Russian alphabet, so no reallocation happens. +void to_lower(std::string &s); +void to_upper(std::string &s); +void to_lower(char *s); +void to_upper(char *s); + +// Bring text stored on disk in KOI8-R (world files, configs, saves) into the engine's native +// encoding. Identity under KOI8-R, a transcode under UTF-8. Having it here keeps the loaders +// free of #ifdefs and gives one place to revisit when the data files themselves move. +std::string from_koi8(const std::string &text); + +// Transliterate `name` into the ASCII form used for save-file names: Russian letters become +// Latin ones, ASCII is lowercased. The mapping is fixed by what the byte-wise implementation +// produced before the migration and MUST NOT drift -- the result is the on-disk file name of a +// player, so a change would orphan every existing character. Pinned by a test in both encodings. +std::string translit_to_filename(std::string_view name); + +// Does the single character `ch` occur in `list`? The replacement for strchr() over a literal +// list of letters: `list` is walked one whole character at a time, so a multibyte character can +// never match on a partial byte sequence. Comparison is exact (case-sensitive), like strchr. +bool list_contains_char(std::string_view list, std::string_view ch); + +} // namespace native_text + +#endif // BYLINS_SRC_UTILS_NATIVE_TEXT_H_ + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/russian_keys.h b/src/utils/russian_keys.h new file mode 100644 index 0000000000..85f21c3683 --- /dev/null +++ b/src/utils/russian_keys.h @@ -0,0 +1,125 @@ +/** +\file russian_keys.h - a part of the Bylins engine. +\brief Russian letters as switch-able constants, valid in both native encodings (issue #3681). + +Menus and OLC editors dispatch on a single letter the player typed: + + switch (*arg) { + case 'y': case 'Y': case 'd': case 'D': ... + +A Cyrillic *character* literal cannot survive the encoding flip: with UTF-8 sources 'd' (Cyrillic) +is a multi-character constant, so the compiler folds it to an implementation-defined value and the +menu silently stops responding. Nor can such a literal be written portably for both encodings. + +The way out is to keep the switch but dispatch on a number instead of a literal. Under KOI8-R a +letter is one byte, under UTF-8 it is a code point, so each letter is spelled out numerically for +both and the constants below are what the switch compares against: + + switch (native_text::first_char_code(arg)) { + case 'y': case 'Y': case rus::kDa: case rus::kDaUpper: ... + +Note that ASCII cases stay ordinary character literals -- those are identical in both encodings. + +Once the flip is permanent (track D) the KOI8-R half goes away and these can collapse into plain +U'...' literals. +*/ + +#ifndef BYLINS_SRC_UTILS_RUSSIAN_KEYS_H_ +#define BYLINS_SRC_UTILS_RUSSIAN_KEYS_H_ + +namespace rus { + +#ifdef INTERNAL_ENCODING_UTF8 + +// Unicode code points: U+0410..U+042F (upper), U+0430..U+044F (lower), U+0401/U+0451 (Yo). +#define BYLINS_RUS_LETTER(name, upper_cp, lower_cp) \ + constexpr char32_t name##Upper = upper_cp; \ + constexpr char32_t name = lower_cp + +#else + +// KOI8-R byte values. The alphabet is not laid out alphabetically in KOI8-R, so every letter is +// spelled out rather than derived from an offset. +#define BYLINS_RUS_LETTER(name, upper_byte, lower_byte) \ + constexpr char32_t name##Upper = upper_byte; \ + constexpr char32_t name = lower_byte + +#endif + +#ifdef INTERNAL_ENCODING_UTF8 +BYLINS_RUS_LETTER(kA, 0x0410, 0x0430); // А а +BYLINS_RUS_LETTER(kBe, 0x0411, 0x0431); // Б б +BYLINS_RUS_LETTER(kVe, 0x0412, 0x0432); // В в +BYLINS_RUS_LETTER(kGe, 0x0413, 0x0433); // Г г +BYLINS_RUS_LETTER(kDe, 0x0414, 0x0434); // Д д +BYLINS_RUS_LETTER(kIe, 0x0415, 0x0435); // Е е +BYLINS_RUS_LETTER(kYo, 0x0401, 0x0451); // Ё ё +BYLINS_RUS_LETTER(kZhe, 0x0416, 0x0436); // Ж ж +BYLINS_RUS_LETTER(kZe, 0x0417, 0x0437); // З з +BYLINS_RUS_LETTER(kI, 0x0418, 0x0438); // И и +BYLINS_RUS_LETTER(kIi, 0x0419, 0x0439); // Й й +BYLINS_RUS_LETTER(kKa, 0x041A, 0x043A); // К к +BYLINS_RUS_LETTER(kEl, 0x041B, 0x043B); // Л л +BYLINS_RUS_LETTER(kEm, 0x041C, 0x043C); // М м +BYLINS_RUS_LETTER(kEn, 0x041D, 0x043D); // Н н +BYLINS_RUS_LETTER(kO, 0x041E, 0x043E); // О о +BYLINS_RUS_LETTER(kPe, 0x041F, 0x043F); // П п +BYLINS_RUS_LETTER(kEr, 0x0420, 0x0440); // Р р +BYLINS_RUS_LETTER(kEs, 0x0421, 0x0441); // С с +BYLINS_RUS_LETTER(kTe, 0x0422, 0x0442); // Т т +BYLINS_RUS_LETTER(kU, 0x0423, 0x0443); // У у +BYLINS_RUS_LETTER(kEf, 0x0424, 0x0444); // Ф ф +BYLINS_RUS_LETTER(kHa, 0x0425, 0x0445); // Х х +BYLINS_RUS_LETTER(kTse, 0x0426, 0x0446); // Ц ц +BYLINS_RUS_LETTER(kChe, 0x0427, 0x0447); // Ч ч +BYLINS_RUS_LETTER(kSha, 0x0428, 0x0448); // Ш ш +BYLINS_RUS_LETTER(kScha,0x0429, 0x0449); // Щ щ +BYLINS_RUS_LETTER(kHard,0x042A, 0x044A); // Ъ ъ +BYLINS_RUS_LETTER(kYery,0x042B, 0x044B); // Ы ы +BYLINS_RUS_LETTER(kSoft,0x042C, 0x044C); // Ь ь +BYLINS_RUS_LETTER(kE, 0x042D, 0x044D); // Э э +BYLINS_RUS_LETTER(kYu, 0x042E, 0x044E); // Ю ю +BYLINS_RUS_LETTER(kYa, 0x042F, 0x044F); // Я я +#else +BYLINS_RUS_LETTER(kA, 0xE1, 0xC1); // А а +BYLINS_RUS_LETTER(kBe, 0xE2, 0xC2); // Б б +BYLINS_RUS_LETTER(kVe, 0xF7, 0xD7); // В в +BYLINS_RUS_LETTER(kGe, 0xE7, 0xC7); // Г г +BYLINS_RUS_LETTER(kDe, 0xE4, 0xC4); // Д д +BYLINS_RUS_LETTER(kIe, 0xE5, 0xC5); // Е е +BYLINS_RUS_LETTER(kYo, 0xB3, 0xA3); // Ё ё +BYLINS_RUS_LETTER(kZhe, 0xF6, 0xD6); // Ж ж +BYLINS_RUS_LETTER(kZe, 0xFA, 0xDA); // З з +BYLINS_RUS_LETTER(kI, 0xE9, 0xC9); // И и +BYLINS_RUS_LETTER(kIi, 0xEA, 0xCA); // Й й +BYLINS_RUS_LETTER(kKa, 0xEB, 0xCB); // К к +BYLINS_RUS_LETTER(kEl, 0xEC, 0xCC); // Л л +BYLINS_RUS_LETTER(kEm, 0xED, 0xCD); // М м +BYLINS_RUS_LETTER(kEn, 0xEE, 0xCE); // Н н +BYLINS_RUS_LETTER(kO, 0xEF, 0xCF); // О о +BYLINS_RUS_LETTER(kPe, 0xF0, 0xD0); // П п +BYLINS_RUS_LETTER(kEr, 0xF2, 0xD2); // Р р +BYLINS_RUS_LETTER(kEs, 0xF3, 0xD3); // С с +BYLINS_RUS_LETTER(kTe, 0xF4, 0xD4); // Т т +BYLINS_RUS_LETTER(kU, 0xF5, 0xD5); // У у +BYLINS_RUS_LETTER(kEf, 0xE6, 0xC6); // Ф ф +BYLINS_RUS_LETTER(kHa, 0xE8, 0xC8); // Х х +BYLINS_RUS_LETTER(kTse, 0xE3, 0xC3); // Ц ц +BYLINS_RUS_LETTER(kChe, 0xFE, 0xDE); // Ч ч +BYLINS_RUS_LETTER(kSha, 0xFB, 0xDB); // Ш ш +BYLINS_RUS_LETTER(kScha,0xFD, 0xDD); // Щ щ +BYLINS_RUS_LETTER(kHard,0xFF, 0xDF); // Ъ ъ +BYLINS_RUS_LETTER(kYery,0xF9, 0xD9); // Ы ы +BYLINS_RUS_LETTER(kSoft,0xF8, 0xD8); // Ь ь +BYLINS_RUS_LETTER(kE, 0xFC, 0xDC); // Э э +BYLINS_RUS_LETTER(kYu, 0xE0, 0xC0); // Ю ю +BYLINS_RUS_LETTER(kYa, 0xF1, 0xD1); // Я я +#endif + +#undef BYLINS_RUS_LETTER + +} // namespace rus + +#endif // BYLINS_SRC_UTILS_RUSSIAN_KEYS_H_ + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/utf8.cpp b/src/utils/utf8.cpp new file mode 100644 index 0000000000..8941786da3 --- /dev/null +++ b/src/utils/utf8.cpp @@ -0,0 +1,286 @@ +/** +\file utf8.cpp - a part of the Bylins engine. +\brief Implementation of the character-semantic UTF-8 helpers declared in utf8.h (issue #3681). +*/ + +#include "utf8.h" + +namespace utf8 { + +namespace { + +// Result of decoding one position: the code point, the byte length consumed, and whether the +// sequence was well-formed. On a malformed byte, `valid` is false, `len` is 1 and `cp` is the +// raw byte -- so scanners always advance and lenient callers can pass the byte through untouched. +struct Decoded { + char32_t cp; + std::size_t len; + bool valid; +}; + +// Decode per the Unicode 3-7 grammar: the first continuation byte has a lead-specific range +// (which is what rejects overlong forms and surrogates), the rest are plain 0x80..0xBF. +Decoded decode_core(std::string_view s, std::size_t pos) { + const std::size_t n = s.size(); + const unsigned char c0 = static_cast(s[pos]); + if (c0 < 0x80) { + return {c0, 1, true}; + } + + int len = 0; + char32_t cp = 0; + unsigned char b1_lo = 0x80; + unsigned char b1_hi = 0xBF; + if (c0 >= 0xC2 && c0 <= 0xDF) { + len = 2; + cp = c0 & 0x1F; + } else if (c0 == 0xE0) { + len = 3; + cp = c0 & 0x0F; + b1_lo = 0xA0; + } else if (c0 >= 0xE1 && c0 <= 0xEC) { + len = 3; + cp = c0 & 0x0F; + } else if (c0 == 0xED) { + len = 3; + cp = c0 & 0x0F; + b1_hi = 0x9F; + } else if (c0 >= 0xEE && c0 <= 0xEF) { + len = 3; + cp = c0 & 0x0F; + } else if (c0 == 0xF0) { + len = 4; + cp = c0 & 0x07; + b1_lo = 0x90; + } else if (c0 >= 0xF1 && c0 <= 0xF3) { + len = 4; + cp = c0 & 0x07; + } else if (c0 == 0xF4) { + len = 4; + cp = c0 & 0x07; + b1_hi = 0x8F; + } else { + // 0xC0, 0xC1, 0xF5..0xFF or a stray continuation byte: cannot start a sequence. + return {c0, 1, false}; + } + + if (pos + static_cast(len) > n) { + return {c0, 1, false}; + } + + const unsigned char b1 = static_cast(s[pos + 1]); + if (b1 < b1_lo || b1 > b1_hi) { + return {c0, 1, false}; + } + cp = (cp << 6) | (b1 & 0x3F); + + for (int k = 2; k < len; ++k) { + const unsigned char b = static_cast(s[pos + static_cast(k)]); + if (b < 0x80 || b > 0xBF) { + return {c0, 1, false}; + } + cp = (cp << 6) | (b & 0x3F); + } + + return {cp, static_cast(len), true}; +} + +} // namespace + +int sequence_length(unsigned char c) { + if (c < 0x80) { + return 1; + } + if (c >= 0xC0 && c <= 0xDF) { + return 2; + } + if (c >= 0xE0 && c <= 0xEF) { + return 3; + } + if (c >= 0xF0 && c <= 0xF7) { + return 4; + } + // Continuation byte (0x80..0xBF) or an out-of-range lead (0xF8..0xFF): not a valid start. + return 1; +} + +std::size_t decode(std::string_view s, std::size_t pos, char32_t &cp) { + if (pos >= s.size()) { + cp = 0; + return 0; + } + const Decoded d = decode_core(s, pos); + cp = d.cp; + return d.len; +} + +std::size_t encode(char32_t cp, std::string &out) { + if (cp <= 0x7F) { + out.push_back(static_cast(cp)); + return 1; + } + if (cp <= 0x7FF) { + out.push_back(static_cast(0xC0 | (cp >> 6))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + return 2; + } + if (cp >= 0xD800 && cp <= 0xDFFF) { + return 0; // surrogate half: not a Unicode scalar value + } + if (cp <= 0xFFFF) { + out.push_back(static_cast(0xE0 | (cp >> 12))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + return 3; + } + if (cp <= 0x10FFFF) { + out.push_back(static_cast(0xF0 | (cp >> 18))); + out.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + return 4; + } + return 0; +} + +std::size_t encode(char32_t cp, char *out) { + if (cp <= 0x7F) { + out[0] = static_cast(cp); + return 1; + } + if (cp <= 0x7FF) { + out[0] = static_cast(0xC0 | (cp >> 6)); + out[1] = static_cast(0x80 | (cp & 0x3F)); + return 2; + } + if (cp >= 0xD800 && cp <= 0xDFFF) { + return 0; + } + if (cp <= 0xFFFF) { + out[0] = static_cast(0xE0 | (cp >> 12)); + out[1] = static_cast(0x80 | ((cp >> 6) & 0x3F)); + out[2] = static_cast(0x80 | (cp & 0x3F)); + return 3; + } + if (cp <= 0x10FFFF) { + out[0] = static_cast(0xF0 | (cp >> 18)); + out[1] = static_cast(0x80 | ((cp >> 12) & 0x3F)); + out[2] = static_cast(0x80 | ((cp >> 6) & 0x3F)); + out[3] = static_cast(0x80 | (cp & 0x3F)); + return 4; + } + return 0; +} + +bool is_valid(std::string_view s) { + std::size_t pos = 0; + const std::size_t n = s.size(); + while (pos < n) { + const Decoded d = decode_core(s, pos); + if (!d.valid) { + return false; + } + pos += d.len; + } + return true; +} + +std::size_t length(std::string_view s) { + std::size_t count = 0; + std::size_t pos = 0; + const std::size_t n = s.size(); + while (pos < n) { + pos += decode_core(s, pos).len; + ++count; + } + return count; +} + +std::size_t byte_offset(std::string_view s, std::size_t index) { + std::size_t pos = 0; + const std::size_t n = s.size(); + while (index > 0 && pos < n) { + pos += decode_core(s, pos).len; + --index; + } + return pos; +} + +std::string_view char_at(std::string_view s, std::size_t index) { + const std::size_t start = byte_offset(s, index); + if (start >= s.size()) { + return {}; + } + const std::size_t len = decode_core(s, start).len; + return s.substr(start, len); +} + +std::string substr(std::string_view s, std::size_t pos, std::size_t count) { + const std::size_t start = byte_offset(s, pos); + if (count == std::string_view::npos) { + return std::string(s.substr(start)); + } + const std::size_t stop = byte_offset(s, pos + count); + return std::string(s.substr(start, stop - start)); +} + +char32_t to_lower(char32_t cp) { + if (cp >= 'A' && cp <= 'Z') { + return cp + 0x20; + } + if (cp >= 0x0410 && cp <= 0x042F) { // U+0410..U+042F (upper) -> U+0430..U+044F (lower) + return cp + 0x20; + } + if (cp == 0x0401) { // U+0401 (Yo) -> U+0451 (yo) + return 0x0451; + } + return cp; +} + +char32_t to_upper(char32_t cp) { + if (cp >= 'a' && cp <= 'z') { + return cp - 0x20; + } + if (cp >= 0x0430 && cp <= 0x044F) { // U+0430..U+044F (lower) -> U+0410..U+042F (upper) + return cp - 0x20; + } + if (cp == 0x0451) { // U+0451 (yo) -> U+0401 (Yo) + return 0x0401; + } + return cp; +} + +namespace { + +// Shared body for the whole-string case folders: decode, fold each code point, re-encode. +// Malformed bytes (valid == false) are copied through verbatim so nothing is silently dropped. +std::string fold_string(std::string_view s, char32_t (*fold)(char32_t)) { + std::string out; + out.reserve(s.size()); + std::size_t pos = 0; + const std::size_t n = s.size(); + while (pos < n) { + const Decoded d = decode_core(s, pos); + if (d.valid) { + encode(fold(d.cp), out); + } else { + out.push_back(s[pos]); + } + pos += d.len; + } + return out; +} + +} // namespace + +std::string to_lower(std::string_view s) { + return fold_string(s, to_lower); +} + +std::string to_upper(std::string_view s) { + return fold_string(s, to_upper); +} + +} // namespace utf8 + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/utf8.h b/src/utils/utf8.h new file mode 100644 index 0000000000..f65f32ec3d --- /dev/null +++ b/src/utils/utf8.h @@ -0,0 +1,72 @@ +/** +\file utf8.h - a part of the Bylins engine. +\brief Character-semantic helpers over UTF-8 strings (issue #3681, "Plan Napoleon"). + +This is the encoding-agnostic building block for the KOI8-R -> UTF-8 migration: code that +must reason about *characters* (code points) rather than bytes -- length, substring, indexed +access, case folding -- lives here. Everything is plain UTF-8 in / UTF-8 out with no external +dependency (no iconv/ICU). ASCII passes through untouched, so the helpers are also correct for +pure-ASCII input regardless of the ambient encoding. + +Case folding covers exactly what the legacy a_ucc/a_lcc tables covered: ASCII A-Z and the +Russian Cyrillic block (U+0410..U+044F plus Yo, U+0401/U+0451). Any other code point passes +through unchanged. +*/ + +#ifndef BYLINS_SRC_UTILS_UTF8_H_ +#define BYLINS_SRC_UTILS_UTF8_H_ + +#include +#include +#include + +namespace utf8 { + +// Number of bytes the UTF-8 sequence starting with lead byte `c` claims to span. +// Returns 1 for ASCII and for any byte that cannot start a sequence, so a caller that +// advances by the result always makes forward progress. +int sequence_length(unsigned char c); + +// Decode the code point starting at byte position `pos` in `s`. +// Writes the code point (or the raw byte, on a malformed sequence) to `cp` and returns the +// number of bytes consumed: >=1 while `pos` is in range, 0 once `pos >= s.size()`. +// Malformed sequences never stall: they yield the single offending byte and a length of 1. +std::size_t decode(std::string_view s, std::size_t pos, char32_t &cp); + +// Append `cp` to `out` as UTF-8. Returns the number of bytes written, or 0 for a value that is +// not a valid Unicode scalar (surrogate half or > U+10FFFF), in which case `out` is untouched. +std::size_t encode(char32_t cp, std::string &out); + +// Same, but writes into a caller-supplied buffer of at least 4 bytes and never allocates. +// Returns the number of bytes written, or 0 for a value that is not a Unicode scalar. +std::size_t encode(char32_t cp, char *out); + +// Strict, whole-string well-formedness check per the Unicode Table 3-7 byte-sequence grammar +// (rejects overlong forms, surrogates, code points above U+10FFFF and stray continuation bytes). +bool is_valid(std::string_view s); + +// Number of code points. Malformed bytes count as one code point each; never throws. +std::size_t length(std::string_view s); + +// Byte offset of the `index`-th code point, or `s.size()` when `index` is past the end. +std::size_t byte_offset(std::string_view s, std::size_t index); + +// The bytes of the `index`-th code point, as a view into `s`. Empty when `index` is out of range. +std::string_view char_at(std::string_view s, std::size_t index); + +// std::string::substr, but `pos`/`count` are counted in code points instead of bytes. +std::string substr(std::string_view s, std::size_t pos, std::size_t count = std::string_view::npos); + +// Single-code-point case folding (ASCII + Russian Cyrillic incl. Yo); other values pass through. +char32_t to_lower(char32_t cp); +char32_t to_upper(char32_t cp); + +// Whole-string case folding. Malformed bytes are copied through verbatim. +std::string to_lower(std::string_view s); +std::string to_upper(std::string_view s); + +} // namespace utf8 + +#endif // BYLINS_SRC_UTILS_UTF8_H_ + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index 21371ef08d..c9c1727548 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -13,6 +13,7 @@ ************************************************************************ */ #include "utils.h" +#include "native_text.h" #include "utils/grammar/declensions.h" #include @@ -266,7 +267,7 @@ void format_text(const utils::AbstractStringWriter::shared_ptr &writer, flow++; } - if ((total_chars + (flow - start) + 1) > 79) { + if ((total_chars + native_text::char_count(start, flow) + 1) > 79) { strcpy(pos, "\r\n"); total_chars = 0; pos += 2; @@ -280,11 +281,11 @@ void format_text(const utils::AbstractStringWriter::shared_ptr &writer, } } - total_chars += flow - start; + total_chars += native_text::char_count(start, flow); strncpy(pos, start, flow - start); if (cap_next) { cap_next = false; - *pos = UPPER(*pos); + native_text::capitalize_first(pos); } pos += flow - start; } @@ -304,17 +305,19 @@ void format_text(const utils::AbstractStringWriter::shared_ptr &writer, strcpy(pos, "\r\n"); if (static_cast(pos - formatted) > maxlen) { - formatted[maxlen] = '\0'; + formatted[native_text::truncate_offset(formatted, maxlen)] = '\0'; } writer->set_string(formatted); } char *rustime(const struct tm *timeptr) { - static char mon_name[12][10] = + // Массив указателей, а не char[12][10]: ширина ячейки зависела бы от кодировки + // (в UTF-8 русская буква занимает два байта, и названия перестают влезать). + static const char *const mon_name[12] = { - "Января\0", "Февраля\0", "Марта\0", "Апреля\0", "Мая\0", "Июня\0", - "Июля\0", "Августа\0", "Сентября\0", "Октября\0", "Ноября\0", "Декабря\0" + "Января", "Февраля", "Марта", "Апреля", "Мая", "Июня", + "Июля", "Августа", "Сентября", "Октября", "Ноября", "Декабря" }; static char result[100]; diff --git a/src/utils/utils_parse.cpp b/src/utils/utils_parse.cpp index 0105e2bd1d..622ddd0fd6 100644 --- a/src/utils/utils_parse.cpp +++ b/src/utils/utils_parse.cpp @@ -4,7 +4,8 @@ #include "utils_parse.h" #include "third_party_libs/pugixml/pugixml.h" -#include "utils/parser_wrapper.h" // issue.xml-parse-cleaning: AttrInt/AttrStr over DataNode +#include "utils/parser_wrapper.h" +#include "utils/native_text.h" // issue.xml-parse-cleaning: AttrInt/AttrStr over DataNode #include "engine/db/obj_prototypes.h" #include "engine/db/db.h" @@ -393,10 +394,21 @@ int get_number(std::string &name) { // issue.handler-cleaning: first keyword of a name list (moved from handler). char *fname(const char *namelist) { static char holder[30]; - char *point; - - for (point = holder; a_isalpha(*namelist); namelist++, point++) - *point = *namelist; + char *point = holder; + + // Copy the leading word one whole character at a time (issue #3681): a byte-wise copy stops + // in the middle of a multibyte letter. The bounds check is new -- the previous loop could + // already run past holder[] on a long keyword, and multibyte text reaches the end twice as + // fast, so leave room for the terminator. + while (native_text::is_alpha_char(namelist)) { + const size_t bytes = native_text::char_bytes(namelist); + if (point + bytes >= holder + sizeof(holder)) { + break; + } + for (size_t i = 0; i < bytes; ++i) { + *point++ = *namelist++; + } + } *point = '\0'; diff --git a/src/utils/utils_string.cpp b/src/utils/utils_string.cpp index cad0ed3443..e4adbbe044 100644 --- a/src/utils/utils_string.cpp +++ b/src/utils/utils_string.cpp @@ -1,6 +1,7 @@ //#include "utils_string.h" #include "utils.h" +#include "utils/native_text.h" #include "utils/utils_encoding.h" #include "gameplay/core/constants.h" @@ -116,10 +117,13 @@ bool IsAbbr(const char *arg1, const char *arg2) { return false; } - for (; *arg1 && *arg2; arg1++, arg2++) { - if (LOWER(*arg1) != LOWER(*arg2)) { + // Посимвольно (issue #3681): побайтное сравнение под UTF-8 теряет регистронезависимость. + while (*arg1 && *arg2) { + if (!native_text::chars_equal_ci(arg1, arg2)) { return false; } + arg1 += native_text::char_bytes(arg1); + arg2 += native_text::char_bytes(arg2); } if (!*arg1) { @@ -228,9 +232,7 @@ std::string ExtractFirstArgument(const std::string &s, std::string &remains) { } std::string SubstToLow(std::string s) { - for (char &it: s) { - it = LOWER(it); - } + ConvertToLow(s); return s; } @@ -257,29 +259,22 @@ std::string SubstWtoK(std::string s) { } void ConvertToLow(std::string &text) { - for (char &it: text) { - it = LOWER(it); - } + native_text::to_lower(text); } void ConvertToLow(char *text) { - while (*text) { - *text = LOWER(*text); - text++; - } + native_text::to_lower(text); } std::string SubstStrToLow(std::string s) { - for (char &it: s) { - it = UPPER(it); - } + // NB: имя говорит "ToLow", а тело поднимает регистр. Расхождение предсуществующее, + // поведение сохранено намеренно -- меняется только байтовая семантика на символьную. + native_text::to_upper(s); return s; } std::string SubstStrToUpper(std::string s) { - for (char &it: s) { - it = UPPER(it); - } + native_text::to_upper(s); return s; } @@ -476,14 +471,14 @@ const char *first_letter(const char *txt) { char *colorCAP(char *txt) { char *letter = const_cast(first_letter(txt)); if (letter && *letter) { - *letter = UPPER(*letter); + native_text::capitalize_first(letter); } return txt; } std::string &colorCAP(std::string &txt) { size_t pos = first_letter(txt.c_str()) - txt.c_str(); - txt[pos] = UPPER(txt[pos]); + native_text::capitalize_first(&txt[pos]); return txt; } @@ -495,14 +490,14 @@ std::string &colorCAP(std::string &&txt) { char *colorLOW(char *txt) { char *letter = const_cast(first_letter(txt)); if (letter && *letter) { - *letter = LOWER(*letter); + native_text::copy_lower_char(letter, letter); } return txt; } std::string &colorLOW(std::string &txt) { size_t pos = first_letter(txt.c_str()) - txt.c_str(); - txt[pos] = LOWER(txt[pos]); + native_text::copy_lower_char(&txt[pos], &txt[pos]); return txt; } @@ -512,13 +507,13 @@ std::string &colorLOW(std::string &&txt) { } char *CAP(char *txt) { - *txt = UPPER(*txt); + native_text::capitalize_first(txt); return (txt); } std::string CAP(const std::string txt) { std::string tmp_str = txt; - tmp_str[0] = UPPER(tmp_str[0]); + native_text::capitalize_first(tmp_str); return (tmp_str); } @@ -579,12 +574,18 @@ char *delete_doubledollar(char *string) { } // Moved from utils.cpp +// The str_cmp/strn_cmp family folds case per *character*: under KOI8-R that is the original +// byte-wise LOWER() loop kept verbatim below, under UTF-8 it is native_text's code-point fold +// (issue #3681). The KOI8-R path is untouched so behaviour is bit-identical until the flip. int str_cmp(const char *arg1, const char *arg2) { int chk, i; if (arg1 == nullptr || arg2 == nullptr) { log("SYSERR: str_cmp() passed a nullptr pointer, %p or %p.", arg1, arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; arg1[i] || arg2[i]; i++) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); @@ -598,6 +599,9 @@ int str_cmp(const std::string &arg1, const char *arg2) { log("SYSERR: str_cmp() passed a NULL pointer, %p.", arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; i != arg1.length() && *arg2; i++, arg2++) if ((chk = LOWER(arg1[i]) - LOWER(*arg2)) != 0) return (chk); @@ -616,6 +620,9 @@ int str_cmp(const char *arg1, const std::string &arg2) { log("SYSERR: str_cmp() passed a NULL pointer, %p.", arg1); return (0); } + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; *arg1 && i != arg2.length(); i++, arg1++) if ((chk = LOWER(*arg1) - LOWER(arg2[i])) != 0) return (chk); @@ -630,6 +637,9 @@ int str_cmp(const char *arg1, const std::string &arg2) { int str_cmp(const std::string &arg1, const std::string &arg2) { int chk; std::string::size_type i; + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; i != arg1.length() && i != arg2.length(); i++) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); @@ -647,6 +657,9 @@ int strn_cmp(const char *arg1, const char *arg2, size_t n) { log("SYSERR: strn_cmp() passed a NULL pointer, %p or %p.", arg1, arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; (arg1[i] || arg2[i]) && (n > 0); i++, n--) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); @@ -660,6 +673,9 @@ int strn_cmp(const std::string &arg1, const char *arg2, size_t n) { log("SYSERR: strn_cmp() passed a NULL pointer, %p.", arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; i != arg1.length() && *arg2 && (n > 0); i++, arg2++, n--) if ((chk = LOWER(arg1[i]) - LOWER(*arg2)) != 0) return (chk); @@ -678,6 +694,9 @@ int strn_cmp(const char *arg1, const std::string &arg2, size_t n) { log("SYSERR: strn_cmp() passed a NULL pointer, %p.", arg1); return (0); } + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; *arg1 && i != arg2.length() && (n > 0); i++, arg1++, n--) if ((chk = LOWER(*arg1) - LOWER(arg2[i])) != 0) return (chk); @@ -692,6 +711,9 @@ int strn_cmp(const char *arg1, const std::string &arg2, size_t n) { int strn_cmp(const std::string &arg1, const std::string &arg2, size_t n) { int chk; std::string::size_type i; + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; i != arg1.length() && i != arg2.length() && (n > 0); i++, n--) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); @@ -850,13 +872,15 @@ void cut_one_word(std::string &str, std::string &word) { } bool process = false; unsigned begin = 0, end = 0; - for (unsigned i = 0; i < str.size(); ++i) { - if (!process && a_isalnum(str.at(i))) { + // Word boundaries are looked for one whole character at a time (issue #3681); a byte-wise + // scan finds a "boundary" inside a multibyte letter and cuts the word in half. + for (unsigned i = 0; i < str.size(); i += native_text::char_bytes(str.c_str() + i)) { + if (!process && native_text::is_alnum_char(str.c_str() + i)) { process = true; begin = i; continue; } - if (process && !a_isalnum(str.at(i))) { + if (process && !native_text::is_alnum_char(str.c_str() + i)) { end = i; break; } @@ -915,13 +939,18 @@ bool IsValidEmail(const char *address) { return true; } +// Walks both strings one *character* at a time (issue #3681): the classification, the +// case-insensitive match and every advance go through native_text, so a multibyte letter is one +// unit instead of a lead byte plus trail bytes that the byte tables would read as punctuation. +// Under KOI8-R every helper is the original byte operation and char_bytes() == 1, so the state +// machine below -- including each `curstr = laststr` backtrack -- behaves exactly as before. bool isname(const char *str, const char *namelist) { bool once_ok = false; const char *curname, *curstr, *laststr; if (!namelist || !*namelist || !str) { return false; } - for (curstr = str; !a_isalnum(*curstr); curstr++) { + for (curstr = str; !native_text::is_alnum_char(curstr); curstr += native_text::char_bytes(curstr)) { if (!*curstr) { return once_ok; } @@ -930,18 +959,18 @@ bool isname(const char *str, const char *namelist) { curname = namelist; for (;;) { once_ok = false; - for (;; curstr++, curname++) { + for (;; curstr += native_text::char_bytes(curstr), curname += native_text::char_bytes(curname)) { if (!*curstr) { return once_ok; } if (*curstr == '!') { - if (a_isalnum(*curname)) { + if (native_text::is_alnum_char(curname)) { curstr = laststr; break; } } - if (!a_isalnum(*curstr)) { - for (; !a_isalnum(*curstr); curstr++) { + if (!native_text::is_alnum_char(curstr)) { + for (; !native_text::is_alnum_char(curstr); curstr += native_text::char_bytes(curstr)) { if (!*curstr) { return once_ok; } @@ -952,19 +981,19 @@ bool isname(const char *str, const char *namelist) { if (!*curname) { return false; } - if (!a_isalnum(*curname)) { + if (!native_text::is_alnum_char(curname)) { curstr = laststr; break; } - if (LOWER(*curstr) != LOWER(*curname)) { + if (!native_text::chars_equal_ci(curstr, curname)) { curstr = laststr; break; } else { once_ok = true; } } - for (; a_isalnum(*curname); curname++); - for (; !a_isalnum(*curname); curname++) { + for (; native_text::is_alnum_char(curname); curname += native_text::char_bytes(curname)); + for (; !native_text::is_alnum_char(curname); curname += native_text::char_bytes(curname)) { if (!*curname) { return false; } @@ -976,17 +1005,21 @@ const char *one_word(const char *argument, char *first_arg) { char *begin = first_arg; skip_spaces(&argument); first_arg = begin; + // Lowercase whole characters (issue #3681); the '"' and a_isspace() tests stay byte-based + // since they only run at a character boundary and both delimiters are ASCII. if (*argument == '\"') { argument++; while (*argument && *argument != '\"') { - *(first_arg++) = a_lcc(*argument); - argument++; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; } argument++; } else { while (*argument && !a_isspace(*argument)) { - *(first_arg++) = a_lcc(*argument); - argument++; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; } } *first_arg = '\0'; diff --git a/tests/meson.build b/tests/meson.build index 7bdee932c6..2493ba0373 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -39,6 +39,9 @@ test_sources = files( 'utils.editor.cpp', 'utils.string.cpp', 'utils.encoding.cpp', + 'utf8.cpp', + 'native_text.cpp', + 'text_semantics.cpp', 'fight.penalties.cpp', 'bonus.command.parser.cpp', 'quested.cpp', diff --git a/tests/native_text.cpp b/tests/native_text.cpp new file mode 100644 index 0000000000..c351efa161 --- /dev/null +++ b/tests/native_text.cpp @@ -0,0 +1,436 @@ +// Unit tests for the native-encoding character helpers (src/utils/native_text.*, issue #3681). +// +// Pure ASCII: non-ASCII fixtures are spelled as UTF-8 byte escapes. Expectations that differ +// between the KOI8-R and UTF-8 builds branch on native_text::native_is_utf8() (which reflects the +// flag the library was built with), so this one test file is correct under either build. + +#include "utils/native_text.h" +#include "utils/russian_keys.h" + +#include + +#include +#include +#include +#include + +namespace { + +// "Privet": 6 Cyrillic code points, 12 UTF-8 bytes. (Name-prefixed: the test files are +// unity-built, so a plain kPrivet would clash with the one in tests/utf8.cpp.) +const char *const kNtPrivet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; + +} // namespace + +// The legacy KOI8-R lowercase table (utils.cpp). Declared here rather than including utils.h, +// which pulls in fmt/ and the rest of the engine headers. +extern const char a_lcc_table[]; + +namespace { + +int legacy_lower(unsigned char c) { + return a_lcc_table[c]; +} + +} // namespace + +TEST(NativeText, CharCountAscii) { + EXPECT_EQ(native_text::char_count("Hello"), 5u); + EXPECT_EQ(native_text::char_count(kNtPrivet, kNtPrivet + 4), native_text::native_is_utf8() ? 2u : 4u); +} + +TEST(NativeText, CharCountReflectsEncoding) { + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::char_count(kNtPrivet), 6u); + EXPECT_EQ(native_text::char_count(kNtPrivet, kNtPrivet + 12), 6u); + } else { + EXPECT_EQ(native_text::char_count(kNtPrivet), 12u); + EXPECT_EQ(native_text::char_count(kNtPrivet, kNtPrivet + 12), 12u); + } +} + +TEST(NativeText, CapitalizeAscii) { + char buf[] = "hello"; + native_text::capitalize_first(buf); + EXPECT_STREQ(buf, "Hello"); + + char empty[] = ""; + native_text::capitalize_first(empty); // must not touch the terminator + EXPECT_STREQ(empty, ""); + + char already[] = "X"; + native_text::capitalize_first(already); + EXPECT_STREQ(already, "X"); +} + +TEST(NativeText, CapitalizeCyrillicUtf8Only) { + if (!native_text::native_is_utf8()) { + GTEST_SKIP() << "Cyrillic-as-UTF-8 fixtures are only meaningful under the UTF-8 build"; + } + char buf[] = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; // "privet" + native_text::capitalize_first(buf); + EXPECT_STREQ(buf, "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"); // "Privet" +} + +TEST(NativeText, CharBytes) { + EXPECT_EQ(native_text::char_bytes("A"), 1u); + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::char_bytes(kNtPrivet), 2u); // Cyrillic lead -> 2 bytes + EXPECT_EQ(native_text::char_bytes("\xF0\x9F\x98\x80"), 4u); // 4-byte code point + EXPECT_EQ(native_text::char_bytes("\xD0"), 1u); // truncated lead: 1 byte present + } else { + EXPECT_EQ(native_text::char_bytes(kNtPrivet), 1u); // KOI8-R: every byte is a char + } +} + +TEST(NativeText, TruncateOffset) { + const std::string_view p(kNtPrivet, 12); + EXPECT_EQ(native_text::truncate_offset(p, 100), 12u); // past end -> full size + EXPECT_EQ(native_text::truncate_offset(p, 0), 0u); + if (native_text::native_is_utf8()) { + // 5 bytes lands mid-character; back up to the boundary after 2 code points (4 bytes). + EXPECT_EQ(native_text::truncate_offset(p, 5), 4u); + EXPECT_EQ(native_text::truncate_offset(p, 4), 4u); + } else { + EXPECT_EQ(native_text::truncate_offset(p, 5), 5u); + } +} + +TEST(NativeText, CompareCiAscii) { + EXPECT_EQ(native_text::compare_ci("abc", "abc"), 0); + EXPECT_EQ(native_text::compare_ci("ABC", "abc"), 0); // case-insensitive + EXPECT_EQ(native_text::compare_ci("AbC", "aBc"), 0); + EXPECT_LT(native_text::compare_ci("abc", "abd"), 0); // ordering by first mismatch + EXPECT_GT(native_text::compare_ci("abd", "abc"), 0); + EXPECT_LT(native_text::compare_ci("ab", "abc"), 0); // prefix orders first + EXPECT_GT(native_text::compare_ci("abc", "ab"), 0); + EXPECT_EQ(native_text::compare_ci("", ""), 0); + EXPECT_LT(native_text::compare_ci("", "a"), 0); +} + +TEST(NativeText, NCompareCiAscii) { + EXPECT_EQ(native_text::ncompare_ci("abcdef", "abcXXX", 3), 0); // only first 3 bytes matter + EXPECT_NE(native_text::ncompare_ci("abcdef", "abXXXX", 3), 0); + EXPECT_EQ(native_text::ncompare_ci("ABC", "abc", 3), 0); + EXPECT_EQ(native_text::ncompare_ci("anything", "other", 0), 0); // zero budget: equal + EXPECT_LT(native_text::ncompare_ci("ab", "abc", 10), 0); // budget beyond the strings +} + +TEST(NativeText, CompareCiCyrillicIsCaseInsensitive) { + // "PRIVET" vs "privet" in Cyrillic: must compare equal in BOTH encodings -- under KOI8-R via + // the byte table, under UTF-8 via the code-point fold. This is the property that a naive + // "just compare bytes" UTF-8 migration would silently lose. + const char *const upper = "\xD0\x9F\xD0\xA0\xD0\x98\xD0\x92\xD0\x95\xD0\xA2"; + const char *const lower = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::compare_ci(upper, lower), 0); + EXPECT_EQ(native_text::compare_ci(upper, upper), 0); + EXPECT_NE(native_text::compare_ci(upper, "\xD0\xBF\xD1\x80\xD0\xB8"), 0); // prefix differs + } else { + // Under KOI8-R these UTF-8 bytes are not Cyrillic; just assert self-equality and that + // the comparison stays reflexive/antisymmetric on arbitrary high bytes. + EXPECT_EQ(native_text::compare_ci(upper, upper), 0); + EXPECT_EQ(native_text::compare_ci(lower, lower), 0); + } +} + +TEST(NativeText, CompareCiMatchesLegacyByteLoopUnderKoi8r) { + if (native_text::native_is_utf8()) { + GTEST_SKIP() << "this pins the KOI8-R branch against the original LOWER() byte loop"; + } + // Reference implementation: the exact loop str_cmp() used before the migration. + auto legacy = [](const char *a, const char *b) { + for (int i = 0;; ++i) { + if (!a[i] && !b[i]) { + return 0; + } + const int chk = legacy_lower(static_cast(a[i])) + - legacy_lower(static_cast(b[i])); + if (chk != 0) { + return chk; + } + } + }; + const char *const samples[] = {"", "a", "A", "abc", "ABC", "abd", "ab", "zzz", "\xC1\xC2", "\xE1\xE2"}; + for (const char *x : samples) { + for (const char *y : samples) { + EXPECT_EQ(native_text::compare_ci(x, y), legacy(x, y)) + << "mismatch for \"" << x << "\" vs \"" << y << "\""; + } + } +} + +TEST(NativeText, IsAlnumChar) { + EXPECT_TRUE(native_text::is_alnum_char("a")); + EXPECT_TRUE(native_text::is_alnum_char("Z")); + EXPECT_TRUE(native_text::is_alnum_char("7")); + EXPECT_FALSE(native_text::is_alnum_char(" ")); + EXPECT_FALSE(native_text::is_alnum_char("!")); + EXPECT_FALSE(native_text::is_alnum_char(".")); + EXPECT_FALSE(native_text::is_alnum_char("")); // terminator is not alphanumeric + if (native_text::native_is_utf8()) { + // A Cyrillic letter is ONE alphanumeric character; its trail byte must not be read as + // punctuation (which is what the raw byte table would do and what breaks tokenisation). + EXPECT_TRUE(native_text::is_alnum_char(kNtPrivet)); + EXPECT_TRUE(native_text::is_alnum_char("\xD0\x81")); // Yo + EXPECT_TRUE(native_text::is_alnum_char("\xD1\x91")); // yo + } +} + +TEST(NativeText, IsAlphaChar) { + EXPECT_TRUE(native_text::is_alpha_char("a")); + EXPECT_TRUE(native_text::is_alpha_char("Z")); + EXPECT_FALSE(native_text::is_alpha_char("7")); // digit: alnum but not alpha + EXPECT_FALSE(native_text::is_alpha_char(" ")); + EXPECT_FALSE(native_text::is_alpha_char("")); + if (native_text::native_is_utf8()) { + EXPECT_TRUE(native_text::is_alpha_char(kNtPrivet)); + EXPECT_TRUE(native_text::is_alpha_char("\xD0\x81")); // Yo + } +} + +TEST(NativeText, IsUpperChar) { + EXPECT_TRUE(native_text::is_upper_char("A")); + EXPECT_FALSE(native_text::is_upper_char("a")); + EXPECT_FALSE(native_text::is_upper_char("7")); + EXPECT_FALSE(native_text::is_upper_char("")); + if (native_text::native_is_utf8()) { + // The byte table cannot see these: a UTF-8 Cyrillic lead byte is outside its uppercase + // range, which is why the anti-caps filter stopped working for Russian. + EXPECT_TRUE(native_text::is_upper_char("\xD0\x9F")); // P + EXPECT_FALSE(native_text::is_upper_char("\xD0\xBF")); // p + EXPECT_TRUE(native_text::is_upper_char("\xD0\x81")); // Yo + EXPECT_FALSE(native_text::is_upper_char("\xD1\x91")); // yo + } +} + +TEST(NativeText, CharsEqualCi) { + EXPECT_TRUE(native_text::chars_equal_ci("a", "a")); + EXPECT_TRUE(native_text::chars_equal_ci("a", "A")); + EXPECT_TRUE(native_text::chars_equal_ci("Z", "z")); + EXPECT_FALSE(native_text::chars_equal_ci("a", "b")); + EXPECT_FALSE(native_text::chars_equal_ci("a", "")); + if (native_text::native_is_utf8()) { + // The regression this whole step exists for: with the raw KOI8-R byte table the lead + // bytes of "P"/"p" fold equal but the trail bytes differ, so the match was lost. + EXPECT_TRUE(native_text::chars_equal_ci("\xD0\x9F", "\xD0\xBF")); // P vs p + EXPECT_TRUE(native_text::chars_equal_ci("\xD0\x81", "\xD1\x91")); // Yo vs yo + EXPECT_FALSE(native_text::chars_equal_ci("\xD0\x9F", "\xD1\x80")); // P vs r + } +} + +TEST(NativeText, CopyLowerChar) { + char buf[8] = {0}; + EXPECT_EQ(native_text::copy_lower_char("A", buf), 1u); + EXPECT_STREQ(buf, "a"); + EXPECT_EQ(native_text::copy_lower_char("z", buf), 1u); + EXPECT_STREQ(buf, "z"); + EXPECT_EQ(native_text::copy_lower_char("7", buf), 1u); + EXPECT_STREQ(buf, "7"); + if (native_text::native_is_utf8()) { + std::memset(buf, 0, sizeof(buf)); + EXPECT_EQ(native_text::copy_lower_char("\xD0\x9F", buf), 2u); // P -> p + EXPECT_STREQ(buf, "\xD0\xBF"); + std::memset(buf, 0, sizeof(buf)); + EXPECT_EQ(native_text::copy_lower_char("\xD0\x81", buf), 2u); // Yo -> yo + EXPECT_STREQ(buf, "\xD1\x91"); + // In-place folding must be safe (the lowercase form keeps the byte length). + char inplace[] = "\xD0\x9F"; + EXPECT_EQ(native_text::copy_lower_char(inplace, inplace), 2u); + EXPECT_STREQ(inplace, "\xD0\xBF"); + } +} + +TEST(NativeText, LastCharOffset) { + EXPECT_EQ(native_text::last_char_offset(""), 0u); + EXPECT_EQ(native_text::last_char_offset("a"), 0u); + EXPECT_EQ(native_text::last_char_offset("abc"), 2u); + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::last_char_offset(kNtPrivet), 10u); // 6 chars, last starts at byte 10 + const std::string_view s(kNtPrivet, 12); + EXPECT_EQ(s.substr(native_text::last_char_offset(s)), "\xD1\x82"); // final char "t" + EXPECT_EQ(native_text::last_char_offset("\xF0\x9F\x98\x80"), 0u); // single 4-byte char + } else { + EXPECT_EQ(native_text::last_char_offset(kNtPrivet), 11u); // 12 bytes -> last byte + } +} + +TEST(NativeText, ListContainsChar) { + EXPECT_TRUE(native_text::list_contains_char("abc", "b")); + EXPECT_FALSE(native_text::list_contains_char("abc", "d")); + EXPECT_FALSE(native_text::list_contains_char("abc", "")); + EXPECT_FALSE(native_text::list_contains_char("", "a")); + EXPECT_FALSE(native_text::list_contains_char("abc", "A")); // case-sensitive, like strchr + if (native_text::native_is_utf8()) { + // A multibyte character must match as a whole and never on a partial byte sequence. + const char *const list = "\xD1\x88\xD1\x89\xD0\xB6\xD1\x87"; // sh shch zh ch + EXPECT_TRUE(native_text::list_contains_char(list, "\xD1\x89")); + EXPECT_TRUE(native_text::list_contains_char(list, "\xD0\xB6")); + EXPECT_FALSE(native_text::list_contains_char(list, "\xD1\x82")); + EXPECT_FALSE(native_text::list_contains_char(list, "\xD1")); // lead byte alone + } +} + +TEST(NativeText, CharRangeIteratesWholeCharacters) { + std::vector got; + for (auto c : native_text::chars(kNtPrivet)) { + got.emplace_back(c); + } + if (native_text::native_is_utf8()) { + ASSERT_EQ(got.size(), 6u); // 6 letters, not 12 bytes + EXPECT_EQ(got.front(), "\xD0\x9F"); // whole "P", both bytes + EXPECT_EQ(got.back(), "\xD1\x82"); // whole "t" + } else { + ASSERT_EQ(got.size(), 12u); // KOI8-R: every byte is a character + } + + got.clear(); + for (auto c : native_text::chars("abc")) { + got.emplace_back(c); + } + EXPECT_EQ(got, (std::vector{"a", "b", "c"})); + + got.clear(); + for (auto c : native_text::chars("")) { + got.emplace_back(c); + } + EXPECT_TRUE(got.empty()); +} + +TEST(NativeText, WholeStringCaseTransforms) { + std::string s = "Hello World"; + native_text::to_lower(s); + EXPECT_EQ(s, "hello world"); + native_text::to_upper(s); + EXPECT_EQ(s, "HELLO WORLD"); + + char buf[] = "MiXeD"; + native_text::to_lower(buf); + EXPECT_STREQ(buf, "mixed"); + + if (native_text::native_is_utf8()) { + std::string ru = "\xD0\x9F\xD0\xA0\xD0\x98"; // "PRI" in Cyrillic + native_text::to_lower(ru); + EXPECT_EQ(ru, "\xD0\xBF\xD1\x80\xD0\xB8"); // "pri" + native_text::to_upper(ru); + EXPECT_EQ(ru, "\xD0\x9F\xD0\xA0\xD0\x98"); + } +} + +TEST(NativeText, RussianKeysMatchFirstCharCode) { + // The switch-dispatch contract: for every Russian letter, the constant in russian_keys.h must + // equal what first_char_code() returns for that letter in the build's native encoding. If this + // ever drifts, menus and OLC editors silently stop responding to that key. + struct Case { const char *koi8; const char *utf8; char32_t expected; }; + const Case cases[] = { + {"\xC1", "\xD0\xB0", rus::kA}, {"\xE1", "\xD0\x90", rus::kAUpper}, + {"\xC4", "\xD0\xB4", rus::kDe}, {"\xE4", "\xD0\x94", rus::kDeUpper}, + {"\xCE", "\xD0\xBD", rus::kEn}, {"\xEE", "\xD0\x9D", rus::kEnUpper}, + {"\xD1", "\xD1\x8F", rus::kYa}, {"\xF1", "\xD0\xAF", rus::kYaUpper}, + {"\xA3", "\xD1\x91", rus::kYo}, {"\xB3", "\xD0\x81", rus::kYoUpper}, + {"\xD7", "\xD0\xB2", rus::kVe}, {"\xC8", "\xD1\x85", rus::kHa}, + }; + for (const auto &c : cases) { + const char *const input = native_text::native_is_utf8() ? c.utf8 : c.koi8; + EXPECT_EQ(native_text::first_char_code(input), c.expected); + } + + // ASCII keys are unchanged by the flip and stay ordinary character literals in the switches. + EXPECT_EQ(native_text::first_char_code("y"), static_cast('y')); + EXPECT_EQ(native_text::first_char_code("N"), static_cast('N')); + EXPECT_EQ(native_text::first_char_code(""), 0u); + EXPECT_EQ(native_text::first_char_code(nullptr), 0u); +} + +TEST(NativeText, TransliterationIsStableAcrossTheFlip) { + // A player's save file is named after the transliterated character name, so this mapping is + // on-disk state: if it ever changes, every existing character stops being found. Each row is + // the letter in both encodings and the single ASCII character it must always produce -- the + // values were taken from what the byte-wise implementation produced before the migration. + struct Row { const char *koi8; const char *utf8; char expected; }; + static const Row kRows[] = { + {"\xC1", "\xD0\xB0", 'a'}, + {"\xE1", "\xD0\x90", 'a'}, + {"\xC2", "\xD0\xB1", 'b'}, + {"\xE2", "\xD0\x91", 'b'}, + {"\xD7", "\xD0\xB2", 'v'}, + {"\xF7", "\xD0\x92", 'v'}, + {"\xC7", "\xD0\xB3", 'g'}, + {"\xE7", "\xD0\x93", 'g'}, + {"\xC4", "\xD0\xB4", 'd'}, + {"\xE4", "\xD0\x94", 'd'}, + {"\xC5", "\xD0\xB5", 'e'}, + {"\xE5", "\xD0\x95", 'e'}, + {"\xA3", "\xD1\x91", '9'}, + {"\xB3", "\xD0\x81", '9'}, + {"\xD6", "\xD0\xB6", '1'}, + {"\xF6", "\xD0\x96", '1'}, + {"\xDA", "\xD0\xB7", 'z'}, + {"\xFA", "\xD0\x97", 'z'}, + {"\xC9", "\xD0\xB8", 'i'}, + {"\xE9", "\xD0\x98", 'i'}, + {"\xCA", "\xD0\xB9", 'j'}, + {"\xEA", "\xD0\x99", 'j'}, + {"\xCB", "\xD0\xBA", 'k'}, + {"\xEB", "\xD0\x9A", 'k'}, + {"\xCC", "\xD0\xBB", 'l'}, + {"\xEC", "\xD0\x9B", 'l'}, + {"\xCD", "\xD0\xBC", 'm'}, + {"\xED", "\xD0\x9C", 'm'}, + {"\xCE", "\xD0\xBD", 'n'}, + {"\xEE", "\xD0\x9D", 'n'}, + {"\xCF", "\xD0\xBE", 'o'}, + {"\xEF", "\xD0\x9E", 'o'}, + {"\xD0", "\xD0\xBF", 'p'}, + {"\xF0", "\xD0\x9F", 'p'}, + {"\xD2", "\xD1\x80", 'r'}, + {"\xF2", "\xD0\xA0", 'r'}, + {"\xD3", "\xD1\x81", 's'}, + {"\xF3", "\xD0\xA1", 's'}, + {"\xD4", "\xD1\x82", 't'}, + {"\xF4", "\xD0\xA2", 't'}, + {"\xD5", "\xD1\x83", 'y'}, + {"\xF5", "\xD0\xA3", 'y'}, + {"\xC6", "\xD1\x84", 'f'}, + {"\xE6", "\xD0\xA4", 'f'}, + {"\xC8", "\xD1\x85", 'h'}, + {"\xE8", "\xD0\xA5", 'h'}, + {"\xC3", "\xD1\x86", 'c'}, + {"\xE3", "\xD0\xA6", 'c'}, + {"\xDE", "\xD1\x87", '7'}, + {"\xFE", "\xD0\xA7", '7'}, + {"\xDB", "\xD1\x88", '4'}, + {"\xFB", "\xD0\xA8", '4'}, + {"\xDD", "\xD1\x89", '6'}, + {"\xFD", "\xD0\xA9", '6'}, + {"\xDF", "\xD1\x8A", '8'}, + {"\xFF", "\xD0\xAA", '8'}, + {"\xD9", "\xD1\x8B", '3'}, + {"\xF9", "\xD0\xAB", '3'}, + {"\xD8", "\xD1\x8C", '2'}, + {"\xF8", "\xD0\xAC", '2'}, + {"\xDC", "\xD1\x8D", '5'}, + {"\xFC", "\xD0\xAD", '5'}, + {"\xC0", "\xD1\x8E", '0'}, + {"\xE0", "\xD0\xAE", '0'}, + {"\xD1", "\xD1\x8F", 'q'}, + {"\xF1", "\xD0\xAF", 'q'}, + }; + for (const auto &r : kRows) { + const char *const input = native_text::native_is_utf8() ? r.utf8 : r.koi8; + EXPECT_EQ(native_text::translit_to_filename(input), std::string(1, r.expected)) + << "transliteration drifted for " << (native_text::native_is_utf8() ? r.utf8 : r.koi8); + } + + // ASCII is lowercased and digits pass through, as before. + EXPECT_EQ(native_text::translit_to_filename("Vasya"), "vasya"); + EXPECT_EQ(native_text::translit_to_filename("Abc123"), "abc123"); + EXPECT_EQ(native_text::translit_to_filename(""), ""); + + // A whole name: "Vasya" in Cyrillic must give the same file name in both encodings. + const char *const name = native_text::native_is_utf8() + ? "\xD0\x92\xD0\xB0\xD1\x81\xD1\x8F" : "\xF7\xC1\xD3\xD1"; + EXPECT_EQ(native_text::translit_to_filename(name), "vasq"); +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/text_semantics.cpp b/tests/text_semantics.cpp new file mode 100644 index 0000000000..35aa6dd78f --- /dev/null +++ b/tests/text_semantics.cpp @@ -0,0 +1,195 @@ +// Regression tests for the byte-vs-character migration (issue #3681). +// +// These pin the *observable behaviour* of the text routines that used to assume 1 byte == 1 +// character: name matching, case-insensitive comparison, argument splitting and Russian name +// declension. Several of these paths had no coverage at all before the migration touched them. +// +// The Russian literals below are deliberately written as literals rather than byte escapes: the +// file is compiled in whatever encoding the engine is built with (KOI8-R today, UTF-8 after the +// flip), and the routines under test operate in that same native encoding. The expectations are +// therefore valid in both, and this file doubles as the guard that the flip did not change +// user-visible behaviour. + +#include "utils/utils_string.h" +#include "utils/utils_parse.h" +#include "utils/mud_string.h" +#include "gameplay/core/genchar.h" +#include "utils/grammar/gender.h" +#include "engine/structs/structs.h" + +#include + +#include + +namespace { + +std::string declension(const char *name, EGender sex, int case_num) { + char buf[128] = {0}; + GetCase(name, sex, case_num, buf); + return std::string(buf); +} + +std::string first_argument(const char *line) { + char buf[kMaxInputLength] = {0}; + one_argument(line, buf); + return std::string(buf); +} + +} // namespace + +// ---------------------------------------------------------------------------- isname + +TEST(TextSemantics, IsnameMatchesAsciiKeywords) { + EXPECT_TRUE(isname("sword", "a long sword")); + EXPECT_TRUE(isname("long", "a long sword")); + EXPECT_TRUE(isname("SWORD", "a long sword")); // case-insensitive + EXPECT_TRUE(isname("swo", "a long sword")); // prefix + EXPECT_FALSE(isname("xyzzy", "a long sword")); +} + +TEST(TextSemantics, IsnameMatchesRussianKeywords) { + EXPECT_TRUE(isname("меч", "меч длинный")); + EXPECT_TRUE(isname("длинный", "меч длинный")); + EXPECT_TRUE(isname("ме", "меч длинный")); // prefix +} + +TEST(TextSemantics, IsnameIsCaseInsensitiveForRussian) { + // Regression: with byte-wise folding under UTF-8 the lead bytes of an upper/lower Cyrillic + // letter fold equal but the trail bytes do not, so this match was silently lost. + EXPECT_TRUE(isname("МЕЧ", "меч")); + EXPECT_TRUE(isname("Меч", "меч длинный")); + EXPECT_TRUE(isname("меч", "МЕЧ ДЛИННЫЙ")); +} + +TEST(TextSemantics, IsnameRejectsUnrelatedRussianWords) { + // Regression: byte-wise matching under UTF-8 compared only the shared leading byte of two + // different Cyrillic letters, so unrelated words matched each other. + EXPECT_FALSE(isname("щит", "меч длинный")); + EXPECT_FALSE(isname("меч", "щит деревянный")); + EXPECT_FALSE(isname("кольцо", "меч")); +} + +// ---------------------------------------------------------------------------- str_cmp + +TEST(TextSemantics, StrCmpIgnoresCase) { + EXPECT_EQ(str_cmp("abc", "ABC"), 0); + EXPECT_EQ(str_cmp("меч", "МЕЧ"), 0); + EXPECT_EQ(str_cmp(std::string("меч"), "МЕЧ"), 0); + EXPECT_NE(str_cmp("меч", "щит"), 0); +} + +TEST(TextSemantics, StrCmpOrdersConsistently) { + EXPECT_LT(str_cmp("abc", "abd"), 0); + EXPECT_GT(str_cmp("abd", "abc"), 0); + EXPECT_LT(str_cmp("ab", "abc"), 0); // prefix sorts first + EXPECT_GT(str_cmp("abc", "ab"), 0); +} + +TEST(TextSemantics, StrnCmpComparesPrefixOnly) { + EXPECT_EQ(strn_cmp("abcdef", "abcXXX", 3), 0); + EXPECT_NE(strn_cmp("abcdef", "abXXXX", 3), 0); + EXPECT_EQ(strn_cmp("МЕЧ", "меч", 6), 0); +} + +// ---------------------------------------------------------------------------- argument splitting + +TEST(TextSemantics, OneArgumentLowercasesAscii) { + EXPECT_EQ(first_argument("LOOK north"), "look"); + EXPECT_EQ(first_argument(" Kill orc"), "kill"); +} + +TEST(TextSemantics, OneArgumentLowercasesRussian) { + // Regression: the byte table leaves UTF-8 Cyrillic untouched, so a Russian command argument + // would reach the command lookup unfolded and fail to match. + EXPECT_EQ(first_argument("СМОТРЕТЬ север"), "смотреть"); + EXPECT_EQ(first_argument("Убить орка"), "убить"); + EXPECT_EQ(first_argument("меч"), "меч"); +} + +TEST(TextSemantics, HalfChopSplitsAndLowercasesFirstWord) { + char arg1[kMaxInputLength] = {0}; + char arg2[kMaxInputLength] = {0}; + half_chop("СКАЗАТЬ привет всем", arg1, arg2); + EXPECT_STREQ(arg1, "сказать"); + EXPECT_STREQ(arg2, "привет всем"); +} + +// ---------------------------------------------------------------------------- GetCase + +TEST(TextSemantics, DeclensionOfFeminineNameEndingInYa) { + // Regression: under UTF-8 the byte-wise last-letter test never matched, so names stopped + // declining entirely and every case returned the nominative. + EXPECT_EQ(declension("Аня", EGender::kFemale, 1), "Ани"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 2), "Ане"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 3), "Аню"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 4), "Аней"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 5), "Ане"); +} + +TEST(TextSemantics, DeclensionOfMasculineNameEndingInConsonant) { + EXPECT_EQ(declension("Иван", EGender::kMale, 1), "Ивана"); + EXPECT_EQ(declension("Иван", EGender::kMale, 2), "Ивану"); + EXPECT_EQ(declension("Иван", EGender::kMale, 4), "Иваном"); + EXPECT_EQ(declension("Иван", EGender::kMale, 5), "Иване"); +} + +TEST(TextSemantics, DeclensionOfNameEndingInA) { + // The genitive/instrumental endings depend on the letter *before* the final one. + EXPECT_EQ(declension("Маша", EGender::kFemale, 1), "Маши"); // after ш -> и + EXPECT_EQ(declension("Анна", EGender::kFemale, 1), "Анны"); // otherwise -> ы + EXPECT_EQ(declension("Маша", EGender::kFemale, 4), "Машей"); // after ш -> ей + EXPECT_EQ(declension("Анна", EGender::kFemale, 4), "Анной"); // otherwise -> ой +} + +TEST(TextSemantics, DeclensionOfMasculineNameEndingInIShort) { + EXPECT_EQ(declension("Дрегвий", EGender::kMale, 1), "Дрегвия"); + EXPECT_EQ(declension("Дрегвий", EGender::kMale, 4), "Дрегвием"); +} + +// ---------------------------------------------------------------------------- fname + +TEST(TextSemantics, FnameExtractsFirstKeyword) { + EXPECT_STREQ(fname("sword long blade"), "sword"); + EXPECT_STREQ(fname("меч длинный"), "меч"); + EXPECT_STREQ(fname("кольцо"), "кольцо"); + EXPECT_STREQ(fname(""), ""); + EXPECT_STREQ(fname(" leading space"), ""); // stops at the very first non-letter +} + +TEST(TextSemantics, FnameStaysInsideItsBuffer) { + // fname() returns a fixed 30-byte buffer and used to copy without any bounds check; a long + // keyword (twice as many bytes per letter once the text is multibyte) ran past its end. + const char *const very_long = "оченьдлинноеключевоесловокотороенепомещается прочее"; + const char *const got = fname(very_long); + EXPECT_LT(std::strlen(got), 30u); + // Whatever was copied must be a prefix of the input, never mangled bytes. + EXPECT_EQ(std::string(very_long).compare(0, std::strlen(got), got), 0); +} + +// ---------------------------------------------------------------------------- cut_one_word + +TEST(TextSemantics, CutOneWordSplitsOnWordBoundaries) { + std::string rest = "меч длинный острый"; + std::string word; + cut_one_word(rest, word); + EXPECT_EQ(word, "меч"); + cut_one_word(rest, word); + EXPECT_EQ(word, "длинный"); + cut_one_word(rest, word); + EXPECT_EQ(word, "острый"); +} + +TEST(TextSemantics, CutOneWordHandlesAsciiAndEmpty) { + std::string rest = "take all"; + std::string word; + cut_one_word(rest, word); + EXPECT_EQ(word, "take"); + cut_one_word(rest, word); + EXPECT_EQ(word, "all"); + + std::string empty; + cut_one_word(empty, word); + EXPECT_TRUE(word.empty()); +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/utf8.cpp b/tests/utf8.cpp new file mode 100644 index 0000000000..d31f8be8f0 --- /dev/null +++ b/tests/utf8.cpp @@ -0,0 +1,166 @@ +// Unit tests for the character-semantic UTF-8 helpers (src/utils/utf8.*, issue #3681). +// +// This file is intentionally pure ASCII: every non-ASCII string is spelled out as explicit +// UTF-8 byte escapes so the test data is independent of the source file's ambient encoding +// (which is KOI8-R today and UTF-8 after the migration flip). Adjacent string literals are +// concatenated so an \xNN escape is never followed by a literal hex digit. + +#include "utils/utf8.h" + +#include + +#include +#include + +namespace { + +// "Privet" (Cyrillic) -- 6 letters, 12 bytes. +const char *const kPrivet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; +// "PRIVET" (all upper) and "privet" (all lower). +const char *const kPrivetUpper = "\xD0\x9F\xD0\xA0\xD0\x98\xD0\x92\xD0\x95\xD0\xA2"; +const char *const kPrivetLower = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; + +std::string S(std::string_view v) { + return std::string(v); +} + +} // namespace + +TEST(Utf8, EmptyString) { + EXPECT_EQ(utf8::length(""), 0u); + EXPECT_TRUE(utf8::is_valid("")); + EXPECT_EQ(utf8::substr("", 0), ""); + EXPECT_EQ(utf8::substr("", 3, 5), ""); + EXPECT_EQ(S(utf8::char_at("", 0)), ""); + EXPECT_EQ(utf8::to_lower(std::string_view("")), ""); + EXPECT_EQ(utf8::byte_offset("", 4), 0u); +} + +TEST(Utf8, AsciiSemanticsMatchBytes) { + EXPECT_EQ(utf8::length("Hello"), 5u); + EXPECT_TRUE(utf8::is_valid("Hello, world!")); + EXPECT_EQ(utf8::to_lower(std::string_view("HeLLo")), "hello"); + EXPECT_EQ(utf8::to_upper(std::string_view("HeLLo")), "HELLO"); + EXPECT_EQ(utf8::substr("Hello", 1, 3), "ell"); + EXPECT_EQ(S(utf8::char_at("Hello", 1)), "e"); + EXPECT_EQ(S(utf8::char_at("Hello", 5)), ""); +} + +TEST(Utf8, CyrillicLengthIsCodePointsNotBytes) { + EXPECT_EQ(std::strlen(kPrivet), 12u); // sanity: the fixture really is 12 bytes + EXPECT_EQ(utf8::length(kPrivet), 6u); + EXPECT_TRUE(utf8::is_valid(kPrivet)); +} + +TEST(Utf8, CyrillicCaseFolding) { + EXPECT_EQ(utf8::to_lower(std::string_view(kPrivetUpper)), kPrivetLower); + EXPECT_EQ(utf8::to_upper(std::string_view(kPrivetLower)), kPrivetUpper); + // "Privet, Mir" -> "privet, mir" (mixed Cyrillic + ASCII punctuation). + const char *const mixed = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82" ", " "\xD0\x9C\xD0\xB8\xD1\x80"; + const char *const mixed_lower = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82" ", " "\xD0\xBC\xD0\xB8\xD1\x80"; + EXPECT_EQ(utf8::to_lower(std::string_view(mixed)), mixed_lower); +} + +TEST(Utf8, YoLetter) { + const char *const kYoUpper = "\xD0\x81"; // U+0401 (Yo) + const char *const kYoLower = "\xD1\x91"; // U+0451 (yo) + EXPECT_EQ(utf8::length(kYoUpper), 1u); + EXPECT_EQ(std::strlen(kYoUpper), 2u); + EXPECT_EQ(utf8::to_lower(std::string_view(kYoUpper)), kYoLower); + EXPECT_EQ(utf8::to_upper(std::string_view(kYoLower)), kYoUpper); + EXPECT_EQ(utf8::to_lower(0x0401u), 0x0451u); + EXPECT_EQ(utf8::to_upper(0x0451u), 0x0401u); +} + +TEST(Utf8, SubstrAndIndexOnCyrillic) { + // chars: [0]P [1]r [2]i [3]v [4]e [5]t + EXPECT_EQ(utf8::substr(kPrivet, 1, 3), "\xD1\x80\xD0\xB8\xD0\xB2"); // "riv" (chars 1..3) + EXPECT_EQ(utf8::substr(kPrivet, 4), "\xD0\xB5\xD1\x82"); // "et" to end + EXPECT_EQ(utf8::substr(kPrivet, 10), ""); // pos past end + EXPECT_EQ(S(utf8::char_at(kPrivet, 0)), "\xD0\x9F"); // char "P" + EXPECT_EQ(S(utf8::char_at(kPrivet, 5)), "\xD1\x82"); // char "t" + EXPECT_EQ(S(utf8::char_at(kPrivet, 6)), ""); // out of range + EXPECT_EQ(utf8::byte_offset(kPrivet, 2), 4u); + EXPECT_EQ(utf8::byte_offset(kPrivet, 6), 12u); + EXPECT_EQ(utf8::byte_offset(kPrivet, 100), 12u); +} + +TEST(Utf8, FourByteAndBom) { + const char *const kGrin = "\xF0\x9F\x98\x80"; // U+1F600 + EXPECT_EQ(utf8::length(kGrin), 1u); + EXPECT_EQ(std::strlen(kGrin), 4u); + EXPECT_TRUE(utf8::is_valid(kGrin)); + EXPECT_EQ(S(utf8::char_at(kGrin, 0)), kGrin); + EXPECT_EQ(utf8::sequence_length(0xF0), 4); + + const char *const kBom = "\xEF\xBB\xBF"; // U+FEFF + EXPECT_EQ(utf8::length(kBom), 1u); + EXPECT_TRUE(utf8::is_valid(kBom)); + char32_t cp = 0; + EXPECT_EQ(utf8::decode(kBom, 0, cp), 3u); + EXPECT_EQ(cp, 0xFEFFu); +} + +TEST(Utf8, SequenceLength) { + EXPECT_EQ(utf8::sequence_length(0x41), 1); // 'A' + EXPECT_EQ(utf8::sequence_length(0xD0), 2); + EXPECT_EQ(utf8::sequence_length(0xE0), 3); + EXPECT_EQ(utf8::sequence_length(0xF0), 4); + EXPECT_EQ(utf8::sequence_length(0x80), 1); // stray continuation + EXPECT_EQ(utf8::sequence_length(0xFF), 1); // out-of-range lead +} + +TEST(Utf8, DecodeBoundaries) { + char32_t cp = 0xABCD; + EXPECT_EQ(utf8::decode("", 0, cp), 0u); // empty + EXPECT_EQ(cp, 0u); + EXPECT_EQ(utf8::decode("A", 1, cp), 0u); // pos at end + EXPECT_EQ(utf8::decode("A", 0, cp), 1u); + EXPECT_EQ(cp, static_cast('A')); +} + +TEST(Utf8, EncodeRoundTrip) { + std::string out; + EXPECT_EQ(utf8::encode('A', out), 1u); + EXPECT_EQ(out, "A"); + out.clear(); + EXPECT_EQ(utf8::encode(0x041Fu, out), 2u); // U+041F (P) + EXPECT_EQ(out, "\xD0\x9F"); + out.clear(); + EXPECT_EQ(utf8::encode(0x1F600u, out), 4u); // U+1F600 + EXPECT_EQ(out, "\xF0\x9F\x98\x80"); + out.clear(); + EXPECT_EQ(utf8::encode(0xD800u, out), 0u); // surrogate rejected + EXPECT_TRUE(out.empty()); + EXPECT_EQ(utf8::encode(0x110000u, out), 0u); // above U+10FFFF + EXPECT_TRUE(out.empty()); +} + +TEST(Utf8, RejectsMalformed) { + EXPECT_FALSE(utf8::is_valid("\x80")); // lone continuation + EXPECT_FALSE(utf8::is_valid("\xD0")); // truncated 2-byte + EXPECT_FALSE(utf8::is_valid("Hi\xD0")); // truncated at end + EXPECT_FALSE(utf8::is_valid("\xC0\x80")); // overlong NUL (0xC0 lead) + EXPECT_FALSE(utf8::is_valid("\xC0\xAF")); // overlong '/' + EXPECT_FALSE(utf8::is_valid("\xE0\x80\xAF")); // overlong 3-byte + EXPECT_FALSE(utf8::is_valid("\xED\xA0\x80")); // U+D800 surrogate + EXPECT_FALSE(utf8::is_valid("\xF4\x90\x80\x80")); // U+110000, above range + EXPECT_FALSE(utf8::is_valid("\xF5\x80\x80\x80")); // 0xF5 lead +} + +TEST(Utf8, AcceptsRangeEdges) { + EXPECT_TRUE(utf8::is_valid("\xED\x9F\xBF")); // U+D7FF, just below surrogates + EXPECT_TRUE(utf8::is_valid("\xEE\x80\x80")); // U+E000, just above surrogates + EXPECT_TRUE(utf8::is_valid("\xF4\x8F\xBF\xBF")); // U+10FFFF, top of range + EXPECT_TRUE(utf8::is_valid("\xC2\x80")); // U+0080, smallest 2-byte +} + +TEST(Utf8, LenientCountingAndFolding) { + // Malformed bytes are counted as one code point each and passed through by the folders, + // so nothing is dropped when the helpers meet non-UTF-8 (e.g. legacy KOI8-R) data. + EXPECT_EQ(utf8::length("\x80\x80"), 2u); + EXPECT_EQ(utf8::to_lower(std::string_view("\x80\x80")), "\x80\x80"); + EXPECT_EQ(utf8::to_upper(std::string_view("A\xFF" "Z")), "A\xFF" "Z"); +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/where.format.cpp b/tests/where.format.cpp index 99810445ba..e0f2b6d3ad 100644 --- a/tests/where.format.cpp +++ b/tests/where.format.cpp @@ -6,6 +6,7 @@ // строки == ширине колонки, и индексная арифметика ниже корректна. #include "engine/ui/cmd/do_where.h" +#include "utils/native_text.h" #include @@ -92,14 +93,23 @@ TEST(WhereFormat, LocationColumnAligned) { const auto lines = SplitLines(FormatWhere(SampleRows())); ASSERT_GE(lines.size(), 5u); - const auto ref = lines[1].rfind(" - "); + // Колонка разделителя измеряется в СИМВОЛАХ, а не в байтах: под UTF-8 байтовое смещение + // зависит от того, сколько в имени кириллицы, и "выровнено" перестаёт значить "в одной + // колонке" (issue #3681). + const auto column_of_separator = [](const std::string &line) { + const auto at = line.rfind(" - "); + return at == std::string::npos + ? std::string::npos : native_text::char_count(std::string_view(line).substr(0, at)); + }; + + const auto ref = column_of_separator(lines[1]); ASSERT_NE(ref, std::string::npos); - EXPECT_EQ(lines[2].rfind(" - "), ref) << "предмет с именем ровно 25 симв."; - EXPECT_EQ(lines[3].rfind(" - "), ref) << "строка-продолжение контейнера"; - EXPECT_EQ(lines[4].rfind(" - "), ref) << "короткое имя"; + EXPECT_EQ(column_of_separator(lines[2]), ref) << "предмет с именем ровно 25 симв."; + EXPECT_EQ(column_of_separator(lines[3]), ref) << "строка-продолжение контейнера"; + EXPECT_EQ(column_of_separator(lines[4]), ref) << "короткое имя"; // Имя моба (29 симв.) длиннее поля в 25 -> разделитель уезжает вправо на 4. - EXPECT_EQ(lines[0].rfind(" - "), ref + 4); + EXPECT_EQ(column_of_separator(lines[0]), ref + 4); } // Предмет в контейнере даёт ровно две строки: первая оканчивается на diff --git a/tools/audit_utf8_migration.py b/tools/audit_utf8_migration.py new file mode 100755 index 0000000000..02cdd798cf --- /dev/null +++ b/tools/audit_utf8_migration.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Audit byte-vs-char assumptions ahead of the KOI8-R -> UTF-8 migration (issue #3681). + +The engine currently assumes "1 byte == 1 character" in three shapes: length/size used as a +character count, fixed byte-offset access/truncation, and per-byte case/classification. Under +UTF-8 every one of those breaks on multibyte (Cyrillic) text. This script scans the C++ sources, +classifies the suspect call sites into categories, and *prioritises files that actually contain +non-ASCII (Russian) bytes* -- those are where a byte-vs-char bug is observable. + +It is a triage aid, not a linter: every hit needs a human to decide whether it means bytes +(fine) or characters (needs utf8::). Source files are KOI8-R today, so snippets are decoded from +KOI8-R for readable output. + +Usage: + tools/audit_utf8_migration.py [PATHS ...] # default: src + tools/audit_utf8_migration.py --full # list hits in ASCII-only files too + tools/audit_utf8_migration.py --category substr # only one category (repeatable) + tools/audit_utf8_migration.py --list-categories +""" + +import argparse +import os +import re +import sys + +# Each category: (regex over a latin-1-decoded line, one-line description). +# The regexes are deliberately broad -- false positives are cheap, missed sites are not. +CATEGORIES = { + "printf-width": ( + re.compile(r"%[-+ 0#]*(?:\d+|\*)(?:\.(?:\d+|\*))?s|%\.(?:\d+|\*)s"), + "printf width/precision on a string (%-20s, %.*s) -- pads/truncates by bytes", + ), + "substr": ( + re.compile(r"\.substr\s*\("), + "substr() -- byte offsets, can cut a code point in half", + ), + "rel-index": ( + re.compile(r"\[[^\]]*-\s*[123]\s*\]"), + "indexing relative to length (str[len-1]) -- last byte, not last character", + ), + "strchr-cyr": ( + re.compile(r'strchr\s*\(\s*"([^"]*)"'), + "strchr() over a literal that contains Cyrillic -- matches a byte, not a letter", + ), + "case-deref": ( + re.compile(r"\b(?:UPPER|LOWER)\s*\(\s*(?:\*|\w+\s*\[)"), + "UPPER/LOWER on a dereferenced/indexed byte -- mangles a UTF-8 lead byte", + ), + "fixed-copy": ( + re.compile(r"\b(?:strn?cpy|strncat)\s*\("), + "strcpy/strncpy/strncat into a fixed buffer -- byte length may split a character", + ), + "strlen": ( + re.compile(r"\bstrlen\s*\("), + "strlen() -- byte count; suspect only when used as a character/display count", + ), + "size-length": ( + re.compile(r"\.(?:size|length)\s*\(\s*\)"), + "std::string size()/length() -- byte count used as a character count", + ), +} + +# High-volume categories: reported in the summary, but only listed for Cyrillic-bearing files +# unless --full, to keep the output actionable. +HIGH_VOLUME = {"strlen", "size-length"} + +SOURCE_EXTENSIONS = (".cpp", ".h", ".hpp", ".cc", ".cxx") + + +def has_cyrillic(raw: bytes) -> bool: + """True if the file carries any high-bit byte (KOI8-R Russian text lives at >= 0x80).""" + return any(b >= 0x80 for b in raw) + + +def literal_has_high_byte(fragment: str) -> bool: + return any(ord(c) >= 0x80 for c in fragment) + + +def decode_snippet(line: str) -> str: + """`line` is latin-1 (bytes 1:1); re-render it from KOI8-R so Russian reads correctly.""" + return line.encode("latin-1", "replace").decode("koi8-r", "replace").rstrip() + + +def iter_source_files(paths): + for path in paths: + if os.path.isfile(path): + yield path + continue + for root, _dirs, files in os.walk(path): + if "third_party_libs" in root: + continue + for name in files: + if name.endswith(SOURCE_EXTENSIONS): + yield os.path.join(root, name) + + +def scan_file(path, wanted): + with open(path, "rb") as handle: + raw = handle.read() + cyrillic = has_cyrillic(raw) + text = raw.decode("latin-1") + hits = [] # (category, lineno, snippet) + for lineno, line in enumerate(text.splitlines(), 1): + for category, (pattern, _desc) in CATEGORIES.items(): + if category not in wanted: + continue + match = pattern.search(line) + if not match: + continue + if category == "strchr-cyr" and not literal_has_high_byte(match.group(1)): + continue + hits.append((category, lineno, decode_snippet(line))) + return cyrillic, hits + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Audit byte-vs-char sites for the UTF-8 migration.") + parser.add_argument("paths", nargs="*", default=["src"], help="files or directories (default: src)") + parser.add_argument("--category", action="append", dest="categories", + help="restrict to a category (repeatable); default: all") + parser.add_argument("--full", action="store_true", + help="also list hits in ASCII-only files and high-volume categories") + parser.add_argument("--list-categories", action="store_true", help="print category names and exit") + args = parser.parse_args(argv) + + if args.list_categories: + for name, (_re, desc) in CATEGORIES.items(): + print(f"{name:<14} {desc}") + return 0 + + wanted = set(args.categories) if args.categories else set(CATEGORIES) + unknown = wanted - set(CATEGORIES) + if unknown: + parser.error(f"unknown category: {', '.join(sorted(unknown))}") + + # counts[category] = [hits_in_cyrillic_files, hits_in_ascii_files] + counts = {name: [0, 0] for name in CATEGORIES} + listing = [] # (priority, path, category, lineno, snippet) + + for path in sorted(iter_source_files(args.paths)): + cyrillic, hits = scan_file(path, wanted) + for category, lineno, snippet in hits: + counts[category][0 if cyrillic else 1] += 1 + show = cyrillic or args.full + if category in HIGH_VOLUME and not args.full: + show = False + if show: + listing.append((0 if cyrillic else 1, path, category, lineno, snippet)) + + print("=" * 78) + print("byte-vs-char audit (Cyr = files with Russian text -> where bugs are observable)") + print("=" * 78) + print(f"{'category':<14}{'Cyr':>8}{'ASCII':>8} description") + print("-" * 78) + for name, (_re, desc) in CATEGORIES.items(): + if name not in wanted: + continue + c_cyr, c_ascii = counts[name] + flag = " [high-volume]" if name in HIGH_VOLUME else "" + print(f"{name:<14}{c_cyr:>8}{c_ascii:>8} {desc}{flag}") + print("-" * 78) + total_cyr = sum(c[0] for c in counts.values()) + total_ascii = sum(c[1] for c in counts.values()) + print(f"{'TOTAL':<14}{total_cyr:>8}{total_ascii:>8}") + print() + + listing.sort(key=lambda row: (row[0], row[1], row[3])) + current_file = None + for _priority, path, category, lineno, snippet in listing: + if path != current_file: + current_file = path + print(f"\n### {path}") + print(f" {lineno:>6} [{category}] {snippet}") + + if not args.full: + print("\n(high-volume categories and ASCII-only files hidden; re-run with --full)") + return 0 + + +if __name__ == "__main__": + sys.exit(main())