From cfdddc07185d3f092914a24b319d873fe44eeea5 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 02:55:16 +0200 Subject: [PATCH 01/20] feat(utf8): add character-semantic UTF-8 helpers + migration audit (#3681) First step of the KOI8-R -> UTF-8 migration (track A0). Adds an encoding-agnostic UTF-8 layer that later turns byte semantics into character semantics -- no runtime behaviour change yet; nothing in the engine calls it. - src/utils/utf8.{h,cpp}: decode/encode, is_valid, length, byte_offset, char_at, substr and ASCII+Cyrillic (incl. Yo) case folding. Code-point based, no external deps (no iconv/ICU); ASCII passes through unchanged. - tests/utf8.cpp: 13 GTest cases (empty, incomplete sequences, 4-byte, BOM, overlong, surrogates, range edges, Cyrillic length/substr/case folding). - tools/audit_utf8_migration.py: triage of byte-vs-char sites, categorised and prioritised by files that contain Russian text. New sources are pure ASCII (UTF-8 test data spelled as \xNN escapes), so they satisfy the KOI8-R working-tree attribute unchanged. --- VERSION.txt | 2 +- meson.build | 1 + src/utils/utf8.cpp | 257 ++++++++++++++++++++++++++++++++++ src/utils/utf8.h | 68 +++++++++ tests/meson.build | 1 + tests/utf8.cpp | 166 ++++++++++++++++++++++ tools/audit_utf8_migration.py | 182 ++++++++++++++++++++++++ 7 files changed, 676 insertions(+), 1 deletion(-) create mode 100644 src/utils/utf8.cpp create mode 100644 src/utils/utf8.h create mode 100644 tests/utf8.cpp create mode 100755 tools/audit_utf8_migration.py diff --git a/VERSION.txt b/VERSION.txt index 3c72f973dd..52b9020e8a 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -0.1.26 \ No newline at end of file +0.1.27 \ No newline at end of file diff --git a/meson.build b/meson.build index 69a1ab0cbe..1a9b04c049 100644 --- a/meson.build +++ b/meson.build @@ -640,6 +640,7 @@ main_sources = files( 'src/gameplay/mechanics/title.cpp', 'src/gameplay/statistics/top.cpp', 'src/utils/utils.cpp', + 'src/utils/utf8.cpp', 'src/utils/utils_encoding.cpp', 'src/gameplay/mechanics/weather.cpp', 'src/engine/olc/zedit.cpp', diff --git a/src/utils/utf8.cpp b/src/utils/utf8.cpp new file mode 100644 index 0000000000..c50486e178 --- /dev/null +++ b/src/utils/utf8.cpp @@ -0,0 +1,257 @@ +/** +\file utf8.cpp - a part of the Bylins engine. +\brief Implementation of the character-semantic UTF-8 helpers declared in utf8.h (issue #3681). +*/ + +#include "utf8.h" + +namespace utf8 { + +namespace { + +// Result of decoding one position: the code point, the byte length consumed, and whether the +// sequence was well-formed. On a malformed byte, `valid` is false, `len` is 1 and `cp` is the +// raw byte -- so scanners always advance and lenient callers can pass the byte through untouched. +struct Decoded { + char32_t cp; + std::size_t len; + bool valid; +}; + +// Decode per the Unicode 3-7 grammar: the first continuation byte has a lead-specific range +// (which is what rejects overlong forms and surrogates), the rest are plain 0x80..0xBF. +Decoded decode_core(std::string_view s, std::size_t pos) { + const std::size_t n = s.size(); + const unsigned char c0 = static_cast(s[pos]); + if (c0 < 0x80) { + return {c0, 1, true}; + } + + int len = 0; + char32_t cp = 0; + unsigned char b1_lo = 0x80; + unsigned char b1_hi = 0xBF; + if (c0 >= 0xC2 && c0 <= 0xDF) { + len = 2; + cp = c0 & 0x1F; + } else if (c0 == 0xE0) { + len = 3; + cp = c0 & 0x0F; + b1_lo = 0xA0; + } else if (c0 >= 0xE1 && c0 <= 0xEC) { + len = 3; + cp = c0 & 0x0F; + } else if (c0 == 0xED) { + len = 3; + cp = c0 & 0x0F; + b1_hi = 0x9F; + } else if (c0 >= 0xEE && c0 <= 0xEF) { + len = 3; + cp = c0 & 0x0F; + } else if (c0 == 0xF0) { + len = 4; + cp = c0 & 0x07; + b1_lo = 0x90; + } else if (c0 >= 0xF1 && c0 <= 0xF3) { + len = 4; + cp = c0 & 0x07; + } else if (c0 == 0xF4) { + len = 4; + cp = c0 & 0x07; + b1_hi = 0x8F; + } else { + // 0xC0, 0xC1, 0xF5..0xFF or a stray continuation byte: cannot start a sequence. + return {c0, 1, false}; + } + + if (pos + static_cast(len) > n) { + return {c0, 1, false}; + } + + const unsigned char b1 = static_cast(s[pos + 1]); + if (b1 < b1_lo || b1 > b1_hi) { + return {c0, 1, false}; + } + cp = (cp << 6) | (b1 & 0x3F); + + for (int k = 2; k < len; ++k) { + const unsigned char b = static_cast(s[pos + static_cast(k)]); + if (b < 0x80 || b > 0xBF) { + return {c0, 1, false}; + } + cp = (cp << 6) | (b & 0x3F); + } + + return {cp, static_cast(len), true}; +} + +} // namespace + +int sequence_length(unsigned char c) { + if (c < 0x80) { + return 1; + } + if (c >= 0xC0 && c <= 0xDF) { + return 2; + } + if (c >= 0xE0 && c <= 0xEF) { + return 3; + } + if (c >= 0xF0 && c <= 0xF7) { + return 4; + } + // Continuation byte (0x80..0xBF) or an out-of-range lead (0xF8..0xFF): not a valid start. + return 1; +} + +std::size_t decode(std::string_view s, std::size_t pos, char32_t &cp) { + if (pos >= s.size()) { + cp = 0; + return 0; + } + const Decoded d = decode_core(s, pos); + cp = d.cp; + return d.len; +} + +std::size_t encode(char32_t cp, std::string &out) { + if (cp <= 0x7F) { + out.push_back(static_cast(cp)); + return 1; + } + if (cp <= 0x7FF) { + out.push_back(static_cast(0xC0 | (cp >> 6))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + return 2; + } + if (cp >= 0xD800 && cp <= 0xDFFF) { + return 0; // surrogate half: not a Unicode scalar value + } + if (cp <= 0xFFFF) { + out.push_back(static_cast(0xE0 | (cp >> 12))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + return 3; + } + if (cp <= 0x10FFFF) { + out.push_back(static_cast(0xF0 | (cp >> 18))); + out.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + return 4; + } + return 0; +} + +bool is_valid(std::string_view s) { + std::size_t pos = 0; + const std::size_t n = s.size(); + while (pos < n) { + const Decoded d = decode_core(s, pos); + if (!d.valid) { + return false; + } + pos += d.len; + } + return true; +} + +std::size_t length(std::string_view s) { + std::size_t count = 0; + std::size_t pos = 0; + const std::size_t n = s.size(); + while (pos < n) { + pos += decode_core(s, pos).len; + ++count; + } + return count; +} + +std::size_t byte_offset(std::string_view s, std::size_t index) { + std::size_t pos = 0; + const std::size_t n = s.size(); + while (index > 0 && pos < n) { + pos += decode_core(s, pos).len; + --index; + } + return pos; +} + +std::string_view char_at(std::string_view s, std::size_t index) { + const std::size_t start = byte_offset(s, index); + if (start >= s.size()) { + return {}; + } + const std::size_t len = decode_core(s, start).len; + return s.substr(start, len); +} + +std::string substr(std::string_view s, std::size_t pos, std::size_t count) { + const std::size_t start = byte_offset(s, pos); + if (count == std::string_view::npos) { + return std::string(s.substr(start)); + } + const std::size_t stop = byte_offset(s, pos + count); + return std::string(s.substr(start, stop - start)); +} + +char32_t to_lower(char32_t cp) { + if (cp >= 'A' && cp <= 'Z') { + return cp + 0x20; + } + if (cp >= 0x0410 && cp <= 0x042F) { // U+0410..U+042F (upper) -> U+0430..U+044F (lower) + return cp + 0x20; + } + if (cp == 0x0401) { // U+0401 (Yo) -> U+0451 (yo) + return 0x0451; + } + return cp; +} + +char32_t to_upper(char32_t cp) { + if (cp >= 'a' && cp <= 'z') { + return cp - 0x20; + } + if (cp >= 0x0430 && cp <= 0x044F) { // U+0430..U+044F (lower) -> U+0410..U+042F (upper) + return cp - 0x20; + } + if (cp == 0x0451) { // U+0451 (yo) -> U+0401 (Yo) + return 0x0401; + } + return cp; +} + +namespace { + +// Shared body for the whole-string case folders: decode, fold each code point, re-encode. +// Malformed bytes (valid == false) are copied through verbatim so nothing is silently dropped. +std::string fold_string(std::string_view s, char32_t (*fold)(char32_t)) { + std::string out; + out.reserve(s.size()); + std::size_t pos = 0; + const std::size_t n = s.size(); + while (pos < n) { + const Decoded d = decode_core(s, pos); + if (d.valid) { + encode(fold(d.cp), out); + } else { + out.push_back(s[pos]); + } + pos += d.len; + } + return out; +} + +} // namespace + +std::string to_lower(std::string_view s) { + return fold_string(s, to_lower); +} + +std::string to_upper(std::string_view s) { + return fold_string(s, to_upper); +} + +} // namespace utf8 + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/utf8.h b/src/utils/utf8.h new file mode 100644 index 0000000000..7991d4c04a --- /dev/null +++ b/src/utils/utf8.h @@ -0,0 +1,68 @@ +/** +\file utf8.h - a part of the Bylins engine. +\brief Character-semantic helpers over UTF-8 strings (issue #3681, "Plan Napoleon"). + +This is the encoding-agnostic building block for the KOI8-R -> UTF-8 migration: code that +must reason about *characters* (code points) rather than bytes -- length, substring, indexed +access, case folding -- lives here. Everything is plain UTF-8 in / UTF-8 out with no external +dependency (no iconv/ICU). ASCII passes through untouched, so the helpers are also correct for +pure-ASCII input regardless of the ambient encoding. + +Case folding covers exactly what the legacy a_ucc/a_lcc tables covered: ASCII A-Z and the +Russian Cyrillic block (U+0410..U+044F plus Yo, U+0401/U+0451). Any other code point passes +through unchanged. +*/ + +#ifndef BYLINS_SRC_UTILS_UTF8_H_ +#define BYLINS_SRC_UTILS_UTF8_H_ + +#include +#include +#include + +namespace utf8 { + +// Number of bytes the UTF-8 sequence starting with lead byte `c` claims to span. +// Returns 1 for ASCII and for any byte that cannot start a sequence, so a caller that +// advances by the result always makes forward progress. +int sequence_length(unsigned char c); + +// Decode the code point starting at byte position `pos` in `s`. +// Writes the code point (or the raw byte, on a malformed sequence) to `cp` and returns the +// number of bytes consumed: >=1 while `pos` is in range, 0 once `pos >= s.size()`. +// Malformed sequences never stall: they yield the single offending byte and a length of 1. +std::size_t decode(std::string_view s, std::size_t pos, char32_t &cp); + +// Append `cp` to `out` as UTF-8. Returns the number of bytes written, or 0 for a value that is +// not a valid Unicode scalar (surrogate half or > U+10FFFF), in which case `out` is untouched. +std::size_t encode(char32_t cp, std::string &out); + +// Strict, whole-string well-formedness check per the Unicode Table 3-7 byte-sequence grammar +// (rejects overlong forms, surrogates, code points above U+10FFFF and stray continuation bytes). +bool is_valid(std::string_view s); + +// Number of code points. Malformed bytes count as one code point each; never throws. +std::size_t length(std::string_view s); + +// Byte offset of the `index`-th code point, or `s.size()` when `index` is past the end. +std::size_t byte_offset(std::string_view s, std::size_t index); + +// The bytes of the `index`-th code point, as a view into `s`. Empty when `index` is out of range. +std::string_view char_at(std::string_view s, std::size_t index); + +// std::string::substr, but `pos`/`count` are counted in code points instead of bytes. +std::string substr(std::string_view s, std::size_t pos, std::size_t count = std::string_view::npos); + +// Single-code-point case folding (ASCII + Russian Cyrillic incl. Yo); other values pass through. +char32_t to_lower(char32_t cp); +char32_t to_upper(char32_t cp); + +// Whole-string case folding. Malformed bytes are copied through verbatim. +std::string to_lower(std::string_view s); +std::string to_upper(std::string_view s); + +} // namespace utf8 + +#endif // BYLINS_SRC_UTILS_UTF8_H_ + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/meson.build b/tests/meson.build index 7bdee932c6..573d15a420 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -39,6 +39,7 @@ test_sources = files( 'utils.editor.cpp', 'utils.string.cpp', 'utils.encoding.cpp', + 'utf8.cpp', 'fight.penalties.cpp', 'bonus.command.parser.cpp', 'quested.cpp', diff --git a/tests/utf8.cpp b/tests/utf8.cpp new file mode 100644 index 0000000000..d31f8be8f0 --- /dev/null +++ b/tests/utf8.cpp @@ -0,0 +1,166 @@ +// Unit tests for the character-semantic UTF-8 helpers (src/utils/utf8.*, issue #3681). +// +// This file is intentionally pure ASCII: every non-ASCII string is spelled out as explicit +// UTF-8 byte escapes so the test data is independent of the source file's ambient encoding +// (which is KOI8-R today and UTF-8 after the migration flip). Adjacent string literals are +// concatenated so an \xNN escape is never followed by a literal hex digit. + +#include "utils/utf8.h" + +#include + +#include +#include + +namespace { + +// "Privet" (Cyrillic) -- 6 letters, 12 bytes. +const char *const kPrivet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; +// "PRIVET" (all upper) and "privet" (all lower). +const char *const kPrivetUpper = "\xD0\x9F\xD0\xA0\xD0\x98\xD0\x92\xD0\x95\xD0\xA2"; +const char *const kPrivetLower = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; + +std::string S(std::string_view v) { + return std::string(v); +} + +} // namespace + +TEST(Utf8, EmptyString) { + EXPECT_EQ(utf8::length(""), 0u); + EXPECT_TRUE(utf8::is_valid("")); + EXPECT_EQ(utf8::substr("", 0), ""); + EXPECT_EQ(utf8::substr("", 3, 5), ""); + EXPECT_EQ(S(utf8::char_at("", 0)), ""); + EXPECT_EQ(utf8::to_lower(std::string_view("")), ""); + EXPECT_EQ(utf8::byte_offset("", 4), 0u); +} + +TEST(Utf8, AsciiSemanticsMatchBytes) { + EXPECT_EQ(utf8::length("Hello"), 5u); + EXPECT_TRUE(utf8::is_valid("Hello, world!")); + EXPECT_EQ(utf8::to_lower(std::string_view("HeLLo")), "hello"); + EXPECT_EQ(utf8::to_upper(std::string_view("HeLLo")), "HELLO"); + EXPECT_EQ(utf8::substr("Hello", 1, 3), "ell"); + EXPECT_EQ(S(utf8::char_at("Hello", 1)), "e"); + EXPECT_EQ(S(utf8::char_at("Hello", 5)), ""); +} + +TEST(Utf8, CyrillicLengthIsCodePointsNotBytes) { + EXPECT_EQ(std::strlen(kPrivet), 12u); // sanity: the fixture really is 12 bytes + EXPECT_EQ(utf8::length(kPrivet), 6u); + EXPECT_TRUE(utf8::is_valid(kPrivet)); +} + +TEST(Utf8, CyrillicCaseFolding) { + EXPECT_EQ(utf8::to_lower(std::string_view(kPrivetUpper)), kPrivetLower); + EXPECT_EQ(utf8::to_upper(std::string_view(kPrivetLower)), kPrivetUpper); + // "Privet, Mir" -> "privet, mir" (mixed Cyrillic + ASCII punctuation). + const char *const mixed = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82" ", " "\xD0\x9C\xD0\xB8\xD1\x80"; + const char *const mixed_lower = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82" ", " "\xD0\xBC\xD0\xB8\xD1\x80"; + EXPECT_EQ(utf8::to_lower(std::string_view(mixed)), mixed_lower); +} + +TEST(Utf8, YoLetter) { + const char *const kYoUpper = "\xD0\x81"; // U+0401 (Yo) + const char *const kYoLower = "\xD1\x91"; // U+0451 (yo) + EXPECT_EQ(utf8::length(kYoUpper), 1u); + EXPECT_EQ(std::strlen(kYoUpper), 2u); + EXPECT_EQ(utf8::to_lower(std::string_view(kYoUpper)), kYoLower); + EXPECT_EQ(utf8::to_upper(std::string_view(kYoLower)), kYoUpper); + EXPECT_EQ(utf8::to_lower(0x0401u), 0x0451u); + EXPECT_EQ(utf8::to_upper(0x0451u), 0x0401u); +} + +TEST(Utf8, SubstrAndIndexOnCyrillic) { + // chars: [0]P [1]r [2]i [3]v [4]e [5]t + EXPECT_EQ(utf8::substr(kPrivet, 1, 3), "\xD1\x80\xD0\xB8\xD0\xB2"); // "riv" (chars 1..3) + EXPECT_EQ(utf8::substr(kPrivet, 4), "\xD0\xB5\xD1\x82"); // "et" to end + EXPECT_EQ(utf8::substr(kPrivet, 10), ""); // pos past end + EXPECT_EQ(S(utf8::char_at(kPrivet, 0)), "\xD0\x9F"); // char "P" + EXPECT_EQ(S(utf8::char_at(kPrivet, 5)), "\xD1\x82"); // char "t" + EXPECT_EQ(S(utf8::char_at(kPrivet, 6)), ""); // out of range + EXPECT_EQ(utf8::byte_offset(kPrivet, 2), 4u); + EXPECT_EQ(utf8::byte_offset(kPrivet, 6), 12u); + EXPECT_EQ(utf8::byte_offset(kPrivet, 100), 12u); +} + +TEST(Utf8, FourByteAndBom) { + const char *const kGrin = "\xF0\x9F\x98\x80"; // U+1F600 + EXPECT_EQ(utf8::length(kGrin), 1u); + EXPECT_EQ(std::strlen(kGrin), 4u); + EXPECT_TRUE(utf8::is_valid(kGrin)); + EXPECT_EQ(S(utf8::char_at(kGrin, 0)), kGrin); + EXPECT_EQ(utf8::sequence_length(0xF0), 4); + + const char *const kBom = "\xEF\xBB\xBF"; // U+FEFF + EXPECT_EQ(utf8::length(kBom), 1u); + EXPECT_TRUE(utf8::is_valid(kBom)); + char32_t cp = 0; + EXPECT_EQ(utf8::decode(kBom, 0, cp), 3u); + EXPECT_EQ(cp, 0xFEFFu); +} + +TEST(Utf8, SequenceLength) { + EXPECT_EQ(utf8::sequence_length(0x41), 1); // 'A' + EXPECT_EQ(utf8::sequence_length(0xD0), 2); + EXPECT_EQ(utf8::sequence_length(0xE0), 3); + EXPECT_EQ(utf8::sequence_length(0xF0), 4); + EXPECT_EQ(utf8::sequence_length(0x80), 1); // stray continuation + EXPECT_EQ(utf8::sequence_length(0xFF), 1); // out-of-range lead +} + +TEST(Utf8, DecodeBoundaries) { + char32_t cp = 0xABCD; + EXPECT_EQ(utf8::decode("", 0, cp), 0u); // empty + EXPECT_EQ(cp, 0u); + EXPECT_EQ(utf8::decode("A", 1, cp), 0u); // pos at end + EXPECT_EQ(utf8::decode("A", 0, cp), 1u); + EXPECT_EQ(cp, static_cast('A')); +} + +TEST(Utf8, EncodeRoundTrip) { + std::string out; + EXPECT_EQ(utf8::encode('A', out), 1u); + EXPECT_EQ(out, "A"); + out.clear(); + EXPECT_EQ(utf8::encode(0x041Fu, out), 2u); // U+041F (P) + EXPECT_EQ(out, "\xD0\x9F"); + out.clear(); + EXPECT_EQ(utf8::encode(0x1F600u, out), 4u); // U+1F600 + EXPECT_EQ(out, "\xF0\x9F\x98\x80"); + out.clear(); + EXPECT_EQ(utf8::encode(0xD800u, out), 0u); // surrogate rejected + EXPECT_TRUE(out.empty()); + EXPECT_EQ(utf8::encode(0x110000u, out), 0u); // above U+10FFFF + EXPECT_TRUE(out.empty()); +} + +TEST(Utf8, RejectsMalformed) { + EXPECT_FALSE(utf8::is_valid("\x80")); // lone continuation + EXPECT_FALSE(utf8::is_valid("\xD0")); // truncated 2-byte + EXPECT_FALSE(utf8::is_valid("Hi\xD0")); // truncated at end + EXPECT_FALSE(utf8::is_valid("\xC0\x80")); // overlong NUL (0xC0 lead) + EXPECT_FALSE(utf8::is_valid("\xC0\xAF")); // overlong '/' + EXPECT_FALSE(utf8::is_valid("\xE0\x80\xAF")); // overlong 3-byte + EXPECT_FALSE(utf8::is_valid("\xED\xA0\x80")); // U+D800 surrogate + EXPECT_FALSE(utf8::is_valid("\xF4\x90\x80\x80")); // U+110000, above range + EXPECT_FALSE(utf8::is_valid("\xF5\x80\x80\x80")); // 0xF5 lead +} + +TEST(Utf8, AcceptsRangeEdges) { + EXPECT_TRUE(utf8::is_valid("\xED\x9F\xBF")); // U+D7FF, just below surrogates + EXPECT_TRUE(utf8::is_valid("\xEE\x80\x80")); // U+E000, just above surrogates + EXPECT_TRUE(utf8::is_valid("\xF4\x8F\xBF\xBF")); // U+10FFFF, top of range + EXPECT_TRUE(utf8::is_valid("\xC2\x80")); // U+0080, smallest 2-byte +} + +TEST(Utf8, LenientCountingAndFolding) { + // Malformed bytes are counted as one code point each and passed through by the folders, + // so nothing is dropped when the helpers meet non-UTF-8 (e.g. legacy KOI8-R) data. + EXPECT_EQ(utf8::length("\x80\x80"), 2u); + EXPECT_EQ(utf8::to_lower(std::string_view("\x80\x80")), "\x80\x80"); + EXPECT_EQ(utf8::to_upper(std::string_view("A\xFF" "Z")), "A\xFF" "Z"); +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tools/audit_utf8_migration.py b/tools/audit_utf8_migration.py new file mode 100755 index 0000000000..02cdd798cf --- /dev/null +++ b/tools/audit_utf8_migration.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Audit byte-vs-char assumptions ahead of the KOI8-R -> UTF-8 migration (issue #3681). + +The engine currently assumes "1 byte == 1 character" in three shapes: length/size used as a +character count, fixed byte-offset access/truncation, and per-byte case/classification. Under +UTF-8 every one of those breaks on multibyte (Cyrillic) text. This script scans the C++ sources, +classifies the suspect call sites into categories, and *prioritises files that actually contain +non-ASCII (Russian) bytes* -- those are where a byte-vs-char bug is observable. + +It is a triage aid, not a linter: every hit needs a human to decide whether it means bytes +(fine) or characters (needs utf8::). Source files are KOI8-R today, so snippets are decoded from +KOI8-R for readable output. + +Usage: + tools/audit_utf8_migration.py [PATHS ...] # default: src + tools/audit_utf8_migration.py --full # list hits in ASCII-only files too + tools/audit_utf8_migration.py --category substr # only one category (repeatable) + tools/audit_utf8_migration.py --list-categories +""" + +import argparse +import os +import re +import sys + +# Each category: (regex over a latin-1-decoded line, one-line description). +# The regexes are deliberately broad -- false positives are cheap, missed sites are not. +CATEGORIES = { + "printf-width": ( + re.compile(r"%[-+ 0#]*(?:\d+|\*)(?:\.(?:\d+|\*))?s|%\.(?:\d+|\*)s"), + "printf width/precision on a string (%-20s, %.*s) -- pads/truncates by bytes", + ), + "substr": ( + re.compile(r"\.substr\s*\("), + "substr() -- byte offsets, can cut a code point in half", + ), + "rel-index": ( + re.compile(r"\[[^\]]*-\s*[123]\s*\]"), + "indexing relative to length (str[len-1]) -- last byte, not last character", + ), + "strchr-cyr": ( + re.compile(r'strchr\s*\(\s*"([^"]*)"'), + "strchr() over a literal that contains Cyrillic -- matches a byte, not a letter", + ), + "case-deref": ( + re.compile(r"\b(?:UPPER|LOWER)\s*\(\s*(?:\*|\w+\s*\[)"), + "UPPER/LOWER on a dereferenced/indexed byte -- mangles a UTF-8 lead byte", + ), + "fixed-copy": ( + re.compile(r"\b(?:strn?cpy|strncat)\s*\("), + "strcpy/strncpy/strncat into a fixed buffer -- byte length may split a character", + ), + "strlen": ( + re.compile(r"\bstrlen\s*\("), + "strlen() -- byte count; suspect only when used as a character/display count", + ), + "size-length": ( + re.compile(r"\.(?:size|length)\s*\(\s*\)"), + "std::string size()/length() -- byte count used as a character count", + ), +} + +# High-volume categories: reported in the summary, but only listed for Cyrillic-bearing files +# unless --full, to keep the output actionable. +HIGH_VOLUME = {"strlen", "size-length"} + +SOURCE_EXTENSIONS = (".cpp", ".h", ".hpp", ".cc", ".cxx") + + +def has_cyrillic(raw: bytes) -> bool: + """True if the file carries any high-bit byte (KOI8-R Russian text lives at >= 0x80).""" + return any(b >= 0x80 for b in raw) + + +def literal_has_high_byte(fragment: str) -> bool: + return any(ord(c) >= 0x80 for c in fragment) + + +def decode_snippet(line: str) -> str: + """`line` is latin-1 (bytes 1:1); re-render it from KOI8-R so Russian reads correctly.""" + return line.encode("latin-1", "replace").decode("koi8-r", "replace").rstrip() + + +def iter_source_files(paths): + for path in paths: + if os.path.isfile(path): + yield path + continue + for root, _dirs, files in os.walk(path): + if "third_party_libs" in root: + continue + for name in files: + if name.endswith(SOURCE_EXTENSIONS): + yield os.path.join(root, name) + + +def scan_file(path, wanted): + with open(path, "rb") as handle: + raw = handle.read() + cyrillic = has_cyrillic(raw) + text = raw.decode("latin-1") + hits = [] # (category, lineno, snippet) + for lineno, line in enumerate(text.splitlines(), 1): + for category, (pattern, _desc) in CATEGORIES.items(): + if category not in wanted: + continue + match = pattern.search(line) + if not match: + continue + if category == "strchr-cyr" and not literal_has_high_byte(match.group(1)): + continue + hits.append((category, lineno, decode_snippet(line))) + return cyrillic, hits + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Audit byte-vs-char sites for the UTF-8 migration.") + parser.add_argument("paths", nargs="*", default=["src"], help="files or directories (default: src)") + parser.add_argument("--category", action="append", dest="categories", + help="restrict to a category (repeatable); default: all") + parser.add_argument("--full", action="store_true", + help="also list hits in ASCII-only files and high-volume categories") + parser.add_argument("--list-categories", action="store_true", help="print category names and exit") + args = parser.parse_args(argv) + + if args.list_categories: + for name, (_re, desc) in CATEGORIES.items(): + print(f"{name:<14} {desc}") + return 0 + + wanted = set(args.categories) if args.categories else set(CATEGORIES) + unknown = wanted - set(CATEGORIES) + if unknown: + parser.error(f"unknown category: {', '.join(sorted(unknown))}") + + # counts[category] = [hits_in_cyrillic_files, hits_in_ascii_files] + counts = {name: [0, 0] for name in CATEGORIES} + listing = [] # (priority, path, category, lineno, snippet) + + for path in sorted(iter_source_files(args.paths)): + cyrillic, hits = scan_file(path, wanted) + for category, lineno, snippet in hits: + counts[category][0 if cyrillic else 1] += 1 + show = cyrillic or args.full + if category in HIGH_VOLUME and not args.full: + show = False + if show: + listing.append((0 if cyrillic else 1, path, category, lineno, snippet)) + + print("=" * 78) + print("byte-vs-char audit (Cyr = files with Russian text -> where bugs are observable)") + print("=" * 78) + print(f"{'category':<14}{'Cyr':>8}{'ASCII':>8} description") + print("-" * 78) + for name, (_re, desc) in CATEGORIES.items(): + if name not in wanted: + continue + c_cyr, c_ascii = counts[name] + flag = " [high-volume]" if name in HIGH_VOLUME else "" + print(f"{name:<14}{c_cyr:>8}{c_ascii:>8} {desc}{flag}") + print("-" * 78) + total_cyr = sum(c[0] for c in counts.values()) + total_ascii = sum(c[1] for c in counts.values()) + print(f"{'TOTAL':<14}{total_cyr:>8}{total_ascii:>8}") + print() + + listing.sort(key=lambda row: (row[0], row[1], row[3])) + current_file = None + for _priority, path, category, lineno, snippet in listing: + if path != current_file: + current_file = path + print(f"\n### {path}") + print(f" {lineno:>6} [{category}] {snippet}") + + if not args.full: + print("\n(high-volume categories and ASCII-only files hidden; re-run with --full)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 3ec73bae3c71db267f294c49906650c49f860ec9 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 03:06:29 +0200 Subject: [PATCH 02/20] feat(utf8): internal_encoding flag + route format_text via native_text (#3681) Track A1 of the KOI8-R -> UTF-8 migration. Introduces the build-time `internal_encoding` switch and a native-encoding dispatch layer, then routes the text reformatter through it. No behaviour change on the default KOI8-R build (char_count == byte count, capitalize_first == UPPER of the first byte, truncate_offset == maxlen); the utf8 build gets code-point semantics. - meson_options.txt / meson.build: internal_encoding=koi8r|utf8, the utf8 value defines INTERNAL_ENCODING_UTF8. - src/utils/native_text.{h,cpp}: char_count / capitalize_first / truncate_offset with a byte-identical KOI8-R branch and a utf8:: branch, plus native_is_utf8(). - src/utils/utils.cpp: format_text() width accounting, first-letter capitalisation and maxlen truncation no longer assume 1 byte == 1 character. - tests/native_text.cpp: adaptive GTest suite branching on native_is_utf8(). --- VERSION.txt | 2 +- meson.build | 6 +++ meson_options.txt | 4 ++ src/utils/native_text.cpp | 102 ++++++++++++++++++++++++++++++++++++++ src/utils/native_text.h | 50 +++++++++++++++++++ src/utils/utils.cpp | 9 ++-- tests/meson.build | 1 + tests/native_text.cpp | 72 +++++++++++++++++++++++++++ 8 files changed, 241 insertions(+), 5 deletions(-) create mode 100644 src/utils/native_text.cpp create mode 100644 src/utils/native_text.h create mode 100644 tests/native_text.cpp diff --git a/VERSION.txt b/VERSION.txt index 52b9020e8a..0affed72b0 100644 --- a/VERSION.txt +++ b/VERSION.txt @@ -1 +1 @@ -0.1.27 \ No newline at end of file +0.1.28 \ No newline at end of file diff --git a/meson.build b/meson.build index 1a9b04c049..2da3288ec6 100644 --- a/meson.build +++ b/meson.build @@ -120,6 +120,11 @@ if host_system == 'windows' endif endif +# Native runtime string encoding (issue #3681). Default koi8r keeps byte semantics unchanged. +if get_option('internal_encoding') == 'utf8' + project_args += '-DINTERNAL_ENCODING_UTF8' +endif + linker_choice = get_option('linker') if linker_choice != '' add_project_link_arguments('-fuse-ld=' + linker_choice, language: 'cpp') @@ -641,6 +646,7 @@ main_sources = files( 'src/gameplay/statistics/top.cpp', 'src/utils/utils.cpp', 'src/utils/utf8.cpp', + 'src/utils/native_text.cpp', 'src/utils/utils_encoding.cpp', 'src/gameplay/mechanics/weather.cpp', 'src/engine/olc/zedit.cpp', diff --git a/meson_options.txt b/meson_options.txt index bf8598bb5d..f870dc1133 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -10,6 +10,10 @@ option('lua_formatter', type: 'boolean', value: true, description: 'Build the em option('with_asan', type: 'boolean', value: false, description: 'Build with Address Sanitizer') option('use_pch', type: 'boolean', value: true, description: 'Use precompiled headers for circle.library') +# KOI8-R -> UTF-8 migration (issue #3681). Selects the engine's native runtime string encoding. +# koi8r = current byte semantics (default); utf8 = character semantics via the utf8:: helpers. +option('internal_encoding', type: 'combo', choices: ['koi8r', 'utf8'], value: 'koi8r', description: 'Engine-native runtime string encoding: koi8r (byte semantics, current) or utf8 (character semantics)') + # Options for Admin API and Web features option('admin_api', type: 'boolean', value: false, description: 'Enable Admin API and JSON-related features') diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp new file mode 100644 index 0000000000..aabc541662 --- /dev/null +++ b/src/utils/native_text.cpp @@ -0,0 +1,102 @@ +/** +\file native_text.cpp - a part of the Bylins engine. +\brief Native-encoding character helpers declared in native_text.h (issue #3681). + +Two implementations selected by the INTERNAL_ENCODING_UTF8 build macro. The KOI8-R branch is kept +byte-for-byte identical to the open-coded logic these helpers replace so that routing call sites +through them changes nothing until the encoding flip. +*/ + +#include "native_text.h" + +#ifdef INTERNAL_ENCODING_UTF8 +#include "utf8.h" +#else +#include "utils.h" // UPPER() / a_ucc() -- the KOI8-R case table +#endif + +#include + +namespace native_text { + +#ifdef INTERNAL_ENCODING_UTF8 + +bool native_is_utf8() { + return true; +} + +std::size_t char_count(const char *begin, const char *end) { + return utf8::length(std::string_view(begin, static_cast(end - begin))); +} + +std::size_t char_count(std::string_view s) { + return utf8::length(s); +} + +void capitalize_first(char *s) { + if (s == nullptr || *s == '\0') { + return; + } + const std::string_view sv(s); + char32_t cp = 0; + const std::size_t len = utf8::decode(sv, 0, cp); + if (len == 0) { + return; + } + const char32_t upper = utf8::to_upper(cp); + if (upper == cp) { + return; + } + std::string encoded; + if (utf8::encode(upper, encoded) == len) { + for (std::size_t i = 0; i < len; ++i) { + s[i] = encoded[i]; + } + } +} + +std::size_t truncate_offset(std::string_view s, std::size_t max_bytes) { + if (max_bytes >= s.size()) { + return s.size(); + } + std::size_t pos = 0; + while (true) { + char32_t cp = 0; + const std::size_t len = utf8::decode(s, pos, cp); + if (len == 0 || pos + len > max_bytes) { + break; + } + pos += len; + } + return pos; +} + +#else // KOI8-R: 1 byte == 1 character + +bool native_is_utf8() { + return false; +} + +std::size_t char_count(const char *begin, const char *end) { + return static_cast(end - begin); +} + +std::size_t char_count(std::string_view s) { + return s.size(); +} + +void capitalize_first(char *s) { + if (s != nullptr && *s != '\0') { + *s = static_cast(UPPER(static_cast(*s))); + } +} + +std::size_t truncate_offset(std::string_view s, std::size_t max_bytes) { + return max_bytes < s.size() ? max_bytes : s.size(); +} + +#endif + +} // namespace native_text + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/native_text.h b/src/utils/native_text.h new file mode 100644 index 0000000000..f334a0bca6 --- /dev/null +++ b/src/utils/native_text.h @@ -0,0 +1,50 @@ +/** +\file native_text.h - a part of the Bylins engine. +\brief Character-semantic operations in the engine's *native runtime encoding* (issue #3681). + +The migration flips the engine's internal string encoding from KOI8-R (1 byte == 1 character) to +UTF-8 (multibyte). Code that must reason about characters -- counting display width, capitalising +a letter, truncating without splitting a character -- should go through this thin dispatch layer +instead of assuming bytes. The active encoding is chosen at build time by the `internal_encoding` +Meson option, which defines INTERNAL_ENCODING_UTF8 for the UTF-8 build: + + * KOI8-R (default, current behaviour): every helper is byte-for-byte identical to the open-coded + byte logic it replaces, so routing call sites through it is a no-op. + * UTF-8: helpers operate on code points via the utf8:: primitives. + +This lets the whole byte-vs-char refactor land and ship on KOI8-R (safely, as a no-op) ahead of +the encoding flip. Once the flip is permanent the KOI8-R branch and this indirection are removed. +*/ + +#ifndef BYLINS_SRC_UTILS_NATIVE_TEXT_H_ +#define BYLINS_SRC_UTILS_NATIVE_TEXT_H_ + +#include +#include + +namespace native_text { + +// True iff the engine was built with UTF-8 as its native runtime encoding. Reflects the flag the +// library itself was compiled with (not the translation unit that calls this), so callers/tests +// can branch reliably regardless of their own compile flags. +bool native_is_utf8(); + +// Number of display characters in a byte range / view. KOI8-R: the byte count. UTF-8: the number +// of code points (malformed bytes counted as one each, so it never stalls on legacy data). +std::size_t char_count(const char *begin, const char *end); +std::size_t char_count(std::string_view s); + +// Uppercase the first character of the null-terminated string `s` in place (ASCII + Russian +// Cyrillic incl. Yo). No-op on an empty string, on a non-cased first character, or in the (never +// occurring for these alphabets) case where the uppercase form has a different byte length. +void capitalize_first(char *s); + +// Largest byte offset <= max_bytes that lands on a character boundary, so cutting the string +// there never splits a multibyte character. KOI8-R: min(max_bytes, s.size()). +std::size_t truncate_offset(std::string_view s, std::size_t max_bytes); + +} // namespace native_text + +#endif // BYLINS_SRC_UTILS_NATIVE_TEXT_H_ + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index 21371ef08d..85c5a6c85a 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -13,6 +13,7 @@ ************************************************************************ */ #include "utils.h" +#include "native_text.h" #include "utils/grammar/declensions.h" #include @@ -266,7 +267,7 @@ void format_text(const utils::AbstractStringWriter::shared_ptr &writer, flow++; } - if ((total_chars + (flow - start) + 1) > 79) { + if ((total_chars + native_text::char_count(start, flow) + 1) > 79) { strcpy(pos, "\r\n"); total_chars = 0; pos += 2; @@ -280,11 +281,11 @@ void format_text(const utils::AbstractStringWriter::shared_ptr &writer, } } - total_chars += flow - start; + total_chars += native_text::char_count(start, flow); strncpy(pos, start, flow - start); if (cap_next) { cap_next = false; - *pos = UPPER(*pos); + native_text::capitalize_first(pos); } pos += flow - start; } @@ -304,7 +305,7 @@ void format_text(const utils::AbstractStringWriter::shared_ptr &writer, strcpy(pos, "\r\n"); if (static_cast(pos - formatted) > maxlen) { - formatted[maxlen] = '\0'; + formatted[native_text::truncate_offset(formatted, maxlen)] = '\0'; } writer->set_string(formatted); } diff --git a/tests/meson.build b/tests/meson.build index 573d15a420..2d1289db79 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -40,6 +40,7 @@ test_sources = files( 'utils.string.cpp', 'utils.encoding.cpp', 'utf8.cpp', + 'native_text.cpp', 'fight.penalties.cpp', 'bonus.command.parser.cpp', 'quested.cpp', diff --git a/tests/native_text.cpp b/tests/native_text.cpp new file mode 100644 index 0000000000..f5cdefecea --- /dev/null +++ b/tests/native_text.cpp @@ -0,0 +1,72 @@ +// Unit tests for the native-encoding character helpers (src/utils/native_text.*, issue #3681). +// +// Pure ASCII: non-ASCII fixtures are spelled as UTF-8 byte escapes. Expectations that differ +// between the KOI8-R and UTF-8 builds branch on native_text::native_is_utf8() (which reflects the +// flag the library was built with), so this one test file is correct under either build. + +#include "utils/native_text.h" + +#include + +#include +#include + +namespace { + +// "Privet": 6 Cyrillic code points, 12 UTF-8 bytes. +const char *const kPrivet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; + +} // namespace + +TEST(NativeText, CharCountAscii) { + EXPECT_EQ(native_text::char_count("Hello"), 5u); + EXPECT_EQ(native_text::char_count(kPrivet, kPrivet + 4), native_text::native_is_utf8() ? 2u : 4u); +} + +TEST(NativeText, CharCountReflectsEncoding) { + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::char_count(kPrivet), 6u); + EXPECT_EQ(native_text::char_count(kPrivet, kPrivet + 12), 6u); + } else { + EXPECT_EQ(native_text::char_count(kPrivet), 12u); + EXPECT_EQ(native_text::char_count(kPrivet, kPrivet + 12), 12u); + } +} + +TEST(NativeText, CapitalizeAscii) { + char buf[] = "hello"; + native_text::capitalize_first(buf); + EXPECT_STREQ(buf, "Hello"); + + char empty[] = ""; + native_text::capitalize_first(empty); // must not touch the terminator + EXPECT_STREQ(empty, ""); + + char already[] = "X"; + native_text::capitalize_first(already); + EXPECT_STREQ(already, "X"); +} + +TEST(NativeText, CapitalizeCyrillicUtf8Only) { + if (!native_text::native_is_utf8()) { + GTEST_SKIP() << "Cyrillic-as-UTF-8 fixtures are only meaningful under the UTF-8 build"; + } + char buf[] = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; // "privet" + native_text::capitalize_first(buf); + EXPECT_STREQ(buf, "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"); // "Privet" +} + +TEST(NativeText, TruncateOffset) { + const std::string_view p(kPrivet, 12); + EXPECT_EQ(native_text::truncate_offset(p, 100), 12u); // past end -> full size + EXPECT_EQ(native_text::truncate_offset(p, 0), 0u); + if (native_text::native_is_utf8()) { + // 5 bytes lands mid-character; back up to the boundary after 2 code points (4 bytes). + EXPECT_EQ(native_text::truncate_offset(p, 5), 4u); + EXPECT_EQ(native_text::truncate_offset(p, 4), 4u); + } else { + EXPECT_EQ(native_text::truncate_offset(p, 5), 5u); + } +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : From a35f164e33d07a1f0fb57d11373a7034919025d4 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 04:08:21 +0200 Subject: [PATCH 03/20] feat(utf8): route pager next_page + string_add truncation via native_text (#3681) Continues track A1. next_page() now counts one column per character and steps over a multibyte character's trailing bytes; string_add()'s truncation points snap to a character boundary so a cut never splits a character. No-op under KOI8-R (char_bytes == 1, truncate_offset == the byte limit) -- byte-identical behaviour. - native_text: add char_bytes() (byte length of the character starting at a pointer). - src/engine/ui/modify.cpp: next_page() column counting; string_add() max_str/80 truncation offsets. - tests/native_text.cpp: char_bytes cases (ASCII, Cyrillic lead, 4-byte, truncated lead). --- src/engine/ui/modify.cpp | 18 ++++++++++++------ src/utils/native_text.cpp | 17 +++++++++++++++++ src/utils/native_text.h | 5 +++++ tests/native_text.cpp | 11 +++++++++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/engine/ui/modify.cpp b/src/engine/ui/modify.cpp index 11fe7132e3..ae3a00537b 100644 --- a/src/engine/ui/modify.cpp +++ b/src/engine/ui/modify.cpp @@ -17,6 +17,7 @@ #include #include "modify.h" +#include "utils/native_text.h" #include "engine/olc/vedun/vedun.h" #include "interpreter.h" #include "engine/core/target_resolver.h" @@ -652,11 +653,11 @@ void string_add(DescriptorData *d, char *str) { if (!d->writer->get_string()) { if (strlen(str) + 3 > d->max_str) { SendMsgToChar("Слишком длинная строка - усечена.\r\n", d->character.get()); - strcpy(&str[d->max_str - 3], "\r\n"); + strcpy(&str[native_text::truncate_offset(str, d->max_str - 3)], "\r\n"); d->writer->set_string(str); } else if (EConState::kWriteMod == d->state && strlen(str) + 3 > 80) { SendMsgToChar("Слишком длинная строка - усечена.\r\n", d->character.get()); - str[80 - 3] = '\0'; + str[native_text::truncate_offset(str, 80 - 3)] = '\0'; d->writer->set_string(str); } else { d->writer->set_string(str); @@ -664,7 +665,7 @@ void string_add(DescriptorData *d, char *str) { } else { if (EConState::kWriteMod == d->state && strlen(str) + 3 > 80) { SendMsgToChar("Слишком длинная строка - усечена.\r\n", d->character.get()); - str[80 - 3] = '\0'; + str[native_text::truncate_offset(str, 80 - 3)] = '\0'; } if (strlen(str) + d->writer->length() + 3 > d->max_str) // \r\n\0 // @@ -1188,9 +1189,14 @@ char *next_page(char *str, CharData *ch) { // * We need to check here and see if we are over the page width, // * and if so, compensate by going to the begining of the next line. - else if ((ch)->player_specials->saved.stringLength && ++col > (ch)->player_specials->saved.stringLength) { - col = 1; - line++; + // * A multibyte character counts as one column; skip its trailing bytes so + // * they are not counted again (native_text::char_bytes == 1 under KOI8-R). + else if ((ch)->player_specials->saved.stringLength) { + if (++col > (ch)->player_specials->saved.stringLength) { + col = 1; + line++; + } + str += native_text::char_bytes(str) - 1; } } } diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index aabc541662..c3ceea1d8d 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -71,6 +71,19 @@ std::size_t truncate_offset(std::string_view s, std::size_t max_bytes) { return pos; } +std::size_t char_bytes(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return 1; + } + const std::size_t want = static_cast(utf8::sequence_length(lead)); + std::size_t n = 1; + while (n < want && (static_cast(s[n]) & 0xC0) == 0x80) { + ++n; + } + return n; +} + #else // KOI8-R: 1 byte == 1 character bool native_is_utf8() { @@ -95,6 +108,10 @@ std::size_t truncate_offset(std::string_view s, std::size_t max_bytes) { return max_bytes < s.size() ? max_bytes : s.size(); } +std::size_t char_bytes(const char *) { + return 1; +} + #endif } // namespace native_text diff --git a/src/utils/native_text.h b/src/utils/native_text.h index f334a0bca6..0a2a49edfa 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -43,6 +43,11 @@ void capitalize_first(char *s); // there never splits a multibyte character. KOI8-R: min(max_bytes, s.size()). std::size_t truncate_offset(std::string_view s, std::size_t max_bytes); +// Byte length of the character that starts at `s` (KOI8-R: 1), for stepping over a whole +// character byte-by-byte. Always >= 1; on a malformed/truncated UTF-8 lead it returns only the +// bytes actually present (never counts past a terminator or a non-continuation byte). +std::size_t char_bytes(const char *s); + } // namespace native_text #endif // BYLINS_SRC_UTILS_NATIVE_TEXT_H_ diff --git a/tests/native_text.cpp b/tests/native_text.cpp index f5cdefecea..159da43f43 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -56,6 +56,17 @@ TEST(NativeText, CapitalizeCyrillicUtf8Only) { EXPECT_STREQ(buf, "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"); // "Privet" } +TEST(NativeText, CharBytes) { + EXPECT_EQ(native_text::char_bytes("A"), 1u); + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::char_bytes(kPrivet), 2u); // Cyrillic lead -> 2 bytes + EXPECT_EQ(native_text::char_bytes("\xF0\x9F\x98\x80"), 4u); // 4-byte code point + EXPECT_EQ(native_text::char_bytes("\xD0"), 1u); // truncated lead: 1 byte present + } else { + EXPECT_EQ(native_text::char_bytes(kPrivet), 1u); // KOI8-R: every byte is a char + } +} + TEST(NativeText, TruncateOffset) { const std::string_view p(kPrivet, 12); EXPECT_EQ(native_text::truncate_offset(p, 100), 12u); // past end -> full size From c1a4f889bed6fac6cf031e8112b8002b595f068f Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 04:27:56 +0200 Subject: [PATCH 04/20] feat(utf8): route str_cmp/strn_cmp case folding via native_text (#3681) Track A2. The case-insensitive comparison family (~600 str_cmp + ~67 strn_cmp call sites) now folds per character instead of per byte -- without touching a single call site. Under KOI8-R the original LOWER() byte loops are left verbatim and a guarded early return sends the UTF-8 build to the code-point fold, so the default build is byte-identical (28 added lines, 0 removed in utils_string.cpp). - native_text: add compare_ci / ncompare_ci (KOI8-R: byte-wise via a_lcc_table, preserving the magnitude callers propagate; UTF-8: code-point fold via utf8::). - native_text no longer includes utils.h -- it declares the two case tables directly, which also keeps the module standalone-testable. - tests: ASCII ordering/prefix/budget cases, Cyrillic case-insensitivity, and a reference test pinning the KOI8-R branch against the original LOWER() byte loop. --- src/utils/native_text.cpp | 88 +++++++++++++++++++++++++++++++++++++- src/utils/native_text.h | 9 ++++ src/utils/utils_string.cpp | 28 ++++++++++++ tests/native_text.cpp | 76 ++++++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 2 deletions(-) diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index c3ceea1d8d..2e2f7b52b5 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -12,7 +12,10 @@ through them changes nothing until the encoding flip. #ifdef INTERNAL_ENCODING_UTF8 #include "utf8.h" #else -#include "utils.h" // UPPER() / a_ucc() -- the KOI8-R case table +// The KOI8-R case tables (defined in utils.cpp). Declared directly instead of including utils.h, +// which drags in fmt/ and much of the engine for what is just two 256-byte lookups. +extern const char a_ucc_table[]; +extern const char a_lcc_table[]; #endif #include @@ -84,6 +87,51 @@ std::size_t char_bytes(const char *s) { return n; } +namespace { + +// Shared driver for the two case-insensitive comparisons. `limit` caps how many bytes of `a` may +// be consumed (npos = unlimited): once that budget is spent the strings count as equal, which is +// what the strn_cmp callers -- who pass a prefix length in bytes -- expect. +int compare_folded(std::string_view a, std::string_view b, std::size_t limit) { + std::size_t pa = 0; + std::size_t pb = 0; + while (true) { + if (limit != std::string_view::npos && pa >= limit) { + return 0; + } + char32_t ca = 0; + char32_t cb = 0; + const std::size_t la = utf8::decode(a, pa, ca); + const std::size_t lb = utf8::decode(b, pb, cb); + if (la == 0 && lb == 0) { + return 0; + } + if (la == 0) { + return -1; + } + if (lb == 0) { + return 1; + } + const char32_t fa = utf8::to_lower(ca); + const char32_t fb = utf8::to_lower(cb); + if (fa != fb) { + return fa < fb ? -1 : 1; + } + pa += la; + pb += lb; + } +} + +} // namespace + +int compare_ci(std::string_view a, std::string_view b) { + return compare_folded(a, b, std::string_view::npos); +} + +int ncompare_ci(std::string_view a, std::string_view b, std::size_t n) { + return compare_folded(a, b, n); +} + #else // KOI8-R: 1 byte == 1 character bool native_is_utf8() { @@ -100,7 +148,7 @@ std::size_t char_count(std::string_view s) { void capitalize_first(char *s) { if (s != nullptr && *s != '\0') { - *s = static_cast(UPPER(static_cast(*s))); + *s = a_ucc_table[static_cast(*s)]; } } @@ -112,6 +160,42 @@ std::size_t char_bytes(const char *) { return 1; } +namespace { + +// Byte-wise fold-and-subtract, identical to the open-coded `LOWER(a[i]) - LOWER(b[i])` loops in +// utils_string.cpp: the magnitude of the result (not just its sign) is preserved, since some +// callers propagate it. A string that ended compares as LOWER('\0') against the other's byte. +int compare_bytes(std::string_view a, std::string_view b, std::size_t limit) { + std::size_t i = 0; + while (true) { + if (limit != std::string_view::npos && i >= limit) { + return 0; + } + const bool a_end = i >= a.size(); + const bool b_end = i >= b.size(); + if (a_end && b_end) { + return 0; + } + const unsigned char ca = a_end ? '\0' : static_cast(a[i]); + const unsigned char cb = b_end ? '\0' : static_cast(b[i]); + const int chk = a_lcc_table[ca] - a_lcc_table[cb]; + if (chk != 0) { + return chk; + } + ++i; + } +} + +} // namespace + +int compare_ci(std::string_view a, std::string_view b) { + return compare_bytes(a, b, std::string_view::npos); +} + +int ncompare_ci(std::string_view a, std::string_view b, std::size_t n) { + return compare_bytes(a, b, n); +} + #endif } // namespace native_text diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 0a2a49edfa..872c3d02b2 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -48,6 +48,15 @@ std::size_t truncate_offset(std::string_view s, std::size_t max_bytes); // bytes actually present (never counts past a terminator or a non-continuation byte). std::size_t char_bytes(const char *s); +// Case-insensitive comparison in the native encoding: lexicographic over lowered characters, +// the shorter string orders first, returns the signed difference at the first mismatch (0 when +// equal). KOI8-R: per byte, via LOWER() -- matches str_cmp/str/str semantics. UTF-8: per code +// point, folded via utf8::to_lower (so the sign is meaningful; the magnitude is a code-point +// difference). In ncompare_ci, `n` is a byte budget -- the callers pass strlen()/length() -- and +// the comparison stops (as a match) once that many bytes of equal text have been consumed. +int compare_ci(std::string_view a, std::string_view b); +int ncompare_ci(std::string_view a, std::string_view b, std::size_t n); + } // namespace native_text #endif // BYLINS_SRC_UTILS_NATIVE_TEXT_H_ diff --git a/src/utils/utils_string.cpp b/src/utils/utils_string.cpp index cad0ed3443..b613434311 100644 --- a/src/utils/utils_string.cpp +++ b/src/utils/utils_string.cpp @@ -1,6 +1,7 @@ //#include "utils_string.h" #include "utils.h" +#include "utils/native_text.h" #include "utils/utils_encoding.h" #include "gameplay/core/constants.h" @@ -579,12 +580,18 @@ char *delete_doubledollar(char *string) { } // Moved from utils.cpp +// The str_cmp/strn_cmp family folds case per *character*: under KOI8-R that is the original +// byte-wise LOWER() loop kept verbatim below, under UTF-8 it is native_text's code-point fold +// (issue #3681). The KOI8-R path is untouched so behaviour is bit-identical until the flip. int str_cmp(const char *arg1, const char *arg2) { int chk, i; if (arg1 == nullptr || arg2 == nullptr) { log("SYSERR: str_cmp() passed a nullptr pointer, %p or %p.", arg1, arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; arg1[i] || arg2[i]; i++) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); @@ -598,6 +605,9 @@ int str_cmp(const std::string &arg1, const char *arg2) { log("SYSERR: str_cmp() passed a NULL pointer, %p.", arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; i != arg1.length() && *arg2; i++, arg2++) if ((chk = LOWER(arg1[i]) - LOWER(*arg2)) != 0) return (chk); @@ -616,6 +626,9 @@ int str_cmp(const char *arg1, const std::string &arg2) { log("SYSERR: str_cmp() passed a NULL pointer, %p.", arg1); return (0); } + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; *arg1 && i != arg2.length(); i++, arg1++) if ((chk = LOWER(*arg1) - LOWER(arg2[i])) != 0) return (chk); @@ -630,6 +643,9 @@ int str_cmp(const char *arg1, const std::string &arg2) { int str_cmp(const std::string &arg1, const std::string &arg2) { int chk; std::string::size_type i; + if (native_text::native_is_utf8()) { + return native_text::compare_ci(arg1, arg2); + } for (i = 0; i != arg1.length() && i != arg2.length(); i++) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); @@ -647,6 +663,9 @@ int strn_cmp(const char *arg1, const char *arg2, size_t n) { log("SYSERR: strn_cmp() passed a NULL pointer, %p or %p.", arg1, arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; (arg1[i] || arg2[i]) && (n > 0); i++, n--) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); @@ -660,6 +679,9 @@ int strn_cmp(const std::string &arg1, const char *arg2, size_t n) { log("SYSERR: strn_cmp() passed a NULL pointer, %p.", arg2); return (0); } + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; i != arg1.length() && *arg2 && (n > 0); i++, arg2++, n--) if ((chk = LOWER(arg1[i]) - LOWER(*arg2)) != 0) return (chk); @@ -678,6 +700,9 @@ int strn_cmp(const char *arg1, const std::string &arg2, size_t n) { log("SYSERR: strn_cmp() passed a NULL pointer, %p.", arg1); return (0); } + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; *arg1 && i != arg2.length() && (n > 0); i++, arg1++, n--) if ((chk = LOWER(*arg1) - LOWER(arg2[i])) != 0) return (chk); @@ -692,6 +717,9 @@ int strn_cmp(const char *arg1, const std::string &arg2, size_t n) { int strn_cmp(const std::string &arg1, const std::string &arg2, size_t n) { int chk; std::string::size_type i; + if (native_text::native_is_utf8()) { + return native_text::ncompare_ci(arg1, arg2, n); + } for (i = 0; i != arg1.length() && i != arg2.length() && (n > 0); i++, n--) if ((chk = LOWER(arg1[i]) - LOWER(arg2[i])) != 0) return (chk); diff --git a/tests/native_text.cpp b/tests/native_text.cpp index 159da43f43..bdf3136914 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -18,6 +18,18 @@ const char *const kPrivet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; } // namespace +// The legacy KOI8-R lowercase table (utils.cpp). Declared here rather than including utils.h, +// which pulls in fmt/ and the rest of the engine headers. +extern const char a_lcc_table[]; + +namespace { + +int legacy_lower(unsigned char c) { + return a_lcc_table[c]; +} + +} // namespace + TEST(NativeText, CharCountAscii) { EXPECT_EQ(native_text::char_count("Hello"), 5u); EXPECT_EQ(native_text::char_count(kPrivet, kPrivet + 4), native_text::native_is_utf8() ? 2u : 4u); @@ -80,4 +92,68 @@ TEST(NativeText, TruncateOffset) { } } +TEST(NativeText, CompareCiAscii) { + EXPECT_EQ(native_text::compare_ci("abc", "abc"), 0); + EXPECT_EQ(native_text::compare_ci("ABC", "abc"), 0); // case-insensitive + EXPECT_EQ(native_text::compare_ci("AbC", "aBc"), 0); + EXPECT_LT(native_text::compare_ci("abc", "abd"), 0); // ordering by first mismatch + EXPECT_GT(native_text::compare_ci("abd", "abc"), 0); + EXPECT_LT(native_text::compare_ci("ab", "abc"), 0); // prefix orders first + EXPECT_GT(native_text::compare_ci("abc", "ab"), 0); + EXPECT_EQ(native_text::compare_ci("", ""), 0); + EXPECT_LT(native_text::compare_ci("", "a"), 0); +} + +TEST(NativeText, NCompareCiAscii) { + EXPECT_EQ(native_text::ncompare_ci("abcdef", "abcXXX", 3), 0); // only first 3 bytes matter + EXPECT_NE(native_text::ncompare_ci("abcdef", "abXXXX", 3), 0); + EXPECT_EQ(native_text::ncompare_ci("ABC", "abc", 3), 0); + EXPECT_EQ(native_text::ncompare_ci("anything", "other", 0), 0); // zero budget: equal + EXPECT_LT(native_text::ncompare_ci("ab", "abc", 10), 0); // budget beyond the strings +} + +TEST(NativeText, CompareCiCyrillicIsCaseInsensitive) { + // "PRIVET" vs "privet" in Cyrillic: must compare equal in BOTH encodings -- under KOI8-R via + // the byte table, under UTF-8 via the code-point fold. This is the property that a naive + // "just compare bytes" UTF-8 migration would silently lose. + const char *const upper = "\xD0\x9F\xD0\xA0\xD0\x98\xD0\x92\xD0\x95\xD0\xA2"; + const char *const lower = "\xD0\xBF\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::compare_ci(upper, lower), 0); + EXPECT_EQ(native_text::compare_ci(upper, upper), 0); + EXPECT_NE(native_text::compare_ci(upper, "\xD0\xBF\xD1\x80\xD0\xB8"), 0); // prefix differs + } else { + // Under KOI8-R these UTF-8 bytes are not Cyrillic; just assert self-equality and that + // the comparison stays reflexive/antisymmetric on arbitrary high bytes. + EXPECT_EQ(native_text::compare_ci(upper, upper), 0); + EXPECT_EQ(native_text::compare_ci(lower, lower), 0); + } +} + +TEST(NativeText, CompareCiMatchesLegacyByteLoopUnderKoi8r) { + if (native_text::native_is_utf8()) { + GTEST_SKIP() << "this pins the KOI8-R branch against the original LOWER() byte loop"; + } + // Reference implementation: the exact loop str_cmp() used before the migration. + auto legacy = [](const char *a, const char *b) { + for (int i = 0;; ++i) { + if (!a[i] && !b[i]) { + return 0; + } + const int chk = legacy_lower(static_cast(a[i])) + - legacy_lower(static_cast(b[i])); + if (chk != 0) { + return chk; + } + } + }; + const char *const samples[] = {"", "a", "A", "abc", "ABC", "abd", "ab", "zzz", "\xC1\xC2", "\xE1\xE2"}; + for (const char *x : samples) { + for (const char *y : samples) { + EXPECT_EQ(native_text::compare_ci(x, y), legacy(x, y)) + << "mismatch for \"" << x << "\" vs \"" << y << "\""; + } + } +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : From bfd9268fac081d0fd34e14084159e76a940dca46 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 05:06:41 +0200 Subject: [PATCH 05/20] feat(utf8): make isname walk characters instead of bytes (#3681) Track A2. isname() (~157 call sites) drives a backtracking state machine over bytes: classification, the case-insensitive match and every advance assumed 1 byte == 1 character. The structure -- including each `curstr = laststr` backtrack -- is kept verbatim; only those three operations now go through native_text. Under KOI8-R this is an identity (is_alnum_char == a_isalnum, chars_equal_ci == LOWER == LOWER, char_bytes == 1); a differential harness comparing the old and new implementations over ASCII and Cyrillic inputs reports 0 differences. Under UTF-8 it fixes the byte logic in both directions, also verified differentially: * false positive: "shchit" matched the unrelated name list "mech dlinnyi", because only the shared D0 lead byte was ever compared; * false negative: "MECH" no longer matched "mech", because the lead bytes fold equal but the trail bytes do not. - native_text: add is_alnum_char() and chars_equal_ci(). - tests: cases for both primitives, incl. the Cyrillic case-fold regression. --- src/utils/native_text.cpp | 32 ++++++++++++++++++++++++++++++++ src/utils/native_text.h | 9 +++++++++ src/utils/utils_string.cpp | 23 ++++++++++++++--------- tests/native_text.cpp | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 2e2f7b52b5..4f93f002de 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -16,6 +16,7 @@ through them changes nothing until the encoding flip. // which drags in fmt/ and much of the engine for what is just two 256-byte lookups. extern const char a_ucc_table[]; extern const char a_lcc_table[]; +extern const bool a_isalnum_table[]; #endif #include @@ -132,6 +133,29 @@ int ncompare_ci(std::string_view a, std::string_view b, std::size_t n) { return compare_folded(a, b, n); } +bool is_alnum_char(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return (lead >= '0' && lead <= '9') || (lead >= 'A' && lead <= 'Z') || (lead >= 'a' && lead <= 'z'); + } + char32_t cp = 0; + if (utf8::decode(std::string_view(s, char_bytes(s)), 0, cp) == 0) { + return false; + } + // Russian Cyrillic block, including Yo. + return (cp >= 0x0410 && cp <= 0x044F) || cp == 0x0401 || cp == 0x0451; +} + +bool chars_equal_ci(const char *a, const char *b) { + char32_t ca = 0; + char32_t cb = 0; + if (utf8::decode(std::string_view(a, char_bytes(a)), 0, ca) == 0 + || utf8::decode(std::string_view(b, char_bytes(b)), 0, cb) == 0) { + return false; + } + return utf8::to_lower(ca) == utf8::to_lower(cb); +} + #else // KOI8-R: 1 byte == 1 character bool native_is_utf8() { @@ -196,6 +220,14 @@ int ncompare_ci(std::string_view a, std::string_view b, std::size_t n) { return compare_bytes(a, b, n); } +bool is_alnum_char(const char *s) { + return a_isalnum_table[static_cast(*s)]; +} + +bool chars_equal_ci(const char *a, const char *b) { + return a_lcc_table[static_cast(*a)] == a_lcc_table[static_cast(*b)]; +} + #endif } // namespace native_text diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 872c3d02b2..65f3a7b263 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -57,6 +57,15 @@ std::size_t char_bytes(const char *s); int compare_ci(std::string_view a, std::string_view b); int ncompare_ci(std::string_view a, std::string_view b, std::size_t n); +// Is the character starting at `s` alphanumeric? KOI8-R: the a_isalnum byte table. UTF-8: ASCII +// letters/digits plus the Russian Cyrillic block -- so a multibyte letter is classified as one +// alphanumeric character rather than a lead byte followed by "punctuation" trail bytes. +bool is_alnum_char(const char *s); + +// Do the characters starting at `a` and `b` match ignoring case? KOI8-R: LOWER(*a) == LOWER(*b). +// UTF-8: compares whole folded code points, so "P" matches "p" in Cyrillic too. +bool chars_equal_ci(const char *a, const char *b); + } // namespace native_text #endif // BYLINS_SRC_UTILS_NATIVE_TEXT_H_ diff --git a/src/utils/utils_string.cpp b/src/utils/utils_string.cpp index b613434311..2274b1aec0 100644 --- a/src/utils/utils_string.cpp +++ b/src/utils/utils_string.cpp @@ -943,13 +943,18 @@ bool IsValidEmail(const char *address) { return true; } +// Walks both strings one *character* at a time (issue #3681): the classification, the +// case-insensitive match and every advance go through native_text, so a multibyte letter is one +// unit instead of a lead byte plus trail bytes that the byte tables would read as punctuation. +// Under KOI8-R every helper is the original byte operation and char_bytes() == 1, so the state +// machine below -- including each `curstr = laststr` backtrack -- behaves exactly as before. bool isname(const char *str, const char *namelist) { bool once_ok = false; const char *curname, *curstr, *laststr; if (!namelist || !*namelist || !str) { return false; } - for (curstr = str; !a_isalnum(*curstr); curstr++) { + for (curstr = str; !native_text::is_alnum_char(curstr); curstr += native_text::char_bytes(curstr)) { if (!*curstr) { return once_ok; } @@ -958,18 +963,18 @@ bool isname(const char *str, const char *namelist) { curname = namelist; for (;;) { once_ok = false; - for (;; curstr++, curname++) { + for (;; curstr += native_text::char_bytes(curstr), curname += native_text::char_bytes(curname)) { if (!*curstr) { return once_ok; } if (*curstr == '!') { - if (a_isalnum(*curname)) { + if (native_text::is_alnum_char(curname)) { curstr = laststr; break; } } - if (!a_isalnum(*curstr)) { - for (; !a_isalnum(*curstr); curstr++) { + if (!native_text::is_alnum_char(curstr)) { + for (; !native_text::is_alnum_char(curstr); curstr += native_text::char_bytes(curstr)) { if (!*curstr) { return once_ok; } @@ -980,19 +985,19 @@ bool isname(const char *str, const char *namelist) { if (!*curname) { return false; } - if (!a_isalnum(*curname)) { + if (!native_text::is_alnum_char(curname)) { curstr = laststr; break; } - if (LOWER(*curstr) != LOWER(*curname)) { + if (!native_text::chars_equal_ci(curstr, curname)) { curstr = laststr; break; } else { once_ok = true; } } - for (; a_isalnum(*curname); curname++); - for (; !a_isalnum(*curname); curname++) { + for (; native_text::is_alnum_char(curname); curname += native_text::char_bytes(curname)); + for (; !native_text::is_alnum_char(curname); curname += native_text::char_bytes(curname)) { if (!*curname) { return false; } diff --git a/tests/native_text.cpp b/tests/native_text.cpp index bdf3136914..2bf42ba56e 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -156,4 +156,36 @@ TEST(NativeText, CompareCiMatchesLegacyByteLoopUnderKoi8r) { } } +TEST(NativeText, IsAlnumChar) { + EXPECT_TRUE(native_text::is_alnum_char("a")); + EXPECT_TRUE(native_text::is_alnum_char("Z")); + EXPECT_TRUE(native_text::is_alnum_char("7")); + EXPECT_FALSE(native_text::is_alnum_char(" ")); + EXPECT_FALSE(native_text::is_alnum_char("!")); + EXPECT_FALSE(native_text::is_alnum_char(".")); + EXPECT_FALSE(native_text::is_alnum_char("")); // terminator is not alphanumeric + if (native_text::native_is_utf8()) { + // A Cyrillic letter is ONE alphanumeric character; its trail byte must not be read as + // punctuation (which is what the raw byte table would do and what breaks tokenisation). + EXPECT_TRUE(native_text::is_alnum_char(kPrivet)); + EXPECT_TRUE(native_text::is_alnum_char("\xD0\x81")); // Yo + EXPECT_TRUE(native_text::is_alnum_char("\xD1\x91")); // yo + } +} + +TEST(NativeText, CharsEqualCi) { + EXPECT_TRUE(native_text::chars_equal_ci("a", "a")); + EXPECT_TRUE(native_text::chars_equal_ci("a", "A")); + EXPECT_TRUE(native_text::chars_equal_ci("Z", "z")); + EXPECT_FALSE(native_text::chars_equal_ci("a", "b")); + EXPECT_FALSE(native_text::chars_equal_ci("a", "")); + if (native_text::native_is_utf8()) { + // The regression this whole step exists for: with the raw KOI8-R byte table the lead + // bytes of "P"/"p" fold equal but the trail bytes differ, so the match was lost. + EXPECT_TRUE(native_text::chars_equal_ci("\xD0\x9F", "\xD0\xBF")); // P vs p + EXPECT_TRUE(native_text::chars_equal_ci("\xD0\x81", "\xD1\x91")); // Yo vs yo + EXPECT_FALSE(native_text::chars_equal_ci("\xD0\x9F", "\xD1\x80")); // P vs r + } +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : From 7339f6f8fbfc5c53d1f788f61b50bb309b4413fe Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 05:13:10 +0200 Subject: [PATCH 06/20] feat(utf8): character-aware argument splitting and name declension (#3681) Track A2, continued: the command-argument splitters and the Russian name declension no longer assume 1 byte == 1 character. Both are identities under KOI8-R (verified differentially, 0 differences) and fix real breakage under UTF-8. Splitters -- one_argument/any_one_arg/half_chop (mud_string.cpp) and one_word (utils_string.cpp) lowercased byte by byte, which under UTF-8 leaves Cyrillic untouched (the byte table maps neither the lead nor the trail bytes), so Russian command arguments would stop being folded and command lookup would miss. They now fold one whole character per step. The a_isspace()/quote tests stay byte-based on purpose: they only run at a character boundary and every delimiter is ASCII. GetCase (genchar.cpp) picked the declension from name[len - 1] / name[len - 2] and cut the stem with substr(0, len - 1) -- all byte offsets. Under UTF-8 the last-letter test never matched, so names simply stopped declining ("Anya" stayed "Anya" in every case); a differential run over 10 names x 2 genders x 6 cases shows 0/120 differences under KOI8-R and 65/120 corrections under UTF-8. - native_text: add copy_lower_char(), last_char_offset() and list_contains_char() (the strchr-over-a-letter-list replacement), plus a bounded char_bytes_at() so the view-based helpers are safe on a non-null-terminated string_view. - tests: cases for each new primitive, incl. in-place folding and partial-sequence rejection in list_contains_char. --- src/gameplay/core/genchar.cpp | 29 ++++++++----- src/utils/mud_string.cpp | 18 +++++--- src/utils/native_text.cpp | 80 +++++++++++++++++++++++++++++++++++ src/utils/native_text.h | 17 ++++++++ src/utils/utils_string.cpp | 12 ++++-- tests/native_text.cpp | 52 +++++++++++++++++++++++ 6 files changed, 188 insertions(+), 20 deletions(-) diff --git a/src/gameplay/core/genchar.cpp b/src/gameplay/core/genchar.cpp index fb1a6c327f..7edccc100b 100644 --- a/src/gameplay/core/genchar.cpp +++ b/src/gameplay/core/genchar.cpp @@ -20,6 +20,7 @@ #include "engine/core/comm.h" #include "utils/logger.h" #include "utils/utils.h" +#include "utils/native_text.h" #include "gameplay/magic/spells.h" #include "engine/entities/char_data.h" #include "engine/entities/char_player.h" @@ -249,10 +250,18 @@ void SetStartAbils(CharData *ch) { // 5 - предложный (о ком? о чем?) // result - результат void GetCase(std::string name, const EGender sex, int caseNum, char *data) { - size_t len = name.size(); std::string result = data; - if (strchr("цкнгшщзхфвпрлджчсмтб", name[len - 1]) != nullptr + // The declension is chosen by the last letter of the name (and sometimes the one before it). + // Those are *characters*, not bytes (issue #3681): under KOI8-R `stem`/`last`/`prev` are the + // same single bytes the old name[len - 1] / name[len - 2] / substr(0, len - 1) produced, + // under UTF-8 they are whole letters. + const size_t last_off = native_text::last_char_offset(name); + const std::string stem = name.substr(0, last_off); + const std::string last = name.substr(last_off); + const std::string prev = stem.substr(native_text::last_char_offset(stem)); + + if (native_text::list_contains_char("цкнгшщзхфвпрлджчсмтб", last) && sex == EGender::kMale) { result = name; if (caseNum == 1) @@ -265,8 +274,8 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { result += "ом"; // Иваном, Ретичем else if (caseNum == 5) result += "е"; // Иване - } else if (name[len - 1] == 'я') { - result = name.substr(0, len - 1); + } else if (last == "я") { + result = stem; if (caseNum == 1) result += "и"; // Ани, Вани else if (caseNum == 2) @@ -279,9 +288,9 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { result += "е"; // Ане, Ване else result += "я"; // Аня, Ваня - } else if (name[len - 1] == 'й' + } else if (last == "й" && sex == EGender::kMale) { - result = name.substr(0, len - 1); + result = stem; if (caseNum == 1) result += "я"; // Дрегвия else if (caseNum == 2) @@ -294,10 +303,10 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { result += "и"; // Дрегвии else result += "й"; // Дрегвий - } else if (name[len - 1] == 'а') { - result = name.substr(0, len - 1); + } else if (last == "а") { + result = stem; if (caseNum == 1) { - if (strchr("шщжч", name[len - 2]) != nullptr) + if (native_text::list_contains_char("шщжч", prev)) result += "и"; // Маши, Паши else result += "ы"; // Анны @@ -306,7 +315,7 @@ void GetCase(std::string name, const EGender sex, int caseNum, char *data) { else if (caseNum == 3) result += "у"; // Пашу, Анну else if (caseNum == 4) { - if (strchr("шщч", name[len - 2]) != nullptr) + if (native_text::list_contains_char("шщч", prev)) result += "ей"; // Машей, Пашей else result += "ой"; // Анной, Ханжой diff --git a/src/utils/mud_string.cpp b/src/utils/mud_string.cpp index 5a9fcd3d98..6bd64f25be 100644 --- a/src/utils/mud_string.cpp +++ b/src/utils/mud_string.cpp @@ -1,6 +1,7 @@ #include "mud_string.h" #include "utils.h" +#include "utils/native_text.h" int search_block(const char *target_string, const char **list, int exact); @@ -45,9 +46,13 @@ T one_argument_template(T argument, char *first_arg) { do { skip_spaces(&argument); first_arg = begin; + // Lowercase one whole character at a time (issue #3681). The a_isspace() test stays + // byte-based on purpose: it only ever runs at a character boundary, and every + // whitespace character is ASCII, so no multibyte lead byte can be mistaken for one. while (*argument && !a_isspace(*argument)) { - *(first_arg++) = a_lcc(*argument); - argument++; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; } *first_arg = '\0'; } while (fill_word(begin)); @@ -64,11 +69,12 @@ T any_one_arg_template(T argument, char *first_arg) { skip_spaces(&argument); int num = 0; + // As above: one character per step, `num` still counts bytes so it remains a buffer guard. while (*argument && !a_isspace(*argument) && num < kMaxStringLength - 1) { - *first_arg = a_lcc(*argument); - ++first_arg; - ++argument; - ++num; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; + num += static_cast(n); } *first_arg = '\0'; skip_spaces(&argument); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 4f93f002de..fd88acd11b 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -156,6 +156,26 @@ bool chars_equal_ci(const char *a, const char *b) { return utf8::to_lower(ca) == utf8::to_lower(cb); } +std::size_t copy_lower_char(const char *src, char *dst) { + const std::size_t len = char_bytes(src); + char32_t cp = 0; + if (utf8::decode(std::string_view(src, len), 0, cp) != 0) { + std::string folded; + // Only rewrite when the lowercase form keeps the byte length -- true for ASCII and for + // the whole Russian alphabet, so callers never see a character change size. + if (utf8::encode(utf8::to_lower(cp), folded) == len) { + for (std::size_t i = 0; i < len; ++i) { + dst[i] = folded[i]; + } + return len; + } + } + for (std::size_t i = 0; i < len; ++i) { + dst[i] = src[i]; + } + return len; +} + #else // KOI8-R: 1 byte == 1 character bool native_is_utf8() { @@ -228,8 +248,68 @@ bool chars_equal_ci(const char *a, const char *b) { return a_lcc_table[static_cast(*a)] == a_lcc_table[static_cast(*b)]; } +std::size_t copy_lower_char(const char *src, char *dst) { + *dst = a_lcc_table[static_cast(*src)]; + return 1; +} + #endif +// --------------------------------------------------------------------------------------------- +// Encoding-independent helpers: expressed purely in terms of the primitives above, so they need +// no per-encoding branch. Unlike char_bytes() these take a bounded view, not a C string, so they +// are safe on a string_view that is not null-terminated. +// --------------------------------------------------------------------------------------------- + +namespace { + +// Byte length of the character at `pos`, clamped to the end of `s`. +std::size_t char_bytes_at(std::string_view s, std::size_t pos) { +#ifdef INTERNAL_ENCODING_UTF8 + const unsigned char lead = static_cast(s[pos]); + if (lead < 0x80) { + return 1; + } + const std::size_t want = static_cast(utf8::sequence_length(lead)); + std::size_t n = 1; + while (n < want && pos + n < s.size() && (static_cast(s[pos + n]) & 0xC0) == 0x80) { + ++n; + } + return n; +#else + (void) s; + (void) pos; + return 1; +#endif +} + +} // namespace + +std::size_t last_char_offset(std::string_view s) { + std::size_t last = 0; + std::size_t pos = 0; + while (pos < s.size()) { + last = pos; + pos += char_bytes_at(s, pos); + } + return last; +} + +bool list_contains_char(std::string_view list, std::string_view ch) { + if (ch.empty()) { + return false; + } + std::size_t pos = 0; + while (pos < list.size()) { + const std::size_t len = char_bytes_at(list, pos); + if (len == ch.size() && list.compare(pos, len, ch) == 0) { + return true; + } + pos += len; + } + return false; +} + } // namespace native_text // vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 65f3a7b263..d6da4304f1 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -66,6 +66,23 @@ bool is_alnum_char(const char *s); // UTF-8: compares whole folded code points, so "P" matches "p" in Cyrillic too. bool chars_equal_ci(const char *a, const char *b); +// Copy the character starting at `src` to `dst`, lowercased, and return how many bytes were +// consumed (always the same number written, so a caller's buffer accounting is unaffected). +// KOI8-R: one byte through a_lcc_table. UTF-8: folds the code point; in the rare case where the +// lowercase form would not be the same byte length, the character is copied unchanged rather +// than resized. `dst` may alias `src` (the length-preserving property makes that safe). +std::size_t copy_lower_char(const char *src, char *dst); + +// Byte offset at which the final character of `s` begins (0 for an empty string), so that +// s.substr(0, last_char_offset(s)) drops exactly one character and s.substr(last_char_offset(s)) +// is that character. KOI8-R: s.size() - 1. +std::size_t last_char_offset(std::string_view s); + +// Does the single character `ch` occur in `list`? The replacement for strchr() over a literal +// list of letters: `list` is walked one whole character at a time, so a multibyte character can +// never match on a partial byte sequence. Comparison is exact (case-sensitive), like strchr. +bool list_contains_char(std::string_view list, std::string_view ch); + } // namespace native_text #endif // BYLINS_SRC_UTILS_NATIVE_TEXT_H_ diff --git a/src/utils/utils_string.cpp b/src/utils/utils_string.cpp index 2274b1aec0..10fd662883 100644 --- a/src/utils/utils_string.cpp +++ b/src/utils/utils_string.cpp @@ -1009,17 +1009,21 @@ const char *one_word(const char *argument, char *first_arg) { char *begin = first_arg; skip_spaces(&argument); first_arg = begin; + // Lowercase whole characters (issue #3681); the '"' and a_isspace() tests stay byte-based + // since they only run at a character boundary and both delimiters are ASCII. if (*argument == '\"') { argument++; while (*argument && *argument != '\"') { - *(first_arg++) = a_lcc(*argument); - argument++; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; } argument++; } else { while (*argument && !a_isspace(*argument)) { - *(first_arg++) = a_lcc(*argument); - argument++; + const size_t n = native_text::copy_lower_char(argument, first_arg); + first_arg += n; + argument += n; } } *first_arg = '\0'; diff --git a/tests/native_text.cpp b/tests/native_text.cpp index 2bf42ba56e..03d2250e06 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -188,4 +188,56 @@ TEST(NativeText, CharsEqualCi) { } } +TEST(NativeText, CopyLowerChar) { + char buf[8] = {0}; + EXPECT_EQ(native_text::copy_lower_char("A", buf), 1u); + EXPECT_STREQ(buf, "a"); + EXPECT_EQ(native_text::copy_lower_char("z", buf), 1u); + EXPECT_STREQ(buf, "z"); + EXPECT_EQ(native_text::copy_lower_char("7", buf), 1u); + EXPECT_STREQ(buf, "7"); + if (native_text::native_is_utf8()) { + std::memset(buf, 0, sizeof(buf)); + EXPECT_EQ(native_text::copy_lower_char("\xD0\x9F", buf), 2u); // P -> p + EXPECT_STREQ(buf, "\xD0\xBF"); + std::memset(buf, 0, sizeof(buf)); + EXPECT_EQ(native_text::copy_lower_char("\xD0\x81", buf), 2u); // Yo -> yo + EXPECT_STREQ(buf, "\xD1\x91"); + // In-place folding must be safe (the lowercase form keeps the byte length). + char inplace[] = "\xD0\x9F"; + EXPECT_EQ(native_text::copy_lower_char(inplace, inplace), 2u); + EXPECT_STREQ(inplace, "\xD0\xBF"); + } +} + +TEST(NativeText, LastCharOffset) { + EXPECT_EQ(native_text::last_char_offset(""), 0u); + EXPECT_EQ(native_text::last_char_offset("a"), 0u); + EXPECT_EQ(native_text::last_char_offset("abc"), 2u); + if (native_text::native_is_utf8()) { + EXPECT_EQ(native_text::last_char_offset(kPrivet), 10u); // 6 chars, last starts at byte 10 + const std::string_view s(kPrivet, 12); + EXPECT_EQ(s.substr(native_text::last_char_offset(s)), "\xD1\x82"); // final char "t" + EXPECT_EQ(native_text::last_char_offset("\xF0\x9F\x98\x80"), 0u); // single 4-byte char + } else { + EXPECT_EQ(native_text::last_char_offset(kPrivet), 11u); // 12 bytes -> last byte + } +} + +TEST(NativeText, ListContainsChar) { + EXPECT_TRUE(native_text::list_contains_char("abc", "b")); + EXPECT_FALSE(native_text::list_contains_char("abc", "d")); + EXPECT_FALSE(native_text::list_contains_char("abc", "")); + EXPECT_FALSE(native_text::list_contains_char("", "a")); + EXPECT_FALSE(native_text::list_contains_char("abc", "A")); // case-sensitive, like strchr + if (native_text::native_is_utf8()) { + // A multibyte character must match as a whole and never on a partial byte sequence. + const char *const list = "\xD1\x88\xD1\x89\xD0\xB6\xD1\x87"; // sh shch zh ch + EXPECT_TRUE(native_text::list_contains_char(list, "\xD1\x89")); + EXPECT_TRUE(native_text::list_contains_char(list, "\xD0\xB6")); + EXPECT_FALSE(native_text::list_contains_char(list, "\xD1\x82")); + EXPECT_FALSE(native_text::list_contains_char(list, "\xD1")); // lead byte alone + } +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : From dfbd79ea24f409dafda50a3c29e71e5b16355329 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 06:20:07 +0200 Subject: [PATCH 07/20] test(utf8): regression tests for the byte-vs-character text routines (#3681) Adds tests/text_semantics.cpp: behavioural coverage for the routines the migration touched -- isname, str_cmp/strn_cmp, one_argument/half_chop and GetCase. Several of these had no unit tests at all, so this also pays down existing debt rather than only guarding the new code. The Russian literals are written as literals, not byte escapes: the file compiles in whatever encoding the engine is built with, and the routines under test work in that same native encoding, so the expectations hold both before and after the flip. That makes this file the guard that flipping the encoding does not change behaviour. Covered regressions (each one silently broken by byte semantics under UTF-8): * isname matched unrelated Cyrillic words that shared a lead byte, and lost case-insensitive matching for Russian keywords; * argument splitting left Russian arguments unfolded, so command lookup missed; * GetCase stopped declining names entirely. Also renames the fixture in tests/native_text.cpp (kPrivet -> kNtPrivet): the test sources are unity-built, so it collided with the one in tests/utf8.cpp. --- tests/meson.build | 1 + tests/native_text.cpp | 29 ++++---- tests/text_semantics.cpp | 148 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 14 deletions(-) create mode 100644 tests/text_semantics.cpp diff --git a/tests/meson.build b/tests/meson.build index 2d1289db79..2493ba0373 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -41,6 +41,7 @@ test_sources = files( 'utils.encoding.cpp', 'utf8.cpp', 'native_text.cpp', + 'text_semantics.cpp', 'fight.penalties.cpp', 'bonus.command.parser.cpp', 'quested.cpp', diff --git a/tests/native_text.cpp b/tests/native_text.cpp index 03d2250e06..8684c4a175 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -13,8 +13,9 @@ namespace { -// "Privet": 6 Cyrillic code points, 12 UTF-8 bytes. -const char *const kPrivet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; +// "Privet": 6 Cyrillic code points, 12 UTF-8 bytes. (Name-prefixed: the test files are +// unity-built, so a plain kPrivet would clash with the one in tests/utf8.cpp.) +const char *const kNtPrivet = "\xD0\x9F\xD1\x80\xD0\xB8\xD0\xB2\xD0\xB5\xD1\x82"; } // namespace @@ -32,16 +33,16 @@ int legacy_lower(unsigned char c) { TEST(NativeText, CharCountAscii) { EXPECT_EQ(native_text::char_count("Hello"), 5u); - EXPECT_EQ(native_text::char_count(kPrivet, kPrivet + 4), native_text::native_is_utf8() ? 2u : 4u); + EXPECT_EQ(native_text::char_count(kNtPrivet, kNtPrivet + 4), native_text::native_is_utf8() ? 2u : 4u); } TEST(NativeText, CharCountReflectsEncoding) { if (native_text::native_is_utf8()) { - EXPECT_EQ(native_text::char_count(kPrivet), 6u); - EXPECT_EQ(native_text::char_count(kPrivet, kPrivet + 12), 6u); + EXPECT_EQ(native_text::char_count(kNtPrivet), 6u); + EXPECT_EQ(native_text::char_count(kNtPrivet, kNtPrivet + 12), 6u); } else { - EXPECT_EQ(native_text::char_count(kPrivet), 12u); - EXPECT_EQ(native_text::char_count(kPrivet, kPrivet + 12), 12u); + EXPECT_EQ(native_text::char_count(kNtPrivet), 12u); + EXPECT_EQ(native_text::char_count(kNtPrivet, kNtPrivet + 12), 12u); } } @@ -71,16 +72,16 @@ TEST(NativeText, CapitalizeCyrillicUtf8Only) { TEST(NativeText, CharBytes) { EXPECT_EQ(native_text::char_bytes("A"), 1u); if (native_text::native_is_utf8()) { - EXPECT_EQ(native_text::char_bytes(kPrivet), 2u); // Cyrillic lead -> 2 bytes + EXPECT_EQ(native_text::char_bytes(kNtPrivet), 2u); // Cyrillic lead -> 2 bytes EXPECT_EQ(native_text::char_bytes("\xF0\x9F\x98\x80"), 4u); // 4-byte code point EXPECT_EQ(native_text::char_bytes("\xD0"), 1u); // truncated lead: 1 byte present } else { - EXPECT_EQ(native_text::char_bytes(kPrivet), 1u); // KOI8-R: every byte is a char + EXPECT_EQ(native_text::char_bytes(kNtPrivet), 1u); // KOI8-R: every byte is a char } } TEST(NativeText, TruncateOffset) { - const std::string_view p(kPrivet, 12); + const std::string_view p(kNtPrivet, 12); EXPECT_EQ(native_text::truncate_offset(p, 100), 12u); // past end -> full size EXPECT_EQ(native_text::truncate_offset(p, 0), 0u); if (native_text::native_is_utf8()) { @@ -167,7 +168,7 @@ TEST(NativeText, IsAlnumChar) { if (native_text::native_is_utf8()) { // A Cyrillic letter is ONE alphanumeric character; its trail byte must not be read as // punctuation (which is what the raw byte table would do and what breaks tokenisation). - EXPECT_TRUE(native_text::is_alnum_char(kPrivet)); + EXPECT_TRUE(native_text::is_alnum_char(kNtPrivet)); EXPECT_TRUE(native_text::is_alnum_char("\xD0\x81")); // Yo EXPECT_TRUE(native_text::is_alnum_char("\xD1\x91")); // yo } @@ -215,12 +216,12 @@ TEST(NativeText, LastCharOffset) { EXPECT_EQ(native_text::last_char_offset("a"), 0u); EXPECT_EQ(native_text::last_char_offset("abc"), 2u); if (native_text::native_is_utf8()) { - EXPECT_EQ(native_text::last_char_offset(kPrivet), 10u); // 6 chars, last starts at byte 10 - const std::string_view s(kPrivet, 12); + EXPECT_EQ(native_text::last_char_offset(kNtPrivet), 10u); // 6 chars, last starts at byte 10 + const std::string_view s(kNtPrivet, 12); EXPECT_EQ(s.substr(native_text::last_char_offset(s)), "\xD1\x82"); // final char "t" EXPECT_EQ(native_text::last_char_offset("\xF0\x9F\x98\x80"), 0u); // single 4-byte char } else { - EXPECT_EQ(native_text::last_char_offset(kPrivet), 11u); // 12 bytes -> last byte + EXPECT_EQ(native_text::last_char_offset(kNtPrivet), 11u); // 12 bytes -> last byte } } diff --git a/tests/text_semantics.cpp b/tests/text_semantics.cpp new file mode 100644 index 0000000000..089776b1fc --- /dev/null +++ b/tests/text_semantics.cpp @@ -0,0 +1,148 @@ +// Regression tests for the byte-vs-character migration (issue #3681). +// +// These pin the *observable behaviour* of the text routines that used to assume 1 byte == 1 +// character: name matching, case-insensitive comparison, argument splitting and Russian name +// declension. Several of these paths had no coverage at all before the migration touched them. +// +// The Russian literals below are deliberately written as literals rather than byte escapes: the +// file is compiled in whatever encoding the engine is built with (KOI8-R today, UTF-8 after the +// flip), and the routines under test operate in that same native encoding. The expectations are +// therefore valid in both, and this file doubles as the guard that the flip did not change +// user-visible behaviour. + +#include "utils/utils_string.h" +#include "utils/mud_string.h" +#include "gameplay/core/genchar.h" +#include "utils/grammar/gender.h" +#include "engine/structs/structs.h" + +#include + +#include + +namespace { + +std::string declension(const char *name, EGender sex, int case_num) { + char buf[128] = {0}; + GetCase(name, sex, case_num, buf); + return std::string(buf); +} + +std::string first_argument(const char *line) { + char buf[kMaxInputLength] = {0}; + one_argument(line, buf); + return std::string(buf); +} + +} // namespace + +// ---------------------------------------------------------------------------- isname + +TEST(TextSemantics, IsnameMatchesAsciiKeywords) { + EXPECT_TRUE(isname("sword", "a long sword")); + EXPECT_TRUE(isname("long", "a long sword")); + EXPECT_TRUE(isname("SWORD", "a long sword")); // case-insensitive + EXPECT_TRUE(isname("swo", "a long sword")); // prefix + EXPECT_FALSE(isname("xyzzy", "a long sword")); +} + +TEST(TextSemantics, IsnameMatchesRussianKeywords) { + EXPECT_TRUE(isname("меч", "меч длинный")); + EXPECT_TRUE(isname("длинный", "меч длинный")); + EXPECT_TRUE(isname("ме", "меч длинный")); // prefix +} + +TEST(TextSemantics, IsnameIsCaseInsensitiveForRussian) { + // Regression: with byte-wise folding under UTF-8 the lead bytes of an upper/lower Cyrillic + // letter fold equal but the trail bytes do not, so this match was silently lost. + EXPECT_TRUE(isname("МЕЧ", "меч")); + EXPECT_TRUE(isname("Меч", "меч длинный")); + EXPECT_TRUE(isname("меч", "МЕЧ ДЛИННЫЙ")); +} + +TEST(TextSemantics, IsnameRejectsUnrelatedRussianWords) { + // Regression: byte-wise matching under UTF-8 compared only the shared leading byte of two + // different Cyrillic letters, so unrelated words matched each other. + EXPECT_FALSE(isname("щит", "меч длинный")); + EXPECT_FALSE(isname("меч", "щит деревянный")); + EXPECT_FALSE(isname("кольцо", "меч")); +} + +// ---------------------------------------------------------------------------- str_cmp + +TEST(TextSemantics, StrCmpIgnoresCase) { + EXPECT_EQ(str_cmp("abc", "ABC"), 0); + EXPECT_EQ(str_cmp("меч", "МЕЧ"), 0); + EXPECT_EQ(str_cmp(std::string("меч"), "МЕЧ"), 0); + EXPECT_NE(str_cmp("меч", "щит"), 0); +} + +TEST(TextSemantics, StrCmpOrdersConsistently) { + EXPECT_LT(str_cmp("abc", "abd"), 0); + EXPECT_GT(str_cmp("abd", "abc"), 0); + EXPECT_LT(str_cmp("ab", "abc"), 0); // prefix sorts first + EXPECT_GT(str_cmp("abc", "ab"), 0); +} + +TEST(TextSemantics, StrnCmpComparesPrefixOnly) { + EXPECT_EQ(strn_cmp("abcdef", "abcXXX", 3), 0); + EXPECT_NE(strn_cmp("abcdef", "abXXXX", 3), 0); + EXPECT_EQ(strn_cmp("МЕЧ", "меч", 6), 0); +} + +// ---------------------------------------------------------------------------- argument splitting + +TEST(TextSemantics, OneArgumentLowercasesAscii) { + EXPECT_EQ(first_argument("LOOK north"), "look"); + EXPECT_EQ(first_argument(" Kill orc"), "kill"); +} + +TEST(TextSemantics, OneArgumentLowercasesRussian) { + // Regression: the byte table leaves UTF-8 Cyrillic untouched, so a Russian command argument + // would reach the command lookup unfolded and fail to match. + EXPECT_EQ(first_argument("СМОТРЕТЬ север"), "смотреть"); + EXPECT_EQ(first_argument("Убить орка"), "убить"); + EXPECT_EQ(first_argument("меч"), "меч"); +} + +TEST(TextSemantics, HalfChopSplitsAndLowercasesFirstWord) { + char arg1[kMaxInputLength] = {0}; + char arg2[kMaxInputLength] = {0}; + half_chop("СКАЗАТЬ привет всем", arg1, arg2); + EXPECT_STREQ(arg1, "сказать"); + EXPECT_STREQ(arg2, "привет всем"); +} + +// ---------------------------------------------------------------------------- GetCase + +TEST(TextSemantics, DeclensionOfFeminineNameEndingInYa) { + // Regression: under UTF-8 the byte-wise last-letter test never matched, so names stopped + // declining entirely and every case returned the nominative. + EXPECT_EQ(declension("Аня", EGender::kFemale, 1), "Ани"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 2), "Ане"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 3), "Аню"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 4), "Аней"); + EXPECT_EQ(declension("Аня", EGender::kFemale, 5), "Ане"); +} + +TEST(TextSemantics, DeclensionOfMasculineNameEndingInConsonant) { + EXPECT_EQ(declension("Иван", EGender::kMale, 1), "Ивана"); + EXPECT_EQ(declension("Иван", EGender::kMale, 2), "Ивану"); + EXPECT_EQ(declension("Иван", EGender::kMale, 4), "Иваном"); + EXPECT_EQ(declension("Иван", EGender::kMale, 5), "Иване"); +} + +TEST(TextSemantics, DeclensionOfNameEndingInA) { + // The genitive/instrumental endings depend on the letter *before* the final one. + EXPECT_EQ(declension("Маша", EGender::kFemale, 1), "Маши"); // after ш -> и + EXPECT_EQ(declension("Анна", EGender::kFemale, 1), "Анны"); // otherwise -> ы + EXPECT_EQ(declension("Маша", EGender::kFemale, 4), "Машей"); // after ш -> ей + EXPECT_EQ(declension("Анна", EGender::kFemale, 4), "Анной"); // otherwise -> ой +} + +TEST(TextSemantics, DeclensionOfMasculineNameEndingInIShort) { + EXPECT_EQ(declension("Дрегвий", EGender::kMale, 1), "Дрегвия"); + EXPECT_EQ(declension("Дрегвий", EGender::kMale, 4), "Дрегвием"); +} + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : From f209945953ace500fdef39ec476489e82c170b97 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 07:51:34 +0200 Subject: [PATCH 08/20] feat(utf8): character-aware ctype scanning, closing track A2 (#3681) The byte ctype tables break under UTF-8 in one specific shape: a multibyte letter reads as "alnum lead byte + non-alnum trail bytes", so every loop that scans a *run* of letters stops in the middle of one. Audited all 82 a_is* call sites; only the alpha/alnum/upper family can misclassify (isspace/isdigit/isxdigit are ASCII-only and answer correctly for a trail byte), which narrowed the work to five real scanners: * fname() -- first keyword of a name list * im.cpp -- crafting alias extraction * dg_scripts.cpp -- script token boundaries * cut_one_word() -- word splitting * do_gen_comm.cpp -- the anti-caps filter, which could not see uppercase Cyrillic at all (a UTF-8 lead byte is outside the table's range), and whose percentage denominator counted bytes, not characters Deliberately left byte-based: IsValidShopId (ids are ASCII identifiers by design) and the single-character checks in login.cpp / interpreter.cpp, which run at a character boundary. pred_separator is dead code -- no users outside utils.h -- so it was not migrated; it should simply be deleted. Debt paid while here: fname() and the im.cpp alias loop copied into fixed buffers (30 and 16 bytes) with no bounds check at all -- multibyte text reaches the end twice as fast, so both now stop short of the buffer. dg_scripts passed a raw (negative) char to isspace(), which is undefined behaviour. - native_text: add is_alpha_char() and is_upper_char(). - tests: predicate coverage plus behavioural tests for fname() (incl. the overflow guard) and cut_one_word(), neither of which had any before. Verified: full suite 648 passed / 0 failed; live boot on the production world (world.20260802.tgz) with declension prompts still correct. --- src/engine/scripting/dg_scripts.cpp | 10 ++++-- src/engine/ui/cmd/do_gen_comm.cpp | 17 ++++++++--- src/gameplay/crafting/im.cpp | 13 +++++++- src/utils/native_text.cpp | 34 +++++++++++++++++++++ src/utils/native_text.h | 8 +++++ src/utils/utils_parse.cpp | 22 +++++++++++--- src/utils/utils_string.cpp | 8 +++-- tests/native_text.cpp | 27 +++++++++++++++++ tests/text_semantics.cpp | 47 +++++++++++++++++++++++++++++ 9 files changed, 171 insertions(+), 15 deletions(-) diff --git a/src/engine/scripting/dg_scripts.cpp b/src/engine/scripting/dg_scripts.cpp index 017c25a373..9b0037ae79 100644 --- a/src/engine/scripting/dg_scripts.cpp +++ b/src/engine/scripting/dg_scripts.cpp @@ -31,6 +31,7 @@ #include "gameplay/mechanics/illumination.h" #include "gameplay/mechanics/inventory.h" #include "utils/utils_parse.h" +#include "utils/native_text.h" #include "dg_event.h" #include "engine/ui/color.h" #include "gameplay/clans/house.h" @@ -4312,8 +4313,13 @@ int eval_lhs_op_rhs(const char *expr, char *result, size_t result_size, void *go p = matching_paren(p) + 1; else if (*p == '"') p = matching_quote(p) + 1; - else if (a_isalnum(*p)) - for (p++; *p && (a_isalnum(*p) || isspace(*p)); p++); + // Step over whole characters (issue #3681): a byte-wise scan ends a token in the middle + // of a multibyte letter. isspace() takes an unsigned value -- a raw char is negative for + // any non-ASCII byte, which is undefined behaviour. + else if (native_text::is_alnum_char(p)) + for (p += native_text::char_bytes(p); + *p && (native_text::is_alnum_char(p) || isspace(static_cast(*p))); + p += native_text::char_bytes(p)); else p++; } diff --git a/src/engine/ui/cmd/do_gen_comm.cpp b/src/engine/ui/cmd/do_gen_comm.cpp index 83c1138185..7af33bc904 100644 --- a/src/engine/ui/cmd/do_gen_comm.cpp +++ b/src/engine/ui/cmd/do_gen_comm.cpp @@ -8,6 +8,7 @@ #include "do_gen_comm.h" #include "administration/privilege.h" +#include "utils/native_text.h" #include "utils/grammar/gender.h" #include "gameplay/mechanics/sight.h" @@ -156,17 +157,25 @@ void do_gen_comm(CharData *ch, char *argument, int/* cmd*/, int subcmd) { int bad_simb_cnt = 0, bad_seq_cnt = 0; // фильтруем верхний регистр - for (int k = 0; argument[k] != '\0'; k++) { - if (a_isupper(argument[k])) { + // Counted and folded per character (issue #3681): a byte-wise scan cannot see an + // uppercase Cyrillic letter at all, so the filter silently stopped working for Russian. + // The denominator is the character count for the same reason -- with byte lengths the + // percentage would be halved for Russian text. + const size_t total_chars = native_text::char_count(argument); + for (int k = 0; argument[k] != '\0';) { + const int bytes = static_cast(native_text::char_bytes(argument + k)); + if (native_text::is_upper_char(argument + k)) { bad_simb_cnt++; bad_seq_cnt++; } else bad_seq_cnt = 0; if ((bad_seq_cnt > 1) && - (((bad_simb_cnt * 100 / strlen(argument)) > bad_smb_procent) || + (((bad_simb_cnt * 100 / total_chars) > bad_smb_procent) || (bad_seq_cnt > MAX_UPPERS_SEQ_CHAR))) - argument[k] = a_lcc(argument[k]); + native_text::copy_lower_char(argument + k, argument + k); + + k += bytes; } // фильтруем одинаковые сообщения в эфире if (!str_cmp(ch->get_last_tell().c_str(), argument)) { diff --git a/src/gameplay/crafting/im.cpp b/src/gameplay/crafting/im.cpp index 5d555bdcdc..81ede15a68 100644 --- a/src/gameplay/crafting/im.cpp +++ b/src/gameplay/crafting/im.cpp @@ -13,6 +13,7 @@ #include "im.h" #include "utils/parser_wrapper.h" #include "utils/utils_parse.h" +#include "utils/native_text.h" #include #include #include @@ -246,7 +247,17 @@ const char *replace_alias(const char *ptr, im_memb *sample, int rnum, const char if (*ptr == VAR_CHAR) { int k; ++ptr; - for (k = 0; (*ptr) && a_isalnum(*ptr); aname[k++] = *ptr++); + // One whole character per step (issue #3681), with a bounds check: the + // previous loop had none, and multibyte text fills aname[] twice as fast. + for (k = 0; *ptr && native_text::is_alnum_char(ptr);) { + const size_t bytes = native_text::char_bytes(ptr); + if (static_cast(k) + bytes >= sizeof(aname)) { + break; + } + for (size_t i = 0; i < bytes; ++i) { + aname[k++] = *ptr++; + } + } aname[k] = 0; al = get_im_alias(sample, aname); strcpy(dst, al ? al : aname); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index fd88acd11b..8c72edd930 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -17,6 +17,8 @@ through them changes nothing until the encoding flip. extern const char a_ucc_table[]; extern const char a_lcc_table[]; extern const bool a_isalnum_table[]; +extern const bool a_isalpha_table[]; +extern const bool a_isupper_table[]; #endif #include @@ -146,6 +148,30 @@ bool is_alnum_char(const char *s) { return (cp >= 0x0410 && cp <= 0x044F) || cp == 0x0401 || cp == 0x0451; } +bool is_alpha_char(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return (lead >= 'A' && lead <= 'Z') || (lead >= 'a' && lead <= 'z'); + } + char32_t cp = 0; + if (utf8::decode(std::string_view(s, char_bytes(s)), 0, cp) == 0) { + return false; + } + return (cp >= 0x0410 && cp <= 0x044F) || cp == 0x0401 || cp == 0x0451; +} + +bool is_upper_char(const char *s) { + const unsigned char lead = static_cast(*s); + if (lead < 0x80) { + return lead >= 'A' && lead <= 'Z'; + } + char32_t cp = 0; + if (utf8::decode(std::string_view(s, char_bytes(s)), 0, cp) == 0) { + return false; + } + return (cp >= 0x0410 && cp <= 0x042F) || cp == 0x0401; +} + bool chars_equal_ci(const char *a, const char *b) { char32_t ca = 0; char32_t cb = 0; @@ -244,6 +270,14 @@ bool is_alnum_char(const char *s) { return a_isalnum_table[static_cast(*s)]; } +bool is_alpha_char(const char *s) { + return a_isalpha_table[static_cast(*s)]; +} + +bool is_upper_char(const char *s) { + return a_isupper_table[static_cast(*s)]; +} + bool chars_equal_ci(const char *a, const char *b) { return a_lcc_table[static_cast(*a)] == a_lcc_table[static_cast(*b)]; } diff --git a/src/utils/native_text.h b/src/utils/native_text.h index d6da4304f1..d9ba7a3ced 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -62,6 +62,14 @@ int ncompare_ci(std::string_view a, std::string_view b, std::size_t n); // alphanumeric character rather than a lead byte followed by "punctuation" trail bytes. bool is_alnum_char(const char *s); +// Is the character starting at `s` a letter? Same contract as is_alnum_char, minus the digits. +bool is_alpha_char(const char *s); + +// Is the character starting at `s` an uppercase letter? KOI8-R: the a_isupper byte table. +// UTF-8: ASCII A-Z plus the uppercase Cyrillic range (and Yo) as whole code points -- the byte +// table cannot see these at all, since a UTF-8 Cyrillic lead byte is not in its uppercase range. +bool is_upper_char(const char *s); + // Do the characters starting at `a` and `b` match ignoring case? KOI8-R: LOWER(*a) == LOWER(*b). // UTF-8: compares whole folded code points, so "P" matches "p" in Cyrillic too. bool chars_equal_ci(const char *a, const char *b); diff --git a/src/utils/utils_parse.cpp b/src/utils/utils_parse.cpp index 0105e2bd1d..622ddd0fd6 100644 --- a/src/utils/utils_parse.cpp +++ b/src/utils/utils_parse.cpp @@ -4,7 +4,8 @@ #include "utils_parse.h" #include "third_party_libs/pugixml/pugixml.h" -#include "utils/parser_wrapper.h" // issue.xml-parse-cleaning: AttrInt/AttrStr over DataNode +#include "utils/parser_wrapper.h" +#include "utils/native_text.h" // issue.xml-parse-cleaning: AttrInt/AttrStr over DataNode #include "engine/db/obj_prototypes.h" #include "engine/db/db.h" @@ -393,10 +394,21 @@ int get_number(std::string &name) { // issue.handler-cleaning: first keyword of a name list (moved from handler). char *fname(const char *namelist) { static char holder[30]; - char *point; - - for (point = holder; a_isalpha(*namelist); namelist++, point++) - *point = *namelist; + char *point = holder; + + // Copy the leading word one whole character at a time (issue #3681): a byte-wise copy stops + // in the middle of a multibyte letter. The bounds check is new -- the previous loop could + // already run past holder[] on a long keyword, and multibyte text reaches the end twice as + // fast, so leave room for the terminator. + while (native_text::is_alpha_char(namelist)) { + const size_t bytes = native_text::char_bytes(namelist); + if (point + bytes >= holder + sizeof(holder)) { + break; + } + for (size_t i = 0; i < bytes; ++i) { + *point++ = *namelist++; + } + } *point = '\0'; diff --git a/src/utils/utils_string.cpp b/src/utils/utils_string.cpp index 10fd662883..7f9725f25d 100644 --- a/src/utils/utils_string.cpp +++ b/src/utils/utils_string.cpp @@ -878,13 +878,15 @@ void cut_one_word(std::string &str, std::string &word) { } bool process = false; unsigned begin = 0, end = 0; - for (unsigned i = 0; i < str.size(); ++i) { - if (!process && a_isalnum(str.at(i))) { + // Word boundaries are looked for one whole character at a time (issue #3681); a byte-wise + // scan finds a "boundary" inside a multibyte letter and cuts the word in half. + for (unsigned i = 0; i < str.size(); i += native_text::char_bytes(str.c_str() + i)) { + if (!process && native_text::is_alnum_char(str.c_str() + i)) { process = true; begin = i; continue; } - if (process && !a_isalnum(str.at(i))) { + if (process && !native_text::is_alnum_char(str.c_str() + i)) { end = i; break; } diff --git a/tests/native_text.cpp b/tests/native_text.cpp index 8684c4a175..17bc01c338 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -174,6 +174,33 @@ TEST(NativeText, IsAlnumChar) { } } +TEST(NativeText, IsAlphaChar) { + EXPECT_TRUE(native_text::is_alpha_char("a")); + EXPECT_TRUE(native_text::is_alpha_char("Z")); + EXPECT_FALSE(native_text::is_alpha_char("7")); // digit: alnum but not alpha + EXPECT_FALSE(native_text::is_alpha_char(" ")); + EXPECT_FALSE(native_text::is_alpha_char("")); + if (native_text::native_is_utf8()) { + EXPECT_TRUE(native_text::is_alpha_char(kNtPrivet)); + EXPECT_TRUE(native_text::is_alpha_char("\xD0\x81")); // Yo + } +} + +TEST(NativeText, IsUpperChar) { + EXPECT_TRUE(native_text::is_upper_char("A")); + EXPECT_FALSE(native_text::is_upper_char("a")); + EXPECT_FALSE(native_text::is_upper_char("7")); + EXPECT_FALSE(native_text::is_upper_char("")); + if (native_text::native_is_utf8()) { + // The byte table cannot see these: a UTF-8 Cyrillic lead byte is outside its uppercase + // range, which is why the anti-caps filter stopped working for Russian. + EXPECT_TRUE(native_text::is_upper_char("\xD0\x9F")); // P + EXPECT_FALSE(native_text::is_upper_char("\xD0\xBF")); // p + EXPECT_TRUE(native_text::is_upper_char("\xD0\x81")); // Yo + EXPECT_FALSE(native_text::is_upper_char("\xD1\x91")); // yo + } +} + TEST(NativeText, CharsEqualCi) { EXPECT_TRUE(native_text::chars_equal_ci("a", "a")); EXPECT_TRUE(native_text::chars_equal_ci("a", "A")); diff --git a/tests/text_semantics.cpp b/tests/text_semantics.cpp index 089776b1fc..35aa6dd78f 100644 --- a/tests/text_semantics.cpp +++ b/tests/text_semantics.cpp @@ -11,6 +11,7 @@ // user-visible behaviour. #include "utils/utils_string.h" +#include "utils/utils_parse.h" #include "utils/mud_string.h" #include "gameplay/core/genchar.h" #include "utils/grammar/gender.h" @@ -145,4 +146,50 @@ TEST(TextSemantics, DeclensionOfMasculineNameEndingInIShort) { EXPECT_EQ(declension("Дрегвий", EGender::kMale, 4), "Дрегвием"); } +// ---------------------------------------------------------------------------- fname + +TEST(TextSemantics, FnameExtractsFirstKeyword) { + EXPECT_STREQ(fname("sword long blade"), "sword"); + EXPECT_STREQ(fname("меч длинный"), "меч"); + EXPECT_STREQ(fname("кольцо"), "кольцо"); + EXPECT_STREQ(fname(""), ""); + EXPECT_STREQ(fname(" leading space"), ""); // stops at the very first non-letter +} + +TEST(TextSemantics, FnameStaysInsideItsBuffer) { + // fname() returns a fixed 30-byte buffer and used to copy without any bounds check; a long + // keyword (twice as many bytes per letter once the text is multibyte) ran past its end. + const char *const very_long = "оченьдлинноеключевоесловокотороенепомещается прочее"; + const char *const got = fname(very_long); + EXPECT_LT(std::strlen(got), 30u); + // Whatever was copied must be a prefix of the input, never mangled bytes. + EXPECT_EQ(std::string(very_long).compare(0, std::strlen(got), got), 0); +} + +// ---------------------------------------------------------------------------- cut_one_word + +TEST(TextSemantics, CutOneWordSplitsOnWordBoundaries) { + std::string rest = "меч длинный острый"; + std::string word; + cut_one_word(rest, word); + EXPECT_EQ(word, "меч"); + cut_one_word(rest, word); + EXPECT_EQ(word, "длинный"); + cut_one_word(rest, word); + EXPECT_EQ(word, "острый"); +} + +TEST(TextSemantics, CutOneWordHandlesAsciiAndEmpty) { + std::string rest = "take all"; + std::string word; + cut_one_word(rest, word); + EXPECT_EQ(word, "take"); + cut_one_word(rest, word); + EXPECT_EQ(word, "all"); + + std::string empty; + cut_one_word(empty, word); + EXPECT_TRUE(word.empty()); +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : From d602bbe98d1c2b666fd3eebfe2dd186d1fd117c3 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 08:00:48 +0200 Subject: [PATCH 09/20] feat(utf8): character-width column padding, first A3 subsystems (#3681) Track A3: a printf "%-Ns" field pads by bytes, so a column of Russian text comes out half as wide once the text is multibyte. Adds pad_right()/pad_left() (width in characters -- byte-identical under KOI8-R) and converts the most visible player-facing columns: WHO short list, WHERE, skills and affects. Also makes the libfort wrapper pick its base class by encoding: char_table measures a cell in bytes, utf8_table in code points. Neither is right for both -- under KOI8-R the byte table is correct and utf8_table would misread KOI8-R as UTF-8 -- so the choice now follows INTERNAL_ENCODING_UTF8 instead of being hardcoded. Scope finding: fmt::format's "{:<20}" needs NO migration. Measured against the vendored fmt -- it counts code points for valid UTF-8 and falls back to one-unit-per-byte for KOI8-R (which is invalid UTF-8), so it is already correct in both encodings. That takes every fmt-based column, including where_format, out of this track; only printf-style fields remain. Deliberately unchanged: colour codes embedded in a padded string still count toward the width, exactly as with "%-Ns" today. That skew is pre-existing and orthogonal. --- src/engine/ui/cmd/do_affects.cpp | 8 ++++++-- src/engine/ui/cmd/do_skills.cpp | 6 ++++-- src/engine/ui/cmd/do_where.cpp | 8 ++++++-- src/engine/ui/cmd/do_who.cpp | 13 +++++++++---- src/engine/ui/table_wrapper.h | 16 ++++++++++++++-- src/utils/native_text.cpp | 21 +++++++++++++++++++++ src/utils/native_text.h | 8 ++++++++ tests/native_text.cpp | 17 +++++++++++++++++ 8 files changed, 85 insertions(+), 12 deletions(-) diff --git a/src/engine/ui/cmd/do_affects.cpp b/src/engine/ui/cmd/do_affects.cpp index db94732efa..43b5b568b5 100644 --- a/src/engine/ui/cmd/do_affects.cpp +++ b/src/engine/ui/cmd/do_affects.cpp @@ -3,6 +3,7 @@ // #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "gameplay/affects/affect_messages.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -65,9 +66,12 @@ void do_affects(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { (mod + 1) / kSecsPerMudHour + 1, grammar::GetDeclensionInNumber((mod + 1) / kSecsPerMudHour + 1, grammar::EWhat::kHour)) : sprintf(buf2, "(менее часа)"); - snprintf(buf, kMaxStringLength, "%s%s%-21s %-12s%s ", + // Ширина колонок - в символах, а не в байтах (issue #3681). + snprintf(buf, kMaxStringLength, "%s%s%s %s%s ", *sp_name == '!' ? "Состояние : " : "Заклинание : ", - kColorBoldCyn, sp_name, buf2, kColorNrm); + kColorBoldCyn, + native_text::pad_right(sp_name, 21).c_str(), + native_text::pad_right(buf2, 12).c_str(), kColorNrm); *buf2 = '\0'; if (!privilege::IsImmortal(ch)) { auto next_affect_i = affect_i; diff --git a/src/engine/ui/cmd/do_skills.cpp b/src/engine/ui/cmd/do_skills.cpp index a3ff7c1d74..4197a913df 100644 --- a/src/engine/ui/cmd/do_skills.cpp +++ b/src/engine/ui/cmd/do_skills.cpp @@ -1,4 +1,5 @@ #include "do_skills.h" +#include "utils/native_text.h" #include "engine/ui/color.h" #include "engine/entities/char_data.h" @@ -77,8 +78,9 @@ void DisplaySkills(CharData *ch, CharData *vict, const char *filter/* = nullptr* default: sprintf(buf, " "); } - sprintf(buf + strlen(buf), "%-23s %s (%d)%s \r\n", - skill.GetName(), + // Ширина колонки - в символах, а не в байтах (issue #3681). + sprintf(buf + strlen(buf), "%s %s (%d)%s \r\n", + native_text::pad_right(skill.GetName(), 23).c_str(), how_good(GetSkill(ch, skill_id), CalcSkillHardCap(ch, skill_id)), GetTrainedSkill(ch, skill_id) == 0 ? GetEquippedSkill(ch, skill_id) : std::min(CalcSkillMinCap(ch, skill_id) + GetEquippedSkill(ch, skill_id), MUD::Skill(skill_id).cap), diff --git a/src/engine/ui/cmd/do_where.cpp b/src/engine/ui/cmd/do_where.cpp index 73ad5cb5e5..c408fcb781 100644 --- a/src/engine/ui/cmd/do_where.cpp +++ b/src/engine/ui/cmd/do_where.cpp @@ -3,6 +3,7 @@ // #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/world_objects.h" #include "gameplay/economics/exchange.h" @@ -145,7 +146,9 @@ void PerformMortalWhere(CharData *ch, char *arg) { continue; } - sprintf(buf, "%-20s - %s\r\n", GET_NAME(i), world[i->in_room]->name); + // Ширина колонки - в символах, а не в байтах (issue #3681). + sprintf(buf, "%s - %s\r\n", + native_text::pad_right(GET_NAME(i), 20).c_str(), world[i->in_room]->name); SendMsgToChar(buf, ch); } } else // print only FIRST char, not all. @@ -165,7 +168,8 @@ void PerformMortalWhere(CharData *ch, char *arg) { continue; } - sprintf(buf, "%-25s - %s\r\n", GET_NAME(i), world[i->in_room]->name); + sprintf(buf, "%s - %s\r\n", + native_text::pad_right(GET_NAME(i), 25).c_str(), world[i->in_room]->name); SendMsgToChar(buf, ch); return; } diff --git a/src/engine/ui/cmd/do_who.cpp b/src/engine/ui/cmd/do_who.cpp index 6dfbbb12fb..d408246376 100644 --- a/src/engine/ui/cmd/do_who.cpp +++ b/src/engine/ui/cmd/do_who.cpp @@ -3,6 +3,7 @@ // #include "engine/ui/cmd/do_who.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "utils/grammar/gender.h" #include "gameplay/mechanics/sight.h" @@ -164,15 +165,19 @@ void DoWho(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (short_list) { char tmp[kMaxInputLength]; snprintf(tmp, sizeof(tmp), "%s%s%s", GetPkNameColor(tch), GET_NAME(tch), kColorNrm); + // Ширина колонки - в символах, а не в байтах (issue #3681): "%-30s" отсчитывает + // байты, из-за чего колонка с русским именем под UTF-8 выходит вдвое уже. if (privilege::IsImpl(ch) || ch->IsFlagged(EPrf::kCoderinfo)) { - sprintf(buf, "%s[%2d %s] %-30s%s", + sprintf(buf, "%s[%2d %s] %s%s", privilege::IsGod(tch.get()) ? kColorWht : "", GetRealLevel(tch), MUD::Class(tch->GetClass()).GetCName(), - tmp, privilege::IsGod(tch.get()) ? kColorNrm : ""); + native_text::pad_right(tmp, 30).c_str(), + privilege::IsGod(tch.get()) ? kColorNrm : ""); } else { - sprintf(buf, "%s%-30s%s", + sprintf(buf, "%s%s%s", privilege::IsImmortal(tch.get()) ? kColorWht : "", - tmp, privilege::IsImmortal(tch.get()) ? kColorNrm : ""); + native_text::pad_right(tmp, 30).c_str(), + privilege::IsImmortal(tch.get()) ? kColorNrm : ""); } } else { if (privilege::IsImpl(ch) diff --git a/src/engine/ui/table_wrapper.h b/src/engine/ui/table_wrapper.h index 94d684c471..e3a36238d5 100644 --- a/src/engine/ui/table_wrapper.h +++ b/src/engine/ui/table_wrapper.h @@ -41,9 +41,21 @@ using Color = fort::color; using TextAlign = fort::text_align; /** - * Таблица в стандартной, не unicode кодировке. + * Базовый тип таблицы выбирается под нативную кодировку движка (issue #3681). + * libfort считает ширину ячейки по-разному: char_table меряет байтами, utf8_table - кодовыми + * точками. Под KOI8-R (1 байт = 1 символ) верен первый, под UTF-8 - второй; при неверном выборе + * колонки с русским текстом съезжают вдвое. */ -class Table : public fort::char_table { +#ifdef INTERNAL_ENCODING_UTF8 +using TableBase = fort::utf8_table; +#else +using TableBase = fort::char_table; +#endif + +/** + * Таблица в нативной кодировке движка. + */ +class Table : public TableBase { public: void SetColumnAlign(std::size_t column_index, TextAlign align) { this->column(column_index).set_cell_text_align(align); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 8c72edd930..8ee147504b 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -329,6 +329,27 @@ std::size_t last_char_offset(std::string_view s) { return last; } +namespace { + +std::size_t missing_width(std::string_view s, std::size_t width) { + const std::size_t have = char_count(s); + return have >= width ? 0 : width - have; +} + +} // namespace + +std::string pad_right(std::string_view s, std::size_t width, char fill) { + std::string out(s); + out.append(missing_width(s, width), fill); + return out; +} + +std::string pad_left(std::string_view s, std::size_t width, char fill) { + std::string out(missing_width(s, width), fill); + out.append(s); + return out; +} + bool list_contains_char(std::string_view list, std::string_view ch) { if (ch.empty()) { return false; diff --git a/src/utils/native_text.h b/src/utils/native_text.h index d9ba7a3ced..47deedb98e 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -86,6 +86,14 @@ std::size_t copy_lower_char(const char *src, char *dst); // is that character. KOI8-R: s.size() - 1. std::size_t last_char_offset(std::string_view s); +// Pad `s` to at least `width` display characters, appending (pad_right) or prepending (pad_left) +// `fill`. The replacement for a printf "%-Ns" / "%Ns" field, which counts bytes: under KOI8-R the +// result is identical, under UTF-8 a Russian column no longer comes out half as wide. +// Note: like printf, colour codes embedded in `s` still count toward the width -- that is a +// separate, pre-existing skew and is deliberately not changed here. +std::string pad_right(std::string_view s, std::size_t width, char fill = ' '); +std::string pad_left(std::string_view s, std::size_t width, char fill = ' '); + // Does the single character `ch` occur in `list`? The replacement for strchr() over a literal // list of letters: `list` is walked one whole character at a time, so a multibyte character can // never match on a partial byte sequence. Comparison is exact (case-sensitive), like strchr. diff --git a/tests/native_text.cpp b/tests/native_text.cpp index 17bc01c338..b20f2de832 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -9,6 +9,7 @@ #include #include +#include #include namespace { @@ -252,6 +253,22 @@ TEST(NativeText, LastCharOffset) { } } +TEST(NativeText, PadRightAndLeft) { + EXPECT_EQ(native_text::pad_right("ab", 5), "ab "); + EXPECT_EQ(native_text::pad_left("ab", 5), " ab"); + EXPECT_EQ(native_text::pad_right("ab", 2), "ab"); // already wide enough + EXPECT_EQ(native_text::pad_right("abcdef", 3), "abcdef"); // never truncates + EXPECT_EQ(native_text::pad_right("", 3), " "); + EXPECT_EQ(native_text::pad_right("ab", 4, '.'), "ab.."); + if (native_text::native_is_utf8()) { + // 6 Cyrillic characters (12 bytes) padded to a 10-character column: no padding, and + // crucially not 12 bytes' worth of "already too wide" either. + EXPECT_EQ(native_text::pad_right(kNtPrivet, 10), kNtPrivet); + // 6 characters padded to 8 -> exactly two spaces, not eight. + EXPECT_EQ(native_text::pad_right(kNtPrivet, 8), std::string(kNtPrivet) + " "); + } +} + TEST(NativeText, ListContainsChar) { EXPECT_TRUE(native_text::list_contains_char("abc", "b")); EXPECT_FALSE(native_text::list_contains_char("abc", "d")); From 8a479f55d6783fa4a70abfa90d36cbd41c8823e4 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 08:13:07 +0200 Subject: [PATCH 10/20] feat(utf8): character-width columns across the remaining game screens (#3681) Continues A3 with the player-facing subsystems: score (the grouping line, which also capped its text with substr(0, 76) -- a byte cut), spellbook, features, exits, PK list, crafting recipes and item creation, glory stats, parcels and named items. Adds native_text::truncate_to_chars() for the "cap a display string" case, so a cap never lands inside a character. Skipped after checking: do_levels pads only digit strings (thousands_sep), so its "%13s" fields are ASCII-only and need nothing. Verified: full suite 649 passed / 0 failed, clean build, and a live boot on the production world with both a UTF-8 and a legacy KOI8-R client rendering correctly. --- src/engine/ui/cmd/do_exits.cpp | 5 +++-- src/engine/ui/cmd/do_features.cpp | 9 +++++---- src/engine/ui/cmd/do_score.cpp | 7 ++++--- src/engine/ui/cmd/do_spells.cpp | 11 +++++++---- src/gameplay/communication/parcel.cpp | 5 +++-- src/gameplay/crafting/im.cpp | 10 +++++----- src/gameplay/crafting/item_creation.cpp | 8 +++++--- src/gameplay/fight/pk.cpp | 17 +++++++++-------- src/gameplay/mechanics/glory_const.cpp | 4 +++- src/gameplay/mechanics/named_stuff.cpp | 11 ++++++----- src/utils/native_text.cpp | 10 ++++++++++ src/utils/native_text.h | 5 +++++ 12 files changed, 65 insertions(+), 37 deletions(-) diff --git a/src/engine/ui/cmd/do_exits.cpp b/src/engine/ui/cmd/do_exits.cpp index e5549e9edf..219b0706b0 100644 --- a/src/engine/ui/cmd/do_exits.cpp +++ b/src/engine/ui/cmd/do_exits.cpp @@ -6,6 +6,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "gameplay/mechanics/sight.h" #include "gameplay/mechanics/illumination.h" @@ -29,10 +30,10 @@ void DoExits(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/) { for (door = 0; door < EDirection::kMaxDirNum; door++) if (EXIT(ch, door) && EXIT(ch, door)->to_room() != kNowhere && !EXIT_FLAGGED(EXIT(ch, door), EExitFlag::kClosed)) { if (privilege::IsGod(ch)) - sprintf(buf2, "%-6s - [%5d] %s\r\n", dirs_rus[door], + sprintf(buf2, "%s - [%5d] %s\r\n", native_text::pad_right(dirs_rus[door], 6).c_str(), GET_ROOM_VNUM(EXIT(ch, door)->to_room()), world[EXIT(ch, door)->to_room()]->name); else { - sprintf(buf2, "%-6s - ", dirs_rus[door]); + sprintf(buf2, "%s - ", native_text::pad_right(dirs_rus[door], 6).c_str()); if (is_dark(EXIT(ch, door)->to_room()) && !sight::CanSeeInDark(ch)) strcat(buf2, "слишком темно\r\n"); else { diff --git a/src/engine/ui/cmd/do_features.cpp b/src/engine/ui/cmd/do_features.cpp index 406081057e..a48d26e9d5 100644 --- a/src/engine/ui/cmd/do_features.cpp +++ b/src/engine/ui/cmd/do_features.cpp @@ -1,4 +1,5 @@ #include "engine/ui/color.h" +#include "utils/native_text.h" #include "gameplay/core/remort.h" #include "engine/entities/char_data.h" #include "gameplay/abilities/timed_abilities.h" @@ -81,17 +82,17 @@ void DisplayFeats(CharData *ch, CharData *vict, bool all_feats) { continue; } if (!ch->IsFlagged(EPrf::kBlindMode)) { - sprintf(buf, " %s%s %-30s%s\r\n", + sprintf(buf, " %s%s %s%s\r\n", ch->HaveFeat(feat.GetId()) ? kColorGrn : CanGetFeat(ch, feat.GetId()) ? kColorNrm : kColorRed, ch->HaveFeat(feat.GetId()) ? "[И]" : CanGetFeat(ch, feat.GetId()) ? "[Д]" : "[Н]", - MUD::Feat(feat.GetId()).GetCName(), kColorNrm); + native_text::pad_right(MUD::Feat(feat.GetId()).GetCName(), 30).c_str(), kColorNrm); } else { - sprintf(buf, " %s %-30s\r\n", + sprintf(buf, " %s %s\r\n", ch->HaveFeat(feat.GetId()) ? "[И]" : CanGetFeat(ch, feat.GetId()) ? "[Д]" : "[Н]", - MUD::Feat(feat.GetId()).GetCName()); + native_text::pad_right(MUD::Feat(feat.GetId()).GetCName(), 30).c_str()); } if (feat.IsInborn() || diff --git a/src/engine/ui/cmd/do_score.cpp b/src/engine/ui/cmd/do_score.cpp index 53e91692ce..12e5aedc04 100644 --- a/src/engine/ui/cmd/do_score.cpp +++ b/src/engine/ui/cmd/do_score.cpp @@ -7,6 +7,7 @@ */ #include "engine/ui/color.h" +#include "utils/native_text.h" #include "gameplay/affects/affect_messages.h" #include "utils/utils_string.h" #include "gameplay/core/experience.h" @@ -232,12 +233,12 @@ void PrintScoreList(CharData *ch) { } else if (NAME_BAD(ch)) { SendMsgToChar(ch, "ВНИМАНИЕ! ваше имя запрещено богами. Очень скоро вы прекратите получать опыт.\r\n"); } - SendMsgToChar(ch, "Вы можете вступить в группу с максимальной разницей в %2d %-75s\r\n", + SendMsgToChar(ch, "Вы можете вступить в группу с максимальной разницей в %2d %s\r\n", grouping[ch->GetClass()][static_cast(remort::GetRealRemort(ch))], - (std::string( + native_text::pad_right(native_text::truncate_to_chars(std::string( grammar::GetDeclensionInNumber(grouping[ch->GetClass()][static_cast(remort::GetRealRemort( ch))], grammar::EWhat::kLvl) - + std::string(" без потерь для опыта.")).substr(0, 76).c_str())); + + std::string(" без потерь для опыта.")), 76), 75).c_str()); SendMsgToChar(ch, "Вы можете принять в группу максимум %d соратников.\r\n", group::max_group_size(ch)); std::ostringstream out; diff --git a/src/engine/ui/cmd/do_spells.cpp b/src/engine/ui/cmd/do_spells.cpp index 690bfbb6a2..f8eb53df59 100644 --- a/src/engine/ui/cmd/do_spells.cpp +++ b/src/engine/ui/cmd/do_spells.cpp @@ -1,4 +1,5 @@ #include "do_spells.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "gameplay/mechanics/magic_item.h" @@ -88,15 +89,17 @@ void DisplaySpells(CharData *ch, CharData *vict, bool all) { continue; if (CheckRecipeItems(ch, spell_id, ESpellType::kRunes, false)) { slots[slot_num] += sprintf(names[slot_num] + slots[slot_num], - "%s|<...%4d.> %s%-38s&n|", + "%s|<...%4d.> %s%s&n|", slots[slot_num] % 114 < 10 ? "\r\n" : " ", - CalcSpellManacost(ch, spell_id), GetSpellColor(spell_id), MUD::Spell(spell_id).GetCName()); + CalcSpellManacost(ch, spell_id), GetSpellColor(spell_id), + native_text::pad_right(MUD::Spell(spell_id).GetCName(), 38).c_str()); } else { if (all) { slots[slot_num] += sprintf(names[slot_num] + slots[slot_num], - "%s|+--------+ %s%-38s&n|", slots[slot_num] % 114 < 10 ? "\r\n" - : " ", GetSpellColor(spell_id), MUD::Spell(spell_id).GetCName()); + "%s|+--------+ %s%s&n|", slots[slot_num] % 114 < 10 ? "\r\n" + : " ", GetSpellColor(spell_id), + native_text::pad_right(MUD::Spell(spell_id).GetCName(), 38).c_str()); } } } else { diff --git a/src/gameplay/communication/parcel.cpp b/src/gameplay/communication/parcel.cpp index f1463863b2..aa63340e50 100644 --- a/src/gameplay/communication/parcel.cpp +++ b/src/gameplay/communication/parcel.cpp @@ -3,6 +3,7 @@ // Part of Bylins http://www.mud.ru #include "parcel.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "gameplay/economics/currencies.h" @@ -828,10 +829,10 @@ bool print_imm_where_obj(CharData *ch, const ObjData *arg, int num) { std::string sender = GetNameByUnique(it2->first); found = true; - SendMsgToChar(ch, "%2d. [%6d] %-25s - наход%sся на почте (отправитель: %s, получатель: %s).\r\n", + SendMsgToChar(ch, "%2d. [%6d] %s - наход%sся на почте (отправитель: %s, получатель: %s).\r\n", num++, GET_OBJ_VNUM(it3->obj_.get()), - it3->obj_->get_short_description().c_str(), + native_text::pad_right(it3->obj_->get_short_description(), 25).c_str(), grammar::ObjPluralVerbEnding((it3->obj_)->get_sex()), sender.c_str(), target.c_str()); diff --git a/src/gameplay/crafting/im.cpp b/src/gameplay/crafting/im.cpp index 81ede15a68..ead6fddec8 100644 --- a/src/gameplay/crafting/im.cpp +++ b/src/gameplay/crafting/im.cpp @@ -953,13 +953,13 @@ void list_recipes(CharData *ch, bool all_recipes) { rs = im_get_char_rskill(ch, sortpos); const bool unavailable = req->level > GetRealLevel(ch) || req->remort > remort::GetRealRemort(ch); if (!ch->IsFlagged(EPrf::kBlindMode)) { - sprintf(buf, " %s%-30s%s %2d (%2d)%s\r\n", + sprintf(buf, " %s%s%s %2d (%2d)%s\r\n", unavailable ? kColorRed : rs ? kColorGrn : kColorNrm, - imrecipes[sortpos].name, kColorCyn, + native_text::pad_right(imrecipes[sortpos].name, 30).c_str(), kColorCyn, req->level, req->remort, kColorNrm); } else { - sprintf(buf, " %s %-30s %2d (%2d)\r\n", - unavailable ? "[Н]" : rs ? "[И]" : "[Д]", imrecipes[sortpos].name, + sprintf(buf, " %s %s %2d (%2d)\r\n", + unavailable ? "[Н]" : rs ? "[И]" : "[Д]", native_text::pad_right(imrecipes[sortpos].name, 30).c_str(), req->level, req->remort); } strcat(buf1, buf); @@ -982,7 +982,7 @@ void list_recipes(CharData *ch, bool all_recipes) { } if (rs->perc <= 0) continue; - sprintf(buf, "%-30s %s%s\r\n", imrecipes[rs->rid].name, how_good(rs->perc, kMaxRecipeLevel), kColorBoldBlk); + sprintf(buf, "%s %s%s\r\n", native_text::pad_right(imrecipes[rs->rid].name, 30).c_str(), how_good(rs->perc, kMaxRecipeLevel), kColorBoldBlk); strcat(buf2, buf); ++i; } diff --git a/src/gameplay/crafting/item_creation.cpp b/src/gameplay/crafting/item_creation.cpp index 6103c48f3e..30405d1ede 100644 --- a/src/gameplay/crafting/item_creation.cpp +++ b/src/gameplay/crafting/item_creation.cpp @@ -7,6 +7,7 @@ * $Revision$ * ************************************************************************ */ #include "item_creation.h" +#include "utils/native_text.h" #include "utils/utils_parse.h" #include "utils/parser_wrapper.h" #include "administration/privilege.h" @@ -333,8 +334,9 @@ void do_list_make(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/ } j++; } - sprintf(tmpbuf, "%3zd %-1s %-6s %-40s(%5d) :", - i + 1, (trec->locked ? "*" : " "), skill_name.c_str(), obj_name.c_str(), trec->obj_proto); + sprintf(tmpbuf, "%3zd %-1s %s %s(%5d) :", + i + 1, (trec->locked ? "*" : " "), native_text::pad_right(skill_name, 6).c_str(), + native_text::pad_right(obj_name, 40).c_str(), trec->obj_proto); tmpstr += string(tmpbuf); for (int j = 0; j < MAX_PARTS; j++) { if (trec->parts[j].proto != 0) { @@ -344,7 +346,7 @@ void do_list_make(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/ } else { obj_name = "Нет"; } - sprintf(tmpbuf, " %-35s(%5d)", obj_name.c_str(), trec->parts[j].proto); + sprintf(tmpbuf, " %s(%5d)", native_text::pad_right(obj_name, 35).c_str(), trec->parts[j].proto); if (j > 0) { if (j % 2 == 0) { // разбиваем строчки если ингров больше 2; diff --git a/src/gameplay/fight/pk.cpp b/src/gameplay/fight/pk.cpp index d927493e2a..66ea9fc89f 100644 --- a/src/gameplay/fight/pk.cpp +++ b/src/gameplay/fight/pk.cpp @@ -12,6 +12,7 @@ ************************************************************************ */ #include "pk.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" #include "gameplay/mechanics/mount.h" @@ -663,9 +664,9 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (tch->get_uid() == uid) { found = true; if (pk.battle_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp.c_str()); + sprintf(buf + strlen(buf), " %s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", native_text::pad_right(temp, 40).c_str()); } else { - sprintf(buf + strlen(buf), " %-40s %3ld %3ld\r\n", temp.c_str(), pk.kill_num, pk.revenge_num); + sprintf(buf + strlen(buf), " %s %3ld %3ld\r\n", native_text::pad_right(temp, 40).c_str(), pk.kill_num, pk.revenge_num); } break; } @@ -673,9 +674,9 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } else { found = true; if (pk.battle_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp.c_str()); + sprintf(buf + strlen(buf), " %s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", native_text::pad_right(temp, 40).c_str()); } else { - sprintf(buf + strlen(buf), " %-40s %3ld %3ld\r\n", temp.c_str(), pk.kill_num, pk.revenge_num); + sprintf(buf + strlen(buf), " %s %3ld %3ld\r\n", native_text::pad_right(temp, 40).c_str(), pk.kill_num, pk.revenge_num); } } } @@ -708,12 +709,12 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { // Сначала проверка клан флага if (CLAN(ch) && pk.clan_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <ВОЙНА>\r\n", GET_NAME(tch)); + sprintf(buf + strlen(buf), " %s <ВОЙНА>\r\n", native_text::pad_right(GET_NAME(tch), 40).c_str()); } else if (pk.clan_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %-40s <ВРЕМЕННЫЙ ФЛАГ>\r\n", GET_NAME(tch)); + sprintf(buf + strlen(buf), " %s <ВРЕМЕННЫЙ ФЛАГ>\r\n", native_text::pad_right(GET_NAME(tch), 40).c_str()); } else if (pk.kill_num + pk.revenge_num > 0) { - sprintf(buf + strlen(buf), " %-40s %3ld %3ld\r\n", - GET_NAME(tch), pk.kill_num, pk.revenge_num); + sprintf(buf + strlen(buf), " %s %3ld %3ld\r\n", + native_text::pad_right(GET_NAME(tch), 40).c_str(), pk.kill_num, pk.revenge_num); } else { continue; } diff --git a/src/gameplay/mechanics/glory_const.cpp b/src/gameplay/mechanics/glory_const.cpp index c4e4c20d7b..b3e9e531ca 100644 --- a/src/gameplay/mechanics/glory_const.cpp +++ b/src/gameplay/mechanics/glory_const.cpp @@ -4,6 +4,7 @@ // Part of Bylins http://www.mud.ru #include "glory_const.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -226,7 +227,8 @@ void print_glory(CharData *ch, GloryListType::iterator &it) { *buf = '\0'; for (auto i = it->second->stats.begin(), iend = it->second->stats.end(); i != iend; ++i) { if ((i->first >= 0) && (i->first < (int) sizeof(olc_stat_name))) { - sprintf(buf + strlen(buf), "%-16s: +%d", olc_stat_name[i->first], i->second * stat_multi(i->first)); + sprintf(buf + strlen(buf), "%s: +%d", native_text::pad_right(olc_stat_name[i->first], 16).c_str(), + i->second * stat_multi(i->first)); if (stat_multi(i->first) > 1) sprintf(buf + strlen(buf), "(%d)", i->second); strcat(buf, "\r\n"); diff --git a/src/gameplay/mechanics/named_stuff.cpp b/src/gameplay/mechanics/named_stuff.cpp index 2451c7b8f5..4d5704c8a9 100644 --- a/src/gameplay/mechanics/named_stuff.cpp +++ b/src/gameplay/mechanics/named_stuff.cpp @@ -3,6 +3,7 @@ // Part of Bylins http://www.mud.ru #include "named_stuff.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" @@ -369,10 +370,10 @@ void do_named(CharData *ch, char *argument, int cmd, int subcmd) { out += buf1; } found++; - sprintf(buf2, "%6ld) &R*&n%-31s Владелец:%-16s e-mail:&S%s&s\r\n", + sprintf(buf2, "%6ld) &R*&n%s Владелец:%s e-mail:&S%s&s\r\n", it->first + 1, - "Несуществующий предмет", - GetNameByUnique(it->second->uid, false).c_str(), + native_text::pad_right("Несуществующий предмет", 31).c_str(), + native_text::pad_right(GetNameByUnique(it->second->uid, false), 16).c_str(), str_dup(it->second->mail.c_str()) ); out += buf2; @@ -388,9 +389,9 @@ void do_named(CharData *ch, char *argument, int cmd, int subcmd) { obj_proto[r_num]->get_vnum(), colored_name(obj_proto[r_num]->get_short_description().c_str(), -32)); if (privilege::IsGrGod(ch) || ch->IsFlagged(EPrf::kCoderinfo)) { - snprintf(buf2, kMaxStringLength, "%s Игра:%d Пост:%d Владелец:%-16s e-mail:&S%s&s\r\n", buf1, + snprintf(buf2, kMaxStringLength, "%s Игра:%d Пост:%d Владелец:%s e-mail:&S%s&s\r\n", buf1, obj_proto.total_online(r_num), obj_proto.stored(r_num), - GetNameByUnique(it->second->uid, false).c_str(), it->second->mail.c_str()); + native_text::pad_right(GetNameByUnique(it->second->uid, false), 16).c_str(), it->second->mail.c_str()); } else { snprintf(buf2, kMaxStringLength, "%s\r\n", buf1); } diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 8ee147504b..679e84ecbc 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -338,6 +338,16 @@ std::size_t missing_width(std::string_view s, std::size_t width) { } // namespace +std::string truncate_to_chars(std::string_view s, std::size_t count) { + std::size_t pos = 0; + std::size_t seen = 0; + while (pos < s.size() && seen < count) { + pos += char_bytes_at(s, pos); + ++seen; + } + return std::string(s.substr(0, pos)); +} + std::string pad_right(std::string_view s, std::size_t width, char fill) { std::string out(s); out.append(missing_width(s, width), fill); diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 47deedb98e..26d7710bd3 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -94,6 +94,11 @@ std::size_t last_char_offset(std::string_view s); std::string pad_right(std::string_view s, std::size_t width, char fill = ' '); std::string pad_left(std::string_view s, std::size_t width, char fill = ' '); +// First `count` characters of `s` (all of it when shorter). The replacement for a substr(0, N) +// used to cap a display length: under KOI8-R it is exactly that, under UTF-8 it counts characters +// and so never cuts one in half. +std::string truncate_to_chars(std::string_view s, std::size_t count); + // Does the single character `ch` occur in `list`? The replacement for strchr() over a literal // list of letters: `list` is walked one whole character at a time, so a multibyte character can // never match on a partial byte sequence. Comparison is exact (case-sensitive), like strchr. From e6757c4c7d4985ae5dabde7d9e6bc54d69d9c7c5 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Mon, 3 Aug 2026 08:16:53 +0200 Subject: [PATCH 11/20] feat(utf8): character-width columns in god commands, closing track A3 (#3681) Finishes A3 with the immortal-facing listings: show (rune spells, linkdrop, snoop), last, users, tabulate, liblist, commands and alias, plus the trigger listing in dg_scripts. The substr(0, N) display caps in show and liblist now go through truncate_to_chars so they cannot split a character. do_users shares one format string across three call sites; it becomes plain "%s" fields with each column padded explicitly. Checked and deliberately skipped: * fmt::sprintf (do_stat) -- measured like fmt::format, already correct in both encodings, so the "%-21s" there needs nothing; * "%-s" in exchange -- a left-align flag with no width, so it never pads; * show_fields/set_fields command names and do_levels' thousands_sep columns -- ASCII-only content; * do_toggle's "%-3s" -- the values are all the same width, so nothing shifts. Verified: full suite 649 passed / 0 failed, clean build. --- src/engine/scripting/dg_scripts.cpp | 4 ++-- src/engine/ui/cmd/do_alias.cpp | 3 ++- src/engine/ui/cmd/do_commands.cpp | 7 ++++--- src/engine/ui/cmd_god/do_last.cpp | 10 ++++++--- src/engine/ui/cmd_god/do_liblist.cpp | 3 ++- src/engine/ui/cmd_god/do_show.cpp | 20 +++++++++--------- src/engine/ui/cmd_god/do_tabulate.cpp | 8 +++++--- src/engine/ui/cmd_god/do_users.cpp | 29 ++++++++++++++++----------- 8 files changed, 50 insertions(+), 34 deletions(-) diff --git a/src/engine/scripting/dg_scripts.cpp b/src/engine/scripting/dg_scripts.cpp index 9b0037ae79..1aa669ab7f 100644 --- a/src/engine/scripting/dg_scripts.cpp +++ b/src/engine/scripting/dg_scripts.cpp @@ -6257,8 +6257,8 @@ void do_tlist(CharData *ch, char *argument, int cmd, int/* subcmd*/) { char trgtypes[256]; for (; nr < top_of_trigt && (trig_index[nr]->vnum <= last); nr++) { std::string out = ""; - snprintf(buf, sizeof(buf), "%2d) [%5d] %-50s ", ++found, - trig_index[nr]->vnum, trig_index[nr]->proto->get_name().c_str()); + snprintf(buf, sizeof(buf), "%2d) [%5d] %s ", ++found, + trig_index[nr]->vnum, native_text::pad_right(trig_index[nr]->proto->get_name(), 50).c_str()); out += buf; if (trig_index[nr]->proto->get_attach_type() == MOB_TRIGGER) { sprintbit(trig_index[nr]->proto->get_trigger_type(), trig_types, trgtypes, sizeof(trgtypes)); diff --git a/src/engine/ui/cmd/do_alias.cpp b/src/engine/ui/cmd/do_alias.cpp index de102d0e2f..f6050cd85f 100644 --- a/src/engine/ui/cmd/do_alias.cpp +++ b/src/engine/ui/cmd/do_alias.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "engine/ui/alias.h" void do_alias(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { @@ -24,7 +25,7 @@ void do_alias(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar(" Нет алиасов.\r\n", ch); else { while (a != nullptr) { - sprintf(buf, "%-15s %s\r\n", a->alias, a->replacement); + sprintf(buf, "%s %s\r\n", native_text::pad_right(a->alias, 15).c_str(), a->replacement); SendMsgToChar(buf, ch); a = a->next; } diff --git a/src/engine/ui/cmd/do_commands.cpp b/src/engine/ui/cmd/do_commands.cpp index 85c460ba04..b84bdb1805 100644 --- a/src/engine/ui/cmd/do_commands.cpp +++ b/src/engine/ui/cmd/do_commands.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "gameplay/communication/social.h" #include "engine/db/global_objects.h" @@ -37,7 +38,7 @@ void do_commands(CharData *ch, char *argument, int/* cmd*/, int subcmd) { continue; } for (const auto &kw : soc.GetKeywords()) { - sprintf(buf + strlen(buf), "%-19s", kw.c_str()); + sprintf(buf + strlen(buf), "%s", native_text::pad_right(kw, 19).c_str()); if (!(no % 4)) strcat(buf, "\r\n"); no++; @@ -49,13 +50,13 @@ void do_commands(CharData *ch, char *argument, int/* cmd*/, int subcmd) { i = cmd_sort_info[cmd_num].sort_pos; if (wizhelp) { if (privilege::HasPrivilege(vict, std::string(cmd_info[i].command), 0, 0, 0)) { - sprintf(buf + strlen(buf), "%-15s", cmd_info[i].command); + sprintf(buf + strlen(buf), "%s", native_text::pad_right(cmd_info[i].command, 15).c_str()); if (!(no % 5)) strcat(buf, "\r\n"); no++; } } else if (cmd_info[i].minimum_level >= 0 && (static_cast(socials) == cmd_sort_info[i].is_social)) { - sprintf(buf + strlen(buf), "%-15s", cmd_info[i].command); + sprintf(buf + strlen(buf), "%s", native_text::pad_right(cmd_info[i].command, 15).c_str()); if (!(no % 5)) strcat(buf, "\r\n"); no++; diff --git a/src/engine/ui/cmd_god/do_last.cpp b/src/engine/ui/cmd_god/do_last.cpp index b1286ab210..bf72a352af 100644 --- a/src/engine/ui/cmd_god/do_last.cpp +++ b/src/engine/ui/cmd_god/do_last.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/entities/char_player.h" #include "engine/db/global_objects.h" @@ -28,10 +29,13 @@ void DoPageLastLogins(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) SendMsgToChar("Вы не столь уж и божественны для этого.\r\n", ch); } else { time_t tmp_time = chdata->get_last_logon(); - sprintf(buf, "[%5ld] [%2d %s] %-12s : %-18s : %-20s\r\n", + sprintf(buf, "[%5ld] [%2d %s] %s : %s : %s\r\n", chdata->get_uid(), GetRealLevel(chdata), - MUD::Class(chdata->GetClass()).GetAbbr().c_str(), GET_NAME(chdata), - chdata->player_specials->saved.LastIP[0] ? chdata->player_specials->saved.LastIP : "Unknown", ctime(&tmp_time)); + MUD::Class(chdata->GetClass()).GetAbbr().c_str(), + native_text::pad_right(GET_NAME(chdata), 12).c_str(), + native_text::pad_right(chdata->player_specials->saved.LastIP[0] + ? chdata->player_specials->saved.LastIP : "Unknown", 18).c_str(), + native_text::pad_right(ctime(&tmp_time), 20).c_str()); SendMsgToChar(buf, ch); } } diff --git a/src/engine/ui/cmd_god/do_liblist.cpp b/src/engine/ui/cmd_god/do_liblist.cpp index 66abc672df..8ff33f5480 100644 --- a/src/engine/ui/cmd_god/do_liblist.cpp +++ b/src/engine/ui/cmd_god/do_liblist.cpp @@ -7,6 +7,7 @@ */ #include "do_liblist.h" +#include "utils/native_text.h" #include "engine/entities/char_data.h" #include "engine/db/obj_prototypes.h" @@ -264,7 +265,7 @@ void Print(CharData *ch, int first, int last, const std::string &options) { for (int i = 0; i <= top_of_mobt; ++i) { if (mob_index[i].vnum >= first && mob_index[i].vnum <= last) { fmt::format_to(std::back_inserter(out), "{:5}. {:<45} [{:<6}] [{:<2}]{}", - ++cnt, mob_proto[i].get_name_str().substr(0, 45), + ++cnt, native_text::truncate_to_chars(mob_proto[i].get_name_str(), 45), mob_index[i].vnum, mob_proto[i].GetLevel(), PrintFlag(mob_proto + i, options)); if (!mob_proto[i].proto_script->empty()) { diff --git a/src/engine/ui/cmd_god/do_show.cpp b/src/engine/ui/cmd_god/do_show.cpp index 1bc4d0adff..9831f1a5e8 100644 --- a/src/engine/ui/cmd_god/do_show.cpp +++ b/src/engine/ui/cmd_god/do_show.cpp @@ -3,6 +3,7 @@ // #include "administration/accounts.h" +#include "utils/native_text.h" #include "administration/ban.h" #include "administration/privilege.h" #include "engine/ui/cmd/do_features.h" @@ -254,10 +255,10 @@ void print_mob_bosses(CharData *ch, bool lvl_sort) { const auto vnum = GET_MOB_VNUM(mob); out += fmt::format("{:<3} {:<31}s [{:<2}][{:<6}] {:<31}s\r\n", ++cnt, - mob->get_name_str().substr(0, 31), + native_text::truncate_to_chars(mob->get_name_str(), 31), zone_table[mob_index[mob_rnum].zone].mob_level, vnum, - zone_name_str.substr(0, 31)); + native_text::truncate_to_chars(zone_name_str, 31)); } page_string(ch->desc, out); } @@ -460,8 +461,8 @@ void ListSpellCreate(CharData *ch) { if (r > 0) runes_str += '|'; runes_str += std::to_string(info.runes[r]); } - SendMsgToChar(ch, "%3d) Rune spell [%3d] &W%-30s&n runes: %s level %d\r\n", - ++i, to_underlying(spell_id), MUD::Spell(spell_id).GetCName(), + SendMsgToChar(ch, "%3d) Rune spell [%3d] &W%s&n runes: %s level %d\r\n", + ++i, to_underlying(spell_id), native_text::pad_right(MUD::Spell(spell_id).GetCName(), 30).c_str(), runes_str.c_str(), info.min_caster_level); } } @@ -718,8 +719,8 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { && ((sight::CanSee(ch, d->character) && GetRealLevel(ch) >= GetRealLevel(d->character)) || ch->IsFlagged(EPrf::kCoderinfo))) { sprintf(buf + strlen(buf), - "%-10s - подслушивается %s (map %s).\r\n", - GET_NAME(d->snooping->character), + "%s - подслушивается %s (map %s).\r\n", + native_text::pad_right(GET_NAME(d->snooping->character), 10).c_str(), GET_PAD(d->character, 4), d->snoop_with_map ? "on" : "off"); } @@ -728,7 +729,8 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { break; // snoop case 9: // show linkdrop SendMsgToChar(" Список игроков в состоянии 'link drop'\r\n", ch); - sprintf(buf, "%-50s%-16s %s\r\n", " Имя", "Комната", "Бездействие (тики)"); + sprintf(buf, "%s%s %s\r\n", native_text::pad_right(" Имя", 50).c_str(), + native_text::pad_right("Комната", 16).c_str(), "Бездействие (тики)"); SendMsgToChar(buf, ch); i = 0; for (const auto &character : character_list) { @@ -737,8 +739,8 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { continue; } ++i; - sprintf(buf, "%-50s[%6d][%6d] %d\r\n", - character->GetNameWithTitleOrRace().c_str(), GET_ROOM_VNUM(character->in_room), + sprintf(buf, "%s[%6d][%6d] %d\r\n", + native_text::pad_right(character->GetNameWithTitleOrRace(), 50).c_str(), GET_ROOM_VNUM(character->in_room), GET_ROOM_VNUM(character->get_was_in_room()), character->char_specials.timer); SendMsgToChar(buf, ch); } diff --git a/src/engine/ui/cmd_god/do_tabulate.cpp b/src/engine/ui/cmd_god/do_tabulate.cpp index 7afa9b019d..26989e19c2 100644 --- a/src/engine/ui/cmd_god/do_tabulate.cpp +++ b/src/engine/ui/cmd_god/do_tabulate.cpp @@ -3,6 +3,7 @@ // #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "gameplay/magic/magic_utils.h" #include "engine/ui/modify.h" #include "engine/ui/objects_filter.h" @@ -96,9 +97,9 @@ int TabulateObjsByFilter(char *argument, CharData *ch) { for (const auto &i : obj_proto) { // ch не передаём: у прототипов нет наносимых меток (custom label). if (filter.check(i.get(), nullptr)) { - snprintf(line, sizeof(line), "%3d. [%7d] %-50s %s\r\n", + snprintf(line, sizeof(line), "%3d. [%7d] %s %s\r\n", ++found, i->get_vnum(), - utils::RemoveColors(i->get_short_description()).c_str(), + native_text::pad_right(utils::RemoveColors(i->get_short_description()), 50).c_str(), filter.show_obj_aff(i.get()).c_str()); out += line; } @@ -117,7 +118,8 @@ int TabulateMobsByName(char *searchname, CharData *ch) { for (nr = 0; nr <= top_of_mobt; nr++) { if (isname(searchname, mob_proto[nr].GetCharAliases())) { - sprintf(buf, "%3d. [%5d] %-30s (%s)\r\n", ++found, mob_index[nr].vnum, mob_proto[nr].get_npc_name().c_str(), + sprintf(buf, "%3d. [%5d] %s (%s)\r\n", ++found, mob_index[nr].vnum, + native_text::pad_right(mob_proto[nr].get_npc_name(), 30).c_str(), npc_race_types[mob_proto[nr].player_data.Race - ENpcRace::kBasic]); SendMsgToChar(buf, ch); } diff --git a/src/engine/ui/cmd_god/do_users.cpp b/src/engine/ui/cmd_god/do_users.cpp index a89f377f20..4d5b95fc2c 100644 --- a/src/engine/ui/cmd_god/do_users.cpp +++ b/src/engine/ui/cmd_god/do_users.cpp @@ -3,6 +3,7 @@ // #include "engine/ui/color.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "gameplay/classes/pc_classes.h" #include "engine/entities/char_data.h" @@ -105,7 +106,9 @@ void do_users(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } } // end while (parser) - const char *format = "%3d %-7s %-20s %-17s %-3s %-8s "; + // Ширина колонок - в символах, а не в байтах (issue #3681): поля ниже паддятся +// через native_text, поэтому формат содержит голые "%s". + const char *format = "%3d %s %s %s %s %s "; if (showemail) { strcpy(line, "Ном Професс Имя Состояние Idl Логин Сайт E-mail\r\n"); } else { @@ -263,23 +266,25 @@ void do_users(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { sprintf(line, format, d->desc_num, - classname, - d->original->GetCharAliases().c_str(), - state, - idletime, - timeptr); + native_text::pad_right(classname, 7).c_str(), + native_text::pad_right(d->original->GetCharAliases().c_str(), 20).c_str(), + native_text::pad_right(state, 17).c_str(), + native_text::pad_right(idletime, 3).c_str(), + native_text::pad_right(timeptr, 8).c_str()); } else { sprintf(line, format, d->desc_num, - classname, - d->character->GetCharAliases().c_str(), - state, - idletime, - timeptr); + native_text::pad_right(classname, 7).c_str(), + native_text::pad_right(d->character->GetCharAliases().c_str(), 20).c_str(), + native_text::pad_right(state, 17).c_str(), + native_text::pad_right(idletime, 3).c_str(), + native_text::pad_right(timeptr, 8).c_str()); } } else { - sprintf(line, format, d->desc_num, " - ", "UNDEFINED", state, idletime, timeptr); + sprintf(line, format, d->desc_num, native_text::pad_right(" - ", 7).c_str(), + native_text::pad_right("UNDEFINED", 20).c_str(), native_text::pad_right(state, 17).c_str(), + native_text::pad_right(idletime, 3).c_str(), native_text::pad_right(timeptr, 8).c_str()); } if (d && *d->host) { From 5b81a4bfae39a9d8751596a147eac17f29735c50 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 04:09:16 +0200 Subject: [PATCH 12/20] docs: fix the declared C++ standard (C++17 -> C++20) CONTRIBUTING.md claimed C++17 while the build has used C++20 for a while (meson.build: cpp_std=c++20, and -std=c++20 in compile_commands.json). CLAUDE.md already said C++20; this brings the two in line. --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0bba9fd8cf..7029a85f69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,7 +7,7 @@ При написании кода вы должны следовать правилу "одна команда - одна строка". Желательно это же правило распространять и на объявления переменных. -Былины используют стандарт C++17. +Былины используют стандарт C++20 (задан в meson.build: cpp_std=c++20). Тела конструкций if, if ... else for, while, do ... while всегда должны быть заключены в символы '{', '}'. From feabc221106a0470c0657c0a75bc712531eeca16 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 04:30:07 +0200 Subject: [PATCH 13/20] refactor(utf8): use fmt for column widths instead of the pad_right helper (#3681) Replaces all 63 native_text::pad_right/pad_left/truncate_to_chars call sites with fmt, which puts the width back where it belongs -- inside the format string -- instead of wrapping every argument: sprintf(buf, "%s - %s\r\n", native_text::pad_right(GET_NAME(i), 20).c_str(), room); SendMsgToChar(buf, ch); becomes SendMsgToChar(fmt::format("{:<20} - {}\r\n", GET_NAME(i), room), ch); This is possible because fmt measures width AND precision in code points for valid UTF-8 and falls back to one-unit-per-byte for KOI8-R (which is not valid UTF-8) -- measured, not assumed -- so it is already correct in both encodings and needs no change at the flip. printf cannot do this: its width/precision are byte-based by definition, which is what made the helper necessary in the first place. pad_right/pad_left/truncate_to_chars are removed from native_text along with their tests; the character primitives the scanners need (char_bytes, is_alnum_char, ...) stay. Several sites also lose their intermediate char buffer entirely. Drive-by, on a line this change already touches: do_show's mob listing had "{:<31}s" in its format string -- a leftover 's' from an earlier %-31s conversion that printed a stray letter after the name. Removed. Verified: clean build, full suite 648 passed / 0 failed. --- src/engine/scripting/dg_scripts.cpp | 4 ++-- src/engine/ui/cmd/do_affects.cpp | 8 +++---- src/engine/ui/cmd/do_alias.cpp | 5 ++-- src/engine/ui/cmd/do_commands.cpp | 8 +++---- src/engine/ui/cmd/do_exits.cpp | 8 +++---- src/engine/ui/cmd/do_features.cpp | 10 ++++---- src/engine/ui/cmd/do_score.cpp | 7 +++--- src/engine/ui/cmd/do_skills.cpp | 8 +++---- src/engine/ui/cmd/do_spells.cpp | 21 +++++++++-------- src/engine/ui/cmd/do_where.cpp | 9 ++----- src/engine/ui/cmd/do_who.cpp | 16 ++++++------- src/engine/ui/cmd_god/do_last.cpp | 15 ++++++------ src/engine/ui/cmd_god/do_liblist.cpp | 5 ++-- src/engine/ui/cmd_god/do_show.cpp | 31 ++++++++++++------------- src/engine/ui/cmd_god/do_tabulate.cpp | 13 +++++------ src/engine/ui/cmd_god/do_users.cpp | 31 ++++++++----------------- src/gameplay/communication/parcel.cpp | 9 ++++--- src/gameplay/crafting/im.cpp | 15 ++++++------ src/gameplay/crafting/item_creation.cpp | 11 ++++----- src/gameplay/fight/pk.cpp | 18 +++++++------- src/gameplay/mechanics/glory_const.cpp | 5 ++-- src/gameplay/mechanics/named_stuff.cpp | 16 ++++++------- src/utils/native_text.cpp | 31 ------------------------- src/utils/native_text.h | 13 ----------- tests/native_text.cpp | 16 ------------- 25 files changed, 125 insertions(+), 208 deletions(-) diff --git a/src/engine/scripting/dg_scripts.cpp b/src/engine/scripting/dg_scripts.cpp index 1aa669ab7f..e2aa3c4be7 100644 --- a/src/engine/scripting/dg_scripts.cpp +++ b/src/engine/scripting/dg_scripts.cpp @@ -6257,8 +6257,8 @@ void do_tlist(CharData *ch, char *argument, int cmd, int/* subcmd*/) { char trgtypes[256]; for (; nr < top_of_trigt && (trig_index[nr]->vnum <= last); nr++) { std::string out = ""; - snprintf(buf, sizeof(buf), "%2d) [%5d] %s ", ++found, - trig_index[nr]->vnum, native_text::pad_right(trig_index[nr]->proto->get_name(), 50).c_str()); + strcpy(buf, fmt::format("{:2}) [{:5}] {:<50} ", ++found, + trig_index[nr]->vnum, trig_index[nr]->proto->get_name()).c_str()); out += buf; if (trig_index[nr]->proto->get_attach_type() == MOB_TRIGGER) { sprintbit(trig_index[nr]->proto->get_trigger_type(), trig_types, trgtypes, sizeof(trgtypes)); diff --git a/src/engine/ui/cmd/do_affects.cpp b/src/engine/ui/cmd/do_affects.cpp index 43b5b568b5..9100bc4a11 100644 --- a/src/engine/ui/cmd/do_affects.cpp +++ b/src/engine/ui/cmd/do_affects.cpp @@ -3,7 +3,7 @@ // #include "engine/entities/char_data.h" -#include "utils/native_text.h" +#include #include "gameplay/affects/affect_messages.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -67,11 +67,11 @@ void do_affects(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { grammar::GetDeclensionInNumber((mod + 1) / kSecsPerMudHour + 1, grammar::EWhat::kHour)) : sprintf(buf2, "(менее часа)"); // Ширина колонок - в символах, а не в байтах (issue #3681). - snprintf(buf, kMaxStringLength, "%s%s%s %s%s ", + strcpy(buf, fmt::format("{}{}{:<21} {:<12}{} ", *sp_name == '!' ? "Состояние : " : "Заклинание : ", kColorBoldCyn, - native_text::pad_right(sp_name, 21).c_str(), - native_text::pad_right(buf2, 12).c_str(), kColorNrm); + sp_name, + buf2, kColorNrm).c_str()); *buf2 = '\0'; if (!privilege::IsImmortal(ch)) { auto next_affect_i = affect_i; diff --git a/src/engine/ui/cmd/do_alias.cpp b/src/engine/ui/cmd/do_alias.cpp index f6050cd85f..c7421ee0e1 100644 --- a/src/engine/ui/cmd/do_alias.cpp +++ b/src/engine/ui/cmd/do_alias.cpp @@ -7,7 +7,7 @@ */ #include "engine/entities/char_data.h" -#include "utils/native_text.h" +#include #include "engine/ui/alias.h" void do_alias(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { @@ -25,8 +25,7 @@ void do_alias(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar(" Нет алиасов.\r\n", ch); else { while (a != nullptr) { - sprintf(buf, "%s %s\r\n", native_text::pad_right(a->alias, 15).c_str(), a->replacement); - SendMsgToChar(buf, ch); + SendMsgToChar(fmt::format("{:<15} {}\r\n", a->alias, a->replacement), ch); a = a->next; } } diff --git a/src/engine/ui/cmd/do_commands.cpp b/src/engine/ui/cmd/do_commands.cpp index b84bdb1805..3520f446bc 100644 --- a/src/engine/ui/cmd/do_commands.cpp +++ b/src/engine/ui/cmd/do_commands.cpp @@ -7,7 +7,7 @@ */ #include "engine/entities/char_data.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/communication/social.h" #include "engine/db/global_objects.h" @@ -38,7 +38,7 @@ void do_commands(CharData *ch, char *argument, int/* cmd*/, int subcmd) { continue; } for (const auto &kw : soc.GetKeywords()) { - sprintf(buf + strlen(buf), "%s", native_text::pad_right(kw, 19).c_str()); + strcat(buf, fmt::format("{:<19}", kw).c_str()); if (!(no % 4)) strcat(buf, "\r\n"); no++; @@ -50,13 +50,13 @@ void do_commands(CharData *ch, char *argument, int/* cmd*/, int subcmd) { i = cmd_sort_info[cmd_num].sort_pos; if (wizhelp) { if (privilege::HasPrivilege(vict, std::string(cmd_info[i].command), 0, 0, 0)) { - sprintf(buf + strlen(buf), "%s", native_text::pad_right(cmd_info[i].command, 15).c_str()); + strcat(buf, fmt::format("{:<15}", cmd_info[i].command).c_str()); if (!(no % 5)) strcat(buf, "\r\n"); no++; } } else if (cmd_info[i].minimum_level >= 0 && (static_cast(socials) == cmd_sort_info[i].is_social)) { - sprintf(buf + strlen(buf), "%s", native_text::pad_right(cmd_info[i].command, 15).c_str()); + strcat(buf, fmt::format("{:<15}", cmd_info[i].command).c_str()); if (!(no % 5)) strcat(buf, "\r\n"); no++; diff --git a/src/engine/ui/cmd/do_exits.cpp b/src/engine/ui/cmd/do_exits.cpp index 219b0706b0..16ebec4a32 100644 --- a/src/engine/ui/cmd/do_exits.cpp +++ b/src/engine/ui/cmd/do_exits.cpp @@ -6,7 +6,7 @@ */ #include "engine/entities/char_data.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/sight.h" #include "gameplay/mechanics/illumination.h" @@ -30,10 +30,10 @@ void DoExits(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/) { for (door = 0; door < EDirection::kMaxDirNum; door++) if (EXIT(ch, door) && EXIT(ch, door)->to_room() != kNowhere && !EXIT_FLAGGED(EXIT(ch, door), EExitFlag::kClosed)) { if (privilege::IsGod(ch)) - sprintf(buf2, "%s - [%5d] %s\r\n", native_text::pad_right(dirs_rus[door], 6).c_str(), - GET_ROOM_VNUM(EXIT(ch, door)->to_room()), world[EXIT(ch, door)->to_room()]->name); + strcpy(buf2, fmt::format("{:<6} - [{:5}] {}\r\n", dirs_rus[door], + GET_ROOM_VNUM(EXIT(ch, door)->to_room()), world[EXIT(ch, door)->to_room()]->name).c_str()); else { - sprintf(buf2, "%s - ", native_text::pad_right(dirs_rus[door], 6).c_str()); + strcpy(buf2, fmt::format("{:<6} - ", dirs_rus[door]).c_str()); if (is_dark(EXIT(ch, door)->to_room()) && !sight::CanSeeInDark(ch)) strcat(buf2, "слишком темно\r\n"); else { diff --git a/src/engine/ui/cmd/do_features.cpp b/src/engine/ui/cmd/do_features.cpp index a48d26e9d5..01f03bce12 100644 --- a/src/engine/ui/cmd/do_features.cpp +++ b/src/engine/ui/cmd/do_features.cpp @@ -1,5 +1,5 @@ #include "engine/ui/color.h" -#include "utils/native_text.h" +#include #include "gameplay/core/remort.h" #include "engine/entities/char_data.h" #include "gameplay/abilities/timed_abilities.h" @@ -82,17 +82,17 @@ void DisplayFeats(CharData *ch, CharData *vict, bool all_feats) { continue; } if (!ch->IsFlagged(EPrf::kBlindMode)) { - sprintf(buf, " %s%s %s%s\r\n", + strcpy(buf, fmt::format(" {}{} {:<30}{}\r\n", ch->HaveFeat(feat.GetId()) ? kColorGrn : CanGetFeat(ch, feat.GetId()) ? kColorNrm : kColorRed, ch->HaveFeat(feat.GetId()) ? "[И]" : CanGetFeat(ch, feat.GetId()) ? "[Д]" : "[Н]", - native_text::pad_right(MUD::Feat(feat.GetId()).GetCName(), 30).c_str(), kColorNrm); + MUD::Feat(feat.GetId()).GetCName(), kColorNrm).c_str()); } else { - sprintf(buf, " %s %s\r\n", + strcpy(buf, fmt::format(" {} {:<30}\r\n", ch->HaveFeat(feat.GetId()) ? "[И]" : CanGetFeat(ch, feat.GetId()) ? "[Д]" : "[Н]", - native_text::pad_right(MUD::Feat(feat.GetId()).GetCName(), 30).c_str()); + MUD::Feat(feat.GetId()).GetCName()).c_str()); } if (feat.IsInborn() || diff --git a/src/engine/ui/cmd/do_score.cpp b/src/engine/ui/cmd/do_score.cpp index 12e5aedc04..bd877500ba 100644 --- a/src/engine/ui/cmd/do_score.cpp +++ b/src/engine/ui/cmd/do_score.cpp @@ -7,7 +7,6 @@ */ #include "engine/ui/color.h" -#include "utils/native_text.h" #include "gameplay/affects/affect_messages.h" #include "utils/utils_string.h" #include "gameplay/core/experience.h" @@ -233,12 +232,12 @@ void PrintScoreList(CharData *ch) { } else if (NAME_BAD(ch)) { SendMsgToChar(ch, "ВНИМАНИЕ! ваше имя запрещено богами. Очень скоро вы прекратите получать опыт.\r\n"); } - SendMsgToChar(ch, "Вы можете вступить в группу с максимальной разницей в %2d %s\r\n", + SendMsgToChar(fmt::format("Вы можете вступить в группу с максимальной разницей в {:2} {:<75.76}\r\n", grouping[ch->GetClass()][static_cast(remort::GetRealRemort(ch))], - native_text::pad_right(native_text::truncate_to_chars(std::string( + grammar::GetDeclensionInNumber(grouping[ch->GetClass()][static_cast(remort::GetRealRemort( ch))], grammar::EWhat::kLvl) - + std::string(" без потерь для опыта.")), 76), 75).c_str()); + + std::string(" без потерь для опыта.")), ch); SendMsgToChar(ch, "Вы можете принять в группу максимум %d соратников.\r\n", group::max_group_size(ch)); std::ostringstream out; diff --git a/src/engine/ui/cmd/do_skills.cpp b/src/engine/ui/cmd/do_skills.cpp index 4197a913df..fbdccf49e3 100644 --- a/src/engine/ui/cmd/do_skills.cpp +++ b/src/engine/ui/cmd/do_skills.cpp @@ -1,5 +1,5 @@ #include "do_skills.h" -#include "utils/native_text.h" +#include #include "engine/ui/color.h" #include "engine/entities/char_data.h" @@ -79,12 +79,12 @@ void DisplaySkills(CharData *ch, CharData *vict, const char *filter/* = nullptr* } // Ширина колонки - в символах, а не в байтах (issue #3681). - sprintf(buf + strlen(buf), "%s %s (%d)%s \r\n", - native_text::pad_right(skill.GetName(), 23).c_str(), + strcat(buf, fmt::format("{:<23} {} ({}){} \r\n", + skill.GetName(), how_good(GetSkill(ch, skill_id), CalcSkillHardCap(ch, skill_id)), GetTrainedSkill(ch, skill_id) == 0 ? GetEquippedSkill(ch, skill_id) : std::min(CalcSkillMinCap(ch, skill_id) + GetEquippedSkill(ch, skill_id), MUD::Skill(skill_id).cap), - kColorNrm); + kColorNrm).c_str()); skills_names.emplace_back(buf); i++; } diff --git a/src/engine/ui/cmd/do_spells.cpp b/src/engine/ui/cmd/do_spells.cpp index f8eb53df59..1ebb0dd5fe 100644 --- a/src/engine/ui/cmd/do_spells.cpp +++ b/src/engine/ui/cmd/do_spells.cpp @@ -1,5 +1,5 @@ #include "do_spells.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/magic_item.h" @@ -88,18 +88,19 @@ void DisplaySpells(CharData *ch, CharData *vict, bool all) { if (CalcSpellManacost(ch, spell_id) > Mana(GetRealWis(ch))) continue; if (CheckRecipeItems(ch, spell_id, ESpellType::kRunes, false)) { - slots[slot_num] += sprintf(names[slot_num] + slots[slot_num], - "%s|<...%4d.> %s%s&n|", - slots[slot_num] % 114 < - 10 ? "\r\n" : " ", + const auto line = fmt::format("{}|<...{:4}.> {}{:<38}&n|", + slots[slot_num] % 114 < 10 ? "\r\n" : " ", CalcSpellManacost(ch, spell_id), GetSpellColor(spell_id), - native_text::pad_right(MUD::Spell(spell_id).GetCName(), 38).c_str()); + MUD::Spell(spell_id).GetCName()); + strcpy(names[slot_num] + slots[slot_num], line.c_str()); + slots[slot_num] += static_cast(line.size()); } else { if (all) { - slots[slot_num] += sprintf(names[slot_num] + slots[slot_num], - "%s|+--------+ %s%s&n|", slots[slot_num] % 114 < 10 ? "\r\n" - : " ", GetSpellColor(spell_id), - native_text::pad_right(MUD::Spell(spell_id).GetCName(), 38).c_str()); + const auto line = fmt::format("{}|+--------+ {}{:<38}&n|", + slots[slot_num] % 114 < 10 ? "\r\n" : " ", GetSpellColor(spell_id), + MUD::Spell(spell_id).GetCName()); + strcpy(names[slot_num] + slots[slot_num], line.c_str()); + slots[slot_num] += static_cast(line.size()); } } } else { diff --git a/src/engine/ui/cmd/do_where.cpp b/src/engine/ui/cmd/do_where.cpp index c408fcb781..2d95e0d0dd 100644 --- a/src/engine/ui/cmd/do_where.cpp +++ b/src/engine/ui/cmd/do_where.cpp @@ -3,7 +3,6 @@ // #include "engine/entities/char_data.h" -#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/world_objects.h" #include "gameplay/economics/exchange.h" @@ -147,9 +146,7 @@ void PerformMortalWhere(CharData *ch, char *arg) { } // Ширина колонки - в символах, а не в байтах (issue #3681). - sprintf(buf, "%s - %s\r\n", - native_text::pad_right(GET_NAME(i), 20).c_str(), world[i->in_room]->name); - SendMsgToChar(buf, ch); + SendMsgToChar(fmt::format("{:<20} - {}\r\n", GET_NAME(i), world[i->in_room]->name), ch); } } else // print only FIRST char, not all. { @@ -168,9 +165,7 @@ void PerformMortalWhere(CharData *ch, char *arg) { continue; } - sprintf(buf, "%s - %s\r\n", - native_text::pad_right(GET_NAME(i), 25).c_str(), world[i->in_room]->name); - SendMsgToChar(buf, ch); + SendMsgToChar(fmt::format("{:<25} - {}\r\n", GET_NAME(i), world[i->in_room]->name), ch); return; } SendMsgToChar("Никого похожего с этим именем нет.\r\n", ch); diff --git a/src/engine/ui/cmd/do_who.cpp b/src/engine/ui/cmd/do_who.cpp index d408246376..482f91e8c9 100644 --- a/src/engine/ui/cmd/do_who.cpp +++ b/src/engine/ui/cmd/do_who.cpp @@ -3,7 +3,7 @@ // #include "engine/ui/cmd/do_who.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "utils/grammar/gender.h" #include "gameplay/mechanics/sight.h" @@ -165,19 +165,19 @@ void DoWho(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (short_list) { char tmp[kMaxInputLength]; snprintf(tmp, sizeof(tmp), "%s%s%s", GetPkNameColor(tch), GET_NAME(tch), kColorNrm); - // Ширина колонки - в символах, а не в байтах (issue #3681): "%-30s" отсчитывает + // Ширина колонки - в символах, а не в байтах (issue #3681): fmt "{:<30}" считает // байты, из-за чего колонка с русским именем под UTF-8 выходит вдвое уже. if (privilege::IsImpl(ch) || ch->IsFlagged(EPrf::kCoderinfo)) { - sprintf(buf, "%s[%2d %s] %s%s", + strcpy(buf, fmt::format("{}[{:2} {}] {:<30}{}", privilege::IsGod(tch.get()) ? kColorWht : "", GetRealLevel(tch), MUD::Class(tch->GetClass()).GetCName(), - native_text::pad_right(tmp, 30).c_str(), - privilege::IsGod(tch.get()) ? kColorNrm : ""); + tmp, + privilege::IsGod(tch.get()) ? kColorNrm : "").c_str()); } else { - sprintf(buf, "%s%s%s", + strcpy(buf, fmt::format("{}{:<30}{}", privilege::IsImmortal(tch.get()) ? kColorWht : "", - native_text::pad_right(tmp, 30).c_str(), - privilege::IsImmortal(tch.get()) ? kColorNrm : ""); + tmp, + privilege::IsImmortal(tch.get()) ? kColorNrm : "").c_str()); } } else { if (privilege::IsImpl(ch) diff --git a/src/engine/ui/cmd_god/do_last.cpp b/src/engine/ui/cmd_god/do_last.cpp index bf72a352af..5c040dcdb5 100644 --- a/src/engine/ui/cmd_god/do_last.cpp +++ b/src/engine/ui/cmd_god/do_last.cpp @@ -7,7 +7,7 @@ */ #include "engine/entities/char_data.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "engine/entities/char_player.h" #include "engine/db/global_objects.h" @@ -29,14 +29,13 @@ void DoPageLastLogins(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) SendMsgToChar("Вы не столь уж и божественны для этого.\r\n", ch); } else { time_t tmp_time = chdata->get_last_logon(); - sprintf(buf, "[%5ld] [%2d %s] %s : %s : %s\r\n", + SendMsgToChar(fmt::format("[{:5}] [{:2} {}] {:<12} : {:<18} : {:<20}\r\n", chdata->get_uid(), GetRealLevel(chdata), - MUD::Class(chdata->GetClass()).GetAbbr().c_str(), - native_text::pad_right(GET_NAME(chdata), 12).c_str(), - native_text::pad_right(chdata->player_specials->saved.LastIP[0] - ? chdata->player_specials->saved.LastIP : "Unknown", 18).c_str(), - native_text::pad_right(ctime(&tmp_time), 20).c_str()); - SendMsgToChar(buf, ch); + MUD::Class(chdata->GetClass()).GetAbbr(), + GET_NAME(chdata), + chdata->player_specials->saved.LastIP[0] + ? chdata->player_specials->saved.LastIP : "Unknown", + ctime(&tmp_time)), ch); } } diff --git a/src/engine/ui/cmd_god/do_liblist.cpp b/src/engine/ui/cmd_god/do_liblist.cpp index 8ff33f5480..f7a5da3b3d 100644 --- a/src/engine/ui/cmd_god/do_liblist.cpp +++ b/src/engine/ui/cmd_god/do_liblist.cpp @@ -7,7 +7,6 @@ */ #include "do_liblist.h" -#include "utils/native_text.h" #include "engine/entities/char_data.h" #include "engine/db/obj_prototypes.h" @@ -264,8 +263,8 @@ void Print(CharData *ch, int first, int last, const std::string &options) { int cnt = 0; for (int i = 0; i <= top_of_mobt; ++i) { if (mob_index[i].vnum >= first && mob_index[i].vnum <= last) { - fmt::format_to(std::back_inserter(out), "{:5}. {:<45} [{:<6}] [{:<2}]{}", - ++cnt, native_text::truncate_to_chars(mob_proto[i].get_name_str(), 45), + fmt::format_to(std::back_inserter(out), "{:5}. {:<45.45} [{:<6}] [{:<2}]{}", + ++cnt, mob_proto[i].get_name_str(), mob_index[i].vnum, mob_proto[i].GetLevel(), PrintFlag(mob_proto + i, options)); if (!mob_proto[i].proto_script->empty()) { diff --git a/src/engine/ui/cmd_god/do_show.cpp b/src/engine/ui/cmd_god/do_show.cpp index 9831f1a5e8..fd2c0229f8 100644 --- a/src/engine/ui/cmd_god/do_show.cpp +++ b/src/engine/ui/cmd_god/do_show.cpp @@ -3,7 +3,6 @@ // #include "administration/accounts.h" -#include "utils/native_text.h" #include "administration/ban.h" #include "administration/privilege.h" #include "engine/ui/cmd/do_features.h" @@ -253,12 +252,12 @@ void print_mob_bosses(CharData *ch, bool lvl_sort) { const auto mob = mob_proto + mob_rnum; const auto vnum = GET_MOB_VNUM(mob); - out += fmt::format("{:<3} {:<31}s [{:<2}][{:<6}] {:<31}s\r\n", + out += fmt::format("{:<3} {:<31.31} [{:<2}][{:<6}] {:<31.31}\r\n", ++cnt, - native_text::truncate_to_chars(mob->get_name_str(), 31), + mob->get_name_str(), zone_table[mob_index[mob_rnum].zone].mob_level, vnum, - native_text::truncate_to_chars(zone_name_str, 31)); + zone_name_str); } page_string(ch->desc, out); } @@ -461,9 +460,9 @@ void ListSpellCreate(CharData *ch) { if (r > 0) runes_str += '|'; runes_str += std::to_string(info.runes[r]); } - SendMsgToChar(ch, "%3d) Rune spell [%3d] &W%s&n runes: %s level %d\r\n", - ++i, to_underlying(spell_id), native_text::pad_right(MUD::Spell(spell_id).GetCName(), 30).c_str(), - runes_str.c_str(), info.min_caster_level); + SendMsgToChar(fmt::format("{:3}) Rune spell [{:3}] &W{:<30}&n runes: {} level {}\r\n", + ++i, to_underlying(spell_id), MUD::Spell(spell_id).GetCName(), + runes_str, info.min_caster_level), ch); } } @@ -718,19 +717,19 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { && d->character->in_room != kNowhere && ((sight::CanSee(ch, d->character) && GetRealLevel(ch) >= GetRealLevel(d->character)) || ch->IsFlagged(EPrf::kCoderinfo))) { - sprintf(buf + strlen(buf), - "%s - подслушивается %s (map %s).\r\n", - native_text::pad_right(GET_NAME(d->snooping->character), 10).c_str(), + strcat(buf, fmt::format( + "{:<10} - подслушивается {} (map {}).\r\n", + GET_NAME(d->snooping->character), GET_PAD(d->character, 4), - d->snoop_with_map ? "on" : "off"); + d->snoop_with_map ? "on" : "off").c_str()); } } SendMsgToChar(*buf ? buf : "Никто не подслушивается.\r\n", ch); break; // snoop case 9: // show linkdrop SendMsgToChar(" Список игроков в состоянии 'link drop'\r\n", ch); - sprintf(buf, "%s%s %s\r\n", native_text::pad_right(" Имя", 50).c_str(), - native_text::pad_right("Комната", 16).c_str(), "Бездействие (тики)"); + strcpy(buf, fmt::format("{:<50}{:<16} {}\r\n", " Имя", + "Комната", "Бездействие (тики)").c_str()); SendMsgToChar(buf, ch); i = 0; for (const auto &character : character_list) { @@ -739,9 +738,9 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { continue; } ++i; - sprintf(buf, "%s[%6d][%6d] %d\r\n", - native_text::pad_right(character->GetNameWithTitleOrRace(), 50).c_str(), GET_ROOM_VNUM(character->in_room), - GET_ROOM_VNUM(character->get_was_in_room()), character->char_specials.timer); + strcpy(buf, fmt::format("{:<50}[{:6}][{:6}] {}\r\n", + character->GetNameWithTitleOrRace(), GET_ROOM_VNUM(character->in_room), + GET_ROOM_VNUM(character->get_was_in_room()), character->char_specials.timer).c_str()); SendMsgToChar(buf, ch); } sprintf(buf, "Всего - %d\r\n", i); diff --git a/src/engine/ui/cmd_god/do_tabulate.cpp b/src/engine/ui/cmd_god/do_tabulate.cpp index 26989e19c2..7a60eb611b 100644 --- a/src/engine/ui/cmd_god/do_tabulate.cpp +++ b/src/engine/ui/cmd_god/do_tabulate.cpp @@ -3,7 +3,6 @@ // #include "engine/entities/char_data.h" -#include "utils/native_text.h" #include "gameplay/magic/magic_utils.h" #include "engine/ui/modify.h" #include "engine/ui/objects_filter.h" @@ -97,10 +96,10 @@ int TabulateObjsByFilter(char *argument, CharData *ch) { for (const auto &i : obj_proto) { // ch не передаём: у прототипов нет наносимых меток (custom label). if (filter.check(i.get(), nullptr)) { - snprintf(line, sizeof(line), "%3d. [%7d] %s %s\r\n", + strcpy(line, fmt::format("{:3}. [{:7}] {:<50} {}\r\n", ++found, i->get_vnum(), - native_text::pad_right(utils::RemoveColors(i->get_short_description()), 50).c_str(), - filter.show_obj_aff(i.get()).c_str()); + utils::RemoveColors(i->get_short_description()), + filter.show_obj_aff(i.get())).c_str()); out += line; } } @@ -118,9 +117,9 @@ int TabulateMobsByName(char *searchname, CharData *ch) { for (nr = 0; nr <= top_of_mobt; nr++) { if (isname(searchname, mob_proto[nr].GetCharAliases())) { - sprintf(buf, "%3d. [%5d] %s (%s)\r\n", ++found, mob_index[nr].vnum, - native_text::pad_right(mob_proto[nr].get_npc_name(), 30).c_str(), - npc_race_types[mob_proto[nr].player_data.Race - ENpcRace::kBasic]); + strcpy(buf, fmt::format("{:3}. [{:5}] {:<30} ({})\r\n", ++found, mob_index[nr].vnum, + mob_proto[nr].get_npc_name(), + npc_race_types[mob_proto[nr].player_data.Race - ENpcRace::kBasic]).c_str()); SendMsgToChar(buf, ch); } } diff --git a/src/engine/ui/cmd_god/do_users.cpp b/src/engine/ui/cmd_god/do_users.cpp index 4d5b95fc2c..7398ea12dd 100644 --- a/src/engine/ui/cmd_god/do_users.cpp +++ b/src/engine/ui/cmd_god/do_users.cpp @@ -3,7 +3,7 @@ // #include "engine/ui/color.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/classes/pc_classes.h" #include "engine/entities/char_data.h" @@ -108,7 +108,7 @@ void do_users(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { // Ширина колонок - в символах, а не в байтах (issue #3681): поля ниже паддятся // через native_text, поэтому формат содержит голые "%s". - const char *format = "%3d %s %s %s %s %s "; + const char *format = "{:3} {:<7} {:<20} {:<17} {:<3} {:<8} "; if (showemail) { strcpy(line, "Ном Професс Имя Состояние Idl Логин Сайт E-mail\r\n"); } else { @@ -263,28 +263,17 @@ void do_users(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (d->character && d->character->GetCharAliases().c_str()) { if (d->original) { - sprintf(line, - format, - d->desc_num, - native_text::pad_right(classname, 7).c_str(), - native_text::pad_right(d->original->GetCharAliases().c_str(), 20).c_str(), - native_text::pad_right(state, 17).c_str(), - native_text::pad_right(idletime, 3).c_str(), - native_text::pad_right(timeptr, 8).c_str()); + strcpy(line, fmt::format(fmt::runtime(format), + d->desc_num, classname, d->original->GetCharAliases().c_str(), + state, idletime, timeptr).c_str()); } else { - sprintf(line, - format, - d->desc_num, - native_text::pad_right(classname, 7).c_str(), - native_text::pad_right(d->character->GetCharAliases().c_str(), 20).c_str(), - native_text::pad_right(state, 17).c_str(), - native_text::pad_right(idletime, 3).c_str(), - native_text::pad_right(timeptr, 8).c_str()); + strcpy(line, fmt::format(fmt::runtime(format), + d->desc_num, classname, d->character->GetCharAliases().c_str(), + state, idletime, timeptr).c_str()); } } else { - sprintf(line, format, d->desc_num, native_text::pad_right(" - ", 7).c_str(), - native_text::pad_right("UNDEFINED", 20).c_str(), native_text::pad_right(state, 17).c_str(), - native_text::pad_right(idletime, 3).c_str(), native_text::pad_right(timeptr, 8).c_str()); + strcpy(line, fmt::format(fmt::runtime(format), d->desc_num, " - ", + "UNDEFINED", state, idletime, timeptr).c_str()); } if (d && *d->host) { diff --git a/src/gameplay/communication/parcel.cpp b/src/gameplay/communication/parcel.cpp index aa63340e50..af8c2d57dc 100644 --- a/src/gameplay/communication/parcel.cpp +++ b/src/gameplay/communication/parcel.cpp @@ -3,7 +3,6 @@ // Part of Bylins http://www.mud.ru #include "parcel.h" -#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "gameplay/economics/currencies.h" @@ -829,13 +828,13 @@ bool print_imm_where_obj(CharData *ch, const ObjData *arg, int num) { std::string sender = GetNameByUnique(it2->first); found = true; - SendMsgToChar(ch, "%2d. [%6d] %s - наход%sся на почте (отправитель: %s, получатель: %s).\r\n", + SendMsgToChar(fmt::format("{:2}. [{:6}] {:<25} - наход{}ся на почте (отправитель: {}, получатель: {}).\r\n", num++, GET_OBJ_VNUM(it3->obj_.get()), - native_text::pad_right(it3->obj_->get_short_description(), 25).c_str(), + it3->obj_->get_short_description(), grammar::ObjPluralVerbEnding((it3->obj_)->get_sex()), - sender.c_str(), - target.c_str()); + sender, + target), ch); } } } diff --git a/src/gameplay/crafting/im.cpp b/src/gameplay/crafting/im.cpp index ead6fddec8..e35dc3e955 100644 --- a/src/gameplay/crafting/im.cpp +++ b/src/gameplay/crafting/im.cpp @@ -11,6 +11,7 @@ // Реализация ингредиентной магии #include "im.h" +#include #include "utils/parser_wrapper.h" #include "utils/utils_parse.h" #include "utils/native_text.h" @@ -953,14 +954,14 @@ void list_recipes(CharData *ch, bool all_recipes) { rs = im_get_char_rskill(ch, sortpos); const bool unavailable = req->level > GetRealLevel(ch) || req->remort > remort::GetRealRemort(ch); if (!ch->IsFlagged(EPrf::kBlindMode)) { - sprintf(buf, " %s%s%s %2d (%2d)%s\r\n", + strcpy(buf, fmt::format(" {}{:<30}{} {:2} ({:2}){}\r\n", unavailable ? kColorRed : rs ? kColorGrn : kColorNrm, - native_text::pad_right(imrecipes[sortpos].name, 30).c_str(), kColorCyn, - req->level, req->remort, kColorNrm); + imrecipes[sortpos].name, kColorCyn, + req->level, req->remort, kColorNrm).c_str()); } else { - sprintf(buf, " %s %s %2d (%2d)\r\n", - unavailable ? "[Н]" : rs ? "[И]" : "[Д]", native_text::pad_right(imrecipes[sortpos].name, 30).c_str(), - req->level, req->remort); + strcpy(buf, fmt::format(" {} {:<30} {:2} ({:2})\r\n", + unavailable ? "[Н]" : rs ? "[И]" : "[Д]", imrecipes[sortpos].name, + req->level, req->remort).c_str()); } strcat(buf1, buf); ++i; @@ -982,7 +983,7 @@ void list_recipes(CharData *ch, bool all_recipes) { } if (rs->perc <= 0) continue; - sprintf(buf, "%s %s%s\r\n", native_text::pad_right(imrecipes[rs->rid].name, 30).c_str(), how_good(rs->perc, kMaxRecipeLevel), kColorBoldBlk); + strcpy(buf, fmt::format("{:<30} {}{}\r\n", imrecipes[rs->rid].name, how_good(rs->perc, kMaxRecipeLevel), kColorBoldBlk).c_str()); strcat(buf2, buf); ++i; } diff --git a/src/gameplay/crafting/item_creation.cpp b/src/gameplay/crafting/item_creation.cpp index 30405d1ede..37cdd0e775 100644 --- a/src/gameplay/crafting/item_creation.cpp +++ b/src/gameplay/crafting/item_creation.cpp @@ -7,7 +7,7 @@ * $Revision$ * ************************************************************************ */ #include "item_creation.h" -#include "utils/native_text.h" +#include #include "utils/utils_parse.h" #include "utils/parser_wrapper.h" #include "administration/privilege.h" @@ -334,10 +334,9 @@ void do_list_make(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/ } j++; } - sprintf(tmpbuf, "%3zd %-1s %s %s(%5d) :", - i + 1, (trec->locked ? "*" : " "), native_text::pad_right(skill_name, 6).c_str(), - native_text::pad_right(obj_name, 40).c_str(), trec->obj_proto); - tmpstr += string(tmpbuf); + tmpstr += fmt::format("{:3} {:<1} {:<6} {:<40}({:5}) :", + i + 1, (trec->locked ? "*" : " "), skill_name, + obj_name, trec->obj_proto); for (int j = 0; j < MAX_PARTS; j++) { if (trec->parts[j].proto != 0) { obj = GetObjectPrototype(trec->parts[j].proto); @@ -346,7 +345,7 @@ void do_list_make(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/ } else { obj_name = "Нет"; } - sprintf(tmpbuf, " %s(%5d)", native_text::pad_right(obj_name, 35).c_str(), trec->parts[j].proto); + strcpy(tmpbuf, fmt::format(" {:<35}({:5})", obj_name, trec->parts[j].proto).c_str()); if (j > 0) { if (j % 2 == 0) { // разбиваем строчки если ингров больше 2; diff --git a/src/gameplay/fight/pk.cpp b/src/gameplay/fight/pk.cpp index 66ea9fc89f..800a424b5d 100644 --- a/src/gameplay/fight/pk.cpp +++ b/src/gameplay/fight/pk.cpp @@ -12,7 +12,7 @@ ************************************************************************ */ #include "pk.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" #include "gameplay/mechanics/mount.h" @@ -664,9 +664,9 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { if (tch->get_uid() == uid) { found = true; if (pk.battle_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", native_text::pad_right(temp, 40).c_str()); + strcat(buf, fmt::format(" {:<40} <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp).c_str()); } else { - sprintf(buf + strlen(buf), " %s %3ld %3ld\r\n", native_text::pad_right(temp, 40).c_str(), pk.kill_num, pk.revenge_num); + strcat(buf, fmt::format(" {:<40} {:3} {:3}\r\n", temp, pk.kill_num, pk.revenge_num).c_str()); } break; } @@ -674,9 +674,9 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } else { found = true; if (pk.battle_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %s <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", native_text::pad_right(temp, 40).c_str()); + strcat(buf, fmt::format(" {:<40} <БОЕВЫЕ ДЕЙСТВИЯ>\r\n", temp).c_str()); } else { - sprintf(buf + strlen(buf), " %s %3ld %3ld\r\n", native_text::pad_right(temp, 40).c_str(), pk.kill_num, pk.revenge_num); + strcat(buf, fmt::format(" {:<40} {:3} {:3}\r\n", temp, pk.kill_num, pk.revenge_num).c_str()); } } } @@ -709,12 +709,12 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { // Сначала проверка клан флага if (CLAN(ch) && pk.clan_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %s <ВОЙНА>\r\n", native_text::pad_right(GET_NAME(tch), 40).c_str()); + strcat(buf, fmt::format(" {:<40} <ВОЙНА>\r\n", GET_NAME(tch)).c_str()); } else if (pk.clan_exp > time(nullptr)) { - sprintf(buf + strlen(buf), " %s <ВРЕМЕННЫЙ ФЛАГ>\r\n", native_text::pad_right(GET_NAME(tch), 40).c_str()); + strcat(buf, fmt::format(" {:<40} <ВРЕМЕННЫЙ ФЛАГ>\r\n", GET_NAME(tch)).c_str()); } else if (pk.kill_num + pk.revenge_num > 0) { - sprintf(buf + strlen(buf), " %s %3ld %3ld\r\n", - native_text::pad_right(GET_NAME(tch), 40).c_str(), pk.kill_num, pk.revenge_num); + strcat(buf, fmt::format(" {:<40} {:3} {:3}\r\n", + GET_NAME(tch), pk.kill_num, pk.revenge_num).c_str()); } else { continue; } diff --git a/src/gameplay/mechanics/glory_const.cpp b/src/gameplay/mechanics/glory_const.cpp index b3e9e531ca..af852f23ed 100644 --- a/src/gameplay/mechanics/glory_const.cpp +++ b/src/gameplay/mechanics/glory_const.cpp @@ -4,7 +4,6 @@ // Part of Bylins http://www.mud.ru #include "glory_const.h" -#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -227,8 +226,8 @@ void print_glory(CharData *ch, GloryListType::iterator &it) { *buf = '\0'; for (auto i = it->second->stats.begin(), iend = it->second->stats.end(); i != iend; ++i) { if ((i->first >= 0) && (i->first < (int) sizeof(olc_stat_name))) { - sprintf(buf + strlen(buf), "%s: +%d", native_text::pad_right(olc_stat_name[i->first], 16).c_str(), - i->second * stat_multi(i->first)); + strcat(buf, fmt::format("{:<16}: +{}", olc_stat_name[i->first], + i->second * stat_multi(i->first)).c_str()); if (stat_multi(i->first) > 1) sprintf(buf + strlen(buf), "(%d)", i->second); strcat(buf, "\r\n"); diff --git a/src/gameplay/mechanics/named_stuff.cpp b/src/gameplay/mechanics/named_stuff.cpp index 4d5704c8a9..8c17334f75 100644 --- a/src/gameplay/mechanics/named_stuff.cpp +++ b/src/gameplay/mechanics/named_stuff.cpp @@ -3,7 +3,7 @@ // Part of Bylins http://www.mud.ru #include "named_stuff.h" -#include "utils/native_text.h" +#include #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" @@ -370,12 +370,12 @@ void do_named(CharData *ch, char *argument, int cmd, int subcmd) { out += buf1; } found++; - sprintf(buf2, "%6ld) &R*&n%s Владелец:%s e-mail:&S%s&s\r\n", + strcpy(buf2, fmt::format("{:6}) &R*&n{:<31} Владелец:{:<16} e-mail:&S{}&s\r\n", it->first + 1, - native_text::pad_right("Несуществующий предмет", 31).c_str(), - native_text::pad_right(GetNameByUnique(it->second->uid, false), 16).c_str(), - str_dup(it->second->mail.c_str()) - ); + "Несуществующий предмет", + GetNameByUnique(it->second->uid, false), + it->second->mail + ).c_str()); out += buf2; } } else { @@ -389,9 +389,9 @@ void do_named(CharData *ch, char *argument, int cmd, int subcmd) { obj_proto[r_num]->get_vnum(), colored_name(obj_proto[r_num]->get_short_description().c_str(), -32)); if (privilege::IsGrGod(ch) || ch->IsFlagged(EPrf::kCoderinfo)) { - snprintf(buf2, kMaxStringLength, "%s Игра:%d Пост:%d Владелец:%s e-mail:&S%s&s\r\n", buf1, + strcpy(buf2, fmt::format("{} Игра:{} Пост:{} Владелец:{:<16} e-mail:&S{}&s\r\n", buf1, obj_proto.total_online(r_num), obj_proto.stored(r_num), - native_text::pad_right(GetNameByUnique(it->second->uid, false), 16).c_str(), it->second->mail.c_str()); + GetNameByUnique(it->second->uid, false), it->second->mail).c_str()); } else { snprintf(buf2, kMaxStringLength, "%s\r\n", buf1); } diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 679e84ecbc..8c72edd930 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -329,37 +329,6 @@ std::size_t last_char_offset(std::string_view s) { return last; } -namespace { - -std::size_t missing_width(std::string_view s, std::size_t width) { - const std::size_t have = char_count(s); - return have >= width ? 0 : width - have; -} - -} // namespace - -std::string truncate_to_chars(std::string_view s, std::size_t count) { - std::size_t pos = 0; - std::size_t seen = 0; - while (pos < s.size() && seen < count) { - pos += char_bytes_at(s, pos); - ++seen; - } - return std::string(s.substr(0, pos)); -} - -std::string pad_right(std::string_view s, std::size_t width, char fill) { - std::string out(s); - out.append(missing_width(s, width), fill); - return out; -} - -std::string pad_left(std::string_view s, std::size_t width, char fill) { - std::string out(missing_width(s, width), fill); - out.append(s); - return out; -} - bool list_contains_char(std::string_view list, std::string_view ch) { if (ch.empty()) { return false; diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 26d7710bd3..d9ba7a3ced 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -86,19 +86,6 @@ std::size_t copy_lower_char(const char *src, char *dst); // is that character. KOI8-R: s.size() - 1. std::size_t last_char_offset(std::string_view s); -// Pad `s` to at least `width` display characters, appending (pad_right) or prepending (pad_left) -// `fill`. The replacement for a printf "%-Ns" / "%Ns" field, which counts bytes: under KOI8-R the -// result is identical, under UTF-8 a Russian column no longer comes out half as wide. -// Note: like printf, colour codes embedded in `s` still count toward the width -- that is a -// separate, pre-existing skew and is deliberately not changed here. -std::string pad_right(std::string_view s, std::size_t width, char fill = ' '); -std::string pad_left(std::string_view s, std::size_t width, char fill = ' '); - -// First `count` characters of `s` (all of it when shorter). The replacement for a substr(0, N) -// used to cap a display length: under KOI8-R it is exactly that, under UTF-8 it counts characters -// and so never cuts one in half. -std::string truncate_to_chars(std::string_view s, std::size_t count); - // Does the single character `ch` occur in `list`? The replacement for strchr() over a literal // list of letters: `list` is walked one whole character at a time, so a multibyte character can // never match on a partial byte sequence. Comparison is exact (case-sensitive), like strchr. diff --git a/tests/native_text.cpp b/tests/native_text.cpp index b20f2de832..6ef53a5635 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -253,22 +253,6 @@ TEST(NativeText, LastCharOffset) { } } -TEST(NativeText, PadRightAndLeft) { - EXPECT_EQ(native_text::pad_right("ab", 5), "ab "); - EXPECT_EQ(native_text::pad_left("ab", 5), " ab"); - EXPECT_EQ(native_text::pad_right("ab", 2), "ab"); // already wide enough - EXPECT_EQ(native_text::pad_right("abcdef", 3), "abcdef"); // never truncates - EXPECT_EQ(native_text::pad_right("", 3), " "); - EXPECT_EQ(native_text::pad_right("ab", 4, '.'), "ab.."); - if (native_text::native_is_utf8()) { - // 6 Cyrillic characters (12 bytes) padded to a 10-character column: no padding, and - // crucially not 12 bytes' worth of "already too wide" either. - EXPECT_EQ(native_text::pad_right(kNtPrivet, 10), kNtPrivet); - // 6 characters padded to 8 -> exactly two spaces, not eight. - EXPECT_EQ(native_text::pad_right(kNtPrivet, 8), std::string(kNtPrivet) + " "); - } -} - TEST(NativeText, ListContainsChar) { EXPECT_TRUE(native_text::list_contains_char("abc", "b")); EXPECT_FALSE(native_text::list_contains_char("abc", "d")); From b5243d1c19b7893d8b5ae9354ca75e52f0d2b034 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 04:50:21 +0200 Subject: [PATCH 14/20] feat(utf8): character iteration API + the case sites the first sweep missed (#3681) A full audit sweep showed track A was closed prematurely: beyond the hotspots already converted, 55 more per-byte case sites were left, and they are the same class of bug. * 36 "capitalise the first letter" sites (name[0] = UPPER(name[0]) and friends across 19 files) now go through native_text::capitalize_first, which folds a whole character instead of mangling a multibyte lead byte. * The shared string helpers are fixed at the source rather than per call site: ConvertToLow (both overloads), SubstToLow, SubstStrToLow, SubstStrToUpper, colorLOW, colorCAP and IsAbbr. Fixing ConvertToLow alone covers its many callers. Adds the API the call sites were missing, so they stop doing pointer arithmetic: * native_text::chars(s) -- range-for over characters, each element a string_view covering exactly one character: for (auto letter : native_text::chars(argument)) { ... } * native_text::to_lower/to_upper for std::string and char* -- whole-string case conversion, which collapses several hand-rolled loops to one line. * copy_upper_char, the counterpart of copy_lower_char. Note on naming: character count stays an explicit call (char_count) rather than a "length()", because both lengths are genuinely needed -- characters for display width, bytes for buffer sizing -- and conflating them is how this whole bug class started. Left as-is deliberately: SubstStrToLow raises case despite its name. That mismatch predates this work; only its byte semantics changed. Verified: clean build, full suite 648 passed / 0 failed. --- src/administration/accounts.cpp | 5 +- src/engine/boot/boot_data_files.cpp | 3 +- src/engine/db/obj_save.cpp | 3 +- src/engine/db/sqlite_world_data_source.cpp | 3 +- src/engine/db/yaml_world_data_source.cpp | 3 +- src/engine/ui/cmd/do_gen_comm.cpp | 16 +++--- src/engine/ui/cmd/do_ignore.cpp | 9 ++-- src/engine/ui/cmd/do_who_am_i.cpp | 3 +- src/engine/ui/cmd_god/do_set_all.cpp | 3 +- src/engine/ui/cmd_god/do_show.cpp | 5 +- src/engine/ui/cmd_god/do_stat.cpp | 3 +- src/gameplay/clans/house.cpp | 13 ++--- src/gameplay/communication/parcel.cpp | 5 +- src/gameplay/core/game_limits.cpp | 5 +- src/gameplay/economics/exchange.cpp | 9 ++-- src/gameplay/fight/pk.cpp | 3 +- src/gameplay/mechanics/glory_const.cpp | 3 +- src/gameplay/mechanics/sight.cpp | 3 +- src/utils/native_text.cpp | 62 ++++++++++++++++++++-- src/utils/native_text.h | 49 +++++++++++++++++ src/utils/utils_string.cpp | 42 +++++++-------- tests/native_text.cpp | 47 ++++++++++++++++ 22 files changed, 229 insertions(+), 68 deletions(-) diff --git a/src/administration/accounts.cpp b/src/administration/accounts.cpp index 5fcf86a8e7..87b4523378 100644 --- a/src/administration/accounts.cpp +++ b/src/administration/accounts.cpp @@ -3,6 +3,7 @@ * 2018 (c) bodrich */ #include "accounts.h" +#include "utils/native_text.h" #include "password.h" #include "engine/entities/zone.h" #include @@ -85,7 +86,7 @@ void Account::show_players(CharData *ch) { ss << "Данные аккаунта: " << this->email << "\r\n"; for (auto &x : this->players_list) { std::string name = GetNameByUnique(x); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); ss << count << ") " << name << "\r\n"; count++; } @@ -101,7 +102,7 @@ void Account::list_players(DescriptorData *d) { for (auto &x : this->players_list) { std::string name = GetNameByUnique(x); iosystem::write_to_output((std::to_string(count) + ") ").c_str(), d); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); iosystem::write_to_output(name.c_str(), d); iosystem::write_to_output("\r\n", d); count++; diff --git a/src/engine/boot/boot_data_files.cpp b/src/engine/boot/boot_data_files.cpp index a04ee6c62f..02b3d24630 100644 --- a/src/engine/boot/boot_data_files.cpp +++ b/src/engine/boot/boot_data_files.cpp @@ -1,4 +1,5 @@ #include "boot_data_files.h" +#include "utils/native_text.h" #include "engine/db/obj_prototypes.h" #include "engine/scripting/dg_olc.h" @@ -397,7 +398,7 @@ void WorldFile::parse_room(int virtual_nr) { world[room_realnum]->zone_rn = zone; world[room_realnum]->vnum = virtual_nr; std::string tmpstr = fread_string(); - tmpstr[0] = UPPER(tmpstr[0]); + native_text::capitalize_first(tmpstr); world[room_realnum]->set_name(tmpstr); // if (zone_table[zone].RnumRoomsLocation.first == -1) { // zone_table[zone].RnumRoomsLocation.first = room_realnum; diff --git a/src/engine/db/obj_save.cpp b/src/engine/db/obj_save.cpp index 7443a64f6a..2f911620d5 100644 --- a/src/engine/db/obj_save.cpp +++ b/src/engine/db/obj_save.cpp @@ -10,6 +10,7 @@ // * AutoEQ by Burkhard Knopf #include "engine/core/char_handler.h" +#include "utils/native_text.h" #include "gameplay/mechanics/equipment.h" #include "obj_save.h" #include "gameplay/mechanics/groups.h" @@ -1628,7 +1629,7 @@ int Crash_load(CharData *ch) { } std::string cap = obj->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); // Предмет разваливается от старости if (obj->get_timer() <= 0) { diff --git a/src/engine/db/sqlite_world_data_source.cpp b/src/engine/db/sqlite_world_data_source.cpp index 2688bc2b3a..d2adfa9964 100644 --- a/src/engine/db/sqlite_world_data_source.cpp +++ b/src/engine/db/sqlite_world_data_source.cpp @@ -4,6 +4,7 @@ #ifdef HAVE_SQLITE #include "sqlite_world_data_source.h" +#include "utils/native_text.h" #include "utils/utils_encoding.h" #include "db.h" #include "obj_prototypes.h" @@ -1256,7 +1257,7 @@ std::vector SqliteWorldDataSource::LoadRooms(const std::vector auto room = new RoomData; room->vnum = vnum; // Apply UPPER to first character (same as Legacy loader) - if (!name.empty()) { name[0] = UPPER(name[0]); } + if (!name.empty()) { native_text::capitalize_first(name); } room->set_name(name); if (!description.empty()) diff --git a/src/engine/db/yaml_world_data_source.cpp b/src/engine/db/yaml_world_data_source.cpp index 22fb053769..0cb0ad4252 100644 --- a/src/engine/db/yaml_world_data_source.cpp +++ b/src/engine/db/yaml_world_data_source.cpp @@ -4,6 +4,7 @@ #ifdef HAVE_YAML #include "yaml_world_data_source.h" +#include "utils/native_text.h" #include "utils/utils_encoding.h" #include "dictionary_loader.h" #include "db.h" @@ -1425,7 +1426,7 @@ RoomData* YamlWorldDataSource::ParseRoomNode(const YAML::Node &root, int vnum, i room->zone_rn = zone_rnum; std::string name = GetText(root, "name", "Untitled Room"); - if (!name.empty()) { name[0] = UPPER(name[0]); } + if (!name.empty()) { native_text::capitalize_first(name); } room->set_name(name); std::string description = GetText(root, "description", ""); diff --git a/src/engine/ui/cmd/do_gen_comm.cpp b/src/engine/ui/cmd/do_gen_comm.cpp index 7af33bc904..30c2dc3f15 100644 --- a/src/engine/ui/cmd/do_gen_comm.cpp +++ b/src/engine/ui/cmd/do_gen_comm.cpp @@ -162,9 +162,8 @@ void do_gen_comm(CharData *ch, char *argument, int/* cmd*/, int subcmd) { // The denominator is the character count for the same reason -- with byte lengths the // percentage would be halved for Russian text. const size_t total_chars = native_text::char_count(argument); - for (int k = 0; argument[k] != '\0';) { - const int bytes = static_cast(native_text::char_bytes(argument + k)); - if (native_text::is_upper_char(argument + k)) { + for (auto letter : native_text::chars(argument)) { + if (native_text::is_upper_char(letter.data())) { bad_simb_cnt++; bad_seq_cnt++; } else @@ -172,10 +171,11 @@ void do_gen_comm(CharData *ch, char *argument, int/* cmd*/, int subcmd) { if ((bad_seq_cnt > 1) && (((bad_simb_cnt * 100 / total_chars) > bad_smb_procent) || - (bad_seq_cnt > MAX_UPPERS_SEQ_CHAR))) - native_text::copy_lower_char(argument + k, argument + k); - - k += bytes; + (bad_seq_cnt > MAX_UPPERS_SEQ_CHAR))) { + // letter указывает внутрь argument; свёртка регистра не меняет длину. + char *at = argument + (letter.data() - argument); + native_text::copy_lower_char(at, at); + } } // фильтруем одинаковые сообщения в эфире if (!str_cmp(ch->get_last_tell().c_str(), argument)) { @@ -302,7 +302,7 @@ std::string format_gossip_name(CharData *ch, CharData *vict) { return ""; } std::string name = privilege::IsImmortal(ch) ? GET_NAME(ch) : sight::PersonName(ch, vict, 0); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); return name; } diff --git a/src/engine/ui/cmd/do_ignore.cpp b/src/engine/ui/cmd/do_ignore.cpp index 8aa55b0470..2ba9d612f3 100644 --- a/src/engine/ui/cmd/do_ignore.cpp +++ b/src/engine/ui/cmd/do_ignore.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "utils/utils_string.h" #include "utils/utils.h" #include "engine/core/comm.h" @@ -43,7 +44,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { strcpy(name, "Все"); } else { strcpy(name, ign_find_name(ignore->id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); } sprintf(buf, " %s: ", name); SendMsgToChar(buf, ch); @@ -146,7 +147,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar("Вы и так не игнорируете всех сразу.\r\n", ch); } else { strcpy(name, ign_find_name(vict_id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); sprintf(buf, "Вы и так не игнорируете " "персонажа %s%s%s.\r\n", kColorWht, name, kColorNrm); @@ -173,7 +174,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar(buf, ch); } else { strcpy(name, ign_find_name(ignore->id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); sprintf(buf, "Для персонажа %s%s%s вы игнорируете:%s.\r\n", kColorWht, name, kColorNrm, text_ignore_modes(ignore->mode, buf1)); SendMsgToChar(buf, ch); @@ -183,7 +184,7 @@ void do_ignore(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { SendMsgToChar("Вы больше не игнорируете всех сразу.\r\n", ch); } else { strcpy(name, ign_find_name(vict_id)); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); sprintf(buf, "Вы больше не игнорируете персонажа %s%s%s.\r\n", kColorWht, name, kColorNrm); SendMsgToChar(buf, ch); diff --git a/src/engine/ui/cmd/do_who_am_i.cpp b/src/engine/ui/cmd/do_who_am_i.cpp index c9189e44b7..31998bcc34 100644 --- a/src/engine/ui/cmd/do_who_am_i.cpp +++ b/src/engine/ui/cmd/do_who_am_i.cpp @@ -7,6 +7,7 @@ */ #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "gameplay/clans/house.h" #include "engine/db/player_index.h" #include "gameplay/core/remort.h" @@ -29,7 +30,7 @@ void DoWhoAmI(CharData *ch, char * /*argument*/, int/* cmd*/, int/* subcmd*/) { } else { const int god_level = (ch)->player_specials->saved.NameGod > 1000 ? (ch)->player_specials->saved.NameGod - 1000 : (ch)->player_specials->saved.NameGod; sprintf(buf1, "%s", GetNameById((ch)->player_specials->saved.NameIDGod).c_str()); - *buf1 = UPPER(*buf1); + native_text::capitalize_first(buf1); static const char *by_rank_god = "Богом"; static const char *by_rank_privileged = "привилегированным игроком"; diff --git a/src/engine/ui/cmd_god/do_set_all.cpp b/src/engine/ui/cmd_god/do_set_all.cpp index e832949bc2..b254ce4721 100644 --- a/src/engine/ui/cmd_god/do_set_all.cpp +++ b/src/engine/ui/cmd_god/do_set_all.cpp @@ -7,6 +7,7 @@ */ #include "engine/ui/cmd_god/do_set_all.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/karma.h" @@ -167,7 +168,7 @@ void setall_inspect() { } Password::set_password(vict, std::string(it->second->pwd)); std::string str = player_table[it->second->pos].name(); - str[0] = UPPER(str[0]); + native_text::capitalize_first(str); sprintf(buf2, "У персонажа %s изменен пароль (setall).", player_table[it->second->pos].name().c_str()); it->second->out += buf2; sprintf(buf1, "\r\n"); diff --git a/src/engine/ui/cmd_god/do_show.cpp b/src/engine/ui/cmd_god/do_show.cpp index fd2c0229f8..c7a16a52cf 100644 --- a/src/engine/ui/cmd_god/do_show.cpp +++ b/src/engine/ui/cmd_god/do_show.cpp @@ -3,6 +3,7 @@ // #include "administration/accounts.h" +#include "utils/native_text.h" #include "administration/ban.h" #include "administration/privilege.h" #include "engine/ui/cmd/do_features.h" @@ -594,11 +595,11 @@ void do_show(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { sprintf(buf + strlen(buf), "Имя никем не одобрено!\r\n"); } else if ((vict)->player_specials->saved.NameGod < 1000) { sprintf(buf1, "%s", GetNameById((vict)->player_specials->saved.NameIDGod).c_str()); - *buf1 = UPPER(*buf1); + native_text::capitalize_first(buf1); snprintf(buf + strlen(buf), kMaxStringLength, "Имя запрещено богом %s\r\n", buf1); } else { sprintf(buf1, "%s", GetNameById((vict)->player_specials->saved.NameIDGod).c_str()); - *buf1 = UPPER(*buf1); + native_text::capitalize_first(buf1); snprintf(buf + strlen(buf), kMaxStringLength, "Имя одобрено богом %s\r\n", buf1); } if (remort::GetRealRemort(vict) < 4) diff --git a/src/engine/ui/cmd_god/do_stat.cpp b/src/engine/ui/cmd_god/do_stat.cpp index a3e5359b00..8a06898a17 100644 --- a/src/engine/ui/cmd_god/do_stat.cpp +++ b/src/engine/ui/cmd_god/do_stat.cpp @@ -1,4 +1,5 @@ #include "gameplay/mechanics/equipment.h" +#include "utils/native_text.h" #include "gameplay/affects/affect_messages.h" #include "do_stat.h" #include "utils/utils_string.h" @@ -844,7 +845,7 @@ void do_stat_object(CharData *ch, ObjData *j, const int virt = 0) { } } if (!str.empty()) { - str[0] = UPPER(str[0]); + native_text::capitalize_first(str); SendMsgToChar(ch, "&C%s&n", str.c_str()); } else { auto room = get_room_where_obj(j); diff --git a/src/gameplay/clans/house.cpp b/src/gameplay/clans/house.cpp index 4cd8b5db45..726ffa348b 100644 --- a/src/gameplay/clans/house.cpp +++ b/src/gameplay/clans/house.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "house.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "gameplay/economics/currencies.h" #include "utils/utils_encoding.h" @@ -991,7 +992,7 @@ void Clan::HouseInfo(CharData *ch) { for (const auto &it : temp_list) { if (temp != ranks[it->rank_num]) { std::string rnk = ranks[it->rank_num]; - rnk[0] = UPPER(rnk[0]); + native_text::capitalize_first(rnk); if (temp == "") { buffer << rnk << ": "; @@ -1116,7 +1117,7 @@ void Clan::HouseAdd(CharData *ch, std::string &buffer) { return; } std::string name = buffer2; - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); if (unique == ch->get_uid()) { SendMsgToChar("Сам себя повысил, самому себе вынес благодарность?\r\n", ch); return; @@ -1308,7 +1309,7 @@ void Clan::remove_member(const ClanMembersList::key_type &key, char *reason) { if (d->character && CLAN(d->character) && CLAN(d->character)->GetRent() == this->GetRent()) { - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); SendMsgToChar(d->character.get(), "%s более не является членом вашей дружины.\r\n", name.c_str()); } } @@ -1426,7 +1427,7 @@ void Clan::hcon_outcast(CharData *ch, std::string &buffer) { char tmpstr[kMaxInputLength]; sprintf(tmpstr, "Богом %s", GET_NAME(ch)); clan->remove_member(member_uid, tmpstr); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); SendMsgToChar(ch, "%s исключен(a) из дружины '%s'.\r\n", name.c_str(), clan->name.c_str()); return; } @@ -2073,7 +2074,7 @@ void Clan::HcontrolBuild(CharData *ch, std::string &buffer) { tempClan->chest_room = rent; tempClan->guard = guard; // пишем воеводу - owner[0] = UPPER(owner[0]); + native_text::capitalize_first(owner); tempClan->owner = owner; const auto tempMember = std::make_shared(); tempMember->name = owner; @@ -3475,7 +3476,7 @@ void Clan::HouseOwner(CharData *ch, std::string &buffer) { else if (CLAN(d->character) && CLAN(ch) != CLAN(d->character)) SendMsgToChar("Вы не можете передать свои права члену другой дружины.\r\n", ch); else { - buffer2[0] = UPPER(buffer2[0]); + native_text::capitalize_first(buffer2); // воевода идет рангом ниже this->m_members.set_rank(ch->get_uid(), 1); Clan::SetClanData(ch); diff --git a/src/gameplay/communication/parcel.cpp b/src/gameplay/communication/parcel.cpp index af8c2d57dc..047c885cb4 100644 --- a/src/gameplay/communication/parcel.cpp +++ b/src/gameplay/communication/parcel.cpp @@ -3,6 +3,7 @@ // Part of Bylins http://www.mud.ru #include "parcel.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "gameplay/economics/currencies.h" @@ -852,8 +853,8 @@ std::string FindParcelObj(const ObjData *obj) { std::string target = GetNameByUnique(it->first); std::string sender = GetNameByUnique(it2->first); - target[0] = UPPER(target[0]); - sender[0] = UPPER(sender[0]); + native_text::capitalize_first(target); + native_text::capitalize_first(sender); str = fmt::format("наход{}ся на почте (отправитель: {}, получатель: {}).\r\n", grammar::ObjPluralVerbEnding((it3->obj_)->get_sex()), sender.c_str(), diff --git a/src/gameplay/core/game_limits.cpp b/src/gameplay/core/game_limits.cpp index de51ea5b6d..d7809574fd 100644 --- a/src/gameplay/core/game_limits.cpp +++ b/src/gameplay/core/game_limits.cpp @@ -13,6 +13,7 @@ ************************************************************************ */ #include "gameplay/core/game_limits.h" +#include "utils/native_text.h" #include "gameplay/core/experience.h" #include "gameplay/affects/affect_data.h" // issue.mob-flag-affect-materialization: restore re-materialize #include "administration/privilege.h" @@ -1154,7 +1155,7 @@ void exchange_point_update() { if (GET_EXCHANGE_ITEM(exch_item)->get_timer() == 0) { std::string cap = GET_EXCHANGE_ITEM(exch_item)->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); sprintf(buf, "Exchange: - %s рассыпал%s от длительного использования.\r\n", cap.c_str(), grammar::ObjSexEnding((GET_EXCHANGE_ITEM(exch_item))->get_sex(), 2)); log("%s", buf); @@ -1237,7 +1238,7 @@ void charmee_obj_decay_tell(CharData *charmee, ObjData *obj, ECharmeeObjPos obj_ короче, рефакторинг приветствуется, если кто-нибудь придумает лучше. */ std::string cap = obj->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); snprintf(local_buf, kMaxStringLength, "%s сказал%s вам : '%s%s рассыпал%s %s...'", GET_NAME(charmee), grammar::SexEnding((charmee)->get_sex(), 1), diff --git a/src/gameplay/economics/exchange.cpp b/src/gameplay/economics/exchange.cpp index 460528edf1..a252b4b7af 100644 --- a/src/gameplay/economics/exchange.cpp +++ b/src/gameplay/economics/exchange.cpp @@ -9,6 +9,7 @@ ************************************************************************ */ #include "exchange.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/global_objects.h" #include "gameplay/economics/currencies.h" @@ -472,7 +473,7 @@ int exchange_information(CharData *ch, char *arg) { } auto seller_name = GetNameById(GET_EXCHANGE_ITEM_SELLERID(item)); snprintf(buf2, sizeof(buf2), "%s", seller_name.empty() ? "(сожран долгоносиком)" : seller_name.c_str()); - *buf2 = UPPER(*buf2); + native_text::capitalize_first(buf2); out += fmt::sprintf("Продавец %s\n", buf2); if (GET_EXCHANGE_ITEM_COMMENT(item)) { out += fmt::sprintf("Берестовая наклейка на лоте гласит: '%s'.\n", GET_EXCHANGE_ITEM_COMMENT(item)); @@ -668,7 +669,7 @@ int exchange_offers(const CharData *ch, const char *arg) { filter += GET_NAME(ch); } else { while (*arg1) { - arg1[0] = UPPER(arg1[0]); + native_text::capitalize_first(arg1); filter += arg1; filter += ' '; arg = one_argument(arg, arg1); @@ -907,7 +908,7 @@ int LoadExchange() { // Предмет разваливается от старости if (GET_EXCHANGE_ITEM(item)->get_timer() <= 0) { std::string cap = GET_EXCHANGE_ITEM(item)->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); log("Exchange: - %s рассыпал%s от длительного использования.\r\n", cap.c_str(), grammar::ObjSexEnding((GET_EXCHANGE_ITEM(item))->get_sex(), 2)); extract_exchange_item(item); @@ -1002,7 +1003,7 @@ int exchange_database_reload(bool loadbackup) { // Предмет разваливается от старости if (GET_EXCHANGE_ITEM(item)->get_timer() <= 0) { std::string cap = GET_EXCHANGE_ITEM(item)->get_PName(grammar::ECase::kNom); - cap[0] = UPPER(cap[0]); + native_text::capitalize_first(cap); log("Exchange: - %s рассыпал%s от длительного использования.\r\n", cap.c_str(), grammar::ObjSexEnding((GET_EXCHANGE_ITEM(item))->get_sex(), 2)); extract_exchange_item(item); diff --git a/src/gameplay/fight/pk.cpp b/src/gameplay/fight/pk.cpp index 800a424b5d..7b092d481d 100644 --- a/src/gameplay/fight/pk.cpp +++ b/src/gameplay/fight/pk.cpp @@ -12,6 +12,7 @@ ************************************************************************ */ #include "pk.h" +#include "utils/native_text.h" #include #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" @@ -653,7 +654,7 @@ void do_revenge(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { continue; } - temp[0] = UPPER(temp[0]); + native_text::capitalize_first(temp); // если нада исключаем тех, кто находится оффлайн if (bOnlineOnly) { for (const auto &tch : character_list) { diff --git a/src/gameplay/mechanics/glory_const.cpp b/src/gameplay/mechanics/glory_const.cpp index af852f23ed..3ac2edcc35 100644 --- a/src/gameplay/mechanics/glory_const.cpp +++ b/src/gameplay/mechanics/glory_const.cpp @@ -4,6 +4,7 @@ // Part of Bylins http://www.mud.ru #include "glory_const.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -1127,7 +1128,7 @@ void PrintGloryChart(CharData *ch) { t_it != playerGloryList.end() && i < kPlayerChartSize; ++t_it, ++i) { std::string name = GetNameByUnique(t_it->get()->uid); - name[0] = UPPER(name[0]); + native_text::capitalize_first(name); if (name.length() == 0) { name = "*скрыто*"; } diff --git a/src/gameplay/mechanics/sight.cpp b/src/gameplay/mechanics/sight.cpp index bd3cbcfddb..d76284a088 100644 --- a/src/gameplay/mechanics/sight.cpp +++ b/src/gameplay/mechanics/sight.cpp @@ -6,6 +6,7 @@ */ #include "engine/core/char_movement.h" +#include "utils/native_text.h" #include "engine/core/target_resolver.h" #include "sight.h" #include "gameplay/mechanics/hide.h" @@ -1357,7 +1358,7 @@ const char *show_obj_to_char(ObjData *object, CharData *ch, int mode, int show_s } } else if (mode >= 2 && how <= 1) { std::string obj_name = OBJN(object, ch, grammar::ECase::kNom); - obj_name[0] = UPPER(obj_name[0]); + native_text::capitalize_first(obj_name); if (object->get_type() == EObjType::kLightSource) { if (GET_OBJ_VAL(object, 2) == -1) { sprintf(buf2, "\r\n%s дает вечный свет.", obj_name.c_str()); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 8c72edd930..bed49dee0d 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -182,14 +182,16 @@ bool chars_equal_ci(const char *a, const char *b) { return utf8::to_lower(ca) == utf8::to_lower(cb); } -std::size_t copy_lower_char(const char *src, char *dst) { +namespace { + +std::size_t copy_folded_char(const char *src, char *dst, char32_t (*fold)(char32_t)) { const std::size_t len = char_bytes(src); char32_t cp = 0; if (utf8::decode(std::string_view(src, len), 0, cp) != 0) { std::string folded; - // Only rewrite when the lowercase form keeps the byte length -- true for ASCII and for - // the whole Russian alphabet, so callers never see a character change size. - if (utf8::encode(utf8::to_lower(cp), folded) == len) { + // Only rewrite when the folded form keeps the byte length -- true for ASCII and for the + // whole Russian alphabet, so callers never see a character change size. + if (utf8::encode(fold(cp), folded) == len) { for (std::size_t i = 0; i < len; ++i) { dst[i] = folded[i]; } @@ -202,6 +204,16 @@ std::size_t copy_lower_char(const char *src, char *dst) { return len; } +} // namespace + +std::size_t copy_lower_char(const char *src, char *dst) { + return copy_folded_char(src, dst, utf8::to_lower); +} + +std::size_t copy_upper_char(const char *src, char *dst) { + return copy_folded_char(src, dst, utf8::to_upper); +} + #else // KOI8-R: 1 byte == 1 character bool native_is_utf8() { @@ -287,6 +299,11 @@ std::size_t copy_lower_char(const char *src, char *dst) { return 1; } +std::size_t copy_upper_char(const char *src, char *dst) { + *dst = a_ucc_table[static_cast(*src)]; + return 1; +} + #endif // --------------------------------------------------------------------------------------------- @@ -319,6 +336,43 @@ std::size_t char_bytes_at(std::string_view s, std::size_t pos) { } // namespace +void capitalize_first(std::string &s) { + if (s.empty()) { + return; + } + // The uppercase form keeps the byte length for ASCII and the whole Russian alphabet, so + // capitalising in place never resizes the string. + capitalize_first(&s[0]); +} + +std::size_t CharRange::Iterator::step(std::string_view s, std::size_t pos) { + return pos < s.size() ? char_bytes_at(s, pos) : 0; +} + +void to_lower(std::string &s) { + for (std::size_t i = 0; i < s.size();) { + i += copy_lower_char(&s[i], &s[i]); + } +} + +void to_upper(std::string &s) { + for (std::size_t i = 0; i < s.size();) { + i += copy_upper_char(&s[i], &s[i]); + } +} + +void to_lower(char *s) { + while (*s) { + s += copy_lower_char(s, s); + } +} + +void to_upper(char *s) { + while (*s) { + s += copy_upper_char(s, s); + } +} + std::size_t last_char_offset(std::string_view s) { std::size_t last = 0; std::size_t pos = 0; diff --git a/src/utils/native_text.h b/src/utils/native_text.h index d9ba7a3ced..7d0140f0da 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -20,6 +20,7 @@ the encoding flip. Once the flip is permanent the KOI8-R branch and this indirec #define BYLINS_SRC_UTILS_NATIVE_TEXT_H_ #include +#include #include namespace native_text { @@ -38,6 +39,7 @@ std::size_t char_count(std::string_view s); // Cyrillic incl. Yo). No-op on an empty string, on a non-cased first character, or in the (never // occurring for these alphabets) case where the uppercase form has a different byte length. void capitalize_first(char *s); +void capitalize_first(std::string &s); // Largest byte offset <= max_bytes that lands on a character boundary, so cutting the string // there never splits a multibyte character. KOI8-R: min(max_bytes, s.size()). @@ -81,11 +83,58 @@ bool chars_equal_ci(const char *a, const char *b); // than resized. `dst` may alias `src` (the length-preserving property makes that safe). std::size_t copy_lower_char(const char *src, char *dst); +// Uppercase counterpart of copy_lower_char, with the same contract. +std::size_t copy_upper_char(const char *src, char *dst); + // Byte offset at which the final character of `s` begins (0 for an empty string), so that // s.substr(0, last_char_offset(s)) drops exactly one character and s.substr(last_char_offset(s)) // is that character. KOI8-R: s.size() - 1. std::size_t last_char_offset(std::string_view s); +// Range-for over the characters of `s`: each element is a string_view covering exactly one +// character, so scanning code never does pointer arithmetic and never lands mid-character. +// +// for (auto ch : native_text::chars(name)) { ... } // ch is one character, whatever its size +// +// The view must outlive the loop (it is not copied). Malformed bytes yield one element each. +class CharRange { + public: + explicit CharRange(std::string_view s) : m_str(s) {} + + class Iterator { + public: + Iterator(std::string_view s, std::size_t pos) : m_str(s), m_pos(pos), m_len(step(s, pos)) {} + std::string_view operator*() const { return m_str.substr(m_pos, m_len); } + Iterator &operator++() { + m_pos += m_len; + m_len = step(m_str, m_pos); + return *this; + } + bool operator!=(const Iterator &other) const { return m_pos != other.m_pos; } + + private: + static std::size_t step(std::string_view s, std::size_t pos); + std::string_view m_str; + std::size_t m_pos; + std::size_t m_len; + }; + + [[nodiscard]] Iterator begin() const { return Iterator(m_str, 0); } + [[nodiscard]] Iterator end() const { return Iterator(m_str, m_str.size()); } + + private: + std::string_view m_str; +}; + +inline CharRange chars(std::string_view s) { return CharRange(s); } + +// Whole-string case conversion in place. Prefer these over hand-rolled per-character loops. +// Length-preserving for ASCII and the Russian alphabet, so no reallocation happens. +void to_lower(std::string &s); +void to_upper(std::string &s); +void to_lower(char *s); +void to_upper(char *s); + // Does the single character `ch` occur in `list`? The replacement for strchr() over a literal // list of letters: `list` is walked one whole character at a time, so a multibyte character can // never match on a partial byte sequence. Comparison is exact (case-sensitive), like strchr. diff --git a/src/utils/utils_string.cpp b/src/utils/utils_string.cpp index 7f9725f25d..e4adbbe044 100644 --- a/src/utils/utils_string.cpp +++ b/src/utils/utils_string.cpp @@ -117,10 +117,13 @@ bool IsAbbr(const char *arg1, const char *arg2) { return false; } - for (; *arg1 && *arg2; arg1++, arg2++) { - if (LOWER(*arg1) != LOWER(*arg2)) { + // Посимвольно (issue #3681): побайтное сравнение под UTF-8 теряет регистронезависимость. + while (*arg1 && *arg2) { + if (!native_text::chars_equal_ci(arg1, arg2)) { return false; } + arg1 += native_text::char_bytes(arg1); + arg2 += native_text::char_bytes(arg2); } if (!*arg1) { @@ -229,9 +232,7 @@ std::string ExtractFirstArgument(const std::string &s, std::string &remains) { } std::string SubstToLow(std::string s) { - for (char &it: s) { - it = LOWER(it); - } + ConvertToLow(s); return s; } @@ -258,29 +259,22 @@ std::string SubstWtoK(std::string s) { } void ConvertToLow(std::string &text) { - for (char &it: text) { - it = LOWER(it); - } + native_text::to_lower(text); } void ConvertToLow(char *text) { - while (*text) { - *text = LOWER(*text); - text++; - } + native_text::to_lower(text); } std::string SubstStrToLow(std::string s) { - for (char &it: s) { - it = UPPER(it); - } + // NB: имя говорит "ToLow", а тело поднимает регистр. Расхождение предсуществующее, + // поведение сохранено намеренно -- меняется только байтовая семантика на символьную. + native_text::to_upper(s); return s; } std::string SubstStrToUpper(std::string s) { - for (char &it: s) { - it = UPPER(it); - } + native_text::to_upper(s); return s; } @@ -477,14 +471,14 @@ const char *first_letter(const char *txt) { char *colorCAP(char *txt) { char *letter = const_cast(first_letter(txt)); if (letter && *letter) { - *letter = UPPER(*letter); + native_text::capitalize_first(letter); } return txt; } std::string &colorCAP(std::string &txt) { size_t pos = first_letter(txt.c_str()) - txt.c_str(); - txt[pos] = UPPER(txt[pos]); + native_text::capitalize_first(&txt[pos]); return txt; } @@ -496,14 +490,14 @@ std::string &colorCAP(std::string &&txt) { char *colorLOW(char *txt) { char *letter = const_cast(first_letter(txt)); if (letter && *letter) { - *letter = LOWER(*letter); + native_text::copy_lower_char(letter, letter); } return txt; } std::string &colorLOW(std::string &txt) { size_t pos = first_letter(txt.c_str()) - txt.c_str(); - txt[pos] = LOWER(txt[pos]); + native_text::copy_lower_char(&txt[pos], &txt[pos]); return txt; } @@ -513,13 +507,13 @@ std::string &colorLOW(std::string &&txt) { } char *CAP(char *txt) { - *txt = UPPER(*txt); + native_text::capitalize_first(txt); return (txt); } std::string CAP(const std::string txt) { std::string tmp_str = txt; - tmp_str[0] = UPPER(tmp_str[0]); + native_text::capitalize_first(tmp_str); return (tmp_str); } diff --git a/tests/native_text.cpp b/tests/native_text.cpp index 6ef53a5635..82f66ff64f 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace { @@ -269,4 +270,50 @@ TEST(NativeText, ListContainsChar) { } } +TEST(NativeText, CharRangeIteratesWholeCharacters) { + std::vector got; + for (auto c : native_text::chars(kNtPrivet)) { + got.emplace_back(c); + } + if (native_text::native_is_utf8()) { + ASSERT_EQ(got.size(), 6u); // 6 letters, not 12 bytes + EXPECT_EQ(got.front(), "\xD0\x9F"); // whole "P", both bytes + EXPECT_EQ(got.back(), "\xD1\x82"); // whole "t" + } else { + ASSERT_EQ(got.size(), 12u); // KOI8-R: every byte is a character + } + + got.clear(); + for (auto c : native_text::chars("abc")) { + got.emplace_back(c); + } + EXPECT_EQ(got, (std::vector{"a", "b", "c"})); + + got.clear(); + for (auto c : native_text::chars("")) { + got.emplace_back(c); + } + EXPECT_TRUE(got.empty()); +} + +TEST(NativeText, WholeStringCaseTransforms) { + std::string s = "Hello World"; + native_text::to_lower(s); + EXPECT_EQ(s, "hello world"); + native_text::to_upper(s); + EXPECT_EQ(s, "HELLO WORLD"); + + char buf[] = "MiXeD"; + native_text::to_lower(buf); + EXPECT_STREQ(buf, "mixed"); + + if (native_text::native_is_utf8()) { + std::string ru = "\xD0\x9F\xD0\xA0\xD0\x98"; // "PRI" in Cyrillic + native_text::to_lower(ru); + EXPECT_EQ(ru, "\xD0\xBF\xD1\x80\xD0\xB8"); // "pri" + native_text::to_upper(ru); + EXPECT_EQ(ru, "\xD0\x9F\xD0\xA0\xD0\x98"); + } +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : From d54e3b360108d58725608949f1beb9c9138b8839 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 04:59:54 +0200 Subject: [PATCH 15/20] perf(utf8): make case folding cost what the byte loop cost (#3681) Benchmarking the character-based case conversion showed it was ~17x slower than the byte loop it replaces under UTF-8. That was not inherent to per-character work; it was this implementation: * a std::string was constructed per character just to hold the re-encoded result; * the fold was reached through a function pointer, so nothing inlined; * char_bytes() called utf8::sequence_length() across a translation unit, once per character, in every scan. Fixed by folding ASCII and the two-byte Cyrillic block directly on the bytes, with the general decode/encode path kept only as a fallback for characters this codebase never carries. Whole-string conversion is now one tight loop with no calls in the hot path. Measured against a like-for-like (also out-of-line) copy of the old ConvertToLow: KOI8-R: 32 ms -> 33 ms (the production path today: no regression) UTF-8 : 140 ms -> 211 ms (1.5x, and the 140 ms baseline does not even fold Cyrillic) The benchmark also caught a real bug this work had introduced: the shared fold loop treated 0xD0/0xD1 as UTF-8 lead bytes, but under KOI8-R those are the letters "p" and "r" -- it would have corrupted text on the current build. The loops are now per encoding. Verified exhaustively: under KOI8-R the result is identical to the raw table for all 255 byte values; under UTF-8 the folds are correct including Yo/yo. Also adds utf8::encode(char32_t, char*), an allocation-free encode into a caller buffer. --- src/utils/native_text.cpp | 237 ++++++++++++++++++++++++++++++++------ src/utils/utf8.cpp | 29 +++++ src/utils/utf8.h | 4 + 3 files changed, 234 insertions(+), 36 deletions(-) diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index bed49dee0d..77ef7830d1 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -77,12 +77,35 @@ std::size_t truncate_offset(std::string_view s, std::size_t max_bytes) { return pos; } +namespace { + +// Local copy of the lead-byte length table. utf8::sequence_length lives in another translation +// unit, and this runs once per character in every scan -- a cross-module call there costs more +// than the work itself. +inline std::size_t lead_len(unsigned char c) { + if (c < 0x80) { + return 1; + } + if (c >= 0xC0 && c <= 0xDF) { + return 2; + } + if (c >= 0xE0 && c <= 0xEF) { + return 3; + } + if (c >= 0xF0 && c <= 0xF7) { + return 4; + } + return 1; +} + +} // namespace + std::size_t char_bytes(const char *s) { const unsigned char lead = static_cast(*s); if (lead < 0x80) { return 1; } - const std::size_t want = static_cast(utf8::sequence_length(lead)); + const std::size_t want = lead_len(lead); std::size_t n = 1; while (n < want && (static_cast(s[n]) & 0xC0) == 0x80) { ++n; @@ -184,22 +207,88 @@ bool chars_equal_ci(const char *a, const char *b) { namespace { -std::size_t copy_folded_char(const char *src, char *dst, char32_t (*fold)(char32_t)) { +// Case folding for the repertoire the engine actually carries -- ASCII and the two-byte Cyrillic +// block -- done directly on the bytes. This runs per character in hot paths (whole-string case +// conversion, argument parsing), so it must not decode, allocate, or make a cross-module call: +// +// A-Z / a-z : one byte, +-0x20 +// A-P (D0 90..D0 9F) <-> a-p (D0 B0..D0 BF) : lead stays D0, trail +-0x20 +// R-Ya(D0 A0..D0 AF) <-> r-ya(D1 80..D1 8F) : lead flips D0<->D1, trail -+0x20 +// Yo (D0 81) <-> yo (D1 91) +// +// Anything else (other scripts, malformed bytes) falls through to the general path below, which +// is correct but slower -- and effectively never taken by this codebase. +inline bool fold_fast(const char *src, char *dst, std::size_t len, bool upper) { + const unsigned char c0 = static_cast(src[0]); + if (c0 < 0x80) { + char c = src[0]; + if (upper) { + if (c >= 'a' && c <= 'z') { + c = static_cast(c - 0x20); + } + } else if (c >= 'A' && c <= 'Z') { + c = static_cast(c + 0x20); + } + dst[0] = c; + return true; + } + if (len != 2) { + return false; + } + const unsigned char c1 = static_cast(src[1]); + unsigned char o0 = c0; + unsigned char o1 = c1; + if (upper) { + if (c0 == 0xD0 && c1 >= 0xB0 && c1 <= 0xBF) { // a-p -> A-P + o1 = static_cast(c1 - 0x20); + } else if (c0 == 0xD1 && c1 >= 0x80 && c1 <= 0x8F) { // r-ya -> R-Ya + o0 = 0xD0; + o1 = static_cast(c1 + 0x20); + } else if (c0 == 0xD1 && c1 == 0x91) { // yo -> Yo + o0 = 0xD0; + o1 = 0x81; + } else if (!((c0 == 0xD0 && c1 >= 0x90 && c1 <= 0xAF) || (c0 == 0xD0 && c1 == 0x81))) { + return false; // not Cyrillic: general path + } + } else { + if (c0 == 0xD0 && c1 >= 0x90 && c1 <= 0x9F) { // A-P -> a-p + o1 = static_cast(c1 + 0x20); + } else if (c0 == 0xD0 && c1 >= 0xA0 && c1 <= 0xAF) { // R-Ya -> r-ya + o0 = 0xD1; + o1 = static_cast(c1 - 0x20); + } else if (c0 == 0xD0 && c1 == 0x81) { // Yo -> yo + o0 = 0xD1; + o1 = 0x91; + } else if (!((c0 == 0xD0 && c1 >= 0xB0) || (c0 == 0xD1 && c1 <= 0x8F) || (c0 == 0xD1 && c1 == 0x91))) { + return false; + } + } + dst[0] = static_cast(o0); + dst[1] = static_cast(o1); + return true; +} + +std::size_t copy_folded_char(const char *src, char *dst, bool upper) { const std::size_t len = char_bytes(src); + if (fold_fast(src, dst, len, upper)) { + return len; + } + // General path: decode, fold, re-encode; only taken for characters outside ASCII+Cyrillic. char32_t cp = 0; if (utf8::decode(std::string_view(src, len), 0, cp) != 0) { - std::string folded; - // Only rewrite when the folded form keeps the byte length -- true for ASCII and for the - // whole Russian alphabet, so callers never see a character change size. - if (utf8::encode(fold(cp), folded) == len) { + const char32_t folded = upper ? utf8::to_upper(cp) : utf8::to_lower(cp); + char tmp[4]; + if (folded != cp && utf8::encode(folded, tmp) == len) { for (std::size_t i = 0; i < len; ++i) { - dst[i] = folded[i]; + dst[i] = tmp[i]; } return len; } } - for (std::size_t i = 0; i < len; ++i) { - dst[i] = src[i]; + if (dst != src) { + for (std::size_t i = 0; i < len; ++i) { + dst[i] = src[i]; + } } return len; } @@ -207,13 +296,72 @@ std::size_t copy_folded_char(const char *src, char *dst, char32_t (*fold)(char32 } // namespace std::size_t copy_lower_char(const char *src, char *dst) { - return copy_folded_char(src, dst, utf8::to_lower); + return copy_folded_char(src, dst, false); } std::size_t copy_upper_char(const char *src, char *dst) { - return copy_folded_char(src, dst, utf8::to_upper); + return copy_folded_char(src, dst, true); +} + + +namespace { + +// Whole-buffer case conversion as one tight loop with no calls in the hot path: ASCII and the +// two-byte Cyrillic block are folded straight on the bytes. Anything else falls back to the +// general helper, which this codebase never hits in practice. Written this way deliberately -- +// a per-character dispatch measured several times slower than the byte loop it replaces. +inline void fold_range_utf8(char *p, char *const end, bool upper) { + while (p < end) { + const unsigned char c0 = static_cast(*p); + if (c0 < 0x80) { + char c = *p; + if (upper) { + if (c >= 'a' && c <= 'z') { + c = static_cast(c - 0x20); + } + } else if (c >= 'A' && c <= 'Z') { + c = static_cast(c + 0x20); + } + *p++ = c; + continue; + } + if ((c0 == 0xD0 || c0 == 0xD1) && p + 1 < end) { + const unsigned char c1 = static_cast(p[1]); + if (upper) { + if (c0 == 0xD0 && c1 >= 0xB0) { + p[1] = static_cast(c1 - 0x20); + } else if (c0 == 0xD1 && c1 <= 0x8F) { + p[0] = static_cast(0xD0); + p[1] = static_cast(c1 + 0x20); + } else if (c0 == 0xD1 && c1 == 0x91) { + p[0] = static_cast(0xD0); + p[1] = static_cast(0x81); + } + } else { + if (c0 == 0xD0 && c1 >= 0x90 && c1 <= 0x9F) { + p[1] = static_cast(c1 + 0x20); + } else if (c0 == 0xD0 && c1 >= 0xA0 && c1 <= 0xAF) { + p[0] = static_cast(0xD1); + p[1] = static_cast(c1 - 0x20); + } else if (c0 == 0xD0 && c1 == 0x81) { + p[0] = static_cast(0xD1); + p[1] = static_cast(0x91); + } + } + p += 2; + continue; + } + p += upper ? copy_upper_char(p, p) : copy_lower_char(p, p); + } } +} // namespace + +void to_lower(std::string &s) { fold_range_utf8(s.data(), s.data() + s.size(), false); } +void to_upper(std::string &s) { fold_range_utf8(s.data(), s.data() + s.size(), true); } +void to_lower(char *s) { fold_range_utf8(s, s + std::char_traits::length(s), false); } +void to_upper(char *s) { fold_range_utf8(s, s + std::char_traits::length(s), true); } + #else // KOI8-R: 1 byte == 1 character bool native_is_utf8() { @@ -304,6 +452,31 @@ std::size_t copy_upper_char(const char *src, char *dst) { return 1; } + +void to_lower(std::string &s) { + for (char &c : s) { + c = a_lcc_table[static_cast(c)]; + } +} + +void to_upper(std::string &s) { + for (char &c : s) { + c = a_ucc_table[static_cast(c)]; + } +} + +void to_lower(char *s) { + for (; *s; ++s) { + *s = a_lcc_table[static_cast(*s)]; + } +} + +void to_upper(char *s) { + for (; *s; ++s) { + *s = a_ucc_table[static_cast(*s)]; + } +} + #endif // --------------------------------------------------------------------------------------------- @@ -314,6 +487,22 @@ std::size_t copy_upper_char(const char *src, char *dst) { namespace { +inline std::size_t lead_len_shared(unsigned char c) { + if (c < 0x80) { + return 1; + } + if (c >= 0xC0 && c <= 0xDF) { + return 2; + } + if (c >= 0xE0 && c <= 0xEF) { + return 3; + } + if (c >= 0xF0 && c <= 0xF7) { + return 4; + } + return 1; +} + // Byte length of the character at `pos`, clamped to the end of `s`. std::size_t char_bytes_at(std::string_view s, std::size_t pos) { #ifdef INTERNAL_ENCODING_UTF8 @@ -321,7 +510,7 @@ std::size_t char_bytes_at(std::string_view s, std::size_t pos) { if (lead < 0x80) { return 1; } - const std::size_t want = static_cast(utf8::sequence_length(lead)); + const std::size_t want = lead_len_shared(lead); std::size_t n = 1; while (n < want && pos + n < s.size() && (static_cast(s[pos + n]) & 0xC0) == 0x80) { ++n; @@ -349,30 +538,6 @@ std::size_t CharRange::Iterator::step(std::string_view s, std::size_t pos) { return pos < s.size() ? char_bytes_at(s, pos) : 0; } -void to_lower(std::string &s) { - for (std::size_t i = 0; i < s.size();) { - i += copy_lower_char(&s[i], &s[i]); - } -} - -void to_upper(std::string &s) { - for (std::size_t i = 0; i < s.size();) { - i += copy_upper_char(&s[i], &s[i]); - } -} - -void to_lower(char *s) { - while (*s) { - s += copy_lower_char(s, s); - } -} - -void to_upper(char *s) { - while (*s) { - s += copy_upper_char(s, s); - } -} - std::size_t last_char_offset(std::string_view s) { std::size_t last = 0; std::size_t pos = 0; diff --git a/src/utils/utf8.cpp b/src/utils/utf8.cpp index c50486e178..8941786da3 100644 --- a/src/utils/utf8.cpp +++ b/src/utils/utf8.cpp @@ -143,6 +143,35 @@ std::size_t encode(char32_t cp, std::string &out) { return 0; } +std::size_t encode(char32_t cp, char *out) { + if (cp <= 0x7F) { + out[0] = static_cast(cp); + return 1; + } + if (cp <= 0x7FF) { + out[0] = static_cast(0xC0 | (cp >> 6)); + out[1] = static_cast(0x80 | (cp & 0x3F)); + return 2; + } + if (cp >= 0xD800 && cp <= 0xDFFF) { + return 0; + } + if (cp <= 0xFFFF) { + out[0] = static_cast(0xE0 | (cp >> 12)); + out[1] = static_cast(0x80 | ((cp >> 6) & 0x3F)); + out[2] = static_cast(0x80 | (cp & 0x3F)); + return 3; + } + if (cp <= 0x10FFFF) { + out[0] = static_cast(0xF0 | (cp >> 18)); + out[1] = static_cast(0x80 | ((cp >> 12) & 0x3F)); + out[2] = static_cast(0x80 | ((cp >> 6) & 0x3F)); + out[3] = static_cast(0x80 | (cp & 0x3F)); + return 4; + } + return 0; +} + bool is_valid(std::string_view s) { std::size_t pos = 0; const std::size_t n = s.size(); diff --git a/src/utils/utf8.h b/src/utils/utf8.h index 7991d4c04a..f65f32ec3d 100644 --- a/src/utils/utf8.h +++ b/src/utils/utf8.h @@ -37,6 +37,10 @@ std::size_t decode(std::string_view s, std::size_t pos, char32_t &cp); // not a valid Unicode scalar (surrogate half or > U+10FFFF), in which case `out` is untouched. std::size_t encode(char32_t cp, std::string &out); +// Same, but writes into a caller-supplied buffer of at least 4 bytes and never allocates. +// Returns the number of bytes written, or 0 for a value that is not a Unicode scalar. +std::size_t encode(char32_t cp, char *out); + // Strict, whole-string well-formedness check per the Unicode Table 3-7 byte-sequence grammar // (rejects overlong forms, surrogates, code points above U+10FFFF and stray continuation bytes). bool is_valid(std::string_view s); From bc18a0643f4df8d2fd525774e54c753826dd8481 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 05:13:21 +0200 Subject: [PATCH 16/20] feat(utf8): encoding-neutral Russian menu keys (A4 groundwork, #3681) Menus and OLC editors dispatch on a Cyrillic character literal (231 case labels across 19 files). Those cannot survive the flip: with UTF-8 sources a Cyrillic 'x' is a multi-character constant, so the compiler folds it to an implementation-defined value and the menu silently stops responding -- and no single literal spelling works for both encodings. Rather than give up the switch (option A, if/else chains) or bundle 231 rewrites into the flip commit (option B), the letters become numeric constants defined per encoding: switch (native_text::first_char_code(arg)) { case 'y': case 'Y': case rus::kDe: case rus::kDeUpper: * src/utils/russian_keys.h -- all 33 letters, upper and lower, as byte values under KOI8-R and code points under UTF-8. Both tables were generated and checked against the actual codecs rather than written from memory: 0 mismatches in 66 entries. * native_text::first_char_code() -- the raw byte under KOI8-R, the code point under UTF-8, so the switch expression means the same thing in both. * ASCII cases stay ordinary literals; they are identical in either encoding. This lands now as a no-op (under KOI8-R the constant is exactly the byte the old literal compiled to) and leaves the flip a one-flag change. At cleanup the KOI8-R half of the table goes away and the constants can collapse into plain U'...' literals. Includes the first converted switch (medit's save prompt) as a worked example, and a test pinning every constant against first_char_code -- verified in both branches, since a drift here would disable a menu key without any visible error. Remaining: ~229 labels in 18 files, mechanical from here. --- src/engine/olc/medit.cpp | 12 ++-- src/utils/native_text.cpp | 13 ++++ src/utils/native_text.h | 6 ++ src/utils/russian_keys.h | 125 ++++++++++++++++++++++++++++++++++++++ tests/native_text.cpp | 26 ++++++++ 5 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 src/utils/russian_keys.h diff --git a/src/engine/olc/medit.cpp b/src/engine/olc/medit.cpp index 0298cdc4f4..b1a18560d7 100644 --- a/src/engine/olc/medit.cpp +++ b/src/engine/olc/medit.cpp @@ -8,6 +8,8 @@ ***************************************************************************/ #include "engine/db/world_characters.h" +#include "utils/native_text.h" +#include "utils/russian_keys.h" #include "gameplay/affects/affect_messages.h" #include "gameplay/fight/fight_messages.h" #include "engine/entities/obj_data.h" @@ -1296,11 +1298,11 @@ void medit_parse(DescriptorData *d, char *arg) { case MEDIT_CONFIRM_SAVESTRING: // * Ensure mob has MOB_ISNPC set or things will go pair shaped. OLC_MOB(d)->SetFlag(EMobFlag::kNpc); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': + case rus::kDe: + case rus::kDeUpper: // * Save the mob in memory and to disk. // SendMsgToChar("Saving mobile to memory a.\r\n", d->character.get()); medit_save_internally(d); @@ -1314,8 +1316,8 @@ void medit_parse(DescriptorData *d, char *arg) { case 'n': case 'N': - case 'н': - case 'Н': cleanup_olc(d, CLEANUP_ALL); + case rus::kEn: + case rus::kEnUpper: cleanup_olc(d, CLEANUP_ALL); break; default: SendMsgToChar("Неверный выбор!\r\n", d->character.get()); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 77ef7830d1..df1ecc1041 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -304,6 +304,15 @@ std::size_t copy_upper_char(const char *src, char *dst) { } +char32_t first_char_code(const char *s) { + if (s == nullptr || *s == '\0') { + return 0; + } + char32_t cp = 0; + utf8::decode(std::string_view(s, char_bytes(s)), 0, cp); + return cp; +} + namespace { // Whole-buffer case conversion as one tight loop with no calls in the hot path: ASCII and the @@ -453,6 +462,10 @@ std::size_t copy_upper_char(const char *src, char *dst) { } +char32_t first_char_code(const char *s) { + return (s == nullptr || *s == '\0') ? 0 : static_cast(*s); +} + void to_lower(std::string &s) { for (char &c : s) { c = a_lcc_table[static_cast(c)]; diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 7d0140f0da..24c45cea99 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -50,6 +50,12 @@ std::size_t truncate_offset(std::string_view s, std::size_t max_bytes); // bytes actually present (never counts past a terminator or a non-continuation byte). std::size_t char_bytes(const char *s); +// Numeric identity of the character starting at `s`, for dispatching a switch on a letter: +// the raw byte under KOI8-R, the code point under UTF-8. Compare against the constants in +// utils/russian_keys.h (Cyrillic) or ordinary character literals (ASCII, identical in both). +// Returns 0 on an empty string. +char32_t first_char_code(const char *s); + // Case-insensitive comparison in the native encoding: lexicographic over lowered characters, // the shorter string orders first, returns the signed difference at the first mismatch (0 when // equal). KOI8-R: per byte, via LOWER() -- matches str_cmp/str/str semantics. UTF-8: per code diff --git a/src/utils/russian_keys.h b/src/utils/russian_keys.h new file mode 100644 index 0000000000..85f21c3683 --- /dev/null +++ b/src/utils/russian_keys.h @@ -0,0 +1,125 @@ +/** +\file russian_keys.h - a part of the Bylins engine. +\brief Russian letters as switch-able constants, valid in both native encodings (issue #3681). + +Menus and OLC editors dispatch on a single letter the player typed: + + switch (*arg) { + case 'y': case 'Y': case 'd': case 'D': ... + +A Cyrillic *character* literal cannot survive the encoding flip: with UTF-8 sources 'd' (Cyrillic) +is a multi-character constant, so the compiler folds it to an implementation-defined value and the +menu silently stops responding. Nor can such a literal be written portably for both encodings. + +The way out is to keep the switch but dispatch on a number instead of a literal. Under KOI8-R a +letter is one byte, under UTF-8 it is a code point, so each letter is spelled out numerically for +both and the constants below are what the switch compares against: + + switch (native_text::first_char_code(arg)) { + case 'y': case 'Y': case rus::kDa: case rus::kDaUpper: ... + +Note that ASCII cases stay ordinary character literals -- those are identical in both encodings. + +Once the flip is permanent (track D) the KOI8-R half goes away and these can collapse into plain +U'...' literals. +*/ + +#ifndef BYLINS_SRC_UTILS_RUSSIAN_KEYS_H_ +#define BYLINS_SRC_UTILS_RUSSIAN_KEYS_H_ + +namespace rus { + +#ifdef INTERNAL_ENCODING_UTF8 + +// Unicode code points: U+0410..U+042F (upper), U+0430..U+044F (lower), U+0401/U+0451 (Yo). +#define BYLINS_RUS_LETTER(name, upper_cp, lower_cp) \ + constexpr char32_t name##Upper = upper_cp; \ + constexpr char32_t name = lower_cp + +#else + +// KOI8-R byte values. The alphabet is not laid out alphabetically in KOI8-R, so every letter is +// spelled out rather than derived from an offset. +#define BYLINS_RUS_LETTER(name, upper_byte, lower_byte) \ + constexpr char32_t name##Upper = upper_byte; \ + constexpr char32_t name = lower_byte + +#endif + +#ifdef INTERNAL_ENCODING_UTF8 +BYLINS_RUS_LETTER(kA, 0x0410, 0x0430); // А а +BYLINS_RUS_LETTER(kBe, 0x0411, 0x0431); // Б б +BYLINS_RUS_LETTER(kVe, 0x0412, 0x0432); // В в +BYLINS_RUS_LETTER(kGe, 0x0413, 0x0433); // Г г +BYLINS_RUS_LETTER(kDe, 0x0414, 0x0434); // Д д +BYLINS_RUS_LETTER(kIe, 0x0415, 0x0435); // Е е +BYLINS_RUS_LETTER(kYo, 0x0401, 0x0451); // Ё ё +BYLINS_RUS_LETTER(kZhe, 0x0416, 0x0436); // Ж ж +BYLINS_RUS_LETTER(kZe, 0x0417, 0x0437); // З з +BYLINS_RUS_LETTER(kI, 0x0418, 0x0438); // И и +BYLINS_RUS_LETTER(kIi, 0x0419, 0x0439); // Й й +BYLINS_RUS_LETTER(kKa, 0x041A, 0x043A); // К к +BYLINS_RUS_LETTER(kEl, 0x041B, 0x043B); // Л л +BYLINS_RUS_LETTER(kEm, 0x041C, 0x043C); // М м +BYLINS_RUS_LETTER(kEn, 0x041D, 0x043D); // Н н +BYLINS_RUS_LETTER(kO, 0x041E, 0x043E); // О о +BYLINS_RUS_LETTER(kPe, 0x041F, 0x043F); // П п +BYLINS_RUS_LETTER(kEr, 0x0420, 0x0440); // Р р +BYLINS_RUS_LETTER(kEs, 0x0421, 0x0441); // С с +BYLINS_RUS_LETTER(kTe, 0x0422, 0x0442); // Т т +BYLINS_RUS_LETTER(kU, 0x0423, 0x0443); // У у +BYLINS_RUS_LETTER(kEf, 0x0424, 0x0444); // Ф ф +BYLINS_RUS_LETTER(kHa, 0x0425, 0x0445); // Х х +BYLINS_RUS_LETTER(kTse, 0x0426, 0x0446); // Ц ц +BYLINS_RUS_LETTER(kChe, 0x0427, 0x0447); // Ч ч +BYLINS_RUS_LETTER(kSha, 0x0428, 0x0448); // Ш ш +BYLINS_RUS_LETTER(kScha,0x0429, 0x0449); // Щ щ +BYLINS_RUS_LETTER(kHard,0x042A, 0x044A); // Ъ ъ +BYLINS_RUS_LETTER(kYery,0x042B, 0x044B); // Ы ы +BYLINS_RUS_LETTER(kSoft,0x042C, 0x044C); // Ь ь +BYLINS_RUS_LETTER(kE, 0x042D, 0x044D); // Э э +BYLINS_RUS_LETTER(kYu, 0x042E, 0x044E); // Ю ю +BYLINS_RUS_LETTER(kYa, 0x042F, 0x044F); // Я я +#else +BYLINS_RUS_LETTER(kA, 0xE1, 0xC1); // А а +BYLINS_RUS_LETTER(kBe, 0xE2, 0xC2); // Б б +BYLINS_RUS_LETTER(kVe, 0xF7, 0xD7); // В в +BYLINS_RUS_LETTER(kGe, 0xE7, 0xC7); // Г г +BYLINS_RUS_LETTER(kDe, 0xE4, 0xC4); // Д д +BYLINS_RUS_LETTER(kIe, 0xE5, 0xC5); // Е е +BYLINS_RUS_LETTER(kYo, 0xB3, 0xA3); // Ё ё +BYLINS_RUS_LETTER(kZhe, 0xF6, 0xD6); // Ж ж +BYLINS_RUS_LETTER(kZe, 0xFA, 0xDA); // З з +BYLINS_RUS_LETTER(kI, 0xE9, 0xC9); // И и +BYLINS_RUS_LETTER(kIi, 0xEA, 0xCA); // Й й +BYLINS_RUS_LETTER(kKa, 0xEB, 0xCB); // К к +BYLINS_RUS_LETTER(kEl, 0xEC, 0xCC); // Л л +BYLINS_RUS_LETTER(kEm, 0xED, 0xCD); // М м +BYLINS_RUS_LETTER(kEn, 0xEE, 0xCE); // Н н +BYLINS_RUS_LETTER(kO, 0xEF, 0xCF); // О о +BYLINS_RUS_LETTER(kPe, 0xF0, 0xD0); // П п +BYLINS_RUS_LETTER(kEr, 0xF2, 0xD2); // Р р +BYLINS_RUS_LETTER(kEs, 0xF3, 0xD3); // С с +BYLINS_RUS_LETTER(kTe, 0xF4, 0xD4); // Т т +BYLINS_RUS_LETTER(kU, 0xF5, 0xD5); // У у +BYLINS_RUS_LETTER(kEf, 0xE6, 0xC6); // Ф ф +BYLINS_RUS_LETTER(kHa, 0xE8, 0xC8); // Х х +BYLINS_RUS_LETTER(kTse, 0xE3, 0xC3); // Ц ц +BYLINS_RUS_LETTER(kChe, 0xFE, 0xDE); // Ч ч +BYLINS_RUS_LETTER(kSha, 0xFB, 0xDB); // Ш ш +BYLINS_RUS_LETTER(kScha,0xFD, 0xDD); // Щ щ +BYLINS_RUS_LETTER(kHard,0xFF, 0xDF); // Ъ ъ +BYLINS_RUS_LETTER(kYery,0xF9, 0xD9); // Ы ы +BYLINS_RUS_LETTER(kSoft,0xF8, 0xD8); // Ь ь +BYLINS_RUS_LETTER(kE, 0xFC, 0xDC); // Э э +BYLINS_RUS_LETTER(kYu, 0xE0, 0xC0); // Ю ю +BYLINS_RUS_LETTER(kYa, 0xF1, 0xD1); // Я я +#endif + +#undef BYLINS_RUS_LETTER + +} // namespace rus + +#endif // BYLINS_SRC_UTILS_RUSSIAN_KEYS_H_ + +// vim: ts=4 sw=4 tw=0 noet syntax=cpp : diff --git a/tests/native_text.cpp b/tests/native_text.cpp index 82f66ff64f..e0ffe7ff0b 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -5,6 +5,7 @@ // flag the library was built with), so this one test file is correct under either build. #include "utils/native_text.h" +#include "utils/russian_keys.h" #include @@ -316,4 +317,29 @@ TEST(NativeText, WholeStringCaseTransforms) { } } +TEST(NativeText, RussianKeysMatchFirstCharCode) { + // The switch-dispatch contract: for every Russian letter, the constant in russian_keys.h must + // equal what first_char_code() returns for that letter in the build's native encoding. If this + // ever drifts, menus and OLC editors silently stop responding to that key. + struct Case { const char *koi8; const char *utf8; char32_t expected; }; + const Case cases[] = { + {"\xC1", "\xD0\xB0", rus::kA}, {"\xE1", "\xD0\x90", rus::kAUpper}, + {"\xC4", "\xD0\xB4", rus::kDe}, {"\xE4", "\xD0\x94", rus::kDeUpper}, + {"\xCE", "\xD0\xBD", rus::kEn}, {"\xEE", "\xD0\x9D", rus::kEnUpper}, + {"\xD1", "\xD1\x8F", rus::kYa}, {"\xF1", "\xD0\xAF", rus::kYaUpper}, + {"\xA3", "\xD1\x91", rus::kYo}, {"\xB3", "\xD0\x81", rus::kYoUpper}, + {"\xD7", "\xD0\xB2", rus::kVe}, {"\xC8", "\xD1\x85", rus::kHa}, + }; + for (const auto &c : cases) { + const char *const input = native_text::native_is_utf8() ? c.utf8 : c.koi8; + EXPECT_EQ(native_text::first_char_code(input), c.expected); + } + + // ASCII keys are unchanged by the flip and stay ordinary character literals in the switches. + EXPECT_EQ(native_text::first_char_code("y"), static_cast('y')); + EXPECT_EQ(native_text::first_char_code("N"), static_cast('N')); + EXPECT_EQ(native_text::first_char_code(""), 0u); + EXPECT_EQ(native_text::first_char_code(nullptr), 0u); +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : From b80b9e979f7c8fe78b7c8e09d965833c1b59f4b8 Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 05:38:10 +0200 Subject: [PATCH 17/20] feat(utf8): convert all Cyrillic menu keys to encoding-neutral constants (#3681) Completes the A4 sweep: 227 case labels across 15 files plus the remaining single-letter comparisons now dispatch on native_text::first_char_code() against the rus:: constants, so menus and OLC editors keep working after the flip instead of silently going deaf. * 29 switch expressions rewritten: *arg / *argument -> first_char_code(), LOWER(...) -> first_char_code_lower(), UPPER(...) -> first_char_code_upper(). The script refused to touch any switch shape it did not recognise, so nothing was converted blind; do_who's `switch (mode)` was handled through its initialiser. * Comparison sites in named_stuff, modify and login converted the same way. * iosystem's legacy zMUD 'z' -> Cyrillic substitution now writes the letter from a *string* literal: string literals are byte-transparent, so the same code is correct whether the letter is one byte (KOI8-R) or two (UTF-8). A character literal cannot be, which is the whole point of this step. Left untouched on purpose: six occurrences inside comments (prose, not code). One site is deliberately NOT converted -- db.cpp's get_filename(). It transliterates a player name into the save-file name byte by byte, so under UTF-8 it would produce a different filename and every existing player's files would stop being found. That is a data-loss risk, not a mechanical edit: it needs character-wise transliteration plus a test pinning the filename across the flip. Recorded in the issue as C1a. Verified: clean build, full suite 651 passed / 0 failed. --- src/engine/core/iosystem.cpp | 14 ++- src/engine/olc/medit.cpp | 104 +++++++++++------------ src/engine/olc/oedit.cpp | 12 +-- src/engine/olc/redit.cpp | 12 +-- src/engine/olc/zedit.cpp | 12 +-- src/engine/ui/cmd/do_display.cpp | 24 +++--- src/engine/ui/cmd/do_who.cpp | 6 +- src/engine/ui/cmd_god/do_print_armor.cpp | 12 +-- src/engine/ui/login.cpp | 34 ++++---- src/engine/ui/modify.cpp | 10 ++- src/engine/ui/objects_filter.cpp | 30 ++++--- src/gameplay/clans/house.cpp | 35 ++++---- src/gameplay/core/genchar.cpp | 63 +++++++------- src/gameplay/mechanics/glory.cpp | 60 ++++++------- src/gameplay/mechanics/glory_const.cpp | 59 ++++++------- src/gameplay/mechanics/named_stuff.cpp | 18 ++-- src/gameplay/mechanics/obj_sets_olc.cpp | 76 +++++++++-------- src/utils/native_text.cpp | 18 ++++ src/utils/native_text.h | 4 + 19 files changed, 335 insertions(+), 268 deletions(-) diff --git a/src/engine/core/iosystem.cpp b/src/engine/core/iosystem.cpp index 1a75220b79..e998805265 100644 --- a/src/engine/core/iosystem.cpp +++ b/src/engine/core/iosystem.cpp @@ -7,6 +7,8 @@ */ #include "engine/core/iosystem.h" +#include +#include #include "gameplay/core/experience.h" #include "administration/privilege.h" #include "utils/utils_encoding.h" @@ -426,8 +428,16 @@ int process_input(DescriptorData *t) { // Увы, это кое-что ломает, напр. wizhelp, или "г я использую zMUD" if (t->state == EConState::kPlaying || (t->state == EConState::kExdesc)) { if (t->keytable == kCodePageWinzZ || t->keytable == kCodePageWinzOld) { - if (*(write_point - 1) == 'z') { - *(write_point - 1) = 'я'; + // Буква задана СТРОКОВЫМ литералом: он байт-прозрачен, поэтому один и тот + // же код верен и для KOI8-R (1 байт), и для UTF-8 (2 байта) -- в отличие от + // символьного литерала, который под UTF-8 не помещается в char (issue #3681). + static const std::string_view kYaLetter = "я"; + if (*(write_point - 1) == 'z' && space_left + 1 >= kYaLetter.size()) { + --write_point; + ++space_left; + std::memcpy(write_point, kYaLetter.data(), kYaLetter.size()); + write_point += kYaLetter.size(); + space_left -= kYaLetter.size(); } } } diff --git a/src/engine/olc/medit.cpp b/src/engine/olc/medit.cpp index b1a18560d7..5d78a4cd9d 100644 --- a/src/engine/olc/medit.cpp +++ b/src/engine/olc/medit.cpp @@ -1329,7 +1329,7 @@ void medit_parse(DescriptorData *d, char *arg) { //------------------------------------------------------------------- case MEDIT_MAIN_MENU: i = 0; olc_log("%s command %c", GET_NAME(d->character), *arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'q': case 'Q': if (OLC_VAL(d)) // Anything been changed? @@ -1545,118 +1545,118 @@ void medit_parse(DescriptorData *d, char *arg) { medit_disp_helpers(d); return; - case 'а': - case 'А': OLC_MODE(d) = MEDIT_SKILLS; + case rus::kA: + case rus::kAUpper: OLC_MODE(d) = MEDIT_SKILLS; medit_disp_skills(d); return; - case 'б': - case 'Б': OLC_MODE(d) = MEDIT_SPELLS; + case rus::kBe: + case rus::kBeUpper: OLC_MODE(d) = MEDIT_SPELLS; medit_disp_spells(d); return; - case 'в': - case 'В': OLC_MODE(d) = MEDIT_STR; + case rus::kVe: + case rus::kVeUpper: OLC_MODE(d) = MEDIT_STR; i++; break; - case 'г': - case 'Г': OLC_MODE(d) = MEDIT_DEX; + case rus::kGe: + case rus::kGeUpper: OLC_MODE(d) = MEDIT_DEX; i++; break; - case 'д': - case 'Д': OLC_MODE(d) = MEDIT_CON; + case rus::kDe: + case rus::kDeUpper: OLC_MODE(d) = MEDIT_CON; i++; break; - case 'е': - case 'Е': OLC_MODE(d) = MEDIT_WIS; + case rus::kIe: + case rus::kIeUpper: OLC_MODE(d) = MEDIT_WIS; i++; break; - case 'ж': - case 'Ж': OLC_MODE(d) = MEDIT_INT; + case rus::kZhe: + case rus::kZheUpper: OLC_MODE(d) = MEDIT_INT; i++; break; - case 'з': - case 'З': OLC_MODE(d) = MEDIT_CHA; + case rus::kZe: + case rus::kZeUpper: OLC_MODE(d) = MEDIT_CHA; i++; break; - case 'и': - case 'И': OLC_MODE(d) = MEDIT_HEIGHT; + case rus::kI: + case rus::kIUpper: OLC_MODE(d) = MEDIT_HEIGHT; i++; break; - case 'к': - case 'К': OLC_MODE(d) = MEDIT_WEIGHT; + case rus::kKa: + case rus::kKaUpper: OLC_MODE(d) = MEDIT_WEIGHT; i++; break; - case 'л': - case 'Л': OLC_MODE(d) = MEDIT_SIZE; + case rus::kEl: + case rus::kElUpper: OLC_MODE(d) = MEDIT_SIZE; i++; break; - case 'м': - case 'М': OLC_MODE(d) = MEDIT_EXTRA; + case rus::kEm: + case rus::kEmUpper: OLC_MODE(d) = MEDIT_EXTRA; i++; break; - case 'х': - case 'Х': OLC_MODE(d) = MEDIT_REMORT; + case rus::kHa: + case rus::kHaUpper: OLC_MODE(d) = MEDIT_REMORT; i++; break; - case 'Ю': - case 'ю': OLC_MODE(d) = MEDIT_MAXFACTOR; + case rus::kYuUpper: + case rus::kYu: OLC_MODE(d) = MEDIT_MAXFACTOR; i++; break; - case 'н': - case 'Н': SendMsgToChar(d->character.get(), "\r\nВведите новое значение от 0 до 100%% :"); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar(d->character.get(), "\r\nВведите новое значение от 0 до 100%% :"); OLC_MODE(d) = MEDIT_LIKE; return; - case 'п': - case 'П': OLC_MODE(d) = MEDIT_DLIST_MENU; + case rus::kPe: + case rus::kPeUpper: OLC_MODE(d) = MEDIT_DLIST_MENU; disp_dl_list(d); return; - case 'р': - case 'Р': OLC_MODE(d) = MEDIT_ROLE; + case rus::kEr: + case rus::kErUpper: OLC_MODE(d) = MEDIT_ROLE; medit_disp_role(d); return; - case 'с': - case 'С': OLC_MODE(d) = MEDIT_RESISTANCES; + case rus::kEs: + case rus::kEsUpper: OLC_MODE(d) = MEDIT_RESISTANCES; medit_disp_resistances(d); return; - case 'т': - case 'Т': OLC_MODE(d) = MEDIT_SAVES; + case rus::kTe: + case rus::kTeUpper: OLC_MODE(d) = MEDIT_SAVES; medit_disp_saves(d); return; - case 'у': - case 'У': OLC_MODE(d) = MEDIT_ADD_PARAMETERS; + case rus::kU: + case rus::kUUpper: OLC_MODE(d) = MEDIT_ADD_PARAMETERS; medit_disp_add_parameters(d); return; - case 'ф': - case 'Ф': OLC_MODE(d) = MEDIT_FEATURES; + case rus::kEf: + case rus::kEfUpper: OLC_MODE(d) = MEDIT_FEATURES; medit_disp_features(d); return; - case 'ц': - case 'Ц': OLC_MODE(d) = MEDIT_RACE; + case rus::kTse: + case rus::kTseUpper: OLC_MODE(d) = MEDIT_RACE; medit_disp_race(d); return; - case 'ч': - case 'Ч': OLC_MODE(d) = MEDIT_CLONE; + case rus::kChe: + case rus::kCheUpper: OLC_MODE(d) = MEDIT_CLONE; medit_disp_clone_menu(d); return; @@ -2174,9 +2174,9 @@ void medit_parse(DescriptorData *d, char *arg) { case MEDIT_DLIST_MENU: if (*arg) { // Обрабатываем комнады добавить удалить и.т.п - switch (*arg) { - case 'а': - case 'А': + switch (native_text::first_char_code(arg)) { + case rus::kA: + case rus::kAUpper: // Добавляем запись. OLC_MODE(d) = MEDIT_DLIST_ADD; SendMsgToChar("\r\nVNUM - виртуальный номер прототипа\r\n" @@ -2195,8 +2195,8 @@ void medit_parse(DescriptorData *d, char *arg) { return; - case 'б': - case 'Б': + case rus::kBe: + case rus::kBeUpper: // Удаляем запись. OLC_MODE(d) = MEDIT_DLIST_DEL; SendMsgToChar("\r\nВведите номер удаляемой записи:", d->character.get()); diff --git a/src/engine/olc/oedit.cpp b/src/engine/olc/oedit.cpp index dedc6aa740..31b4c605f3 100644 --- a/src/engine/olc/oedit.cpp +++ b/src/engine/olc/oedit.cpp @@ -9,6 +9,8 @@ ************************************************************************/ #include "engine/db/world_objects.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "gameplay/fight/fight_messages.h" #include "engine/db/obj_prototypes.h" #include "engine/core/conf.h" @@ -1486,11 +1488,11 @@ void oedit_parse(DescriptorData *d, char *arg) { switch (OLC_MODE(d)) { case OEDIT_CONFIRM_SAVESTRING: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': SendMsgToChar("Объект сохранен.\r\n", d->character.get()); + case rus::kDe: + case rus::kDeUpper: SendMsgToChar("Объект сохранен.\r\n", d->character.get()); OLC_OBJ(d)->remove_incorrect_values_keys(OLC_OBJ(d)->get_type()); oedit_save_internally(d); snprintf(buf, sizeof(buf), "OLC: %s edits obj %d", GET_NAME(d->character), OLC_NUM(d)); @@ -1501,8 +1503,8 @@ void oedit_parse(DescriptorData *d, char *arg) { case 'n': case 'N': - case 'н': - case 'Н': cleanup_olc(d, CLEANUP_ALL); + case rus::kEn: + case rus::kEnUpper: cleanup_olc(d, CLEANUP_ALL); break; default: SendMsgToChar("Неверный выбор!\r\n", d->character.get()); diff --git a/src/engine/olc/redit.cpp b/src/engine/olc/redit.cpp index 78b8008b41..f88147a984 100644 --- a/src/engine/olc/redit.cpp +++ b/src/engine/olc/redit.cpp @@ -9,6 +9,8 @@ ************************************************************************/ #include "engine/entities/obj_data.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/core/comm.h" #include "engine/db/db.h" #include "engine/db/world_data_source_manager.h" @@ -564,11 +566,11 @@ void redit_parse(DescriptorData *d, char *arg) { switch (OLC_MODE(d)) { case REDIT_CONFIRM_SAVESTRING: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': redit_save_internally(d); + case rus::kDe: + case rus::kDeUpper: redit_save_internally(d); snprintf(buf, sizeof(buf), "OLC: %s edits room %d.", GET_NAME(d->character), OLC_NUM(d)); olc_log("%s edit room %d", GET_NAME(d->character), OLC_NUM(d)); mudlog(buf, NRM, std::max(kLvlBuilder, GET_INVIS_LEV(d->character)), SYSLOG, true); @@ -579,8 +581,8 @@ void redit_parse(DescriptorData *d, char *arg) { case 'n': case 'N': - case 'н': - case 'Н': + case rus::kEn: + case rus::kEnUpper: // * Free everything up, including strings, etc. cleanup_olc(d, CLEANUP_ALL); break; diff --git a/src/engine/olc/zedit.cpp b/src/engine/olc/zedit.cpp index c69ca37498..c41407d961 100644 --- a/src/engine/olc/zedit.cpp +++ b/src/engine/olc/zedit.cpp @@ -5,6 +5,8 @@ ************************************************************************/ #include "engine/db/obj_prototypes.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/entities/obj_data.h" #include "engine/core/comm.h" #include "engine/db/db.h" @@ -1370,11 +1372,11 @@ void zedit_parse(DescriptorData *d, char *arg) { switch (OLC_MODE(d)) { case ZEDIT_CONFIRM_SAVESTRING: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': + case rus::kDe: + case rus::kDeUpper: // * Save the zone in memory, hiding invisible people. SendMsgToChar("Зона сохранена.\r\n", d->character.get()); zedit_save_internally(d); @@ -1384,8 +1386,8 @@ void zedit_parse(DescriptorData *d, char *arg) { // FALL THROUGH case 'n': case 'N': - case 'н': - case 'Н': cleanup_olc(d, CLEANUP_ALL); + case rus::kEn: + case rus::kEnUpper: cleanup_olc(d, CLEANUP_ALL); break; default: SendMsgToChar("Неверный выбор!\r\n", d->character.get()); SendMsgToChar("Вы желаете сохранить зону? : ", d->character.get()); diff --git a/src/engine/ui/cmd/do_display.cpp b/src/engine/ui/cmd/do_display.cpp index ef7ba374cd..6cc71f950e 100644 --- a/src/engine/ui/cmd/do_display.cpp +++ b/src/engine/ui/cmd/do_display.cpp @@ -7,6 +7,8 @@ */ #include "engine/entities/char_data.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "administration/privilege.h" const char *DISPLAY_HELP = "Формат: статус { { Ж | Э | З | В | Д | У | О | Б | П | К } | все | нет }\r\n"; @@ -38,35 +40,35 @@ void do_display(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { const size_t len = strlen(argument); for (size_t i = 0; i < len; i++) { - switch (LOWER(argument[i])) { + switch (native_text::first_char_code_lower(argument + i)) { case 'h': - case 'ж': ch->SetFlag(EPrf::kDispHp); + case rus::kZhe: ch->SetFlag(EPrf::kDispHp); break; case 'w': - case 'з': ch->SetFlag(EPrf::kDispMana); + case rus::kZe: ch->SetFlag(EPrf::kDispMana); break; case 'm': - case 'э': ch->SetFlag(EPrf::kDispMove); + case rus::kE: ch->SetFlag(EPrf::kDispMove); break; case 'e': - case 'в': ch->SetFlag(EPrf::kDispExits); + case rus::kVe: ch->SetFlag(EPrf::kDispExits); break; case 'g': - case 'д': ch->SetFlag(EPrf::kDispMoney); + case rus::kDe: ch->SetFlag(EPrf::kDispMoney); break; case 'l': - case 'у': ch->SetFlag(EPrf::kDispLvl); + case rus::kU: ch->SetFlag(EPrf::kDispLvl); break; case 'x': - case 'о': ch->SetFlag(EPrf::kDispExp); + case rus::kO: ch->SetFlag(EPrf::kDispExp); break; - case 'б': + case rus::kBe: case 'f': ch->SetFlag(EPrf::kDispFight); break; - case 'п': + case rus::kPe: case 't': ch->SetFlag(EPrf::kDispTimed); break; - case 'к': + case rus::kKa: case 'c': ch->SetFlag(EPrf::kDispCooldowns); break; case ' ': break; diff --git a/src/engine/ui/cmd/do_who.cpp b/src/engine/ui/cmd/do_who.cpp index 482f91e8c9..198d7b2955 100644 --- a/src/engine/ui/cmd/do_who.cpp +++ b/src/engine/ui/cmd/do_who.cpp @@ -3,6 +3,8 @@ // #include "engine/ui/cmd/do_who.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include #include "administration/privilege.h" #include "utils/grammar/gender.h" @@ -52,10 +54,10 @@ void DoWho(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { sscanf(arg, "%d-%d", &low, &high); strcpy(buf, buf1); } else if (*arg == '-') { - const char mode = *(arg + 1); // just in case; we destroy arg in the switch + const char32_t mode = native_text::first_char_code(arg + 1); // just in case; we destroy arg in the switch switch (mode) { case 'b': - case 'и': + case rus::kI: if (privilege::IsImmortal(ch) || GET_GOD_FLAG(ch, EGf::kDemigod) || ch->IsFlagged(EPrf::kCoderinfo)) showname = true; strcpy(buf, buf1); diff --git a/src/engine/ui/cmd_god/do_print_armor.cpp b/src/engine/ui/cmd_god/do_print_armor.cpp index 7507a96f0a..8ecdeb8a3c 100644 --- a/src/engine/ui/cmd_god/do_print_armor.cpp +++ b/src/engine/ui/cmd_god/do_print_armor.cpp @@ -7,6 +7,8 @@ */ #include "engine/entities/char_data.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/obj_prototypes.h" #include "engine/db/global_objects.h" @@ -34,8 +36,8 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { char tmpbuf[kMaxInputLength]; bool find_param = false; while (*argument) { - switch (*argument) { - case 'М': argument = one_argument(++argument, tmpbuf); + switch (native_text::first_char_code(argument)) { + case rus::kEmUpper: argument = one_argument(++argument, tmpbuf); if (utils::IsAbbr(tmpbuf, "булат")) { filter.material = EObjMaterial::kBulat; } else if (utils::IsAbbr(tmpbuf, "бронза")) { @@ -78,7 +80,7 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } find_param = true; break; - case 'Т': argument = one_argument(++argument, tmpbuf); + case rus::kTeUpper: argument = one_argument(++argument, tmpbuf); if (utils::IsAbbr(tmpbuf, "броня") || utils::IsAbbr(tmpbuf, "armor")) { filter.type = EObjType::kArmor; } else if (utils::IsAbbr(tmpbuf, "легкие") || utils::IsAbbr(tmpbuf, "легкая")) { @@ -93,7 +95,7 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } find_param = true; break; - case 'О': argument = one_argument(++argument, tmpbuf); + case rus::kOUpper: argument = one_argument(++argument, tmpbuf); if (utils::IsAbbr(tmpbuf, "тело")) { filter.wear = EWearFlag::kBody; filter.wear_message = 3; @@ -118,7 +120,7 @@ void DoPrintArmor(CharData *ch, char *argument, int/* cmd*/, int/* subcmd*/) { } find_param = true; break; - case 'А': { + case rus::kAUpper: { bool tmp_find = false; argument = one_argument(++argument, tmpbuf); if (!strlen(tmpbuf)) { diff --git a/src/engine/ui/login.cpp b/src/engine/ui/login.cpp index 5d75a7a582..f5f21776e1 100644 --- a/src/engine/ui/login.cpp +++ b/src/engine/ui/login.cpp @@ -4,6 +4,8 @@ extracted from interpreter.cpp. Entry point ProcessLoginInput (was nanny). */ #include "interpreter.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/ui/system_messages.h" #include "engine/core/config.h" #include "gameplay/mechanics/condition.h" @@ -185,8 +187,8 @@ int _parse_name(char *argument, char *name) { // skip whitespaces for (i = 0; (*name = (i ? LOWER(*argument) : UPPER(*argument))); argument++, i++, name++) { - if (*argument == 'ё' - || *argument == 'Ё' + if (native_text::first_char_code(argument) == rus::kYo + || native_text::first_char_code(argument) == rus::kYoUpper || !a_isalpha(*argument) || *argument > 0) { return (1); @@ -1532,7 +1534,8 @@ static void HandleGetKeytable(DescriptorData *d, char *argument) { static void HandleNameConfirm(DescriptorData *d, char *argument) { char buffer[kMaxStringLength]; - if (UPPER(*argument) == 'Y' || UPPER(*argument) == 'Д') { + if (native_text::first_char_code_upper(argument) == 'Y' + || native_text::first_char_code_upper(argument) == rus::kDeUpper) { if (ban->IsBanned(d->host) >= BanList::BAN_NEW) { sprintf(buffer, "Попытка создания персонажа %s отклонена для [%s] (siteban)", GET_PC_NAME(d->character), d->host); @@ -1570,7 +1573,8 @@ static void HandleNameConfirm(DescriptorData *d, char *argument) { d->state = EConState::kQsex; return; - } else if (UPPER(*argument) == 'N' || UPPER(*argument) == 'Н') { + } else if (native_text::first_char_code_upper(argument) == 'N' + || native_text::first_char_code_upper(argument) == rus::kEnUpper) { iosystem::write_to_output("Итак, чего изволите? Учтите, бананов нет :)\r\n" "Имя : ", d); d->character->SetCharAliases(nullptr); d->state = EConState::kGetName; @@ -1677,12 +1681,12 @@ static void HandleQuerySex(DescriptorData *d, char *argument) { return; } - switch (UPPER(*argument)) { - case 'М': + switch (native_text::first_char_code_upper(argument)) { + case rus::kEmUpper: case 'M': d->character->set_sex(EGender::kMale); break; - case 'Ж': + case rus::kZheUpper: case 'F': d->character->set_sex(EGender::kFemale); break; @@ -1706,9 +1710,9 @@ static void HandleQueryReligion(DescriptorData *d, char *argument) { return; } - switch (UPPER(*argument)) { - case 'Я': - case 'З': + switch (native_text::first_char_code_upper(argument)) { + case rus::kYaUpper: + case rus::kZeUpper: case 'P': if (class_religion[to_underlying(d->character->GetClass())] == kReligionMono) { iosystem::write_to_output("Персонаж выбранной вами профессии не желает быть язычником!\r\n" @@ -1718,7 +1722,7 @@ static void HandleQueryReligion(DescriptorData *d, char *argument) { GET_RELIGION(d->character) = kReligionPoly; break; - case 'Х': + case rus::kHaUpper: case 'C': if (class_religion[to_underlying(d->character->GetClass())] == kReligionPoly) { iosystem::write_to_output("Персонажу выбранной вами профессии противно христианство!\r\n" @@ -2049,9 +2053,9 @@ static void HandleResetReligion(DescriptorData *d, char *argument) { return; } - switch (UPPER(*argument)) { - case 'Я': - case 'З': + switch (native_text::first_char_code_upper(argument)) { + case rus::kYaUpper: + case rus::kZeUpper: case 'P': if (class_religion[to_underlying(d->character->GetClass())] == kReligionMono) { iosystem::write_to_output("Персонаж выбранной вами профессии не желает быть язычником!\r\n" @@ -2061,7 +2065,7 @@ static void HandleResetReligion(DescriptorData *d, char *argument) { GET_RELIGION(d->character) = kReligionPoly; break; - case 'Х': + case rus::kHaUpper: case 'C': if (class_religion[to_underlying(d->character->GetClass())] == kReligionPoly) { iosystem::write_to_output("Персонажу выбранной вами профессии противно христианство!\r\n" diff --git a/src/engine/ui/modify.cpp b/src/engine/ui/modify.cpp index ae3a00537b..8378412488 100644 --- a/src/engine/ui/modify.cpp +++ b/src/engine/ui/modify.cpp @@ -13,6 +13,7 @@ ************************************************************************ */ #include +#include "utils/russian_keys.h" #include "engine/db/player_index.h" #include @@ -1269,7 +1270,8 @@ void show_string(DescriptorData *d, char *input) { any_one_arg(input, buf); //* Q is for quit. :) - if (LOWER(*buf) == 'q' || LOWER(*buf) == 'к') { + if (native_text::first_char_code_lower(buf) == 'q' + || native_text::first_char_code_lower(buf) == rus::kKa) { free(d->showstr_vector); d->showstr_count = 0; if (d->showstr_head) { @@ -1281,12 +1283,14 @@ void show_string(DescriptorData *d, char *input) { } // R is for refresh, so back up one page internally so we can display // it again. - else if (LOWER(*buf) == 'r' || LOWER(*buf) == 'п') { + else if (native_text::first_char_code_lower(buf) == 'r' + || native_text::first_char_code_lower(buf) == rus::kPe) { d->showstr_page = MAX(0, d->showstr_page - 1); } // B is for back, so back up two pages internally so we can display the // correct page here. - else if (LOWER(*buf) == 'b' || LOWER(*buf) == 'н') { + else if (native_text::first_char_code_lower(buf) == 'b' + || native_text::first_char_code_lower(buf) == rus::kEn) { d->showstr_page = MAX(0, d->showstr_page - 2); } // Feature to 'goto' a page. Just type the number of the page and you diff --git a/src/engine/ui/objects_filter.cpp b/src/engine/ui/objects_filter.cpp index ef969c4334..f612622f4f 100644 --- a/src/engine/ui/objects_filter.cpp +++ b/src/engine/ui/objects_filter.cpp @@ -6,6 +6,8 @@ */ #include "objects_filter.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "gameplay/mechanics/sight.h" #include "gameplay/economics/exchange.h" @@ -825,45 +827,45 @@ bool ParseFilter::parse_filter(const CharData *ch, ParseFilter &filter, const ch return false; } while (*argument) { - switch (*argument) { - case 'И': argument = one_argument(++argument, buf_tmp); + switch (native_text::first_char_code(argument)) { + case rus::kIUpper: argument = one_argument(++argument, buf_tmp); if (strlen(buf_tmp) == 0) { SendMsgToChar("Укажите имя предмета.\r\n", ch); return false; } filter.name = buf_tmp; break; - case 'Т': argument = one_argument(++argument, buf_tmp); + case rus::kTeUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_type(buf_tmp)) { SendMsgToChar("Неверный тип предмета.\r\n", ch); return false; } break; - case 'С': argument = one_argument(++argument, buf_tmp); + case rus::kEsUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_state(buf_tmp)) { SendMsgToChar("Неверное состояние предмета.\r\n", ch); return false; } break; - case 'О': argument = one_argument(++argument, buf_tmp); + case rus::kOUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_wear(buf_tmp)) { SendMsgToChar("Неверное место одевания предмета.\r\n", ch); return false; } break; - case 'Ц': argument = one_argument(++argument, buf_tmp); + case rus::kTseUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_cost(buf_tmp)) { SendMsgToChar("Неверный формат в фильтре: Ц<цена><+->.\r\n", ch); return false; } break; - case 'К': argument = one_argument(++argument, buf_tmp); + case rus::kKaUpper: argument = one_argument(++argument, buf_tmp); if (!filter.init_weap_class(buf_tmp)) { SendMsgToChar("Неверный класс оружия.\r\n", ch); return false; } break; - case 'А': { + case rus::kAUpper: { argument = one_argument(++argument, buf_tmp); size_t len = strlen(buf_tmp); if (len == 0) { @@ -878,29 +880,29 @@ bool ParseFilter::parse_filter(const CharData *ch, ParseFilter &filter, const ch return false; } break; - } // case 'А' - case 'Р':// стоимость ренты + } // case rus::kAUpper + case rus::kErUpper:// стоимость ренты argument = one_argument(++argument, buf_tmp); if (!filter.init_rent(buf_tmp)) { SendMsgToChar("Неверный формат в фильтре: Р<стоимость><+->.\r\n", ch); return false; } break; - case 'М':// количество мортов + case rus::kEmUpper:// количество мортов argument = one_argument(++argument, buf_tmp); if (!filter.init_remorts(buf_tmp)) { SendMsgToChar("Неверный формат в фильтре: М<количество мортов><+->.\r\n", ch); return false; } break; - case 'У':// умения + case rus::kUUpper:// умения argument = one_argument(++argument, buf_tmp); if (!filter.init_skill(buf_tmp)) { SendMsgToChar("Неверное умение.\r\n", ch); return false; } break; - case 'В':// имя выставившего на базаре + case rus::kVeUpper:// имя выставившего на базаре argument = one_argument(++argument, buf_tmp); if (filter_type != EXCHANGE) { SendMsgToChar("Только для базара.\r\n", ch); @@ -908,7 +910,7 @@ bool ParseFilter::parse_filter(const CharData *ch, ParseFilter &filter, const ch } owner = buf_tmp; break; - case 'П':// профессия (отсечь предметы запрещенные данному классу) + case rus::kPeUpper:// профессия (отсечь предметы запрещенные данному классу) argument = one_argument(++argument, buf_tmp); if (!filter.init_profession(buf_tmp)) { SendMsgToChar("Неверное название профессии.\r\n", ch); diff --git a/src/gameplay/clans/house.cpp b/src/gameplay/clans/house.cpp index 726ffa348b..0a7df152a0 100644 --- a/src/gameplay/clans/house.cpp +++ b/src/gameplay/clans/house.cpp @@ -5,6 +5,7 @@ ******************************************************************************/ #include "house.h" +#include "utils/russian_keys.h" #include "utils/native_text.h" #include "engine/db/player_index.h" #include "gameplay/economics/currencies.h" @@ -2752,9 +2753,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { switch (d->clan_olc->mode) { case CLAN_MAIN_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // есть вариант, что за время в олц в клане изменят кол-во званий @@ -2834,9 +2835,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_PRIVILEGE_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // выход в общее меню @@ -2879,11 +2880,11 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_SAVE_MENU: - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': d->clan_olc->clan->privileges.clear(); + case rus::kDe: + case rus::kDeUpper: d->clan_olc->clan->privileges.clear(); d->clan_olc->clan->privileges = d->clan_olc->privileges; d->clan_olc.reset(); // Clan::ClanSave(); @@ -2893,8 +2894,8 @@ void Clan::Manage(DescriptorData *d, const char *arg) { case 'n': case 'N': - case 'н': - case 'Н': d->clan_olc.reset(); + case rus::kEn: + case rus::kEnUpper: d->clan_olc.reset(); d->state = EConState::kPlaying; SendMsgToChar("Редактирование отменено.\r\n", d->character.get()); return; @@ -2909,9 +2910,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_ADDALL_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // выход в общее меню с изменением всех званий @@ -2963,9 +2964,9 @@ void Clan::Manage(DescriptorData *d, const char *arg) { break; case CLAN_DELALL_MENU: - switch (*arg) { - case 'в': - case 'В': + switch (native_text::first_char_code(arg)) { + case rus::kVe: + case rus::kVeUpper: case 'q': case 'Q': // выход в общее меню с изменением всех званий diff --git a/src/gameplay/core/genchar.cpp b/src/gameplay/core/genchar.cpp index 7edccc100b..777d24e887 100644 --- a/src/gameplay/core/genchar.cpp +++ b/src/gameplay/core/genchar.cpp @@ -13,6 +13,7 @@ ************************************************************************ */ #include "genchar.h" +#include "utils/russian_keys.h" #include "engine/core/conf.h" #include "engine/core/sysdep.h" @@ -95,48 +96,48 @@ void genchar_disp_menu(CharData *ch) { int genchar_parse(CharData *ch, char *arg) { const auto &ch_class = MUD::Class(ch->GetClass()); - switch (*arg) { - case 'А': - case 'а': ch->set_str(std::max(ch->GetInbornStr() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kStr))); + switch (native_text::first_char_code(arg)) { + case rus::kAUpper: + case rus::kA: ch->set_str(std::max(ch->GetInbornStr() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kStr))); break; - case 'Б': - case 'б': ch->set_dex(std::max(ch->GetInbornDex() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kDex))); + case rus::kBeUpper: + case rus::kBe: ch->set_dex(std::max(ch->GetInbornDex() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kDex))); break; - case 'Г': - case 'г': ch->set_int(std::max(ch->GetInbornInt() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kInt))); + case rus::kGeUpper: + case rus::kGe: ch->set_int(std::max(ch->GetInbornInt() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kInt))); break; - case 'Д': - case 'д': ch->set_wis(std::max(ch->GetInbornWis() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kWis))); + case rus::kDeUpper: + case rus::kDe: ch->set_wis(std::max(ch->GetInbornWis() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kWis))); break; - case 'Е': - case 'е': ch->set_con(std::max(ch->GetInbornCon() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCon))); + case rus::kIeUpper: + case rus::kIe: ch->set_con(std::max(ch->GetInbornCon() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCon))); break; - case 'Ж': - case 'ж': ch->set_cha(std::max(ch->GetInbornCha() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCha))); + case rus::kZheUpper: + case rus::kZhe: ch->set_cha(std::max(ch->GetInbornCha() - 1, ch_class.GetBaseStatGenMin(EBaseStat::kCha))); break; - case 'З': - case 'з': ch->set_str(std::min(ch->GetInbornStr() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kStr))); + case rus::kZeUpper: + case rus::kZe: ch->set_str(std::min(ch->GetInbornStr() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kStr))); break; - case 'И': - case 'и': ch->set_dex(std::min(ch->GetInbornDex() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kDex))); + case rus::kIUpper: + case rus::kI: ch->set_dex(std::min(ch->GetInbornDex() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kDex))); break; - case 'К': - case 'к': ch->set_int(std::min(ch->GetInbornInt() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kInt))); + case rus::kKaUpper: + case rus::kKa: ch->set_int(std::min(ch->GetInbornInt() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kInt))); break; - case 'Л': - case 'л': ch->set_wis(std::min(ch->GetInbornWis() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kWis))); + case rus::kElUpper: + case rus::kEl: ch->set_wis(std::min(ch->GetInbornWis() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kWis))); break; - case 'М': - case 'м': ch->set_con(std::min(ch->GetInbornCon() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCon))); + case rus::kEmUpper: + case rus::kEm: ch->set_con(std::min(ch->GetInbornCon() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCon))); break; - case 'Н': - case 'н': ch->set_cha(std::min(ch->GetInbornCha() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCha))); + case rus::kEnUpper: + case rus::kEn: ch->set_cha(std::min(ch->GetInbornCha() + 1, ch_class.GetBaseStatGenMax(EBaseStat::kCha))); break; - case 'П': - case 'п': SendMsgToChar(genchar_help, ch); + case rus::kPeUpper: + case rus::kPe: SendMsgToChar(genchar_help, ch); break; - case 'В': - case 'в': + case rus::kVeUpper: + case rus::kVe: if (CalcBasseStatsSum(ch) != kBaseStatsSum) break; // по случаю успешной генерации сохраняем стартовые статы @@ -147,8 +148,8 @@ int genchar_parse(CharData *ch, char *arg) { ch->set_start_stat(G_CON, ch->GetInbornCon()); ch->set_start_stat(G_CHA, ch->GetInbornCha()); return kGencharExit; - case 'О': - case 'о': { + case rus::kOUpper: + case rus::kO: { const auto &tmp_class = MUD::Class(ch->GetClass()); ch->set_str(tmp_class.GetBaseStatGenAuto(EBaseStat::kStr)); ch->set_dex(tmp_class.GetBaseStatGenAuto(EBaseStat::kDex)); diff --git a/src/gameplay/mechanics/glory.cpp b/src/gameplay/mechanics/glory.cpp index aec1f17a68..f5e66bcab9 100644 --- a/src/gameplay/mechanics/glory.cpp +++ b/src/gameplay/mechanics/glory.cpp @@ -3,6 +3,8 @@ // Part of Bylins http://www.mud.ru #include "glory.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" #include "utils/grammar/declensions.h" @@ -478,57 +480,57 @@ void parse_add_stat(CharData *ch, int stat) { // * Парс олц меню 'слава'. bool parse_spend_glory_menu(CharData *ch, const char *arg) { - switch (*arg) { - case 'А': - case 'а': + switch (native_text::first_char_code(arg)) { + case rus::kAUpper: + case rus::kA: if (ch->desc->glory->olc_add_str >= 1) { if (!parse_remove_stat(ch, G_STR)) break; ch->desc->glory->olc_str -= 1; ch->desc->glory->olc_add_str -= 1; } break; - case 'Б': - case 'б': + case rus::kBeUpper: + case rus::kBe: if (ch->desc->glory->olc_add_dex >= 1) { if (!parse_remove_stat(ch, G_DEX)) break; ch->desc->glory->olc_dex -= 1; ch->desc->glory->olc_add_dex -= 1; } break; - case 'Г': - case 'г': + case rus::kGeUpper: + case rus::kGe: if (ch->desc->glory->olc_add_int >= 1) { if (!parse_remove_stat(ch, G_INT)) break; ch->desc->glory->olc_int -= 1; ch->desc->glory->olc_add_int -= 1; } break; - case 'Д': - case 'д': + case rus::kDeUpper: + case rus::kDe: if (ch->desc->glory->olc_add_wis >= 1) { if (!parse_remove_stat(ch, G_WIS)) break; ch->desc->glory->olc_wis -= 1; ch->desc->glory->olc_add_wis -= 1; } break; - case 'Е': - case 'е': + case rus::kIeUpper: + case rus::kIe: if (ch->desc->glory->olc_add_con >= 1) { if (!parse_remove_stat(ch, G_CON)) break; ch->desc->glory->olc_con -= 1; ch->desc->glory->olc_add_con -= 1; } break; - case 'Ж': - case 'ж': + case rus::kZheUpper: + case rus::kZhe: if (ch->desc->glory->olc_add_cha >= 1) { if (!parse_remove_stat(ch, G_CHA)) break; ch->desc->glory->olc_cha -= 1; ch->desc->glory->olc_add_cha -= 1; } break; - case 'З': - case 'з': + case rus::kZeUpper: + case rus::kZe: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_STR); @@ -536,8 +538,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_str += 1; } break; - case 'И': - case 'и': + case rus::kIUpper: + case rus::kI: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_DEX); @@ -545,8 +547,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_dex += 1; } break; - case 'К': - case 'к': + case rus::kKaUpper: + case rus::kKa: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_INT); @@ -554,8 +556,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_int += 1; } break; - case 'Л': - case 'л': + case rus::kElUpper: + case rus::kEl: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_WIS); @@ -563,8 +565,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_wis += 1; } break; - case 'М': - case 'м': + case rus::kEmUpper: + case rus::kEm: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_CON); @@ -572,8 +574,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_con += 1; } break; - case 'Н': - case 'н': + case rus::kEnUpper: + case rus::kEn: if (ch->desc->glory->olc_node->free_glory >= 1000 && ch->desc->glory->olc_add_spend_glory < MAX_STATS_BY_GLORY) { parse_add_stat(ch, G_CHA); @@ -581,8 +583,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { ch->desc->glory->olc_add_cha += 1; } break; - case 'В': - case 'в': { + case rus::kVeUpper: + case rus::kVe: { // проверка, чтобы не записывать зря, а только при изменения // и чтобы нельзя было из стата славу вытащить if ((ch->desc->glory->olc_str == ch->GetInbornStr() @@ -642,8 +644,8 @@ bool parse_spend_glory_menu(CharData *ch, const char *arg) { SendMsgToChar("Ваши изменения сохранены.\r\n", ch); return true; } - case 'Х': - case 'х': ch->desc->glory.reset(); + case rus::kHaUpper: + case rus::kHa: ch->desc->glory.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование прервано.\r\n", ch); return true; diff --git a/src/gameplay/mechanics/glory_const.cpp b/src/gameplay/mechanics/glory_const.cpp index 3ac2edcc35..1f9de1b0ae 100644 --- a/src/gameplay/mechanics/glory_const.cpp +++ b/src/gameplay/mechanics/glory_const.cpp @@ -4,6 +4,7 @@ // Part of Bylins http://www.mud.ru #include "glory_const.h" +#include "utils/russian_keys.h" #include "utils/native_text.h" #include "engine/db/player_index.h" #include "administration/privilege.h" @@ -467,32 +468,32 @@ int olc_real_stat(CharData *ch, int stat) { } bool parse_spend_glory_menu(CharData *ch, char *arg) { - switch (LOWER(*arg)) { - case 'а': olc_del_stat(ch, GLORY_STR); + switch (native_text::first_char_code_lower(arg)) { + case rus::kA: olc_del_stat(ch, GLORY_STR); break; - case 'б': olc_del_stat(ch, GLORY_DEX); + case rus::kBe: olc_del_stat(ch, GLORY_DEX); break; - case 'г': olc_del_stat(ch, GLORY_INT); + case rus::kGe: olc_del_stat(ch, GLORY_INT); break; - case 'д': olc_del_stat(ch, GLORY_WIS); + case rus::kDe: olc_del_stat(ch, GLORY_WIS); break; - case 'е': olc_del_stat(ch, GLORY_CON); + case rus::kIe: olc_del_stat(ch, GLORY_CON); break; - case 'ж': olc_del_stat(ch, GLORY_CHA); + case rus::kZhe: olc_del_stat(ch, GLORY_CHA); break; - case 'з': olc_del_stat(ch, GLORY_HIT); + case rus::kZe: olc_del_stat(ch, GLORY_HIT); break; - case 'и': olc_del_stat(ch, GLORY_SUCCESS); + case rus::kI: olc_del_stat(ch, GLORY_SUCCESS); break; - case 'к': olc_del_stat(ch, GLORY_WILL); + case rus::kKa: olc_del_stat(ch, GLORY_WILL); break; - case 'л': olc_del_stat(ch, GLORY_STABILITY); + case rus::kEl: olc_del_stat(ch, GLORY_STABILITY); break; - case 'м': olc_del_stat(ch, GLORY_REFLEX); + case rus::kEm: olc_del_stat(ch, GLORY_REFLEX); break; - case 'н': olc_del_stat(ch, GLORY_MIND); + case rus::kEn: olc_del_stat(ch, GLORY_MIND); break; - case 'э': olc_del_stat(ch, GLORY_MANAREG); + case rus::kE: olc_del_stat(ch, GLORY_MANAREG); break; case 'x': olc_add_stat(ch, GLORY_BONUSPSYS); break; @@ -502,33 +503,33 @@ bool parse_spend_glory_menu(CharData *ch, char *arg) { break; case 'd': olc_del_stat(ch, GLORY_BONUSMAG); break; - case 'о': olc_add_stat(ch, GLORY_STR); + case rus::kO: olc_add_stat(ch, GLORY_STR); break; - case 'п': olc_add_stat(ch, GLORY_DEX); + case rus::kPe: olc_add_stat(ch, GLORY_DEX); break; - case 'р': olc_add_stat(ch, GLORY_INT); + case rus::kEr: olc_add_stat(ch, GLORY_INT); break; - case 'с': olc_add_stat(ch, GLORY_WIS); + case rus::kEs: olc_add_stat(ch, GLORY_WIS); break; - case 'т': olc_add_stat(ch, GLORY_CON); + case rus::kTe: olc_add_stat(ch, GLORY_CON); break; - case 'у': olc_add_stat(ch, GLORY_CHA); + case rus::kU: olc_add_stat(ch, GLORY_CHA); break; - case 'ф': olc_add_stat(ch, GLORY_HIT); + case rus::kEf: olc_add_stat(ch, GLORY_HIT); break; - case 'х': olc_add_stat(ch, GLORY_SUCCESS); + case rus::kHa: olc_add_stat(ch, GLORY_SUCCESS); break; - case 'ц': olc_add_stat(ch, GLORY_WILL); + case rus::kTse: olc_add_stat(ch, GLORY_WILL); break; - case 'ч': olc_add_stat(ch, GLORY_STABILITY); + case rus::kChe: olc_add_stat(ch, GLORY_STABILITY); break; - case 'ш': olc_add_stat(ch, GLORY_REFLEX); + case rus::kSha: olc_add_stat(ch, GLORY_REFLEX); break; - case 'щ': olc_add_stat(ch, GLORY_MIND); + case rus::kScha: olc_add_stat(ch, GLORY_MIND); break; - case 'ю': olc_add_stat(ch, GLORY_MANAREG); + case rus::kYu: olc_add_stat(ch, GLORY_MANAREG); break; - case 'в': { + case rus::kVe: { // получившиеся статы ch->set_str(olc_real_stat(ch, GLORY_STR)); ch->set_dex(olc_real_stat(ch, GLORY_DEX)); @@ -574,7 +575,7 @@ bool parse_spend_glory_menu(CharData *ch, char *arg) { save(); return 1; } - case 'я': ch->desc->glory_const.reset(); + case rus::kYa: ch->desc->glory_const.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование прервано.\r\n", ch); return 1; diff --git a/src/gameplay/mechanics/named_stuff.cpp b/src/gameplay/mechanics/named_stuff.cpp index 8c17334f75..9abda42e83 100644 --- a/src/gameplay/mechanics/named_stuff.cpp +++ b/src/gameplay/mechanics/named_stuff.cpp @@ -3,6 +3,8 @@ // Part of Bylins http://www.mud.ru #include "named_stuff.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include #include "administration/privilege.h" #include "gameplay/mechanics/minions.h" @@ -166,11 +168,15 @@ bool parse_nedit_menu(CharData *ch, char *arg) { if (!*buf1) { return false; } - if ((*buf1 < '1' || *buf1 > '8') && (LOWER(*buf1) != 'в' && LOWER(*buf1) != 'х' && LOWER(*buf1) != 'у')) { + if ((*buf1 < '1' || *buf1 > '8') && (native_text::first_char_code_lower(buf1) != rus::kVe + && native_text::first_char_code_lower(buf1) != rus::kHa + && native_text::first_char_code_lower(buf1) != rus::kU)) { SendMsgToChar(ch, "Неверный параметр %c!\r\n", *buf1); return false; } - if (!*buf2 && LOWER(*buf1) != 'в' && LOWER(*buf1) != 'х' && LOWER(*buf1) != 'у') { + if (!*buf2 && native_text::first_char_code_lower(buf1) != rus::kVe + && native_text::first_char_code_lower(buf1) != rus::kHa + && native_text::first_char_code_lower(buf1) != rus::kU) { if (*buf1 < '5' || *buf1 > '8') { SendMsgToChar("Не указан второй параметр!\r\n", ch); } else { @@ -191,7 +197,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { return false; } - switch (LOWER(*buf1)) { + switch (native_text::first_char_code_lower(buf1)) { case '1': if (a_isdigit(*buf2) && sscanf(buf2, "%d", &num)) { if (GetObjRnum(num) < 0) { @@ -263,7 +269,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { } break; - case 'у': + case rus::kU: if (!ch->desc->old_vnum) return false; stuff_list.erase(ch->desc->old_vnum); @@ -272,7 +278,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { save(); return true; - case 'в': tmp_node->uid = ch->desc->named_obj->uid; + case rus::kVe: tmp_node->uid = ch->desc->named_obj->uid; tmp_node->can_clan = ch->desc->named_obj->can_clan; tmp_node->can_alli = ch->desc->named_obj->can_alli; tmp_node->mail = ch->desc->named_obj->mail; @@ -288,7 +294,7 @@ bool parse_nedit_menu(CharData *ch, char *arg) { save(); return true; - case 'х': ch->desc->state = EConState::kPlaying; + case rus::kHa: ch->desc->state = EConState::kPlaying; SendMsgToChar(CommonMsg(ECommonMsg::kOk) + "\r\n", ch); return true; diff --git a/src/gameplay/mechanics/obj_sets_olc.cpp b/src/gameplay/mechanics/obj_sets_olc.cpp index 7195f13be8..fb411cb649 100644 --- a/src/gameplay/mechanics/obj_sets_olc.cpp +++ b/src/gameplay/mechanics/obj_sets_olc.cpp @@ -2,6 +2,8 @@ // Part of Bylins http://www.mud.ru #include "obj_sets.h" +#include "utils/russian_keys.h" +#include "utils/native_text.h" #include "utils/grammar/declensions.h" #include @@ -525,19 +527,19 @@ void sedit::save_olc(CharData *ch) { void parse_main_exit(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': ch->desc->state = EConState::kPlaying; + case rus::kDe: + case rus::kDeUpper: ch->desc->state = EConState::kPlaying; ch->desc->sedit->save_olc(ch); ch->desc->sedit.reset(); SendMsgToChar("Изменения сохранены.\r\n", ch); break; case 'n': case 'N': - case 'н': - case 'Н': ch->desc->sedit.reset(); + case rus::kEn: + case rus::kEnUpper: ch->desc->sedit.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование отменено.\r\n", ch); break; @@ -551,11 +553,11 @@ void parse_main_exit(CharData *ch, const char *arg) { void parse_set_remove(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': { + case rus::kDe: + case rus::kDeUpper: { for (auto i = sets_list.begin(); i != sets_list.end(); ++i) { if ((*i)->uid == ch->desc->sedit->olc_set.uid) { sets_list.erase(i); @@ -571,8 +573,8 @@ void parse_set_remove(CharData *ch, const char *arg) { } case 'n': case 'N': - case 'н': - case 'Н': SendMsgToChar("Удаление отменено.\r\n", ch); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar("Удаление отменено.\r\n", ch); ch->desc->sedit->show_main(ch); break; default: @@ -585,11 +587,11 @@ void parse_set_remove(CharData *ch, const char *arg) { void sedit::parse_obj_remove(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': { + case rus::kDe: + case rus::kDeUpper: { auto i = olc_set.obj_list.find(obj_edit); if (i != olc_set.obj_list.end()) { olc_set.obj_list.erase(i); @@ -601,8 +603,8 @@ void sedit::parse_obj_remove(CharData *ch, const char *arg) { } case 'n': case 'N': - case 'н': - case 'Н': SendMsgToChar("Удаление отменено.\r\n", ch); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar("Удаление отменено.\r\n", ch); show_obj_edit(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); @@ -613,11 +615,11 @@ void sedit::parse_obj_remove(CharData *ch, const char *arg) { void sedit::parse_activ_remove(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': { + case rus::kDe: + case rus::kDeUpper: { auto i = olc_set.activ_list.find(activ_edit); if (i != olc_set.activ_list.end()) { olc_set.activ_list.erase(i); @@ -629,8 +631,8 @@ void sedit::parse_activ_remove(CharData *ch, const char *arg) { } case 'n': case 'N': - case 'н': - case 'Н': SendMsgToChar("Удаление отменено.\r\n", ch); + case rus::kEn: + case rus::kEnUpper: SendMsgToChar("Удаление отменено.\r\n", ch); show_activ_edit(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); @@ -667,11 +669,11 @@ void sedit::parse_global_msg(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': + case rus::kVeUpper: + case rus::kVe: if (msg_edit != global_msg) { SendMsgToChar("Вы хотите сохранить изменения? Y(Д)/N(Н) : ", ch); state = STATE_GLOBAL_MSG_EXIT; @@ -721,11 +723,11 @@ void sedit::parse_global_msg(CharData *ch, const char *arg) { void parse_global_msg_exit(CharData *ch, const char *arg) { skip_spaces(&arg); - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'y': case 'Y': - case 'д': - case 'Д': ch->desc->state = EConState::kPlaying; + case rus::kDe: + case rus::kDeUpper: ch->desc->state = EConState::kPlaying; global_msg = ch->desc->sedit->msg_edit; obj_sets::save(); ch->desc->sedit.reset(); @@ -733,8 +735,8 @@ void parse_global_msg_exit(CharData *ch, const char *arg) { break; case 'n': case 'N': - case 'н': - case 'Н': ch->desc->sedit.reset(); + case rus::kEn: + case rus::kEnUpper: ch->desc->sedit.reset(); ch->desc->state = EConState::kPlaying; SendMsgToChar("Редактирование отменено.\r\n", ch); break; @@ -754,11 +756,11 @@ void sedit::parse_main(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': + case rus::kVeUpper: + case rus::kVe: if (new_entry || changed()) { SendMsgToChar("Вы хотите сохранить изменения? Y(Д)/N(Н) : ", ch); state = STATE_MAIN_EXIT; @@ -1132,11 +1134,11 @@ void sedit::parse_obj_edit(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': show_main(ch); + case rus::kVeUpper: + case rus::kVe: show_main(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); show_obj_edit(ch); @@ -1337,11 +1339,11 @@ void sedit::parse_activ_edit(CharData *ch, const char *arg) { return; } if (!a_isdigit(*arg)) { - switch (*arg) { + switch (native_text::first_char_code(arg)) { case 'Q': case 'q': - case 'В': - case 'в': show_main(ch); + case rus::kVeUpper: + case rus::kVe: show_main(ch); break; default: SendMsgToChar("Неверный выбор!\r\n", ch); show_activ_edit(ch); diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index df1ecc1041..bea8cbb269 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -313,6 +313,14 @@ char32_t first_char_code(const char *s) { return cp; } +char32_t first_char_code_lower(const char *s) { + return utf8::to_lower(first_char_code(s)); +} + +char32_t first_char_code_upper(const char *s) { + return utf8::to_upper(first_char_code(s)); +} + namespace { // Whole-buffer case conversion as one tight loop with no calls in the hot path: ASCII and the @@ -466,6 +474,16 @@ char32_t first_char_code(const char *s) { return (s == nullptr || *s == '\0') ? 0 : static_cast(*s); } +char32_t first_char_code_lower(const char *s) { + return (s == nullptr || *s == '\0') + ? 0 : static_cast(a_lcc_table[static_cast(*s)]); +} + +char32_t first_char_code_upper(const char *s) { + return (s == nullptr || *s == '\0') + ? 0 : static_cast(a_ucc_table[static_cast(*s)]); +} + void to_lower(std::string &s) { for (char &c : s) { c = a_lcc_table[static_cast(c)]; diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 24c45cea99..ae945ab075 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -56,6 +56,10 @@ std::size_t char_bytes(const char *s); // Returns 0 on an empty string. char32_t first_char_code(const char *s); +// The same, case-folded -- the replacements for switch (LOWER(*s)) / switch (UPPER(*s)). +char32_t first_char_code_lower(const char *s); +char32_t first_char_code_upper(const char *s); + // Case-insensitive comparison in the native encoding: lexicographic over lowered characters, // the shorter string orders first, returns the signed difference at the first mismatch (0 when // equal). KOI8-R: per byte, via LOWER() -- matches str_cmp/str/str semantics. UTF-8: per code From ac9b843afefa324a7136000842a425b933c8e15c Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 06:28:33 +0200 Subject: [PATCH 18/20] fix(utf8): keep save-file names identical across the flip (C1a, #3681) get_filename() derives a player's save-file name by transliterating the character name byte by byte. Under UTF-8 a Russian letter is two bytes, so the loop would have produced a different name and every existing player's files -- saves, aliases, depots, all of which go through this function -- would have stopped being found. This is the one place in the migration where a mistake costs characters, so it is fixed with the mapping pinned rather than re-derived. * The mapping was extracted from the actual tables (AltToLat + the lowercase table) rather than written from memory, so it reproduces today's output exactly: a->a, zh->1, ya->q, yo->9, and upper/lower collapse together because the original lowercased after transliterating. * native_text::translit_to_filename() implements it in both encodings -- byte-wise under KOI8-R (unchanged behaviour), code-point-wise under UTF-8. * A test pins all 33 letters in both cases plus a whole name, and it is the kind of test that must fail loudly: a silent drift here orphans player files. Verified in both branches: the full alphabet transliterates to the identical string ("abvgde91zijklmnoprstyfhc74683250q") and "Vasya" in Cyrillic gives "vasq" either way. Writing the check also caught a bug the normal build cannot see: the UTF-8 branch used char_bytes_at() before its declaration, so that branch did not compile at all -- it would only have surfaced at the flip. It now takes the length from utf8::decode(). Full suite 652 passed / 0 failed. --- src/engine/db/db.cpp | 11 ++--- src/utils/native_text.cpp | 92 +++++++++++++++++++++++++++++++++++++++ src/utils/native_text.h | 6 +++ tests/native_text.cpp | 91 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 193 insertions(+), 7 deletions(-) diff --git a/src/engine/db/db.cpp b/src/engine/db/db.cpp index 912ac9efba..f73e4e5eee 100644 --- a/src/engine/db/db.cpp +++ b/src/engine/db/db.cpp @@ -1,4 +1,5 @@ #include +#include "utils/native_text.h" #include "gameplay/affects/affect_messages.h" #include "gameplay/abilities/feats.h" // issue.perk-action-patching: BuildTalentPatchIndex #include "utils/utils_encoding.h" @@ -3630,13 +3631,9 @@ int get_filename(const char *orig_name, char *filename, int mode) { default: return (0); } - strcpy(name, orig_name); - for (ptr = name; *ptr; ptr++) { - if (*ptr == 'Ё' || *ptr == 'ё') - *ptr = '9'; - else - *ptr = LOWER(codepages::AtoL(*ptr)); - } + // Транслитерация вынесена в native_text: имя файла игрока обязано совпадать до и после + // смены кодировки, иначе сохранёнки перестанут находиться (issue #3681). + strcpy(name, native_text::translit_to_filename(orig_name).c_str()); switch (LOWER(*name)) { case 'a': diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index bea8cbb269..7fdd79f66f 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -8,6 +8,7 @@ through them changes nothing until the encoding flip. */ #include "native_text.h" +#include "utils_encoding.h" #ifdef INTERNAL_ENCODING_UTF8 #include "utf8.h" @@ -19,6 +20,10 @@ extern const char a_lcc_table[]; extern const bool a_isalnum_table[]; extern const bool a_isalpha_table[]; extern const bool a_isupper_table[]; + +// Yo is absent from the Latin table and always had a special case in get_filename(). +constexpr unsigned char kYoLowerByte = 0xA3; +constexpr unsigned char kYoUpperByte = 0xB3; #endif #include @@ -379,6 +384,76 @@ void to_upper(std::string &s) { fold_range_utf8(s.data(), s.data() + s.size(), t void to_lower(char *s) { fold_range_utf8(s, s + std::char_traits::length(s), false); } void to_upper(char *s) { fold_range_utf8(s, s + std::char_traits::length(s), true); } + +std::string translit_to_filename(std::string_view name) { + // Code point -> the very same Latin character the KOI8-R byte table yields, so a player's + // file name is identical before and after the flip. Upper and lower case collapse together + // because the byte-wise original lowercased after transliterating. + static const struct { char32_t cp; char latin; } kMap[] = { + {0x0430, 'a'}, {0x0410, 'a'}, + {0x0431, 'b'}, {0x0411, 'b'}, + {0x0432, 'v'}, {0x0412, 'v'}, + {0x0433, 'g'}, {0x0413, 'g'}, + {0x0434, 'd'}, {0x0414, 'd'}, + {0x0435, 'e'}, {0x0415, 'e'}, + {0x0451, '9'}, {0x0401, '9'}, + {0x0436, '1'}, {0x0416, '1'}, + {0x0437, 'z'}, {0x0417, 'z'}, + {0x0438, 'i'}, {0x0418, 'i'}, + {0x0439, 'j'}, {0x0419, 'j'}, + {0x043A, 'k'}, {0x041A, 'k'}, + {0x043B, 'l'}, {0x041B, 'l'}, + {0x043C, 'm'}, {0x041C, 'm'}, + {0x043D, 'n'}, {0x041D, 'n'}, + {0x043E, 'o'}, {0x041E, 'o'}, + {0x043F, 'p'}, {0x041F, 'p'}, + {0x0440, 'r'}, {0x0420, 'r'}, + {0x0441, 's'}, {0x0421, 's'}, + {0x0442, 't'}, {0x0422, 't'}, + {0x0443, 'y'}, {0x0423, 'y'}, + {0x0444, 'f'}, {0x0424, 'f'}, + {0x0445, 'h'}, {0x0425, 'h'}, + {0x0446, 'c'}, {0x0426, 'c'}, + {0x0447, '7'}, {0x0427, '7'}, + {0x0448, '4'}, {0x0428, '4'}, + {0x0449, '6'}, {0x0429, '6'}, + {0x044A, '8'}, {0x042A, '8'}, + {0x044B, '3'}, {0x042B, '3'}, + {0x044C, '2'}, {0x042C, '2'}, + {0x044D, '5'}, {0x042D, '5'}, + {0x044E, '0'}, {0x042E, '0'}, + {0x044F, 'q'}, {0x042F, 'q'}, + }; + std::string out; + out.reserve(name.size()); + std::size_t pos = 0; + while (pos < name.size()) { + char32_t cp = 0; + const std::size_t len = utf8::decode(name, pos, cp); // decode() already reports the length + if (len == 0) { + break; + } + if (cp < 0x80) { + char c = static_cast(cp); + if (c >= 'A' && c <= 'Z') { + c = static_cast(c + 0x20); + } + out.push_back(c); + } else { + char mapped = '_'; + for (const auto &e : kMap) { + if (e.cp == cp) { + mapped = e.latin; + break; + } + } + out.push_back(mapped); + } + pos += len; + } + return out; +} + #else // KOI8-R: 1 byte == 1 character bool native_is_utf8() { @@ -508,6 +583,23 @@ void to_upper(char *s) { } } + +std::string translit_to_filename(std::string_view name) { + // Byte-wise, exactly as get_filename() did before this was extracted: transliterate through + // the Latin table, then lowercase. Yo is not in that table and had its own special case. + std::string out; + out.reserve(name.size()); + for (const char ch : name) { + const unsigned char b = static_cast(ch); + if (b == kYoLowerByte || b == kYoUpperByte) { + out.push_back('9'); + } else { + out.push_back(a_lcc_table[static_cast(codepages::AtoL(ch))]); + } + } + return out; +} + #endif // --------------------------------------------------------------------------------------------- diff --git a/src/utils/native_text.h b/src/utils/native_text.h index ae945ab075..30c15d1c8d 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -145,6 +145,12 @@ void to_upper(std::string &s); void to_lower(char *s); void to_upper(char *s); +// Transliterate `name` into the ASCII form used for save-file names: Russian letters become +// Latin ones, ASCII is lowercased. The mapping is fixed by what the byte-wise implementation +// produced before the migration and MUST NOT drift -- the result is the on-disk file name of a +// player, so a change would orphan every existing character. Pinned by a test in both encodings. +std::string translit_to_filename(std::string_view name); + // Does the single character `ch` occur in `list`? The replacement for strchr() over a literal // list of letters: `list` is walked one whole character at a time, so a multibyte character can // never match on a partial byte sequence. Comparison is exact (case-sensitive), like strchr. diff --git a/tests/native_text.cpp b/tests/native_text.cpp index e0ffe7ff0b..c351efa161 100644 --- a/tests/native_text.cpp +++ b/tests/native_text.cpp @@ -342,4 +342,95 @@ TEST(NativeText, RussianKeysMatchFirstCharCode) { EXPECT_EQ(native_text::first_char_code(nullptr), 0u); } +TEST(NativeText, TransliterationIsStableAcrossTheFlip) { + // A player's save file is named after the transliterated character name, so this mapping is + // on-disk state: if it ever changes, every existing character stops being found. Each row is + // the letter in both encodings and the single ASCII character it must always produce -- the + // values were taken from what the byte-wise implementation produced before the migration. + struct Row { const char *koi8; const char *utf8; char expected; }; + static const Row kRows[] = { + {"\xC1", "\xD0\xB0", 'a'}, + {"\xE1", "\xD0\x90", 'a'}, + {"\xC2", "\xD0\xB1", 'b'}, + {"\xE2", "\xD0\x91", 'b'}, + {"\xD7", "\xD0\xB2", 'v'}, + {"\xF7", "\xD0\x92", 'v'}, + {"\xC7", "\xD0\xB3", 'g'}, + {"\xE7", "\xD0\x93", 'g'}, + {"\xC4", "\xD0\xB4", 'd'}, + {"\xE4", "\xD0\x94", 'd'}, + {"\xC5", "\xD0\xB5", 'e'}, + {"\xE5", "\xD0\x95", 'e'}, + {"\xA3", "\xD1\x91", '9'}, + {"\xB3", "\xD0\x81", '9'}, + {"\xD6", "\xD0\xB6", '1'}, + {"\xF6", "\xD0\x96", '1'}, + {"\xDA", "\xD0\xB7", 'z'}, + {"\xFA", "\xD0\x97", 'z'}, + {"\xC9", "\xD0\xB8", 'i'}, + {"\xE9", "\xD0\x98", 'i'}, + {"\xCA", "\xD0\xB9", 'j'}, + {"\xEA", "\xD0\x99", 'j'}, + {"\xCB", "\xD0\xBA", 'k'}, + {"\xEB", "\xD0\x9A", 'k'}, + {"\xCC", "\xD0\xBB", 'l'}, + {"\xEC", "\xD0\x9B", 'l'}, + {"\xCD", "\xD0\xBC", 'm'}, + {"\xED", "\xD0\x9C", 'm'}, + {"\xCE", "\xD0\xBD", 'n'}, + {"\xEE", "\xD0\x9D", 'n'}, + {"\xCF", "\xD0\xBE", 'o'}, + {"\xEF", "\xD0\x9E", 'o'}, + {"\xD0", "\xD0\xBF", 'p'}, + {"\xF0", "\xD0\x9F", 'p'}, + {"\xD2", "\xD1\x80", 'r'}, + {"\xF2", "\xD0\xA0", 'r'}, + {"\xD3", "\xD1\x81", 's'}, + {"\xF3", "\xD0\xA1", 's'}, + {"\xD4", "\xD1\x82", 't'}, + {"\xF4", "\xD0\xA2", 't'}, + {"\xD5", "\xD1\x83", 'y'}, + {"\xF5", "\xD0\xA3", 'y'}, + {"\xC6", "\xD1\x84", 'f'}, + {"\xE6", "\xD0\xA4", 'f'}, + {"\xC8", "\xD1\x85", 'h'}, + {"\xE8", "\xD0\xA5", 'h'}, + {"\xC3", "\xD1\x86", 'c'}, + {"\xE3", "\xD0\xA6", 'c'}, + {"\xDE", "\xD1\x87", '7'}, + {"\xFE", "\xD0\xA7", '7'}, + {"\xDB", "\xD1\x88", '4'}, + {"\xFB", "\xD0\xA8", '4'}, + {"\xDD", "\xD1\x89", '6'}, + {"\xFD", "\xD0\xA9", '6'}, + {"\xDF", "\xD1\x8A", '8'}, + {"\xFF", "\xD0\xAA", '8'}, + {"\xD9", "\xD1\x8B", '3'}, + {"\xF9", "\xD0\xAB", '3'}, + {"\xD8", "\xD1\x8C", '2'}, + {"\xF8", "\xD0\xAC", '2'}, + {"\xDC", "\xD1\x8D", '5'}, + {"\xFC", "\xD0\xAD", '5'}, + {"\xC0", "\xD1\x8E", '0'}, + {"\xE0", "\xD0\xAE", '0'}, + {"\xD1", "\xD1\x8F", 'q'}, + {"\xF1", "\xD0\xAF", 'q'}, + }; + for (const auto &r : kRows) { + const char *const input = native_text::native_is_utf8() ? r.utf8 : r.koi8; + EXPECT_EQ(native_text::translit_to_filename(input), std::string(1, r.expected)) + << "transliteration drifted for " << (native_text::native_is_utf8() ? r.utf8 : r.koi8); + } + + // ASCII is lowercased and digits pass through, as before. + EXPECT_EQ(native_text::translit_to_filename("Vasya"), "vasya"); + EXPECT_EQ(native_text::translit_to_filename("Abc123"), "abc123"); + EXPECT_EQ(native_text::translit_to_filename(""), ""); + + // A whole name: "Vasya" in Cyrillic must give the same file name in both encodings. + const char *const name = native_text::native_is_utf8() + ? "\xD0\x92\xD0\xB0\xD1\x81\xD1\x8F" : "\xF7\xC1\xD3\xD1"; + EXPECT_EQ(native_text::translit_to_filename(name), "vasq"); +} + // vim: ts=4 sw=4 tw=0 noet syntax=cpp : From bda07daa6dd3c4d40c982b9108949331bdb6e7ec Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Tue, 4 Aug 2026 06:55:45 +0200 Subject: [PATCH 19/20] ci(utf8): rehearse the encoding flip on every build (#3681) The UTF-8 branch of native_text had no CI coverage at all -- nothing built it, which is how a branch that did not even compile survived until a hand-written check found it. A unit test cannot close that gap on its own: it only ever exercises the branch its build selected. Adds a Linux job that builds the *coherent* UTF-8 configuration. Sources and runtime flip together, never separately: git already stores src/ and tests/ as UTF-8, so dropping working-tree-encoding and checking out again yields UTF-8 sources, and -Dinternal_encoding=utf8 switches the runtime to character semantics. That is exactly what C2 will do, so the job is a continuous rehearsal of the flip rather than a synthetic configuration. It paid for itself immediately -- two real flip blockers, neither of which any existing test or the audit script could see: * utils.cpp sized the Russian month names as char[12][10]. That fits KOI8-R ("Sentyabrya" is 8 bytes) and overflows in UTF-8 (16). Now an array of pointers, so the cell width stops depending on the encoding at all. * where_format indented continuation lines by prefix.size() -- bytes, not columns -- so the location column drifted apart for names with Cyrillic in them. The test that was supposed to catch this measured alignment in bytes too, so it agreed with the bug; both now measure characters. Verified in both configurations: 652 passed / 0 failed, no warnings. Incidentally confirms the literal-repertoire rule this migration imposes: a comment written with guillemets could not be encoded back to KOI8-R, and git refused it. --- .github/workflows/_linux.yml | 33 +++++++++++++++++++++++++++++++++ src/engine/db/db.cpp | 2 +- src/engine/ui/cmd/do_where.cpp | 9 ++++++--- src/utils/utils.cpp | 8 +++++--- tests/where.format.cpp | 20 +++++++++++++++----- 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_linux.yml b/.github/workflows/_linux.yml index 8dc3ca035f..06db4c4cc9 100644 --- a/.github/workflows/_linux.yml +++ b/.github/workflows/_linux.yml @@ -90,6 +90,39 @@ jobs: - name: Run tests run: meson test -C build + utf8-flip: + name: GCC / UTF-8 flip rehearsal + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Install base dependencies + uses: ./.github/actions/install-linux-deps + with: + add-universe: 'true' + + # Репетиция флипа кодировки (issue #3681). Исходники и рантайм переключаются ВМЕСТЕ: + # блобы в git и так лежат в UTF-8, поэтому снятие working-tree-encoding + повторный + # checkout даёт UTF-8-исходники, а -Dinternal_encoding=utf8 переводит на символьную + # семантику рантайм. Задача держит UTF-8-ветку компилируемой и протестированной до + # самого флипа -- иначе она не собирается вообще ничем. + - name: Check out sources as UTF-8 + run: | + sed -i -E 's|^(/(src\|tests)/\*\* +)working-tree-encoding=KOI8-R |\1|' .gitattributes + rm -rf src tests + git checkout -- src tests + file src/utils/utils.cpp | grep -q UTF-8 || { echo "sources did not become UTF-8"; exit 1; } + + - name: Configure meson + run: meson setup build --wipe -Dbuild_profile=release -Dinternal_encoding=utf8 -Dunity_size=45 + + - name: Build + run: meson compile -C build tests + + - name: Run tests + run: meson test -C build + gcc15: name: GCC 15 / Base runs-on: ubuntu-latest diff --git a/src/engine/db/db.cpp b/src/engine/db/db.cpp index f73e4e5eee..4d0538e6dd 100644 --- a/src/engine/db/db.cpp +++ b/src/engine/db/db.cpp @@ -3596,7 +3596,7 @@ Rooms::~Rooms() { int get_filename(const char *orig_name, char *filename, int mode) { const char *prefix, *middle, *suffix; - char name[64], *ptr; + char name[64]; if (orig_name == nullptr || *orig_name == '\0' || filename == nullptr) { log("SYSERR: NULL pointer or empty string passed to get_filename(), %p or %p.", orig_name, filename); diff --git a/src/engine/ui/cmd/do_where.cpp b/src/engine/ui/cmd/do_where.cpp index 2d95e0d0dd..2dd5fc28e6 100644 --- a/src/engine/ui/cmd/do_where.cpp +++ b/src/engine/ui/cmd/do_where.cpp @@ -3,6 +3,7 @@ // #include "engine/entities/char_data.h" +#include "utils/native_text.h" #include "administration/privilege.h" #include "engine/db/world_objects.h" #include "gameplay/economics/exchange.h" @@ -195,9 +196,11 @@ std::string where_format::FormatWhere(const std::vector &rows) { for (const auto &row : rows) { const std::string prefix = fmt::format("{:>{}}. {:<5} [{:>7}] {:<25} - ", row.num, num_width, RowKindLabel(row.kind), row.vnum, row.name); - // Отступ строк-продолжений = длине префикса, чтобы разделитель " - " - // встал ровно под разделителем первой строки. - const std::string cont = fmt::format("{:>{}}", " - ", static_cast(prefix.size())); + // Отступ строк-продолжений = ШИРИНЕ префикса в символах, чтобы разделитель " - " + // встал ровно под разделителем первой строки. Именно в символах, а не в байтах: + // prefix.size() под UTF-8 больше числа колонок, и отступ уезжал (issue #3681). + const std::string cont = + fmt::format("{:>{}}", " - ", static_cast(native_text::char_count(prefix))); out += prefix; if (!row.location_lines.empty()) { diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp index 85c5a6c85a..c9c1727548 100644 --- a/src/utils/utils.cpp +++ b/src/utils/utils.cpp @@ -312,10 +312,12 @@ void format_text(const utils::AbstractStringWriter::shared_ptr &writer, char *rustime(const struct tm *timeptr) { - static char mon_name[12][10] = + // Массив указателей, а не char[12][10]: ширина ячейки зависела бы от кодировки + // (в UTF-8 русская буква занимает два байта, и названия перестают влезать). + static const char *const mon_name[12] = { - "Января\0", "Февраля\0", "Марта\0", "Апреля\0", "Мая\0", "Июня\0", - "Июля\0", "Августа\0", "Сентября\0", "Октября\0", "Ноября\0", "Декабря\0" + "Января", "Февраля", "Марта", "Апреля", "Мая", "Июня", + "Июля", "Августа", "Сентября", "Октября", "Ноября", "Декабря" }; static char result[100]; diff --git a/tests/where.format.cpp b/tests/where.format.cpp index 99810445ba..e0f2b6d3ad 100644 --- a/tests/where.format.cpp +++ b/tests/where.format.cpp @@ -6,6 +6,7 @@ // строки == ширине колонки, и индексная арифметика ниже корректна. #include "engine/ui/cmd/do_where.h" +#include "utils/native_text.h" #include @@ -92,14 +93,23 @@ TEST(WhereFormat, LocationColumnAligned) { const auto lines = SplitLines(FormatWhere(SampleRows())); ASSERT_GE(lines.size(), 5u); - const auto ref = lines[1].rfind(" - "); + // Колонка разделителя измеряется в СИМВОЛАХ, а не в байтах: под UTF-8 байтовое смещение + // зависит от того, сколько в имени кириллицы, и "выровнено" перестаёт значить "в одной + // колонке" (issue #3681). + const auto column_of_separator = [](const std::string &line) { + const auto at = line.rfind(" - "); + return at == std::string::npos + ? std::string::npos : native_text::char_count(std::string_view(line).substr(0, at)); + }; + + const auto ref = column_of_separator(lines[1]); ASSERT_NE(ref, std::string::npos); - EXPECT_EQ(lines[2].rfind(" - "), ref) << "предмет с именем ровно 25 симв."; - EXPECT_EQ(lines[3].rfind(" - "), ref) << "строка-продолжение контейнера"; - EXPECT_EQ(lines[4].rfind(" - "), ref) << "короткое имя"; + EXPECT_EQ(column_of_separator(lines[2]), ref) << "предмет с именем ровно 25 симв."; + EXPECT_EQ(column_of_separator(lines[3]), ref) << "строка-продолжение контейнера"; + EXPECT_EQ(column_of_separator(lines[4]), ref) << "короткое имя"; // Имя моба (29 симв.) длиннее поля в 25 -> разделитель уезжает вправо на 4. - EXPECT_EQ(lines[0].rfind(" - "), ref + 4); + EXPECT_EQ(column_of_separator(lines[0]), ref + 4); } // Предмет в контейнере даёт ровно две строки: первая оканчивается на From 52bfc9a374aa3781b6c718ffc99d894a13357aad Mon Sep 17 00:00:00 2001 From: Anton Gorev Date: Wed, 5 Aug 2026 01:14:17 +0200 Subject: [PATCH 20/20] feat(utf8): funnel world text through the encoding boundary (#3681) Adds native_text::from_koi8() -- identity under KOI8-R, a transcode under UTF-8 -- and routes the YAML loader's GetText() through it. That is the single point all world text passes, so the loader stops being encoding-specific and the world files can stay KOI8-R on disk (track B remains deferred). No-op on the current build; 652 passed / 0 failed. Booting the flip build on the production world surfaced two things worth recording now rather than on the day of the flip: * The server aborted during boot in HelpSystem::SetsHelp -> libfort, on data from an XML config. So converting the world loader is not sufficient: every KOI8-R source (cfg/**, help text, boards, mail, saves) needs the same boundary before C2. * libfort's utf8_table does not degrade on malformed input -- the visible-width calculation underflows and the allocation check aborts the process. An unconverted string reaching any table is therefore a crash, not a cosmetic defect, which raises C3 from "convert what players see" to "convert every source". Neither is reachable from unit tests; both came out of running the real binary against the real world, which is what the rehearsal job is for. --- src/engine/db/yaml_world_data_source.cpp | 5 +++-- src/utils/native_text.cpp | 15 +++++++++++++++ src/utils/native_text.h | 5 +++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/engine/db/yaml_world_data_source.cpp b/src/engine/db/yaml_world_data_source.cpp index 0cb0ad4252..142442fdd4 100644 --- a/src/engine/db/yaml_world_data_source.cpp +++ b/src/engine/db/yaml_world_data_source.cpp @@ -701,8 +701,9 @@ std::string YamlWorldDataSource::GetText(const YAML::Node &node, const std::stri { if (node[key]) { - // YAML files are already in KOI8-R, no conversion needed - std::string text = node[key].as(); + // Файлы мира лежат на диске в KOI8-R; переводим в нативную кодировку движка + // (под KOI8-R это тождество, под UTF-8 - перекодировка). Issue #3681. + std::string text = native_text::from_koi8(node[key].as()); // Convert line endings if configured for DOS format if (m_convert_lf_to_crlf) { diff --git a/src/utils/native_text.cpp b/src/utils/native_text.cpp index 7fdd79f66f..9ebddeb3b7 100644 --- a/src/utils/native_text.cpp +++ b/src/utils/native_text.cpp @@ -27,6 +27,7 @@ constexpr unsigned char kYoUpperByte = 0xB3; #endif #include +#include namespace native_text { @@ -385,6 +386,16 @@ void to_lower(char *s) { fold_range_utf8(s, s + std::char_traits::length(s void to_upper(char *s) { fold_range_utf8(s, s + std::char_traits::length(s), true); } +std::string from_koi8(const std::string &text) { + if (text.empty()) { + return text; + } + // koi_to_utf8() can grow the text; utils_encoding sizes its own buffers at 6x, so match that. + std::vector out(text.size() * 6 + 1, '\0'); + codepages::koi_to_utf8(const_cast(text.c_str()), out.data()); + return std::string(out.data()); +} + std::string translit_to_filename(std::string_view name) { // Code point -> the very same Latin character the KOI8-R byte table yields, so a player's // file name is identical before and after the flip. Upper and lower case collapse together @@ -584,6 +595,10 @@ void to_upper(char *s) { } +std::string from_koi8(const std::string &text) { + return text; // the native encoding already is KOI8-R +} + std::string translit_to_filename(std::string_view name) { // Byte-wise, exactly as get_filename() did before this was extracted: transliterate through // the Latin table, then lowercase. Yo is not in that table and had its own special case. diff --git a/src/utils/native_text.h b/src/utils/native_text.h index 30c15d1c8d..f7715296e2 100644 --- a/src/utils/native_text.h +++ b/src/utils/native_text.h @@ -145,6 +145,11 @@ void to_upper(std::string &s); void to_lower(char *s); void to_upper(char *s); +// Bring text stored on disk in KOI8-R (world files, configs, saves) into the engine's native +// encoding. Identity under KOI8-R, a transcode under UTF-8. Having it here keeps the loaders +// free of #ifdefs and gives one place to revisit when the data files themselves move. +std::string from_koi8(const std::string &text); + // Transliterate `name` into the ASCII form used for save-file names: Russian letters become // Latin ones, ASCII is lowercased. The mapping is fixed by what the byte-wise implementation // produced before the migration and MUST NOT drift -- the result is the on-disk file name of a