From 0aa4bcdba492d616b003e59f85dae7816184863f Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Fri, 14 Aug 2026 16:20:49 +0800 Subject: [PATCH 1/3] test(json): add adversarial QuickCheck roundtrip properties New property families complementing json/quickcheck_test.mbt: - adversarial strings (every control character, JSON syntax characters, BMP boundaries, astral pairs, and lone surrogates) roundtrip through stringify/parse as both values and object keys, across indent and escape_slash options; - the fully \uXXXX-escaped spelling of those strings parses back to the exact original, including surrogate pairs split across two escapes and lone-surrogate escapes; - every textual spelling of zero preserves the sign of zero bitwise; - Int64/UInt64 literals roundtrip textually (repr preserved past 2^53); - random legal whitespace inserted between tokens never changes the parsed value; - duplicate object keys: last occurrence wins; - single code-unit deletions/replacements of valid documents keep the parser total (parse agrees with valid, no panics) and successfully parsed mutants are fixed points of restringify-and-reparse; - deterministic pins for the default 1024 nesting-depth boundary and for surrogate handling. The AdvString generator shrinks at the UTF-16 code-unit level so counterexamples can minimize to half of a surrogate pair. These properties found the two parser bugs fixed in the previous commit. Co-Authored-By: Claude Fable 5 --- json/quickcheck_adversarial_test.mbt | 339 +++++++++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 json/quickcheck_adversarial_test.mbt diff --git a/json/quickcheck_adversarial_test.mbt b/json/quickcheck_adversarial_test.mbt new file mode 100644 index 000000000..e1b901449 --- /dev/null +++ b/json/quickcheck_adversarial_test.mbt @@ -0,0 +1,339 @@ +// Copyright 2026 International Digital Economy Academy +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Adversarial property-based tests for the json package, complementing +// `quickcheck_test.mbt`: +// +// - strings built from hostile UTF-16 code units (every control character, +// JSON syntax characters, astral pairs, and *lone surrogates*), used both +// as values and as object keys; +// - the fully `\uXXXX`-escaped spelling of those strings; +// - textual zero literals (`-0`, `-0.0e7`, ...) which must preserve the +// IEEE-754 sign of zero; +// - integer literals across the full Int64/UInt64 range, whose text must be +// preserved exactly by `stringify` (via `repr`) beyond 2^53; +// - random legal whitespace inserted between tokens; +// - objects spelled with duplicate keys (last occurrence wins); +// - single-code-unit deletions/replacements of valid documents, which must +// never make the parser panic and must keep `parse` consistent with +// `valid`. + +///| +/// Characters drawn from pools that stress every branch of the escaper and +/// the string lexer: control characters (escaped as `\uXXXX` or short +/// escapes), JSON syntax characters, BMP boundary values, astral code points +/// (surrogate pairs in UTF-16), and lone surrogates, which MoonBit strings +/// can represent and the parser passes through. +fn adversarial_char_gen() -> @quickcheck.Generator[Char] { + @quickcheck.frequency([ + // Every control character U+0000..U+001F. + (3, @quickcheck.int_range(0, 0x20).map(i => i.unsafe_to_char())), + // Characters that interact with JSON syntax and escaping. + ( + 3, + @quickcheck.elements(['"', '\\', '/', '{', '}', '[', ']', ',', ':', ' ']), + ), + // BMP boundaries and astral code points (encoded as surrogate pairs). + ( + 2, + @quickcheck.elements([ + '\u{7F}', '\u{80}', '\u{7FF}', '\u{800}', '\u{D7FF}', '\u{E000}', '\u{FFFD}', + '\u{FFFF}', '\u{10000}', '\u{1F600}', '\u{10FFFF}', + ]), + ), + // Lone surrogates: representable in UTF-16 strings, and the classic + // way to corrupt a JSON stringifier or parser. + (2, @quickcheck.int_range(0xD800, 0xE000).map(i => i.unsafe_to_char())), + // Ordinary ASCII so escapes sit inside unescaped runs. + (3, @quickcheck.char_range('a', 'z')), + ]) +} + +///| +fn adversarial_string_gen() -> @quickcheck.Generator[String] { + @quickcheck.int_range(0, 24) + .flat_map(n => adversarial_char_gen().array_with_size(n)) + .map(chars => String::from_array(chars)) +} + +///| +priv struct AdvString(String) derive(@debug.Debug) + +///| +impl @quickcheck.Arbitrary for AdvString with fn arbitrary(size, state) { + AdvString(adversarial_string_gen().run(size, state)) +} + +///| +/// Shrinks at the UTF-16 code-unit level (dropping one unit at a time), so a +/// counterexample can minimize to half of a surrogate pair if that is what +/// triggers a failure. +impl @shrink.Shrink for AdvString with fn shrink(self) { + let units = self.0.code_units() + let n = units.length() + if n == 0 { + return Iter::empty() + } + Iter::singleton(AdvString("")).concat( + (0) + .until(n) + .map(i => { + let buf = StringBuilder(size_hint=n - 1) + for j in 0.. { + let (key, value, raw_indent, escape_slash) = input + let doc = Json::object(Map([(key.0, Json::array([Json::string(value.0)]))])) + let text = doc.stringify(indent=wrap_index(raw_indent, 5), escape_slash~) + @json.parse(text) == doc + }) +} + +///| +/// Spells every UTF-16 code unit of the string as a `\uXXXX` escape +/// (alternating hex-digit case) and checks the parser reassembles the exact +/// original string — including surrogate pairs split across two escapes and +/// lone surrogates. +test "fully \\uXXXX-escaped strings parse back to the original" { + let hex_lower = "0123456789abcdef".to_array() + let hex_upper = "0123456789ABCDEF".to_array() + @quickcheck.check((s : AdvString) => { + let buf = StringBuilder() + buf.write_char('"') + for i, unit in s.0.code_units() { + let code = unit.to_int() + let hex = if i % 2 == 0 { hex_lower } else { hex_upper } + buf.write_char('\\') + buf.write_char('u') + buf.write_char(hex[(code >> 12) & 0xF]) + buf.write_char(hex[(code >> 8) & 0xF]) + buf.write_char(hex[(code >> 4) & 0xF]) + buf.write_char(hex[code & 0xF]) + } + buf.write_char('"') + @json.parse(buf.to_string()) == Json::string(s.0) + }) +} + +///| +/// Every spelling of zero (`-0`, `-0.00`, `-0e13`, `0.0E-7`, ...) must parse +/// to an IEEE-754 zero whose sign bit matches the literal's sign. The +/// integer spelling `-0` used to lose the sign because the integer fast path +/// negated an `Int64` (where `-0 == 0`) before converting to `Double`. +test "zero literals preserve the sign of zero" { + @quickcheck.check((input : (Bool, Int, Int, Int)) => { + let (negative, raw_frac, raw_exp_kind, raw_exp) = input + let text = StringBuilder() + if negative { + text.write_char('-') + } + text.write_char('0') + let frac_digits = wrap_index(raw_frac, 4) + if frac_digits > 0 { + text.write_char('.') + text.write_string("0".repeat(frac_digits)) + } + match wrap_index(raw_exp_kind, 4) { + 0 => () + 1 => text.write_string("e" + wrap_index(raw_exp, 400).to_string()) + 2 => text.write_string("E+" + wrap_index(raw_exp, 400).to_string()) + _ => text.write_string("e-" + wrap_index(raw_exp, 400).to_string()) + } + guard @json.parse(text.to_string()) is Number(n, ..) else { return false } + n == 0.0 && (n.reinterpret_as_int64() < 0L) == negative + }) +} + +///| +/// Integer literals over the full Int64/UInt64 range: `parse` must produce +/// the correctly rounded double, and `stringify` must reproduce the source +/// text exactly — beyond 2^53 that requires the preserved `repr`. +test "integer literals roundtrip through parse and stringify textually" { + @quickcheck.check((x : Int64) => { + let text = x.to_string() + let parsed = @json.parse(text) + parsed == Json::number(x.to_double()) && parsed.stringify() == text + }) + @quickcheck.check((x : UInt64) => { + let text = x.to_string() + let parsed = @json.parse(text) + parsed == Json::number(x.to_double()) && parsed.stringify() == text + }) +} + +///| +/// Inserting random legal whitespace (space, tab, CR, LF) around structural +/// tokens never changes the parsed value. +test "whitespace between tokens does not change the parsed value" { + @quickcheck.check((input : (ArbJson, UInt64)) => { + let (json, seed) = input + let rng = @splitmix.new(seed~) + let ws : ReadOnlyArray[Char] = [' ', '\t', '\n', '\r'] + let text = json.0.stringify() + let buf = StringBuilder() + fn maybe_ws() { + if rng.next_uint() % 2 == 0 { + let n = (rng.next_uint() % 3).reinterpret_as_int() + for _ in 0..<(n + 1) { + buf.write_char(ws[(rng.next_uint() % 4).reinterpret_as_int()]) + } + } + } + + maybe_ws() + let mut in_string = false + let mut escaped = false + for unit in text.code_units() { + let c = unit.to_int().unsafe_to_char() + buf.write_char(c) + if in_string { + if escaped { + escaped = false + } else if c == '\\' { + escaped = true + } else if c == '"' { + in_string = false + maybe_ws() + } + } else { + match c { + '"' => in_string = true + '[' | ']' | '{' | '}' | ',' | ':' => maybe_ws() + _ => () + } + } + } + maybe_ws() + @json.parse(buf.to_string()) == json.0 + }) +} + +///| +/// Objects spelled with duplicate keys parse with the last occurrence of +/// each key winning, matching `Map` insert semantics. +test "duplicate object keys: last occurrence wins" { + @quickcheck.check((entries : Array[(Int, Int)]) => { + let text = StringBuilder() + text.write_char('{') + let expected : Map[String, Json] = Map([]) + for i, entry in entries { + let (raw_key, value) = entry + // A pool of three keys guarantees duplicates in most runs. + let key = "k" + wrap_index(raw_key, 3).to_string() + if i > 0 { + text.write_char(',') + } + text.write_string("\"" + key + "\":" + value.to_string()) + expected[key] = Json::number(value.to_double()) + } + text.write_char('}') + @json.parse(text.to_string()) == Json::object(expected) + }) +} + +///| +/// Deleting or replacing one code unit of a valid document must keep the +/// parser total: it either succeeds or raises a parse error (`valid` agrees +/// with `parse`), and when the mutant still parses, the parsed value is a +/// fixed point of restringify-and-reparse. +test "parse stays total under single code-unit deletion and replacement" { + @quickcheck.check((input : (ArbJson, Int, Char)) => { + let (json, position, replacement) = input + let chars = json.0.stringify().to_array() + guard chars.length() > 0 else { return true } + let idx = wrap_index(position, chars.length()) + let deleted = chars.copy() + ignore(deleted.remove(idx)) + let replaced = chars.copy() + replaced[idx] = replacement + for mutant in [String::from_array(deleted), String::from_array(replaced)] { + guard parse_succeeds(mutant) == @json.valid(mutant) else { return false } + if @json.valid(mutant) { + let value = @json.parse(mutant) + guard @json.parse(value.stringify()) == value else { return false } + } + } + true + }) +} + +///| +/// The default nesting limit is exactly 1024: a document 1024 levels deep +/// parses (and roundtrips), 1025 levels raises `DepthLimitExceeded`, for +/// both arrays and objects. Also pins that `stringify` itself is iterative +/// and survives a 1024-deep tree on every backend. +test "default nesting limit boundary at depth 1024" { + let deep_array = "[".repeat(1024) + "0" + "]".repeat(1024) + assert_true(@json.valid(deep_array)) + fn outcome(text : String) -> String { + try { + ignore(@json.parse(text)) + "parsed" + } catch { + DepthLimitExceeded => "depth limit" + _ => "other error" + } + } + + let deeper_array = "[".repeat(1025) + "0" + "]".repeat(1025) + assert_eq(outcome(deeper_array), "depth limit") + let deep_object = "{\"k\":".repeat(1025) + "0" + "}".repeat(1025) + assert_eq(outcome(deep_object), "depth limit") + let mut tree : Json = Json::number(0.0) + for _ in 0..<1024 { + tree = Json::array([tree]) + } + assert_true(@json.parse(tree.stringify()) == tree) +} + +///| +/// Deterministic pins for the surrogate cases the properties above explore +/// randomly, so a regression shows up with a readable diff. +test "surrogate handling pins" { + // A lone high surrogate roundtrips raw. + let lone = String::from_array([(0xD800).unsafe_to_char()]) + let json = Json::string(lone) + assert_true(@json.parse(json.stringify()) == json) + // Its escaped spelling parses to the same string. + assert_true(@json.parse("\"\\uD800\"") == json) + // An escaped surrogate pair reassembles to the astral character. + assert_true(@json.parse("\"\\uD83D\\uDE00\"") == Json::string("\u{1F600}")) + // A reversed pair (low then high) roundtrips as two lone surrogates. + let reversed = String::from_array([ + (0xDC00).unsafe_to_char(), + (0xD800).unsafe_to_char(), + ]) + let reversed_json = Json::string(reversed) + assert_true(@json.parse(reversed_json.stringify()) == reversed_json) + // A raw lone trailing surrogate combined with an escape used to abort the + // slow-path string lexer: `flush` sliced with the checked `[start:end]`, + // which panics when the code unit at a boundary is a trailing surrogate. + let lone_low = String::from_array([(0xDC00).unsafe_to_char()]) + assert_true( + @json.parse("\"" + lone_low + "\\n\"") == Json::string(lone_low + "\n"), + ) + assert_true( + @json.parse("\"\\n" + lone_low + "\"") == Json::string("\n" + lone_low), + ) +} From 5a5e003698dbef4cb2b378222ee86ca941074d07 Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 15 Aug 2026 09:21:23 +0800 Subject: [PATCH 2/3] =?UTF-8?q?test(json):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20pin=20depth-1024=20parse=20for=20both=20shapes,=20mutate=20a?= =?UTF-8?q?t=20code-unit=20level?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The nesting-limit test now asserts that documents exactly 1024 levels deep parse successfully for BOTH arrays and objects (previously it only checked valid() for the array and only checked the 1025-deep failure for objects), and the 1025-deep variables are renamed too_deep_array / too_deep_object to reflect their depth. - The mutation-totality test now genuinely mutates single UTF-16 code units via code_units() instead of Char-level to_array(), so deleting or replacing a unit can split an astral surrogate pair, and the replacement unit is drawn from the full 16-bit range (including lone surrogates) — strictly stronger fuzzing that matches the test's name and doc comment. Co-Authored-By: Claude Fable 5 --- json/quickcheck_adversarial_test.mbt | 53 +++++++++++++++++----------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/json/quickcheck_adversarial_test.mbt b/json/quickcheck_adversarial_test.mbt index e1b901449..96ef07b3d 100644 --- a/json/quickcheck_adversarial_test.mbt +++ b/json/quickcheck_adversarial_test.mbt @@ -253,21 +253,32 @@ test "duplicate object keys: last occurrence wins" { } ///| -/// Deleting or replacing one code unit of a valid document must keep the -/// parser total: it either succeeds or raises a parse error (`valid` agrees -/// with `parse`), and when the mutant still parses, the parsed value is a -/// fixed point of restringify-and-reparse. +/// Deleting or replacing one UTF-16 code unit of a valid document must keep +/// the parser total: it either succeeds or raises a parse error (`valid` +/// agrees with `parse`), and when the mutant still parses, the parsed value +/// is a fixed point of restringify-and-reparse. +/// +/// Mutating at the code-unit level (not the `Char` level) means an astral +/// character can lose half of its surrogate pair, and the replacement unit — +/// drawn from the full 16-bit range — can itself be a lone surrogate. test "parse stays total under single code-unit deletion and replacement" { - @quickcheck.check((input : (ArbJson, Int, Char)) => { - let (json, position, replacement) = input - let chars = json.0.stringify().to_array() - guard chars.length() > 0 else { return true } - let idx = wrap_index(position, chars.length()) - let deleted = chars.copy() - ignore(deleted.remove(idx)) - let replaced = chars.copy() - replaced[idx] = replacement - for mutant in [String::from_array(deleted), String::from_array(replaced)] { + @quickcheck.check((input : (ArbJson, Int, Int)) => { + let (json, position, raw_replacement) = input + let units = json.0.stringify().code_units() + guard units.length() > 0 else { return true } + let idx = wrap_index(position, units.length()) + let replacement = wrap_index(raw_replacement, 0x10000) + let deleted = StringBuilder(size_hint=units.length()) + let replaced = StringBuilder(size_hint=units.length()) + for i, unit in units { + if i != idx { + deleted.write_char(unit.to_int().unsafe_to_char()) + replaced.write_char(unit.to_int().unsafe_to_char()) + } else { + replaced.write_char(replacement.unsafe_to_char()) + } + } + for mutant in [deleted.to_string(), replaced.to_string()] { guard parse_succeeds(mutant) == @json.valid(mutant) else { return false } if @json.valid(mutant) { let value = @json.parse(mutant) @@ -284,8 +295,6 @@ test "parse stays total under single code-unit deletion and replacement" { /// both arrays and objects. Also pins that `stringify` itself is iterative /// and survives a 1024-deep tree on every backend. test "default nesting limit boundary at depth 1024" { - let deep_array = "[".repeat(1024) + "0" + "]".repeat(1024) - assert_true(@json.valid(deep_array)) fn outcome(text : String) -> String { try { ignore(@json.parse(text)) @@ -296,10 +305,14 @@ test "default nesting limit boundary at depth 1024" { } } - let deeper_array = "[".repeat(1025) + "0" + "]".repeat(1025) - assert_eq(outcome(deeper_array), "depth limit") - let deep_object = "{\"k\":".repeat(1025) + "0" + "}".repeat(1025) - assert_eq(outcome(deep_object), "depth limit") + let deep_array = "[".repeat(1024) + "0" + "]".repeat(1024) + assert_eq(outcome(deep_array), "parsed") + let too_deep_array = "[".repeat(1025) + "0" + "]".repeat(1025) + assert_eq(outcome(too_deep_array), "depth limit") + let deep_object = "{\"k\":".repeat(1024) + "0" + "}".repeat(1024) + assert_eq(outcome(deep_object), "parsed") + let too_deep_object = "{\"k\":".repeat(1025) + "0" + "}".repeat(1025) + assert_eq(outcome(too_deep_object), "depth limit") let mut tree : Json = Json::number(0.0) for _ in 0..<1024 { tree = Json::array([tree]) From f9a3219bb51c9af693fe7c22061aed15b979ef3c Mon Sep 17 00:00:00 2001 From: Hongbo Zhang Date: Sat, 15 Aug 2026 11:02:52 +0800 Subject: [PATCH 3/3] test(json): assert clean rejection of lone surrogates per unicode-safe policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser now keeps strings Unicode well-formed (#4056): unpaired surrogates — raw or as \uXXXX escapes — are rejected with a ParseError instead of being passed through. Update the adversarial suite to match: - the roundtrip and fully-escaped generators produce only Unicode scalar values (astral pairs still included), and the AdvString shrinker drops whole characters so candidates stay well-formed; - new property: a lone surrogate injected at any position of a hostile string — raw via stringify or spelled as a \uXXXX escape, with escapes/astral pairs/control characters nearby — is always rejected cleanly (parse raises, valid is false, never an abort); - the deterministic surrogate pins now assert rejection for raw, escaped, reversed-pair, and mixed raw/escaped-half spellings, while well-formed pairs (raw or split across two escapes) still parse. Note: "zero literals preserve the sign of zero" requires the parse(-0) fix from #4061 (based on main) and fails until that lands in this branch's history; all other tests are green on wasm-gc, js, and native. Co-Authored-By: Claude Fable 5 --- json/quickcheck_adversarial_test.mbt | 154 ++++++++++++++++----------- 1 file changed, 91 insertions(+), 63 deletions(-) diff --git a/json/quickcheck_adversarial_test.mbt b/json/quickcheck_adversarial_test.mbt index 96ef07b3d..f4d7cc2be 100644 --- a/json/quickcheck_adversarial_test.mbt +++ b/json/quickcheck_adversarial_test.mbt @@ -15,10 +15,13 @@ // Adversarial property-based tests for the json package, complementing // `quickcheck_test.mbt`: // -// - strings built from hostile UTF-16 code units (every control character, -// JSON syntax characters, astral pairs, and *lone surrogates*), used both -// as values and as object keys; +// - strings built from hostile UTF-16 sequences (every control character, +// JSON syntax characters, astral pairs), used both as values and as +// object keys; // - the fully `\uXXXX`-escaped spelling of those strings; +// - *lone surrogates* injected into any position of such strings — raw or +// as `\uXXXX` escapes — which the parser must always reject with a clean +// parse error (strings stay Unicode well-formed) and never abort on; // - textual zero literals (`-0`, `-0.0e7`, ...) which must preserve the // IEEE-754 sign of zero; // - integer literals across the full Int64/UInt64 range, whose text must be @@ -32,9 +35,10 @@ ///| /// Characters drawn from pools that stress every branch of the escaper and /// the string lexer: control characters (escaped as `\uXXXX` or short -/// escapes), JSON syntax characters, BMP boundary values, astral code points -/// (surrogate pairs in UTF-16), and lone surrogates, which MoonBit strings -/// can represent and the parser passes through. +/// escapes), JSON syntax characters, BMP boundary values, and astral code +/// points (surrogate pairs in UTF-16). Only Unicode scalar values appear +/// here — MoonBit strings stay Unicode well-formed, so unpaired surrogates +/// are generated separately and asserted to be *rejected* by the parser. fn adversarial_char_gen() -> @quickcheck.Generator[Char] { @quickcheck.frequency([ // Every control character U+0000..U+001F. @@ -52,9 +56,6 @@ fn adversarial_char_gen() -> @quickcheck.Generator[Char] { '\u{FFFF}', '\u{10000}', '\u{1F600}', '\u{10FFFF}', ]), ), - // Lone surrogates: representable in UTF-16 strings, and the classic - // way to corrupt a JSON stringifier or parser. - (2, @quickcheck.int_range(0xD800, 0xE000).map(i => i.unsafe_to_char())), // Ordinary ASCII so escapes sit inside unescaped runs. (3, @quickcheck.char_range('a', 'z')), ]) @@ -76,12 +77,13 @@ impl @quickcheck.Arbitrary for AdvString with fn arbitrary(size, state) { } ///| -/// Shrinks at the UTF-16 code-unit level (dropping one unit at a time), so a -/// counterexample can minimize to half of a surrogate pair if that is what -/// triggers a failure. +/// Shrinks by dropping one character at a time. Working at the `Char` level +/// keeps every candidate Unicode well-formed, so a shrunk counterexample +/// fails for the same reason as the original instead of tripping the +/// parser's unpaired-surrogate rejection. impl @shrink.Shrink for AdvString with fn shrink(self) { - let units = self.0.code_units() - let n = units.length() + let chars = self.0.to_array() + let n = chars.length() if n == 0 { return Iter::empty() } @@ -89,17 +91,35 @@ impl @shrink.Shrink for AdvString with fn shrink(self) { (0) .until(n) .map(i => { - let buf = StringBuilder(size_hint=n - 1) - for j in 0.. String { + let hex_lower = "0123456789abcdef".to_array() + let hex_upper = "0123456789ABCDEF".to_array() + let buf = StringBuilder() + buf.write_char('"') + for i, unit in s.code_units() { + let code = unit.to_int() + let hex = if i % 2 == 0 { hex_lower } else { hex_upper } + buf.write_char('\\') + buf.write_char('u') + buf.write_char(hex[(code >> 12) & 0xF]) + buf.write_char(hex[(code >> 8) & 0xF]) + buf.write_char(hex[(code >> 4) & 0xF]) + buf.write_char(hex[code & 0xF]) + } + buf.write_char('"') + buf.to_string() +} + ///| test "adversarial strings roundtrip as values and as object keys" { @quickcheck.check((input : (AdvString, AdvString, Int, Bool)) => { @@ -113,26 +133,35 @@ test "adversarial strings roundtrip as values and as object keys" { ///| /// Spells every UTF-16 code unit of the string as a `\uXXXX` escape /// (alternating hex-digit case) and checks the parser reassembles the exact -/// original string — including surrogate pairs split across two escapes and -/// lone surrogates. +/// original string — including surrogate pairs split across two escapes. test "fully \\uXXXX-escaped strings parse back to the original" { - let hex_lower = "0123456789abcdef".to_array() - let hex_upper = "0123456789ABCDEF".to_array() @quickcheck.check((s : AdvString) => { - let buf = StringBuilder() - buf.write_char('"') - for i, unit in s.0.code_units() { - let code = unit.to_int() - let hex = if i % 2 == 0 { hex_lower } else { hex_upper } - buf.write_char('\\') - buf.write_char('u') - buf.write_char(hex[(code >> 12) & 0xF]) - buf.write_char(hex[(code >> 8) & 0xF]) - buf.write_char(hex[(code >> 4) & 0xF]) - buf.write_char(hex[code & 0xF]) + @json.parse(fully_escaped(s.0)) == Json::string(s.0) + }) +} + +///| +/// A lone surrogate — raw or spelled as a `\uXXXX` escape — injected at any +/// position of an otherwise hostile string must always be rejected with a +/// clean parse error (`parse` raises, `valid` is false, nothing aborts): +/// parsed strings stay Unicode well-formed. The surrounding prefix/suffix +/// supply nearby escapes, astral pairs, and control characters, exercising +/// both the escape-free fast path and the slow path of the string lexer. +test "lone surrogates are rejected in every position" { + @quickcheck.check((input : (AdvString, AdvString, Int, Bool)) => { + let (prefix, suffix, raw_unit, escape_spelling) = input + let unit = 0xD800 + wrap_index(raw_unit, 0x800) + let lone = String::from_array([unit.unsafe_to_char()]) + let content = prefix.0 + lone + suffix.0 + let text = if escape_spelling { + fully_escaped(content) + } else { + // `stringify` writes the lone surrogate raw; prefix/suffix contribute + // short escapes and `\uXXXX` escapes when they contain control or + // quote characters. + Json::string(content).stringify() } - buf.write_char('"') - @json.parse(buf.to_string()) == Json::string(s.0) + parse_succeeds(text) == false && @json.valid(text) == false }) } @@ -322,31 +351,30 @@ test "default nesting limit boundary at depth 1024" { ///| /// Deterministic pins for the surrogate cases the properties above explore -/// randomly, so a regression shows up with a readable diff. +/// randomly, so a regression shows up with a readable diff. Parsed strings +/// stay Unicode well-formed: every unpaired surrogate — raw or escaped — is +/// a clean parse error, never an abort and never an ill-formed string. test "surrogate handling pins" { - // A lone high surrogate roundtrips raw. - let lone = String::from_array([(0xD800).unsafe_to_char()]) - let json = Json::string(lone) - assert_true(@json.parse(json.stringify()) == json) - // Its escaped spelling parses to the same string. - assert_true(@json.parse("\"\\uD800\"") == json) - // An escaped surrogate pair reassembles to the astral character. - assert_true(@json.parse("\"\\uD83D\\uDE00\"") == Json::string("\u{1F600}")) - // A reversed pair (low then high) roundtrips as two lone surrogates. - let reversed = String::from_array([ - (0xDC00).unsafe_to_char(), - (0xD800).unsafe_to_char(), - ]) - let reversed_json = Json::string(reversed) - assert_true(@json.parse(reversed_json.stringify()) == reversed_json) - // A raw lone trailing surrogate combined with an escape used to abort the - // slow-path string lexer: `flush` sliced with the checked `[start:end]`, - // which panics when the code unit at a boundary is a trailing surrogate. + let lone_high = String::from_array([(0xD800).unsafe_to_char()]) let lone_low = String::from_array([(0xDC00).unsafe_to_char()]) - assert_true( - @json.parse("\"" + lone_low + "\\n\"") == Json::string(lone_low + "\n"), - ) - assert_true( - @json.parse("\"\\n" + lone_low + "\"") == Json::string("\n" + lone_low), - ) + // Raw lone surrogates, escape-free (fast path). + assert_false(@json.valid("\"" + lone_high + "\"")) + assert_false(@json.valid("\"" + lone_low + "\"")) + // A reversed pair (low then high) is two unpaired surrogates. + assert_false(@json.valid("\"" + lone_low + lone_high + "\"")) + // Raw lone surrogates next to escapes (slow path; used to abort the + // process via checked slicing in `flush`). + assert_false(@json.valid("\"" + lone_low + "\\n\"")) + assert_false(@json.valid("\"\\n" + lone_low + "\"")) + assert_false(@json.valid("\"" + lone_high + "\\t\"")) + // Escaped lone surrogates. + assert_false(@json.valid("\"\\uD800\"")) + assert_false(@json.valid("\"\\uDC00\"")) + assert_false(@json.valid("\"\\uD800\\uD800\"")) + // Mixed raw/escaped halves do not pair up. + assert_false(@json.valid("\"\\uD800" + lone_low + "\"")) + assert_false(@json.valid("\"" + lone_high + "\\uDC00\"")) + // Well-formed pairs still parse, raw or escaped. + assert_true(@json.parse("\"\\uD83D\\uDE00\"") == Json::string("\u{1F600}")) + assert_true(@json.parse("\"\u{1F600}\"") == Json::string("\u{1F600}")) }