From 60a2c9edc530b420f430749ea4334c9add89efec Mon Sep 17 00:00:00 2001 From: linear3735 Date: Mon, 28 Sep 2026 20:33:39 +0800 Subject: [PATCH 1/2] Add Laya request preprocessing --- .github/workflows/ci.yml | 12 + Cargo.lock | 529 +++++++++++++++++++++++++++- src/models/laya/Cargo.toml | 3 +- src/models/laya/README.md | 14 + src/models/laya/src/lib.rs | 1 + src/models/laya/src/preprocess.rs | 342 ++++++++++++++++++ src/models/laya/tests/packing.rs | 62 ++++ src/models/laya/tests/preprocess.rs | 173 +++++++++ 8 files changed, 1129 insertions(+), 7 deletions(-) create mode 100644 src/models/laya/src/preprocess.rs create mode 100644 src/models/laya/tests/packing.rs create mode 100644 src/models/laya/tests/preprocess.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2ab7b6..27f4e28 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,5 +26,17 @@ jobs: run: cargo clippy --workspace --locked --all-targets -- -D warnings - name: Test run: cargo test --workspace --locked + - name: Official Laya packing parity + env: + LAYA_TOKENIZER: ${{ runner.temp }}/laya-tokenizer.json + LAYA_PACKING_ORACLE: ${{ runner.temp }}/laya-packing.json + run: | + curl --fail --location --retry 3 \ + https://huggingface.co/convaiinnovations/laya/resolve/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851/tokenizer/tokenizer.json \ + --output "$LAYA_TOKENIZER" + curl --fail --location --retry 3 \ + https://raw.githubusercontent.com/linear3735/system1-omni/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native/packing-golden.json \ + --output "$LAYA_PACKING_ORACLE" + cargo test --locked -p omni-laya --test packing -- --ignored - name: Build run: cargo build --workspace --release --locked diff --git a/Cargo.lock b/Cargo.lock index 9212bac..4c42f36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,29 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -66,6 +89,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -78,6 +107,21 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.2" @@ -105,6 +149,15 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.4.7" @@ -135,7 +188,22 @@ checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.1", - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", ] [[package]] @@ -156,6 +224,31 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crunchy" version = "0.2.4" @@ -172,6 +265,87 @@ dependencies = [ "typenum", ] +[[package]] +name = "daachorse" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -193,6 +367,18 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -203,6 +389,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -215,6 +418,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -307,6 +516,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -316,8 +537,8 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", - "rand_core", + "r-efi 6.0.0", + "rand_core 0.10.1", "wasm-bindgen", ] @@ -332,6 +553,12 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "http" version = "1.5.0" @@ -521,6 +748,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -542,12 +775,31 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "ipnet" version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -595,6 +847,22 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "matchit" version = "0.8.4" @@ -622,6 +890,12 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "mio" version = "1.2.3" @@ -633,6 +907,38 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "omni-jev" version = "0.1.0" @@ -655,6 +961,7 @@ dependencies = [ "serde_json", "sha2", "tempfile", + "tokenizers", ] [[package]] @@ -663,6 +970,18 @@ version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -684,6 +1003,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -722,7 +1050,7 @@ dependencies = [ "bytes", "getrandom 0.4.3", "lru-slab", - "rand", + "rand 0.10.3", "rand_pcg", "ring", "rustc-hash", @@ -758,12 +1086,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.3" @@ -772,7 +1116,26 @@ checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -787,9 +1150,69 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "rand_core", + "rand_core 0.10.1", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "reqwest" version = "0.12.28" @@ -957,6 +1380,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -1036,12 +1460,36 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -1139,6 +1587,39 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tokenizers" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc" +dependencies = [ + "ahash", + "compact_str", + "daachorse", + "dary_heap", + "derive_builder", + "esaxx-rs", + "fancy-regex", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.53.1" @@ -1273,6 +1754,27 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.9.0" @@ -1318,6 +1820,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.129" @@ -1504,6 +2015,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.4" diff --git a/src/models/laya/Cargo.toml b/src/models/laya/Cargo.toml index d9e73e8..79a459d 100644 --- a/src/models/laya/Cargo.toml +++ b/src/models/laya/Cargo.toml @@ -10,7 +10,8 @@ half = "2" memmap2 = "0.9" safetensors = "0.6" serde = { version = "1", features = ["derive"] } -serde_json = "1" +serde_json = { version = "1", features = ["preserve_order", "arbitrary_precision"] } +tokenizers = { version = "0.23.2", default-features = false, features = ["fancy-regex"] } [dev-dependencies] sha2 = "0.10" diff --git a/src/models/laya/README.md b/src/models/laya/README.md index 858da19..5c1e9d1 100644 --- a/src/models/laya/README.md +++ b/src/models/laya/README.md @@ -8,6 +8,8 @@ The `omni-laya` crate currently reads and checks the English Laya 0.3.20 checkpo Keep checkpoint files unchanged while `Weights` holds a read-only memory mapping. This crate does not yet execute inference. +`Preprocessor::load` reads a tokenizer JSON file. `prepare` packs English `choice`, `score` and `noul` questions into ordered token rows, option-marker positions and type IDs. Rows follow Laya 0.3.20's 512-token limit and 192-token head budget. Conversation lists keep the newest state tokens; other state values keep the beginning. The result includes normalized criteria for later decoding and the total input-token usage. Backends own padding, batching and resource limits. + ## CPU checks The normal workspace tests cover configuration errors, malformed tensors, inventory mismatches and conversion boundaries without downloading weights. @@ -22,3 +24,15 @@ cargo test --release --locked -p omni-laya --test weights -- --ignored ``` These two CPU tests check all 206 tensor names and shapes, 618 conversion hashes, and the legacy temperature buffer. The normal CI job skips them because it does not download the full checkpoint. + +The normal tests also check input validation, question and option order, truncation and JSON rendering with a small test tokenizer. CPU CI separately downloads the [official tokenizer](https://huggingface.co/convaiinnovations/laya/blob/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851/tokenizer/tokenizer.json) and a [frozen 17-case packing reference](https://github.com/linear3735/system1-omni/blob/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native/packing-golden.json). Both files are checked by SHA-256 before comparison. No weights or GPU are needed. + +To run that check locally with the same files: + +```sh +export LAYA_TOKENIZER=/path/to/laya/tokenizer/tokenizer.json +export LAYA_PACKING_ORACLE=/path/to/packing-golden.json +cargo test --locked -p omni-laya --test packing -- --ignored +``` + +The comparison covers every token, marker, question type, row length, question order and usage count. It excludes the reference's backend padding and bucket dimensions. The [reference generator and inputs](https://github.com/linear3735/system1-omni/tree/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native) use `laya==0.3.20`; packing parity does not measure model quality. diff --git a/src/models/laya/src/lib.rs b/src/models/laya/src/lib.rs index fbc136f..ddda161 100644 --- a/src/models/laya/src/lib.rs +++ b/src/models/laya/src/lib.rs @@ -1,2 +1,3 @@ pub mod config; +pub mod preprocess; pub mod weights; diff --git a/src/models/laya/src/preprocess.rs b/src/models/laya/src/preprocess.rs new file mode 100644 index 0000000..113287e --- /dev/null +++ b/src/models/laya/src/preprocess.rs @@ -0,0 +1,342 @@ +//! English Laya 0.3.20 token packing, before backend padding or batching. +use anyhow::{Result, anyhow, bail, ensure}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; +use std::path::Path; +use tokenizers::Tokenizer; + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Request { + pub state: Value, + #[serde(default)] + pub model: Option, + pub questions: Map, + #[serde(default)] + pub lang: Option, +} + +#[derive(Debug, Serialize)] +pub struct Question { + pub id: String, + pub kind: String, + pub criteria: Value, + pub ids: Vec, + pub markers: Vec, + pub qtype: i64, +} + +#[derive(Debug, Serialize)] +pub struct Prepared { + pub questions: Vec, + pub usage: usize, +} + +pub struct Preprocessor { + tokenizer: Tokenizer, + cls: u32, + sep: u32, + mask: u32, +} + +// Python json.dumps(..., ensure_ascii=False) uses spaces after commas/colons. +// Walk values so punctuation inside strings remains untouched. +pub fn render(value: &Value) -> String { + match value { + Value::String(s) => s.clone(), + _ => spaced_json(value), + } +} +fn spaced_json(value: &Value) -> String { + match value { + Value::Array(a) => format!( + "[{}]", + a.iter().map(spaced_json).collect::>().join(", ") + ), + Value::Object(o) => format!( + "{{{}}}", + o.iter() + .map(|(k, v)| format!("{}: {}", serde_json::to_string(k).unwrap(), spaced_json(v))) + .collect::>() + .join(", ") + ), + Value::Number(n) => python_number(n), + _ => serde_json::to_string(value).unwrap(), + } +} + +impl Preprocessor { + pub fn load(tokenizer_path: &Path) -> Result { + let mut tokenizer = Tokenizer::from_file(tokenizer_path).map_err(|e| anyhow!("{e}"))?; + tokenizer.with_padding(None); + tokenizer + .with_truncation(None) + .map_err(|e| anyhow!("{e}"))?; + let token = |s| { + tokenizer + .token_to_id(s) + .ok_or_else(|| anyhow!("missing token {s}")) + }; + Ok(Self { + cls: token("[CLS]")?, + sep: token("[SEP]")?, + mask: token("[MASK]")?, + tokenizer, + }) + } + fn encode(&self, text: &str) -> Result> { + Ok(self + .tokenizer + .encode(text.replace("[MASK]", " "), false) + .map_err(|e| anyhow!("{e}"))? + .get_ids() + .to_vec()) + } + pub fn prepare(&self, request: &Request) -> Result { + validate_numbers(&request.state)?; + for question in request.questions.values() { + validate_numbers(question)?; + } + ensure!( + request.model.as_deref().is_none_or(|s| s == "english"), + "only model=english is supported" + ); + ensure!( + request + .lang + .as_deref() + .is_none_or(|s| s == "en" || s == "english"), + "only English is supported; use lang=en" + ); + let state = self.encode(&render(&request.state))?; + let mut questions = Vec::new(); + for (id, definition) in &request.questions { + let (kind, criteria, opts) = + options(definition).map_err(|e| anyhow!("question {id:?}: {e}"))?; + let ins = definition + .get("instructions") + .ok_or_else(|| anyhow!("question {id:?}: missing instructions"))?; + let mut head = self.encode(&format!("{kind} question: {}", render(ins)))?; + let mut opt_ids = Vec::new(); + for opt in &opts { + let mut ids = vec![self.mask]; + ids.extend(self.encode(&format!(" {opt}"))?.into_iter().take(48)); + opt_ids.push(ids); + } + let mut budget = 192isize - opt_ids.iter().map(Vec::len).sum::() as isize; + if budget < 16 { + let per = (176 / opt_ids.len()).max(4); + for o in &mut opt_ids { + o.truncate(per); + } + budget = 192 - opt_ids.iter().map(Vec::len).sum::() as isize; + } + head.truncate(budget.max(8) as usize); + let mut ids = vec![self.cls]; + ids.extend(head); + ids.push(self.sep); + let mut markers = Vec::new(); + for opt in opt_ids { + markers.push(ids.len()); + ids.extend(opt); + } + ids.push(self.sep); + let room = 512usize.saturating_sub(ids.len() + 1); + if request.state.is_array() { + ids.extend_from_slice(&state[state.len().saturating_sub(room)..]); + } else { + ids.extend_from_slice(&state[..room.min(state.len())]); + } + ids.push(self.sep); + ids.truncate(512); + ensure!( + markers.iter().all(|&m| m < 512), + "question {id:?}: options exceed head_max_len=192" + ); + let qtype = match kind.as_str() { + "choice" => 0, + "score" => 1, + _ => 2, + }; + questions.push(Question { + id: id.clone(), + kind, + criteria, + ids, + markers, + qtype, + }); + } + let usage = questions.iter().map(|q| q.ids.len()).sum(); + Ok(Prepared { questions, usage }) + } +} + +fn options(q: &Value) -> Result<(String, Value, Vec)> { + ensure!(q.is_object(), "definition must be an object"); + let kind = q["type"].as_str().ok_or_else(|| anyhow!("missing type"))?; + ensure!( + ["choice", "score", "noul"].contains(&kind), + "unknown type {kind}" + ); + ensure!( + kind == "noul" || q.get("labels").is_none(), + "labels only apply to noul" + ); + let mut criteria = q.get("criteria").cloned().unwrap_or(Value::Null); + let opts = match kind { + "choice" => { + if let Some(a) = criteria.as_array() { + let mut o = Map::new(); + for key in a { + o.insert( + key.as_str() + .ok_or_else(|| anyhow!("choice labels must be strings"))? + .to_owned(), + Value::Null, + ); + } + criteria = Value::Object(o); + } + let o = criteria + .as_object() + .ok_or_else(|| anyhow!("choice criteria must be object or list"))?; + ensure!(!o.is_empty(), "at least one choice required"); + o.iter() + .map(|(k, v)| { + if v.is_null() || v.as_str() == Some("") { + k.clone() + } else { + format!("{k}: {}", render(v)) + } + }) + .collect() + } + "score" => { + let a = criteria + .as_array() + .ok_or_else(|| anyhow!("score criteria must be a list"))?; + ensure!(!a.is_empty(), "at least one level required"); + a.iter() + .enumerate() + .map(|(i, v)| format!("level {i}: {}", render(v))) + .collect() + } + _ => { + if criteria.is_null() { + criteria = Value::Object(Map::new()); + } + let o = criteria + .as_object() + .ok_or_else(|| anyhow!("noul criteria must be object"))?; + let mut normalized = Map::new(); + for (k, v) in o { + let key = k.to_lowercase(); + ensure!( + key == "true" || key == "false", + "noul criteria keys must be true/false" + ); + normalized.insert(key, v.clone()); + } + criteria = Value::Object(normalized); + let labels = match q.get("labels") { + None | Some(Value::Null) => ["false", "true"], + Some(Value::Object(o)) if o.len() == 2 => [ + python_strip(o.get("false").and_then(Value::as_str).unwrap_or("")), + python_strip(o.get("true").and_then(Value::as_str).unwrap_or("")), + ], + _ => bail!("noul labels must map true/false to distinct non-empty strings"), + }; + ensure!( + !labels[0].is_empty() && !labels[1].is_empty() && labels[0] != labels[1], + "invalid noul labels" + ); + ["false", "true"] + .iter() + .enumerate() + .map(|(i, k)| { + let v = &criteria[*k]; + let desc = if v.is_null() || v.as_str() == Some("") { + if i == 0 { + "no, the statement does not hold".to_owned() + } else { + "yes, the statement holds".to_owned() + } + } else { + render(v) + }; + format!("{}: {desc}", labels[i]) + }) + .collect() + } + }; + Ok((kind.to_owned(), criteria, opts)) +} + +fn python_strip(value: &str) -> &str { + value.trim_matches(|c: char| c.is_whitespace() || ('\u{1c}'..='\u{1f}').contains(&c)) +} + +// Preserve arbitrary-size integers; floats use Python's shortest decimal and notation rules. +fn python_number(n: &serde_json::Number) -> String { + let raw = n.to_string(); + if !raw.contains(['.', 'e', 'E']) { + return raw; + } + let Some(value) = n.as_f64().filter(|v| v.is_finite()) else { + return raw; + }; + let shortest = serde_json::Number::from_f64(value).unwrap().to_string(); + let (mantissa, exponent) = shortest.split_once('e').unwrap_or((&shortest, "0")); + let sign = if value.is_sign_negative() { "-" } else { "" }; + let mantissa = mantissa.trim_start_matches('-'); + let point = mantissa.find('.').unwrap_or(mantissa.len()); + let digits = mantissa.replace('.', ""); + let significant = digits.trim_start_matches('0'); + if significant.is_empty() { + return format!("{sign}0.0"); + } + let e = exponent.parse::().unwrap() + point as i32 + - (digits.len() - significant.len()) as i32 + - 1; + let significant = significant.trim_end_matches('0'); + if !(-4..16).contains(&e) { + let mut mantissa = significant.to_owned(); + if mantissa.len() > 1 { + mantissa.insert(1, '.'); + } + return format!("{sign}{mantissa}e{e:+03}"); + } + let point = e + 1; + let mut plain = significant.to_owned(); + if point <= 0 { + plain = format!("0.{}{plain}", "0".repeat((-point) as usize)); + } else if point as usize >= plain.len() { + plain.push_str(&"0".repeat(point as usize - plain.len())); + plain.push_str(".0"); + } else { + plain.insert(point as usize, '.'); + } + format!("{sign}{plain}") +} + +fn validate_numbers(value: &Value) -> Result<()> { + match value { + Value::Number(n) if n.to_string().contains(['.', 'e', 'E']) => ensure!( + n.as_f64().is_some_and(f64::is_finite), + "floating point value outside supported finite range" + ), + Value::Array(a) => { + for v in a { + validate_numbers(v)?; + } + } + Value::Object(o) => { + for v in o.values() { + validate_numbers(v)?; + } + } + _ => {} + } + Ok(()) +} diff --git a/src/models/laya/tests/packing.rs b/src/models/laya/tests/packing.rs new file mode 100644 index 0000000..9601068 --- /dev/null +++ b/src/models/laya/tests/packing.rs @@ -0,0 +1,62 @@ +use omni_laya::preprocess::{Preprocessor, Request}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::path::PathBuf; + +fn checked_file(variable: &str, sha256: &str) -> PathBuf { + let path = + PathBuf::from(std::env::var_os(variable).unwrap_or_else(|| panic!("set {variable}"))); + let bytes = std::fs::read(&path).unwrap(); + assert_eq!( + format!("{:x}", Sha256::digest(&bytes)), + sha256, + "{variable}" + ); + path +} + +#[test] +#[ignore = "requires pinned tokenizer and packing oracle; run by CPU CI, no GPU"] +fn official_packing_parity() { + let tokenizer = checked_file( + "LAYA_TOKENIZER", + "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30", + ); + let oracle = checked_file( + "LAYA_PACKING_ORACLE", + "8cbaa311a59924ac9f9f2ba6f8438f82dafbd08476f64e86868289c2ede67f60", + ); + let pre = Preprocessor::load(&tokenizer).unwrap(); + let cases: Vec = serde_json::from_slice(&std::fs::read(oracle).unwrap()).unwrap(); + assert_eq!(cases.len(), 17); + for case in cases { + let request: Request = serde_json::from_value(case["request"].clone()).unwrap(); + let got = pre.prepare(&request).unwrap(); + let expected = &case["expected"]; + let items = expected["items"].as_array().unwrap(); + assert_eq!( + got.questions.len(), + items.len(), + "{} row count", + case["name"] + ); + assert_eq!(got.usage, expected["usage"].as_u64().unwrap() as usize); + for (i, ((question, item), id)) in got + .questions + .iter() + .zip(items) + .zip(request.questions.keys()) + .enumerate() + { + assert_eq!(&question.id, id, "{} row order", case["name"]); + let actual = serde_json::to_value(question).unwrap(); + for key in ["ids", "markers", "qtype"] { + assert_eq!(actual[key], item[key], "{} row {i} {key}", case["name"]); + } + assert_eq!( + question.ids.len(), + expected["lens"][i].as_u64().unwrap() as usize + ); + } + } +} diff --git a/src/models/laya/tests/preprocess.rs b/src/models/laya/tests/preprocess.rs new file mode 100644 index 0000000..aeff56b --- /dev/null +++ b/src/models/laya/tests/preprocess.rs @@ -0,0 +1,173 @@ +use omni_laya::preprocess::{Preprocessor, Request, render}; +use serde_json::json; +use tokenizers::{Tokenizer, models::wordlevel::WordLevel, pre_tokenizers::whitespace::Whitespace}; + +// Small tokenizer for validation and packing boundaries; official parity is in packing.rs. +fn preprocessor() -> (tempfile::TempDir, Preprocessor) { + let dir = tempfile::tempdir().unwrap(); + let mut tokenizer = Tokenizer::new( + WordLevel::builder() + .vocab( + ["[UNK]", "[CLS]", "[SEP]", "[MASK]", "old", "new"] + .into_iter() + .enumerate() + .map(|(i, token)| (token.to_owned(), i as u32)) + .collect(), + ) + .unk_token("[UNK]".to_owned()) + .build() + .unwrap(), + ); + tokenizer.with_pre_tokenizer(Some(Whitespace)); + let path = dir.path().join("tokenizer.json"); + tokenizer.save(&path, false).unwrap(); + let pre = Preprocessor::load(&path).unwrap(); + (dir, pre) +} + +#[test] +fn python_json_numbers_and_order() { + for (input, expected) in [ + ("1e-6", "1e-06"), + ("1e20", "1e+20"), + ("1e16", "1e+16"), + ("1e-4", "0.0001"), + ("1.0", "1.0"), + ("-0.0", "-0.0"), + ("1331752170181752.2", "1331752170181752.2"), + ("-243915020125850.12", "-243915020125850.12"), + ("1e-5", "1e-05"), + ("5e-324", "5e-324"), + ("18446744073709551616000", "18446744073709551616000"), + ] { + assert_eq!(render(&serde_json::from_str(input).unwrap()), expected); + } + let v = serde_json::from_str(r#"{"z":1e-6,"a":"你好,x:y"}"#).unwrap(); + assert_eq!(render(&v), r#"{"z": 1e-06, "a": "你好,x:y"}"#); +} + +#[test] +fn strips_python_whitespace_from_noul_labels() { + let (_dir, pre) = preprocessor(); + let packed = |label| { + let request: Request = serde_json::from_value(json!({ + "state":"", "questions":{"q":{"type":"noul","instructions":"New?", + "labels":{"false":label,"true":"new"}}} + })) + .unwrap(); + pre.prepare(&request).unwrap().questions.remove(0).ids + }; + assert_eq!(packed("\u{1c}old\u{1f}"), packed("old")); + let invalid: Request = serde_json::from_value(json!({ + "state":"", "questions":{"q":{"type":"noul","instructions":"New?", + "labels":{"false":"\u{1c}\u{1f}","true":"new"}}} + })) + .unwrap(); + assert!( + pre.prepare(&invalid) + .unwrap_err() + .to_string() + .contains("invalid noul labels") + ); +} + +#[test] +fn validation_names_the_question_and_leaves_preprocessor_usable() { + let (_dir, pre) = preprocessor(); + for definition in [ + json!({"type":"choice","instructions":"Pick","criteria":[]}), + json!({"type":"noul","instructions":"Pick","criteria":{"yes":"ok"}}), + json!({"type":"noul","instructions":"Pick","labels":{"false":"x","true":"x"}}), + json!({"type":"score","criteria":["low","high"]}), + ] { + let request: Request = serde_json::from_value(json!({ + "state":"new", "questions":{"broken":definition} + })) + .unwrap(); + assert!( + pre.prepare(&request) + .unwrap_err() + .to_string() + .contains("broken") + ); + } + let request: Request = serde_json::from_value(json!({ + "state":"new", "questions":{"valid":{"type":"noul","instructions":"New?"}} + })) + .unwrap(); + assert_eq!(pre.prepare(&request).unwrap().questions.len(), 1); +} + +#[test] +fn preserves_question_and_choice_order() { + let (_dir, pre) = preprocessor(); + let request: Request = serde_json::from_str( + r#"{"state":"","questions":{ + "z":{"type":"choice","instructions":"Pick","criteria":["new","old","new"]}, + "a":{"type":"noul","instructions":"New?"} + }}"#, + ) + .unwrap(); + let prepared = pre.prepare(&request).unwrap(); + assert_eq!(prepared.questions[0].id, "z"); + assert_eq!(prepared.questions[1].id, "a"); + let choice = &prepared.questions[0]; + assert_eq!(choice.markers.len(), 2); + assert_eq!(choice.ids[choice.markers[0] + 1], 5); + assert_eq!(choice.ids[choice.markers[1] + 1], 4); + assert_eq!( + prepared.usage, + prepared + .questions + .iter() + .map(|q| q.ids.len()) + .sum::() + ); +} + +#[test] +fn keeps_newest_conversation_and_start_of_plain_text() { + let (_dir, pre) = preprocessor(); + for state in [ + json!(format!("{} new", "old ".repeat(1000))), + json!(["old ".repeat(1000), "new"]), + ] { + let is_conversation = state.is_array(); + let request: Request = serde_json::from_value(json!({ + "state":state, "questions":{"q":{"type":"noul","instructions":"New?"}} + })) + .unwrap(); + let prepared = pre.prepare(&request).unwrap(); + let ids = &prepared.questions[0].ids; + assert_eq!(ids.len(), 512); + assert_eq!(ids.contains(&5), is_conversation); + } +} + +#[test] +fn rejects_truncated_option_markers() { + let (_dir, pre) = preprocessor(); + let criteria: Vec<_> = (0..300).map(|i| i.to_string()).collect(); + let request: Request = serde_json::from_value(json!({ + "state":"", "questions":{"q":{"type":"choice","instructions":"Pick","criteria":criteria}} + })) + .unwrap(); + assert!( + pre.prepare(&request) + .unwrap_err() + .to_string() + .contains("options exceed") + ); +} + +#[test] +fn rejects_unsupported_language_and_nonfinite_numbers() { + let (_dir, pre) = preprocessor(); + for input in [ + r#"{"state":"","lang":"de","questions":{}}"#, + r#"{"state":"","model":"multilingual","questions":{}}"#, + r#"{"state":1e400,"questions":{}}"#, + ] { + assert!(pre.prepare(&serde_json::from_str(input).unwrap()).is_err()); + } +} From 37736f080cdb49db47da8b3de6e6eb6f602e4986 Mon Sep 17 00:00:00 2001 From: linear3735 Date: Sun, 4 Oct 2026 02:33:49 +0800 Subject: [PATCH 2/2] fix(laya): preserve structured JSON request values --- recipe/laya/README.md | 44 ++++++++ src/models/cua_s1/native/src/json.rs | 17 +-- src/models/laya/Cargo.toml | 2 +- src/models/laya/README.md | 22 ++-- src/models/laya/src/preprocess.rs | 83 +++++++++++++- tests/cua_s1/json.rs | 18 +++ tests/laya/packing.rs | 2 +- tests/laya/preprocess.rs | 163 +++++++++++++++++++++++++-- 8 files changed, 311 insertions(+), 40 deletions(-) create mode 100644 tests/cua_s1/json.rs diff --git a/recipe/laya/README.md b/recipe/laya/README.md index 254c12c..609cd7c 100644 --- a/recipe/laya/README.md +++ b/recipe/laya/README.md @@ -56,3 +56,47 @@ if the worker requires a bearer token. See the [frontend documentation](../../src/frontend/README.md) for configuration and transport behavior. + +## Native CPU packing check + +The `omni-laya` preprocessor packs English Laya 0.3.20 requests without weights +or a GPU. + +```sh +cargo test --locked -p omni-laya --test preprocess +``` + +Native callers use `Request::from_json(&str)` for a single top-level JSON request, +or `Request::from_value(Value)` for an existing structured value. `Request` +retains its public fields and `Serialize`; it does not implement generic +`Deserialize`. The JSON entry checks the complete request against serde_json's +default nesting limit. The value entry preserves existing nested values without +reparsing. Both preserve literal private Number/RawValue object keys and reject +unknown request fields. + +The normal tests cover validation, JSON rendering, question and option order, +and truncation with a small tokenizer: +Pass raw JSON directly to `from_json`. + +For the official 17-case comparison, use the same pinned inputs as CPU CI. +The test checks both files by SHA-256 before comparing: + +```sh +LAYA_PACKING_DIR=$(mktemp -d) +export LAYA_TOKENIZER="$LAYA_PACKING_DIR/tokenizer.json" +export LAYA_PACKING_ORACLE="$LAYA_PACKING_DIR/packing-golden.json" +curl --fail --location --retry 3 \ + https://huggingface.co/convaiinnovations/laya/resolve/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851/tokenizer/tokenizer.json \ + --output "$LAYA_TOKENIZER" +curl --fail --location --retry 3 \ + https://raw.githubusercontent.com/linear3735/system1-omni/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native/packing-golden.json \ + --output "$LAYA_PACKING_ORACLE" +cargo test --locked -p omni-laya --test packing -- --ignored +``` + +Existing copies of these pinned files can be supplied through `LAYA_TOKENIZER` +and `LAYA_PACKING_ORACLE` instead. The comparison covers every token, marker, +question type, row length, question order and usage count; it excludes backend +padding and bucket dimensions. The [reference generator and inputs](https://github.com/linear3735/system1-omni/tree/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native) +use `laya==0.3.20`. Packing parity does not measure model quality or execute +native model inference. diff --git a/src/models/cua_s1/native/src/json.rs b/src/models/cua_s1/native/src/json.rs index 0565820..59aaeb0 100644 --- a/src/models/cua_s1/native/src/json.rs +++ b/src/models/cua_s1/native/src/json.rs @@ -243,19 +243,6 @@ mod tests { ); } - #[test] - fn number_encoding_does_not_change_user_objects() { - let input = - r#"{"a": {"$serde_json::private::Number": "1.5"}, "b": -0, "c": 18446744073709551616}"#; - let value = Value::Object(parse(input.as_bytes()).unwrap()); - assert_eq!(value["a"]["$serde_json::private::Number"], "1.5"); - assert_eq!( - dumps(&value), - r#"{"a": {"$serde_json::private::Number": "1.5"}, "b": -0.0, "c": 1.8446744073709552e+19}"# - ); - assert!(err(r#"{"a": {"x": 1, "\u0078": 2}}"#).contains("duplicate key")); - } - #[test] fn rejects_what_the_contract_rejects() { assert_eq!(err("[]"), "request body must be a JSON object"); @@ -285,3 +272,7 @@ mod tests { assert!(parse(b"{\"a\": \"\xff\"}").is_err()); } } + +#[cfg(test)] +#[path = "../../../../../tests/cua_s1/json.rs"] +mod json_regression_tests; diff --git a/src/models/laya/Cargo.toml b/src/models/laya/Cargo.toml index 5ac5268..3ba4850 100644 --- a/src/models/laya/Cargo.toml +++ b/src/models/laya/Cargo.toml @@ -10,7 +10,7 @@ half = "2" memmap2 = "0.9" safetensors = "0.6" serde = { version = "1", features = ["derive"] } -serde_json = { version = "1", features = ["preserve_order", "arbitrary_precision"] } +serde_json = { version = "1", features = ["preserve_order", "arbitrary_precision", "raw_value"] } tokenizers = { version = "0.23.2", default-features = false, features = ["fancy-regex"] } [dev-dependencies] diff --git a/src/models/laya/README.md b/src/models/laya/README.md index 410fb0c..f8b90ed 100644 --- a/src/models/laya/README.md +++ b/src/models/laya/README.md @@ -10,6 +10,16 @@ Keep checkpoint files unchanged while `Weights` holds a read-only memory mapping `Preprocessor::load` reads a tokenizer JSON file. `prepare` packs English `choice`, `score` and `noul` questions into ordered token rows, option-marker positions and type IDs. Rows follow Laya 0.3.20's 512-token limit and 192-token head budget. Conversation lists keep the newest state tokens; other state values keep the beginning. The result includes normalized criteria for later decoding and the total input-token usage. Backends own padding, batching and resource limits. +Build a request with `Request::from_json(&str)` for one top-level JSON object, or +`Request::from_value(Value)` for an already constructed value. The JSON entry +preserves object order and arbitrary-size integers, treats serde_json's private +Number/RawValue keys as ordinary user keys, and applies its default nesting limit +to the complete request. The value entry moves state and questions without +reparsing or adding a depth limit. Both reject unknown request fields and require +state and an object of questions. Public fields and `Serialize` remain available; +`Request` does not implement generic `Deserialize`, so use these explicit entries +instead of `serde_json::from_str::` or `serde_json::from_value::`. + ## CPU checks The normal workspace tests cover configuration errors, malformed tensors, inventory mismatches and conversion boundaries without downloading weights. @@ -25,17 +35,7 @@ cargo test --release --locked -p omni-laya --test weights -- --ignored These two CPU tests check all 206 tensor names and shapes, 618 conversion hashes, and the legacy temperature buffer. The normal CI job skips them because it does not download the full checkpoint. -The normal tests also check input validation, question and option order, truncation and JSON rendering with a small test tokenizer. CPU CI separately downloads the [official tokenizer](https://huggingface.co/convaiinnovations/laya/blob/55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851/tokenizer/tokenizer.json) and a [frozen 17-case packing reference](https://github.com/linear3735/system1-omni/blob/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native/packing-golden.json). Both files are checked by SHA-256 before comparison. No weights or GPU are needed. - -To run that check locally with the same files: - -```sh -export LAYA_TOKENIZER=/path/to/laya/tokenizer/tokenizer.json -export LAYA_PACKING_ORACLE=/path/to/packing-golden.json -cargo test --locked -p omni-laya --test packing -- --ignored -``` - -The comparison covers every token, marker, question type, row length, question order and usage count. It excludes the reference's backend padding and bucket dimensions. The [reference generator and inputs](https://github.com/linear3735/system1-omni/tree/5e4dd4215c925ebd93bb9ce4097b27bd6375f7c0/recipe/laya/native) use `laya==0.3.20`; packing parity does not measure model quality. +The normal tests also check input validation, question and option order, truncation and JSON rendering with a small test tokenizer. The [Laya recipe](../../../recipe/laya/README.md#native-cpu-packing-check) provides the CPU packing validation commands and pinned inputs for the official 17-case comparison. No weights or GPU are needed; packing parity does not measure model quality. ## Python worker diff --git a/src/models/laya/src/preprocess.rs b/src/models/laya/src/preprocess.rs index 113287e..cbdf7e2 100644 --- a/src/models/laya/src/preprocess.rs +++ b/src/models/laya/src/preprocess.rs @@ -1,21 +1,94 @@ //! English Laya 0.3.20 token packing, before backend padding or batching. use anyhow::{Result, anyhow, bail, ensure}; -use serde::{Deserialize, Serialize}; +use serde::de::{self, MapAccess, SeqAccess, Visitor}; +use serde::{Deserializer, Serialize}; +use serde_json::value::RawValue; use serde_json::{Map, Value}; use std::path::Path; use tokenizers::Tokenizer; -#[derive(Debug, Deserialize, Serialize)] -#[serde(deny_unknown_fields)] +/// Use `from_json` for a top-level JSON request or `from_value` for an existing value. +#[derive(Debug, Serialize)] pub struct Request { pub state: Value, - #[serde(default)] pub model: Option, pub questions: Map, - #[serde(default)] pub lang: Option, } +impl Request { + /// Decode one JSON request with serde_json's default nesting limit. + pub fn from_json(raw: &str) -> Result { + let raw: Box = serde_json::from_str(raw)?; + Self::from_value(parse_value(&raw, 0)?) + } + + /// Keep existing structured values without reparsing or imposing a JSON depth limit. + pub fn from_value(value: Value) -> Result { + let Value::Object(mut fields) = value else { + bail!("request must be an object"); + }; + let state = fields + .remove("state") + .ok_or_else(|| anyhow!("missing state"))?; + let questions = fields + .remove("questions") + .ok_or_else(|| anyhow!("missing questions"))?; + let Value::Object(questions) = questions else { + bail!("questions must be an object"); + }; + let model = serde_json::from_value(fields.remove("model").unwrap_or(Value::Null))?; + let lang = serde_json::from_value(fields.remove("lang").unwrap_or(Value::Null))?; + ensure!(fields.is_empty(), "unknown request fields"); + Ok(Self { + state, + model, + questions, + lang, + }) + } +} + +fn parse_value(raw: &RawValue, depth: usize) -> serde_json::Result { + let text = raw.get(); + if !matches!(text.as_bytes()[0], b'{' | b'[') { + return serde_json::from_str(text); + } + if depth >= 127 { + return Err(de::Error::custom("recursion limit exceeded")); + } + // Construct containers explicitly: serde_json's private Number/RawValue + // map encodings must not interpret legitimate user object keys. + struct Containers(usize); + impl<'de> Visitor<'de> for Containers { + type Value = Value; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a JSON object or array") + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut values = Vec::new(); + while let Some(raw) = seq.next_element::>()? { + values.push(parse_value(&raw, self.0 + 1).map_err(de::Error::custom)?); + } + Ok(Value::Array(values)) + } + fn visit_map>(self, mut map: A) -> Result { + let mut values = Map::new(); + while let Some((key, raw)) = map.next_entry::>()? { + if self.0 == 0 && values.contains_key(&key) { + return Err(de::Error::custom(format!("duplicate field {key}"))); + } + values.insert( + key, + parse_value(&raw, self.0 + 1).map_err(de::Error::custom)?, + ); + } + Ok(Value::Object(values)) + } + } + serde_json::Deserializer::from_str(text).deserialize_any(Containers(depth)) +} + #[derive(Debug, Serialize)] pub struct Question { pub id: String, diff --git a/tests/cua_s1/json.rs b/tests/cua_s1/json.rs new file mode 100644 index 0000000..bb0e070 --- /dev/null +++ b/tests/cua_s1/json.rs @@ -0,0 +1,18 @@ +use super::*; + +#[test] +fn number_encoding_does_not_change_user_objects() { + let input = + r#"{"a": {"$serde_json::private::Number": "1.5"}, "b": -0, "c": 18446744073709551616}"#; + let value = Value::Object(parse(input.as_bytes()).unwrap()); + assert_eq!(value["a"]["$serde_json::private::Number"], "1.5"); + assert_eq!( + dumps(&value), + r#"{"a": {"$serde_json::private::Number": "1.5"}, "b": -0.0, "c": 1.8446744073709552e+19}"# + ); + assert!( + parse(br#"{"a": {"x": 1, "\u0078": 2}}"#) + .unwrap_err() + .contains("duplicate key") + ); +} diff --git a/tests/laya/packing.rs b/tests/laya/packing.rs index 9601068..bd753d8 100644 --- a/tests/laya/packing.rs +++ b/tests/laya/packing.rs @@ -30,7 +30,7 @@ fn official_packing_parity() { let cases: Vec = serde_json::from_slice(&std::fs::read(oracle).unwrap()).unwrap(); assert_eq!(cases.len(), 17); for case in cases { - let request: Request = serde_json::from_value(case["request"].clone()).unwrap(); + let request: Request = Request::from_value(case["request"].clone()).unwrap(); let got = pre.prepare(&request).unwrap(); let expected = &case["expected"]; let items = expected["items"].as_array().unwrap(); diff --git a/tests/laya/preprocess.rs b/tests/laya/preprocess.rs index aeff56b..0cbcc05 100644 --- a/tests/laya/preprocess.rs +++ b/tests/laya/preprocess.rs @@ -1,5 +1,6 @@ use omni_laya::preprocess::{Preprocessor, Request, render}; -use serde_json::json; +use serde::Deserialize; +use serde_json::{Map, Value, json}; use tokenizers::{Tokenizer, models::wordlevel::WordLevel, pre_tokenizers::whitespace::Whitespace}; // Small tokenizer for validation and packing boundaries; official parity is in packing.rs. @@ -50,7 +51,7 @@ fn python_json_numbers_and_order() { fn strips_python_whitespace_from_noul_labels() { let (_dir, pre) = preprocessor(); let packed = |label| { - let request: Request = serde_json::from_value(json!({ + let request: Request = Request::from_value(json!({ "state":"", "questions":{"q":{"type":"noul","instructions":"New?", "labels":{"false":label,"true":"new"}}} })) @@ -58,7 +59,7 @@ fn strips_python_whitespace_from_noul_labels() { pre.prepare(&request).unwrap().questions.remove(0).ids }; assert_eq!(packed("\u{1c}old\u{1f}"), packed("old")); - let invalid: Request = serde_json::from_value(json!({ + let invalid: Request = Request::from_value(json!({ "state":"", "questions":{"q":{"type":"noul","instructions":"New?", "labels":{"false":"\u{1c}\u{1f}","true":"new"}}} })) @@ -80,7 +81,7 @@ fn validation_names_the_question_and_leaves_preprocessor_usable() { json!({"type":"noul","instructions":"Pick","labels":{"false":"x","true":"x"}}), json!({"type":"score","criteria":["low","high"]}), ] { - let request: Request = serde_json::from_value(json!({ + let request: Request = Request::from_value(json!({ "state":"new", "questions":{"broken":definition} })) .unwrap(); @@ -91,7 +92,7 @@ fn validation_names_the_question_and_leaves_preprocessor_usable() { .contains("broken") ); } - let request: Request = serde_json::from_value(json!({ + let request: Request = Request::from_value(json!({ "state":"new", "questions":{"valid":{"type":"noul","instructions":"New?"}} })) .unwrap(); @@ -101,7 +102,7 @@ fn validation_names_the_question_and_leaves_preprocessor_usable() { #[test] fn preserves_question_and_choice_order() { let (_dir, pre) = preprocessor(); - let request: Request = serde_json::from_str( + let request: Request = Request::from_json( r#"{"state":"","questions":{ "z":{"type":"choice","instructions":"Pick","criteria":["new","old","new"]}, "a":{"type":"noul","instructions":"New?"} @@ -133,7 +134,7 @@ fn keeps_newest_conversation_and_start_of_plain_text() { json!(["old ".repeat(1000), "new"]), ] { let is_conversation = state.is_array(); - let request: Request = serde_json::from_value(json!({ + let request: Request = Request::from_value(json!({ "state":state, "questions":{"q":{"type":"noul","instructions":"New?"}} })) .unwrap(); @@ -148,7 +149,7 @@ fn keeps_newest_conversation_and_start_of_plain_text() { fn rejects_truncated_option_markers() { let (_dir, pre) = preprocessor(); let criteria: Vec<_> = (0..300).map(|i| i.to_string()).collect(); - let request: Request = serde_json::from_value(json!({ + let request: Request = Request::from_value(json!({ "state":"", "questions":{"q":{"type":"choice","instructions":"Pick","criteria":criteria}} })) .unwrap(); @@ -168,6 +169,150 @@ fn rejects_unsupported_language_and_nonfinite_numbers() { r#"{"state":"","model":"multilingual","questions":{}}"#, r#"{"state":1e400,"questions":{}}"#, ] { - assert!(pre.prepare(&serde_json::from_str(input).unwrap()).is_err()); + assert!(pre.prepare(&Request::from_json(input).unwrap()).is_err()); } } + +#[test] +fn private_json_keys_stay_objects_in_raw_requests() { + let (_dir, pre) = preprocessor(); + for key in [ + "$serde_json::private::Number", + "$serde_json::private::RawValue", + ] { + let state = json!({"outer": [{(key): "1.5"}]}); + let questions = json!({ + "z": {"type": "choice", "instructions": {(key): "2"}, + "criteria": {"new": [{(key): "3"}], "old": null}}, + "a": {"type": "noul", "instructions": "New?"} + }) + .as_object() + .unwrap() + .clone(); + let expected = Request { + state, + model: None, + questions, + lang: None, + }; + let encoded = serde_json::to_string(&expected).unwrap(); + let request: Request = Request::from_json(&encoded).unwrap(); + assert_eq!(request.state, expected.state, "{key}"); + assert_eq!(request.questions, expected.questions, "{key}"); + assert_eq!(serde_json::to_string(&request).unwrap(), encoded); + assert_eq!( + render(&request.state), + format!(r#"{{"outer": [{{"{key}": "1.5"}}]}}"#) + ); + let packed = pre.prepare(&request).unwrap(); + let expected_packed = pre.prepare(&expected).unwrap(); + assert_eq!( + packed.questions[0].criteria, + expected_packed.questions[0].criteria + ); + assert_eq!(packed.questions[0].ids, expected_packed.questions[0].ids); + assert_eq!(packed.questions[0].id, "z"); + assert_eq!(packed.questions[1].id, "a"); + } +} + +#[test] +fn private_json_keys_stay_objects_from_value() { + for key in [ + "$serde_json::private::RawValue", + "$serde_json::private::Number", + ] { + let value = json!({"state": {(key): "1.5"}, "questions": { + "q": {"type": "score", "instructions": "New?", + "criteria": [{"outer": {(key): "2"}}]} + }}); + let request: Request = Request::from_value(value.clone()).unwrap(); + assert_eq!(request.state, value["state"]); + assert_eq!(Value::Object(request.questions), value["questions"]); + } +} + +#[test] +fn request_numbers_keep_arbitrary_precision_and_syntax() { + for number in [ + "18446744073709551616000", + "-18446744073709551616000", + "1e+03", + "1.2300", + "-0", + "1e400", + ] { + let encoded = + format!(r#"{{"state":[{number}],"questions":{{"q":{{"criteria":[{number}]}}}}}}"#); + let request: Request = Request::from_json(&encoded).unwrap(); + let scalar: Value = serde_json::from_str(number).unwrap(); + assert_eq!(request.state[0], scalar); + assert_eq!(request.questions["q"]["criteria"][0], scalar); + let roundtrip: Request = + Request::from_value(serde_json::to_value(&request).unwrap()).unwrap(); + assert_eq!(roundtrip.state, request.state); + assert_eq!(roundtrip.questions, request.questions); + } +} + +#[test] +fn request_keeps_default_json_recursion_limit() { + #[derive(Deserialize)] + struct Reference { + #[serde(rename = "state")] + _state: Value, + #[serde(rename = "questions")] + _questions: Map, + } + for depth in [124, 125, 126, 127, 128, 200] { + let nested = format!("{}0{}", "[".repeat(depth), "]".repeat(depth)); + for encoded in [ + format!(r#"{{"state":{nested},"questions":{{}}}}"#), + format!(r#"{{"state":null,"questions":{{"q":{{"criteria":{nested}}}}}}}"#), + ] { + let expected = serde_json::from_str::(&encoded).is_ok(); + let actual = Request::from_json(&encoded); + assert_eq!(actual.is_ok(), expected, "depth {depth}: {encoded}"); + if !expected { + assert!(actual.unwrap_err().to_string().contains("recursion limit")); + } + } + } +} + +#[test] +fn request_from_value_preserves_deep_values() { + let mut nested = json!({"$serde_json::private::Number": "1.5"}); + for _ in 0..200 { + nested = Value::Array(vec![nested]); + } + let value = json!({"state": nested.clone(), "questions": { + "q": {"criteria": nested.clone()} + }}); + let request = Request::from_value(value).unwrap(); + assert_eq!(request.state, nested); + assert_eq!(request.questions["q"]["criteria"], nested); + let request = Request::from_value(serde_json::to_value(&request).unwrap()).unwrap(); + assert_eq!(request.state, nested); +} + +#[test] +fn request_rejects_invalid_json_values() { + for state in ["NaN", "01", "[1,]", r#""\ud800""#] { + let encoded = format!(r#"{{"state":{state},"questions":{{}}}}"#); + assert!(Request::from_json(&encoded).is_err(), "{state}"); + } + assert!(Request::from_json(r#"{"state":null,"questions":[]}"#).is_err()); + for encoded in [ + r#"{"state":null,"questions":{},"extra":1}"#, + r#"{"state":null,"state":1,"questions":{}}"#, + r#"{"state":null}"#, + r#"{"questions":{}}"#, + r#"{"state":null,"questions":{},"lang":1}"#, + r#"[{"state":null,"questions":{}}]"#, + ] { + assert!(Request::from_json(encoded).is_err(), "{encoded}"); + } + let request = Request::from_json(r#"{"state":{"x":1,"x":2},"questions":{}}"#).unwrap(); + assert_eq!(request.state["x"], 2); +}