From 947f32e09038a693d4124d73216dcbaf46a8c294 Mon Sep 17 00:00:00 2001 From: "Alexey O. Shigarov" Date: Wed, 26 Aug 2026 18:32:26 +0800 Subject: [PATCH] Move the anchor attribute in ANCH(n)/REC(n), not just its values; release 0.5.1 apply_anchor_at_position permuted only the record values while the schema kept its original order. With anonymous $a_i names (position == name) that is invisible, but with named attributes -- the ones AVP produces from a header row -- it broke the attribute-value binding: the column carrying the anchor's name received another attribute's values. The reported case, over a Dato,Tid,Eksamen,Fagkode,Lokaler,Klasse header, came out as Lokaler,Dato,Tid,... with every value shifted. The transformation now moves the anchor attribute itself -- name together with values -- so every record keeps its pairs and only the schema order changes, which makes it the special case of apply_schema_reordering it always was. One rule for named and anonymous attributes alike: an anonymous name is no longer reassigned positionally, it travels with its attribute, so a schema $a_1..$a_4 under ANCH(2) reads $a_2, $a_3, $a_1, $a_4 while the values stay in the same positions as before. All ANCH/REC(n) task fixtures are header-less and compare positionally, so they are unaffected and none was touched. Port of jRegTab 0.5.1 (a092102, merged in 43c1fa9); the reference implementation is AnchorAttributeAtPosition.java. No change was needed in the RTL plumbing: the settings prefix , inline REC(n) on an atomic content specification and inline REC(n) inside a delimited one already converge on the same Transformation -- collect_rec_params descends into delimited specs on the RTL path and actions_of reads d.atom.actions on the ATP path. That is now pinned by a test rather than left to inspection. Tests: a #[cfg(test)] module in src/spec.rs (the file had none) covering named, anonymous, mixed and degenerate schemas, a negative position, and a regression check that value positions are unchanged; tests/test_rtl_anchor_forms.py, where the three forms must agree and REC(n) under {','} yields one record per raw token (" C1.1" keeps its leading space -- the 0.5.0 S_delim rule); AnchorAttributeAtPosition added to test_api.py's transformation coverage. Temporarily restoring the old return value fails 4 core tests and 5 Python ones while the regression check stays green. Corpus re-pinned to jRegTab v0.5.1 (c126337) and synced byte-for-byte: the two new semantic cases anch_named_attrs and anch_named_inline_delim. Both set expectedHasHeader, which is what lets them see the bug at all -- every other semantic case is header-less and positional. Docs: the ANCH(n) row in the settings table described an attribute-naming operation that never existed ("use position n in the first record as the attribute name"); rtl-reference.md now states what the transformation does, including the anonymous-name rule and the fact that an inline REC(n) is picked up anywhere in the pattern, delimited specifications included. Same correction in the REC(n) operation row, atp.md, itm.md and api.md. Release 0.5.1: version bumped in Cargo.toml, Cargo.lock, pyproject.toml, __init__.py, README.md and docs/index.md. The README's differential note still refers to the v0.5.0 run -- that comparison was not repeated here. Co-Authored-By: Claude Opus 5 (1M context) --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 4 +- conformance/UPSTREAM | 4 +- .../semantic/anch_named_attrs/expected.csv | 3 + .../semantic/anch_named_attrs/input.csv | 3 + .../semantic/anch_named_attrs/options.json | 1 + .../semantic/anch_named_attrs/pattern.rtl | 3 + .../anch_named_inline_delim/expected.csv | 4 + .../anch_named_inline_delim/input.csv | 3 + .../anch_named_inline_delim/options.json | 1 + .../anch_named_inline_delim/pattern.rtl | 2 + docs/api.md | 3 + docs/index.md | 2 +- docs/model/atp.md | 2 +- docs/model/itm.md | 5 +- docs/rtl-reference.md | 18 +- plans/ANCH_MOVES_ATTRIBUTE.md | 223 ++++++++++++++++++ plans/INDEX.md | 16 ++ pyproject.toml | 2 +- python/pyregtab/__init__.py | 2 +- src/spec.rs | 124 +++++++++- tests/test_api.py | 5 + tests/test_rtl_anchor_forms.py | 97 ++++++++ 24 files changed, 516 insertions(+), 15 deletions(-) create mode 100644 conformance/semantic/anch_named_attrs/expected.csv create mode 100644 conformance/semantic/anch_named_attrs/input.csv create mode 100644 conformance/semantic/anch_named_attrs/options.json create mode 100644 conformance/semantic/anch_named_attrs/pattern.rtl create mode 100644 conformance/semantic/anch_named_inline_delim/expected.csv create mode 100644 conformance/semantic/anch_named_inline_delim/input.csv create mode 100644 conformance/semantic/anch_named_inline_delim/options.json create mode 100644 conformance/semantic/anch_named_inline_delim/pattern.rtl create mode 100644 plans/ANCH_MOVES_ATTRIBUTE.md create mode 100644 plans/INDEX.md create mode 100644 tests/test_rtl_anchor_forms.py diff --git a/Cargo.lock b/Cargo.lock index cf56bda..0bbf2b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,7 +167,7 @@ dependencies = [ [[package]] name = "pyregtab" -version = "0.5.0" +version = "0.5.1" dependencies = [ "indexmap", "pyo3", diff --git a/Cargo.toml b/Cargo.toml index 632ec95..c4c6724 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyregtab" -version = "0.5.0" +version = "0.5.1" edition = "2021" description = "Native core of pyRegTab: RTL compiler, ATP matcher and table interpreter" license = "MIT" diff --git a/README.md b/README.md index 79017b9..d6c3ad5 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ and interprets the match into a relational **recordset**: TableSyntax → RtlCompiler/TablePattern → AtpMatcher → TableInterpreter → Recordset ``` -**pyRegTab 0.5.0 ≙ jRegTab 0.5.0** (same API, same semantics, same test +**pyRegTab 0.5.1 ≙ jRegTab 0.5.1** (same API, same semantics, same test corpus), including the embedded RTL DSL `pyregtab.dsl` — a port of jRegTab's `ru.icc.regtab.dsl` (added upstream in jRegTab 0.3.0). Python-side extras on top of the Java API: @@ -143,7 +143,7 @@ Rust (`pyregtab._core`, built with [PyO3](https://pyo3.rs) and ## Testing -`pytest tests` runs (1 908 tests): +`pytest tests` runs (1 925 tests): - the full benchmark suite — tasks 001–150 (Foofah, RegTab, Baikal), every fixture variant, **both** via RTL patterns and via ATP patterns diff --git a/conformance/UPSTREAM b/conformance/UPSTREAM index 04a2f76..67f8315 100644 --- a/conformance/UPSTREAM +++ b/conformance/UPSTREAM @@ -1,3 +1,3 @@ -commit: 035ff1a139e885e4cea85aa66a33e89a6b30f8c9 -tag: v0.5.0 +commit: c12633763b309fd00f65d8b236a4ab91795303b4 +tag: v0.5.1 path: conformance/ diff --git a/conformance/semantic/anch_named_attrs/expected.csv b/conformance/semantic/anch_named_attrs/expected.csv new file mode 100644 index 0000000..cc9c821 --- /dev/null +++ b/conformance/semantic/anch_named_attrs/expected.csv @@ -0,0 +1,3 @@ +"Dato","Lokaler","Klasse" +"20.05","AU","0" +"11.06","A2.1","1" diff --git a/conformance/semantic/anch_named_attrs/input.csv b/conformance/semantic/anch_named_attrs/input.csv new file mode 100644 index 0000000..cc9c821 --- /dev/null +++ b/conformance/semantic/anch_named_attrs/input.csv @@ -0,0 +1,3 @@ +"Dato","Lokaler","Klasse" +"20.05","AU","0" +"11.06","A2.1","1" diff --git a/conformance/semantic/anch_named_attrs/options.json b/conformance/semantic/anch_named_attrs/options.json new file mode 100644 index 0000000..41659ef --- /dev/null +++ b/conformance/semantic/anch_named_attrs/options.json @@ -0,0 +1 @@ +{ "expectedHasHeader": true } diff --git a/conformance/semantic/anch_named_attrs/pattern.rtl b/conformance/semantic/anch_named_attrs/pattern.rtl new file mode 100644 index 0000000..a11e492 --- /dev/null +++ b/conformance/semantic/anch_named_attrs/pattern.rtl @@ -0,0 +1,3 @@ + +[ [ATTR]+ ] +[ COL->AVP [VAL] [VAL: ROW*->REC] [VAL] ]+ diff --git a/conformance/semantic/anch_named_inline_delim/expected.csv b/conformance/semantic/anch_named_inline_delim/expected.csv new file mode 100644 index 0000000..6cf2c6d --- /dev/null +++ b/conformance/semantic/anch_named_inline_delim/expected.csv @@ -0,0 +1,4 @@ +"Dato","Lokaler","Klasse" +"20.05","AU","0" +"20.05"," C1.1","0" +"11.06","A2.1","1" diff --git a/conformance/semantic/anch_named_inline_delim/input.csv b/conformance/semantic/anch_named_inline_delim/input.csv new file mode 100644 index 0000000..0e2d61e --- /dev/null +++ b/conformance/semantic/anch_named_inline_delim/input.csv @@ -0,0 +1,3 @@ +"Dato","Lokaler","Klasse" +"20.05","AU, C1.1","0" +"11.06","A2.1","1" diff --git a/conformance/semantic/anch_named_inline_delim/options.json b/conformance/semantic/anch_named_inline_delim/options.json new file mode 100644 index 0000000..41659ef --- /dev/null +++ b/conformance/semantic/anch_named_inline_delim/options.json @@ -0,0 +1 @@ +{ "expectedHasHeader": true } diff --git a/conformance/semantic/anch_named_inline_delim/pattern.rtl b/conformance/semantic/anch_named_inline_delim/pattern.rtl new file mode 100644 index 0000000..d397eb3 --- /dev/null +++ b/conformance/semantic/anch_named_inline_delim/pattern.rtl @@ -0,0 +1,2 @@ +[ [ATTR]+ ] +[ COL->AVP [VAL] [(VAL: ROW*->REC(1)){','}] [VAL] ]+ diff --git a/docs/api.md b/docs/api.md index 10b724b..f0f90dd 100644 --- a/docs/api.md +++ b/docs/api.md @@ -124,6 +124,9 @@ byte-identical to `RtlCompiler.compile` for lambda-free patterns. See the with `pattern.with_transformations(...)` or to an interpreter with `.with_transformations([...])`. RTL settings `` and inline `REC(n)` / `REC('s')` parameters compile to these transformations. +`AnchorAttributeAtPosition(pos)` moves the anchor attribute — its name together with +its values — to 0-based position `pos`; it renames nothing, so every attribute-value +binding survives and only the schema order changes. ## RTL bindings diff --git a/docs/index.md b/docs/index.md index 0d1f73a..9d5ee20 100644 --- a/docs/index.md +++ b/docs/index.md @@ -109,4 +109,4 @@ Requires **Python 3.10+**; binary wheels for Windows, Linux, and macOS. --- !!! note "Status" - Current release: **0.5.0** (feature parity with jRegTab 0.5.0) · License: **MIT** · [PyPI](https://pypi.org/project/pyregtab/) · [GitHub](https://github.com/regtab/pyregtab) + Current release: **0.5.1** (feature parity with jRegTab 0.5.1) · License: **MIT** · [PyPI](https://pypi.org/project/pyregtab/) · [GitHub](https://github.com/regtab/pyregtab) diff --git a/docs/model/atp.md b/docs/model/atp.md index 0e03e37..02d6398 100644 --- a/docs/model/atp.md +++ b/docs/model/atp.md @@ -331,7 +331,7 @@ chosen operation (see [ITM — Interpretation actions](itm.md#interpretation-act | Operation | Python factory | Notes | |---|---|---| | `REC` | `ActionSpec.rec(providers…)` | anchor → first field; providers supply remaining fields | - | `REC(n)` | `ActionSpec.rec(int n, providers…)` | adds `AnchorAttributeAtPosition` post-step | + | `REC(n)` | `ActionSpec.rec(int n, providers…)` | adds `AnchorAttributeAtPosition` post-step — moves the anchor attribute (name with its values) to position *n* | | `REC('s')` | `ActionSpec.rec(String delim, providers…)` | adds `DelimitedFieldSplit` post-step | | `AVP` | `ActionSpec.avp(provider)` | associates VAL anchor with ATTR item | | `AVP "name"` | `ActionSpec.avp("ATTR_NAME")` | context-derived ATTR constant | diff --git a/docs/model/itm.md b/docs/model/itm.md index 7c857ac..4c5d89a 100644 --- a/docs/model/itm.md +++ b/docs/model/itm.md @@ -317,7 +317,8 @@ satisfy the constraints of the chosen operation (Tab. I in the paper): | AVP with literal | `ActionSpec.avp("ATTR_NAME")` | Context-derived ATTR constant | `ActionSpec.rec(int anchorPos, providers…)` adds an `AnchorAttributeAtPosition` - post-processing step (RTL: `REC(n)`). + post-processing step (RTL: `REC(n)`) — it moves the anchor attribute, name and + values together, to position `anchorPos` of the schema. `ActionSpec.rec(String splitDelimiter, providers…)` adds a `DelimitedFieldSplit` step (RTL: `REC('s')`). @@ -431,7 +432,7 @@ The extracted recordset may be further post-processed by optional operations: | Delimited field split | `DelimitedFieldSplit` | `ActionSpec.rec(String delimiter, …)` — RTL `REC('s')` | | Field splitting | `FieldSplitting` | explicit split spec | | Whitespace normalisation | `WhitespaceNormalization` | `with_transformations(…)` | - | Anchor attribute at position | `AnchorAttributeAtPosition` | `ActionSpec.rec(int pos, …)` — RTL `REC(n)` | + | Anchor attribute at position | `AnchorAttributeAtPosition` | `ActionSpec.rec(int pos, …)` — RTL `REC(n)` / ``; moves the anchor attribute (name with its values) to position `pos`, preserving every attribute-value binding | --- diff --git a/docs/rtl-reference.md b/docs/rtl-reference.md index 53a3dfa..2a1d85a 100644 --- a/docs/rtl-reference.md +++ b/docs/rtl-reference.md @@ -131,15 +131,29 @@ post-extraction `RecordsetTransformation`s on the resulting `TablePattern`. | Setting | Effect | |---|---| | `NORM` | Apply whitespace normalisation to all field values after extraction | -| `ANCH(n)` | Use position *n* in the first record as the attribute name for all records | +| `ANCH(n)` | Move the anchor attribute to 0-based position *n* in the schema | | `SPLIT("s")` | Split all field values by delimiter *s* after extraction | Example: ` [ … ]` — normalise and anchor at position 2. +The anchor is the first attribute of the extracted schema, and `ANCH(n)` moves that +**attribute** — its name travels together with its values, so the attribute-value binding +of every record is untouched and only the order of the schema changes. A position of 0, +a position beyond the schema, or a single-attribute schema leaves the recordset as is. + +The rule is the same for named attributes (produced by `AVP`) and for the anonymous +`$a_i` names the interpreter invents: an anonymous name is **not** renumbered, it moves +with its attribute. A schema `$a_1, $a_2, $a_3` under `ANCH(2)` therefore becomes +`$a_2, $a_3, $a_1` — the values sit in the same positions as before, and the name shows +which attribute was moved. + !!! note "Inline equivalents" The same two transformations can be requested *inline* on a `REC` action: `REC(n)` is equivalent to the `ANCH(n)` setting, and `REC('s')` is equivalent to `SPLIT("s")`. Inline forms are by far the more common in practice (see Tasks 02, 03). + The position of the inline form in the pattern does not matter — `REC(n)` is picked up + anywhere, including inside a delimited content specification such as + `[(VAL: ROW*->REC(1)){','}]`, and always yields the same transformation. The compiler merges inline and prefix forms and raises `RtlCompileError` if they conflict (e.g. `ANCH(1)` together with `REC(2)`). @@ -397,7 +411,7 @@ provSpecs -> op |---|---|---| | `REC` | `prov->REC` | Anchor item → record entry; provider supplies additional field values | | `REC` | `()->REC` | Anchor item → single-field record (no additional providers; useful after `SUFFIX`/`PREFIX`/`FILL` has enriched the anchor value) | -| `REC(n)` | `prov->REC(n)` | Same + use attribute at position *n* as the record's attribute name | +| `REC(n)` | `prov->REC(n)` | Same + move the anchor attribute (name with its values) to position *n* | | `REC('s')` | `prov->REC('s')` | Same + split field values by delimiter *s* | | `AVP` | `prov->AVP` | Associate anchor (VAL) with an attribute from the provider (ATTR) | | `JOIN` | `prov->JOIN` | Join item-based records: all items included, then dedup by named attribute (K=∅) | diff --git a/plans/ANCH_MOVES_ATTRIBUTE.md b/plans/ANCH_MOVES_ATTRIBUTE.md new file mode 100644 index 0000000..476c18a --- /dev/null +++ b/plans/ANCH_MOVES_ATTRIBUTE.md @@ -0,0 +1,223 @@ +# План: ANCH(n)/REC(n) переставляет атрибут, а не только значения (паритет с jRegTab 0.5.1) + +**Статус:** РЕАЛИЗОВАН (2026-08-26; результаты и отклонения от плана — в §9) +**Дата:** 2026-08-26 +**Upstream:** `d:\YandexDisk\code2\jregtab` @ v0.5.1 (`c126337`), коммит поведения +`a092102` (merge `43c1fa9`) «Fix ANCH(n)/REC(n): move the anchor attribute, not just +its values» +**Характер:** исправление семантики пост-трансформации; грамматика, парсер, matcher +и сериализатор не затрагиваются + +--- + +## 1. Контекст + +`apply_anchor_at_position` в [src/spec.rs](../src/spec.rs) строит вектор перестановки +`reordered`, применяет его к значениям каждой записи и возвращает **исходную схему**: +`Ok(RecordsetCore { schema: rs.schema, records })`. Для анонимных атрибутов (`$a_i`, +имя = позиция) это незаметно, но для именованных (полученных через `AVP`) связка +имя ↔ значение разъезжается: столбец с именем якоря получает чужие значения. Паттерн + + + [[ATTR]+] + [COL->AVP [VAL]{4}[(VAL: ROW*->REC){','}][VAL]]+ + +на шапке `Dato,Tid,Eksamen,Fagkode,Lokaler,Klasse` даёт схему «Lokaler,Dato,Tid,…» +со сдвинутыми значениями вместо «Dato,Tid,…,Lokaler,Klasse». + +Целевая семантика (принята в апстриме): `ANCH(n)` перемещает **сам атрибут** — имя +вместе со значениями — на 0-based позицию `n`; связка имя ↔ значение в каждой записи +неизменна, меняется только порядок схемы. Правило одно для именованных и анонимных: +анонимные имена **не** перенумеровываются, они переезжают вместе со своим атрибутом +(`$a_1..$a_4` при `ANCH(2)` → `$a_2, $a_3, $a_1, $a_4`). Последовательность значений +по позициям при этом не меняется — все header-less эталоны задач остаются зелёными. + +Эталон: `src/main/java/ru/icc/regtab/interpret/AnchorAttributeAtPosition.java`, +тесты `AnchorAttributeAtPositionTest` и `RtlAnchorPositionFormsTest` в jregtab. + +## 2. Установленные факты (разведка перед реализацией) + +- Образец стиля в проекте — `apply_schema_reordering` ([src/spec.rs](../src/spec.rs)): + собирает `new_attrs`, `Schema::new(new_attrs)?`, переставляет значения по индексам. + Исправленная `apply_anchor_at_position` — его частный случай. +- `Schema::new` ([src/recordset.rs](../src/recordset.rs)) отвергает дубликаты; + перестановка уникальной схемы остаётся уникальной, но `?` оставляем для единообразия. +- Python-обёртка `PyAnchorAttributeAtPosition` ([src/py.rs](../src/py.rs)) уже отвергает + отрицательную позицию и вызывает то же ядро — правок не требует. +- Все три формы уже сходятся к `Transformation::AnchorAttributeAtPosition`: + - префикс `` → [src/rtl/build.rs](../src/rtl/build.rs) и слияние + в [src/rtl/mod.rs](../src/rtl/mod.rs) (там же проверка конфликта `ANCH`/`REC`); + - inline `REC(n)`, RTL-путь → `ast::collect_rec_params` + ([src/rtl/ast.rs](../src/rtl/ast.rs)), который явно спускается в delimited-спецификации + (`CompSegAst::Delim`, `XSpecAst::Delim`, `ContAst::Delim` → `walk_atom(&d.atom, …)`); + - inline `REC(n)`, ATP/Python-путь `TablePattern::of` → `extract_inline_transformations` + → `actions_of`, где `ContentSpec::Delimited(d) => d.atom.actions`. + + Вывод: `[(VAL: ROW*->REC(n)){','}]` виден компилятору по обоим путям, правок кода + здесь не нужно — поведение фиксируется тестом. +- В `src/spec.rs` тестового модуля нет. Прецедент для приватной функции — инлайновый + `#[cfg(test)] mod tests` в конце файла (как в `src/matcher.rs`), `snake_case` без + префикса `test_`, голые `assert_eq!`. +- `conformance/` — байт-в-байт зеркало апстрима, помечено `-text` в `.gitattributes`. + `git diff 035ff1a c126337 -- conformance/` в jregtab даёт ровно 8 файлов из двух новых + каталогов, так что синк исчерпывающий. Все файлы: UTF-8 без BOM, LF, завершающий + перевод строки. +- `tests/test_semantic_conformance.py` находит кейсы листингом каталогов, регистрация + не нужна; по умолчанию `expectedHasHeader: False`, оба новых кейса ставят `true` — + именно это делает их способными поймать баг. +- `CHANGELOG.md` в проекте нет — не заводился. +- Сборка/прогон: `.venv\Scripts\python.exe`, `python -m maturin develop --release` + обязателен перед pytest (иначе гоняется старое ядро); `cargo test --no-default-features` + (фича `python` включена по умолчанию и тянет pyo3-линковку). + +## 3. Ядро + +`src/spec.rs`, `apply_anchor_at_position`: границы, проверка `position < 0` и построение +`reordered` — без изменений. После `reordered` добавляется перестановка схемы: + +```rust +let new_attrs: Vec = reordered.iter().map(|&i| attrs[i].clone()).collect(); +let schema = Schema::new(new_attrs)?; +let records = /* как было */; +Ok(RecordsetCore { schema, records }) +``` + +Doc-комментарий функции переписывается по образцу javadoc эталона: перемещается атрибут, +имя едет со значениями, анонимные имена не перенумеровываются, граничные случаи +(позиция 0, ≥ len, схема из одного атрибута) возвращают recordset как есть. + +## 4. Юнит-тесты ядра (Rust) + +Новый `#[cfg(test)] mod tests` в конце `src/spec.rs`; помощник строит `RecordsetCore` +из списка атрибутов и строк значений (поля `pub`, конструкторов нет). Кейсы — порт +`AnchorAttributeAtPositionTest`: + +1. `named_attributes_keep_their_values` — `Lokaler,Dato,Tid` + `ANCH(2)` → схема + `Dato,Tid,Lokaler`, значения `20.05.2019, 08.30-11.30, AU`. +2. `named_attributes_at_position_one` — `ANCH(1)` → `Dato,Lokaler,Tid`. +3. `anonymous_attributes_are_not_renumbered` — `$a_1..$a_4` + `ANCH(2)` → + `$a_2,$a_3,$a_1,$a_4`, при этом `rs.get(0, "$a_1") == Some("anchor")`. +4. `anonymous_value_order_matches_legacy_behaviour` — **регресс-гарантия**: позиционно + `v2, v3, anchor, v4` (как до правки). Тест обязан быть зелёным и на старом коде. +5. `mixed_schema_keeps_every_name` — `Lokaler,$a_2,Klasse` + `ANCH(1)` → + `$a_2,Lokaler,Klasse`. +6. `degenerate_cases_return_the_input` — позиции 0, `len`, `> len`, схема из одного + атрибута: результат равен входу (в Rust — `assert_eq!` с клоном, `assertSame` + неприменим). +7. `negative_position_is_rejected` — специфика Rust-сигнатуры `i64`: `Err`. + +## 5. Тесты уровня RTL (Python) + +Новый `tests/test_rtl_anchor_forms.py` (порт `RtlAnchorPositionFormsTest`) с локальным +помощником `make_table` (как в `tests/test_api.py`). Таблица `Dato,Lokaler,Klasse` +(шапка + 2 строки, якорь — средний столбец): + +- `test_settings_prefix_moves_the_anchor_attribute` — префикс `` над + `[[ATTR]+] [COL->AVP [VAL] [VAL: ROW*->REC] [VAL]]+` даёт схему + `["Dato","Lokaler","Klasse"]` и значения `20.05,AU,0` / `11.06,A2.1,1` под своими именами. +- `test_all_three_forms_agree` — префикс ``, inline `[VAL: ROW*->REC(1)]` + и inline `[(VAL: ROW*->REC(1)){','}]` дают идентичный дамп (схема + значения записей). +- `test_inline_rec_inside_delimited_specification` — ячейка `"AU, C1.1"` даёт 3 записи; + вторая — `20.05`, `" C1.1"`, `0`, с **ведущим пробелом** (токены сырые, без trim — + правило `S_delim` из 0.5.0). + +Пайплайн как в `tests/task_runner.py`: `AtpMatcher.match` → `TableInterpreter() +.with_strategy(SchemaConstructionStrategy.RECORD_FIRST).interpret(itm)` → +`pattern.transform(rs)`. + +Плюс в `tests/test_api.py::test_transformations` добавляется `AnchorAttributeAtPosition` +(единственная трансформация без покрытия на уровне Python-объектов): перестановка схемы +и сохранение связки имя ↔ значение. + +## 6. Синк conformance-корпуса + +Байт-в-байт из локального jregtab (Git Bash `cp -r`, без EOL-конверсии): + + conformance/semantic/anch_named_attrs/ {pattern.rtl, input.csv, expected.csv, options.json} + conformance/semantic/anch_named_inline_delim/ {pattern.rtl, input.csv, expected.csv, options.json} + +Проверка после копирования: `diff -r ../jregtab/conformance conformance` — расхождений +быть не должно; `git ls-files --eol conformance/semantic/anch_*` — всюду `w/lf`. + +`conformance/UPSTREAM`: пин `035ff1a` / `v0.5.0` → +`c12633763b309fd00f65d8b236a4ab91795303b4` / `v0.5.1`. `conformance/VERSION` +(`generated: 2026-08-26`) не меняется. + +## 7. Документация + +- `docs/rtl-reference.md`, таблица «Settings prefix»: строка `ANCH(n)` — + «Use position *n* in the first record as the attribute name for all records» → + «Move the anchor attribute to 0-based position *n* in the schema». Ниже примера — + два абзаца (формулировки из jregtab): перемещается атрибут, имя едет со значениями, + граничные случаи; анонимные имена не перенумеровываются (`$a_1,$a_2,$a_3` → + `$a_2,$a_3,$a_1`). +- Там же во врезке «Inline equivalents»: позиция inline-формы в паттерне не важна, + `REC(n)` подхватывается где угодно, включая delimited-спецификацию + `[(VAL: ROW*->REC(1)){','}]`, и всегда даёт ту же трансформацию. +- `docs/rtl-reference.md`, таблица операций: строка `REC(n)` — «Same + use attribute at + position *n* as the record's attribute name» → «Same + move the anchor attribute to + position *n*». +- `docs/model/atp.md` — «adds `AnchorAttributeAtPosition` post-step» → «… — moves the + anchor attribute (name with its values) to position *n*». +- `docs/model/itm.md` — в описании `ActionSpec.rec(int anchorPos, …)` и в строке таблицы + трансформаций: «moves the anchor attribute, name and values together, preserving every + attribute-value binding». +- `docs/api.md`, «Recordset transformations» — уточнение про `AnchorAttributeAtPosition(pos)` + (перемещает атрибут, не переименовывает). + +## 8. Версия + +0.5.0 → 0.5.1 в `Cargo.toml`, `Cargo.lock`, `pyproject.toml`, +`python/pyregtab/__init__.py`, `README.md` («pyRegTab 0.5.1 ≙ jRegTab 0.5.1»), +`docs/index.md`. Публикация на PyPI в этот PR не входит. + +## 9. Результат + +``` +cargo test --no-default-features 21 passed (было 14; +7 юнит-тестов §4) +pytest tests -q 1925 passed (было 1918; +3 RTL-теста §5, + +4 от двух conformance-кейсов) +tests/fixtures/ не тронуты — 0 изменённых expected_*.csv +diff -r -x UPSTREAM ../jregtab/conformance conformance — расхождений нет +``` + +Исходный репродьюсер из отчёта (`` на шапке +`Dato,Tid,Eksamen,Fagkode,Lokaler,Klasse`, ячейка `"AU, C1.1"`) даёт схему +`Dato,Tid,Eksamen,Fagkode,Lokaler,Klasse` и три записи с правильной связкой +имя ↔ значение, включая `" C1.1"` с ведущим пробелом. + +**Дискриминирующая проверка (§10):** при временно возвращённом `schema: rs.schema` +падают 4 юнит-теста ядра (`named_attributes_keep_their_values`, +`named_attributes_at_position_one`, `anonymous_attributes_are_not_renumbered`, +`mixed_schema_keeps_every_name`) и 5 тестов Python (`test_transformations`, +`test_settings_prefix_moves_the_anchor_attribute`, +`test_inline_rec_inside_delimited_specification` и оба conformance-кейса) — 9 падений +против 8 в jregtab, лишнее приходится на добавленную проверку в `test_transformations`. +`anonymous_value_order_matches_legacy_behaviour`, `degenerate_cases_return_the_input` +и `negative_position_is_rejected` при этом зелёные, как и задумано. +`test_all_three_forms_agree` на сломанном ядре тоже проходит: он фиксирует +эквивалентность трёх форм, а не корректность самой перестановки. + +**Отклонения от плана (несущественные):** + +1. В §8 дополнительно обновлены `README.md` (число тестов 1 908 → 1 925) и `Cargo.lock` + (перегенерирован `cargo check`). +2. Строка `README.md` про differential-тестирование («zero mismatches against jRegTab + v0.5.0») оставлена как есть: прогон против v0.5.1 в эту работу не входил. + +## 10. Проверка качества тестов + +Дискриминирующая проверка (как в §4 плана S_DELIM): временно вернуть `schema: rs.schema` +и убедиться, что падают новые юнит-тесты §4 (кроме №4 и №7), RTL-тесты §5 и оба +conformance-кейса, а `anonymous_value_order_matches_legacy_behaviour` — проходит. +В jregtab так и вышло: 8 падений, регресс-тест зелёный. + +## 11. Что осознанно не трогается + +- `src/py.rs`, грамматика, лексер, парсер, `build.rs`, matcher, сериализатор — поведение + трёх форм уже единое, меняется только пост-трансформация. +- Эталоны `tests/fixtures/` (задачи с ANCH/REC(n)) — header-less, сравниваются позиционно. +- Асимметрия `extract_inline_transformations` (ATP-путь) и `collect_rec_params` (RTL-путь) + по унаследованным action-спекам на уровнях table/subtable/row/subrow — существующее + расхождение, к этому багу отношения не имеет. +- `conformance/VERSION`, differential-тесты против Java, публикация на PyPI. diff --git a/plans/INDEX.md b/plans/INDEX.md new file mode 100644 index 0000000..24bb9c4 --- /dev/null +++ b/plans/INDEX.md @@ -0,0 +1,16 @@ +# Планы + +Указатель по каталогу `plans/`: один план — одна крупная работа, самый свежий сверху. +Статус и результат каждой работы — в шапке и в разделе «Результат» самого плана. + +- [ANCH_MOVES_ATTRIBUTE.md](ANCH_MOVES_ATTRIBUTE.md) — `ANCH(n)`/`REC(n)` переставляет + сам атрибут (имя вместе со значениями), а не только значения; синк двух семантических + conformance-кейсов и паритет с jRegTab 0.5.1. **РЕАЛИЗОВАН** (2026-08-26). +- [S_DELIM_RAW_SPLIT.md](S_DELIM_RAW_SPLIT.md) — сырое разбиение делимитированной + спецификации: токены передаются в атом дословно, обрезка стала opt-in через `=TRIM`/`=NORM`; + паритет с jRegTab 0.5.0. **РЕАЛИЗОВАН** (2026-08-26). +- [EMBEDDED_RTL_DSL.md](EMBEDDED_RTL_DSL.md) — встроенный DSL `pyregtab.dsl`: fluent-фабрики + паттернов вместо многословного ATP API, порт `ru.icc.regtab.dsl.Rtl`. +- [PYREGTAB_MIGRATION_PLAN.md](PYREGTAB_MIGRATION_PLAN.md) — исходная миграция + jRegTab → pyRegTab (вариант A: нативное ядро на Rust + Python-обёртка). + **РЕАЛИЗОВАН** (сверка 2026-07-10). diff --git a/pyproject.toml b/pyproject.toml index 1e28be1..4b3ff0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "pyregtab" -version = "0.5.0" +version = "0.5.1" description = "pyRegTab: pattern-based extraction of recordsets from tables (RTL / ATP / ITM)" readme = "README.md" license = { text = "MIT" } diff --git a/python/pyregtab/__init__.py b/python/pyregtab/__init__.py index a5834a4..38f3178 100644 --- a/python/pyregtab/__init__.py +++ b/python/pyregtab/__init__.py @@ -73,7 +73,7 @@ from pyregtab import dsl -__version__ = "0.5.0" +__version__ = "0.5.1" __all__ = [ "TableSyntax", "Cell", "Row", "Subrow", "Subtable", "GridPosition", diff --git a/src/spec.rs b/src/spec.rs index 5f898eb..029ecd2 100644 --- a/src/spec.rs +++ b/src/spec.rs @@ -1128,6 +1128,19 @@ impl Transformation { } } +/// Moves the anchor attribute (first in schema) to the given 0-based position. +/// +/// The *attribute* is moved — its name travels together with its values, so the +/// attribute-value binding of every record is preserved and only the order of the +/// schema changes. The rule is the same for named attributes (produced by `AVP`) +/// and for the anonymous `$a_i` names the interpreter invents: an anonymous name is +/// *not* renumbered, it moves with its attribute, so a schema `$a_1, $a_2, $a_3` +/// under `ANCH(2)` becomes `$a_2, $a_3, $a_1` while the values stay in the same +/// positions as before. +/// +/// This is [`apply_schema_reordering`] with the order derived from the anchor +/// position. A position of 0, a position beyond the schema, or a schema of at most +/// one attribute leaves the recordset unchanged. fn apply_anchor_at_position(rs: RecordsetCore, position: i64) -> CoreResult { if position < 0 { return Err(format!("position must be non-negative: {position}").into()); @@ -1147,6 +1160,8 @@ fn apply_anchor_at_position(rs: RecordsetCore, position: i64) -> CoreResult = reordered.iter().map(|&src| attrs[src].clone()).collect(); + let schema = Schema::new(new_attrs)?; let records = rs .records .iter() @@ -1154,7 +1169,7 @@ fn apply_anchor_at_position(rs: RecordsetCore, position: i64) -> CoreResult String { @@ -1318,3 +1333,110 @@ fn apply_schema_reordering(rs: RecordsetCore, order: &[String]) -> CoreResult RecordsetCore { + let schema = Schema::new(attributes.iter().map(|a| a.to_string()).collect()).unwrap(); + let records = rows + .iter() + .map(|row| RecordCore { + values: row.iter().map(|v| Some(v.to_string())).collect(), + }) + .collect(); + RecordsetCore { schema, records } + } + + /// Values of one record in the order of the recordset's own schema. + fn values(rs: &RecordsetCore, record: usize) -> Vec<&str> { + rs.records[record] + .values + .iter() + .map(|v| v.as_deref().unwrap_or("")) + .collect() + } + + #[test] + fn named_attributes_keep_their_values() { + // The anchor attribute moves to position 2; every name keeps its own value. + let rs = recordset( + &["Lokaler", "Dato", "Tid"], + &[ + &["AU", "20.05.2019", "08.30-11.30"], + &["A2.1", "11.06.2019", "0"], + ], + ); + let out = apply_anchor_at_position(rs, 2).unwrap(); + assert_eq!(out.schema.attributes, vec!["Dato", "Tid", "Lokaler"]); + assert_eq!(values(&out, 0), vec!["20.05.2019", "08.30-11.30", "AU"]); + assert_eq!(values(&out, 1), vec!["11.06.2019", "0", "A2.1"]); + } + + #[test] + fn named_attributes_at_position_one() { + let rs = recordset( + &["Lokaler", "Dato", "Tid"], + &[&["AU", "20.05.2019", "08.30-11.30"]], + ); + let out = apply_anchor_at_position(rs, 1).unwrap(); + assert_eq!(out.schema.attributes, vec!["Dato", "Lokaler", "Tid"]); + assert_eq!(values(&out, 0), vec!["20.05.2019", "AU", "08.30-11.30"]); + } + + #[test] + fn anonymous_attributes_are_not_renumbered() { + // The anonymous name travels with its attribute instead of being reassigned + // positionally, so $a_1 still names the anchor after the move. + let rs = recordset( + &["$a_1", "$a_2", "$a_3", "$a_4"], + &[&["anchor", "v2", "v3", "v4"]], + ); + let out = apply_anchor_at_position(rs, 2).unwrap(); + assert_eq!(out.schema.attributes, vec!["$a_2", "$a_3", "$a_1", "$a_4"]); + assert_eq!(out.get(0, "$a_1"), Some("anchor")); + assert_eq!(out.get(0, "$a_2"), Some("v2")); + } + + #[test] + fn anonymous_value_order_matches_legacy_behaviour() { + // Header-less fixtures compare positionally: this order pins the ANCH/REC(n) + // task expectations, which must not move when the schema does. + let rs = recordset( + &["$a_1", "$a_2", "$a_3", "$a_4"], + &[&["anchor", "v2", "v3", "v4"]], + ); + let out = apply_anchor_at_position(rs, 2).unwrap(); + assert_eq!(values(&out, 0), vec!["v2", "v3", "anchor", "v4"]); + } + + #[test] + fn mixed_schema_keeps_every_name() { + // One rule for named and anonymous attributes alike. + let rs = recordset(&["Lokaler", "$a_2", "Klasse"], &[&["AU", "v2", "0"]]); + let out = apply_anchor_at_position(rs, 1).unwrap(); + assert_eq!(out.schema.attributes, vec!["$a_2", "Lokaler", "Klasse"]); + assert_eq!(values(&out, 0), vec!["v2", "AU", "0"]); + } + + #[test] + fn degenerate_cases_return_the_input() { + let three = recordset(&["a", "b", "c"], &[&["1", "2", "3"]]); + assert_eq!(apply_anchor_at_position(three.clone(), 0).unwrap(), three); + assert_eq!(apply_anchor_at_position(three.clone(), 3).unwrap(), three); + assert_eq!(apply_anchor_at_position(three.clone(), 7).unwrap(), three); + + let single = recordset(&["a"], &[&["1"]]); + assert_eq!(apply_anchor_at_position(single.clone(), 1).unwrap(), single); + } + + #[test] + fn negative_position_is_rejected() { + let rs = recordset(&["a", "b"], &[&["1", "2"]]); + assert!(apply_anchor_at_position(rs, -1).is_err()); + } +} diff --git a/tests/test_api.py b/tests/test_api.py index 102c5c0..3614f17 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,6 +4,7 @@ import pytest from pyregtab import ( + AnchorAttributeAtPosition, AtpMatcher, Bindings, CellColor, @@ -123,6 +124,10 @@ def test_transformations(): rs4 = SchemaReordering(["B", "A"]).apply(rs) assert rs4.schema.attributes == ["B", "A"] assert rs4[0]["B"] == "1/2" + # the anchor attribute moves, its name travels with its values + rs5 = AnchorAttributeAtPosition(1).apply(rs) + assert rs5.schema.attributes == ["B", "A"] + assert rs5[0]["A"] == " x y " and rs5[0]["B"] == "1/2" def test_custom_cell_predicate_matching(): diff --git a/tests/test_rtl_anchor_forms.py b/tests/test_rtl_anchor_forms.py new file mode 100644 index 0000000..643ddbb --- /dev/null +++ b/tests/test_rtl_anchor_forms.py @@ -0,0 +1,97 @@ +"""ANCH(n) / REC(n) at the RTL level: the settings prefix, inline REC(n) on an +atomic content specification and inline REC(n) inside a delimited one request the +same transformation, and it moves the anchor *attribute* -- name together with its +values (port of jRegTab's RtlAnchorPositionFormsTest).""" + +from pyregtab import ( + AtpMatcher, + RtlCompiler, + SchemaConstructionStrategy, + TableInterpreter, + TableSyntax, +) + +# Header row + two data rows; the anchor column ("Lokaler") sits in the middle. +TABLE = [ + ["Dato", "Lokaler", "Klasse"], + ["20.05", "AU", "0"], + ["11.06", "A2.1", "1"], +] + +SETTINGS_PREFIX = """\ + +[ [ATTR]+ ] +[ COL->AVP [VAL] [VAL: ROW*->REC] [VAL] ]+ +""" + +INLINE_ATOMIC = """\ +[ [ATTR]+ ] +[ COL->AVP [VAL] [VAL: ROW*->REC(1)] [VAL] ]+ +""" + +INLINE_DELIMITED = """\ +[ [ATTR]+ ] +[ COL->AVP [VAL] [(VAL: ROW*->REC(1)){','}] [VAL] ]+ +""" + + +def make_table(rows): + t = TableSyntax(len(rows), len(rows[0])) + for r, row in enumerate(rows): + for c, v in enumerate(row): + t.cell(r, c).set_text(v) + return t + + +def run(rtl, rows): + pattern = RtlCompiler.compile(rtl) + itm = AtpMatcher.match(pattern, make_table(rows)) + assert itm is not None, f"pattern did not match:\n{rtl}" + return pattern.transform( + TableInterpreter() + .with_strategy(SchemaConstructionStrategy.RECORD_FIRST) + .interpret(itm) + ) + + +def values(rs, record): + return [rs[record][a] for a in rs.schema.attributes] + + +def dump(rs): + return "\n".join( + [str(rs.schema.attributes)] + [str(values(rs, i)) for i in range(len(rs))] + ) + + +def test_settings_prefix_moves_the_anchor_attribute(): + # The extracted schema is anchor-first (Lokaler, Dato, Klasse); ANCH(1) restores + # the column order of the table with every attribute-value binding intact. + rs = run(SETTINGS_PREFIX, TABLE) + assert rs.schema.attributes == ["Dato", "Lokaler", "Klasse"] + assert values(rs, 0) == ["20.05", "AU", "0"] + assert values(rs, 1) == ["11.06", "A2.1", "1"] + + +def test_all_three_forms_agree(): + via_settings = run(SETTINGS_PREFIX, TABLE) + via_atomic = run(INLINE_ATOMIC, TABLE) + via_delimited = run(INLINE_DELIMITED, TABLE) + assert dump(via_settings) == dump(via_atomic) + assert dump(via_settings) == dump(via_delimited) + + +def test_inline_rec_inside_delimited_specification(): + # One record per token, names intact; tokens are raw, so " C1.1" keeps its + # leading space (the S_delim rule). + rows = [ + ["Dato", "Lokaler", "Klasse"], + ["20.05", "AU, C1.1", "0"], + ["11.06", "A2.1", "1"], + ] + rs = run(INLINE_DELIMITED, rows) + assert rs.schema.attributes == ["Dato", "Lokaler", "Klasse"] + assert len(rs) == 3 + assert values(rs, 0) == ["20.05", "AU", "0"] + assert values(rs, 1) == ["20.05", " C1.1", "0"] + assert values(rs, 2) == ["11.06", "A2.1", "1"]