From 1af68100678e433f2723632888d0083845a458e3 Mon Sep 17 00:00:00 2001 From: Curt Hagenlocher Date: Sat, 22 Aug 2026 08:48:23 -0700 Subject: [PATCH] Add variant object examples with differing field ID and offset widths object_wide_offsets and object_wide_field_ids are the first examples in which an object's field_id_size and field_offset_size differ. Every existing object example uses 1-byte field IDs and 1-byte field offsets, where the two value_header fields hold the same value, so a reader that reads them from each other's bit positions still decodes correctly and the corpus cannot detect that class of bug. - object_wide_offsets: two fields (1-byte field IDs) and more than 256 bytes of field data (2-byte field offsets), giving header byte 0x06. - object_wide_field_ids: 262 distinct field names force 2-byte field IDs, while the object under the "nested" field holds a single small value so its own field offsets still fit in 1 byte, giving header byte 0x12. Its enclosing object has 261 fields, which also makes this the first example with is_large set to 1. Verified by running regen.py end to end: it reproduces the four new files byte-identically, along with 49 of the 52 files it generates. The three that differ are pre-existing and unrelated -- primitive_null.metadata and primitive_null.value, which Spark emits empty and which README "Modification 1" documents as hand-corrected, and primitive_timestamp.value, which encodes the generating machine's session timezone. The new examples were also checked to decode to the expected JSON, and to be rejected by a reader that transposes the two value_header fields. object_wide_field_ids also covers the first case listed in #78 (an object with more than 2**8 fields, requiring a 4-byte num_elements). The remaining cases there -- more than 2**16 and 2**24 distinct fields -- are still open, and their TODOs in regen.py now sit under a reference to #78 rather than #77. They are also reworded, since the number of distinct fields drives the field ID width rather than the field offset width. Closes #124. Co-Authored-By: Claude Opus 5 (1M context) --- variant/README.md | 18 ++ variant/data_dictionary.json | 269 +++++++++++++++++++++++++ variant/object_wide_field_ids.metadata | Bin 0 -> 1579 bytes variant/object_wide_field_ids.value | Bin 0 -> 1579 bytes variant/object_wide_offsets.metadata | Bin 0 -> 14 bytes variant/object_wide_offsets.value | Bin 0 -> 317 bytes variant/regen.py | 33 ++- 7 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 variant/object_wide_field_ids.metadata create mode 100644 variant/object_wide_field_ids.value create mode 100644 variant/object_wide_offsets.metadata create mode 100644 variant/object_wide_offsets.value diff --git a/variant/README.md b/variant/README.md index c09c49a..cdb2424 100644 --- a/variant/README.md +++ b/variant/README.md @@ -39,10 +39,28 @@ Each example consists of 2 files: 3. `object_empty` -- Example of object (`basic_type` = 3) with no fields 3. `object_primitive` -- Example of object with only primitive fields 4. `object_nested` -- Example of object with other objects in fields +4. `object_wide_offsets` -- Example of object with 1-byte field IDs and 2-byte field offsets +4. `object_wide_field_ids` -- Example of object with 2-byte field IDs and 1-byte field offsets 5. `array_empty` -- Example of array (`basic_type` = 4) with no elements 5. `array_primitive` -- Example of array with only primitive elements 6. `array_nested` -- Example of an with objects and other arrays in the elements +## Objects whose field ID and field offset widths differ + +`object_wide_offsets` and `object_wide_field_ids` are the only examples in which +an object's `field_id_size` and `field_offset_size` differ. In every other object +example both are 1 byte, so the two `value_header` fields hold the same value and +a reader that reads them from each other's bit positions still decodes correctly. + +* `object_wide_offsets` -- the top-level object has two fields (1-byte field IDs) + and more than 256 bytes of field data (2-byte field offsets), giving header + byte `0x06`. +* `object_wide_field_ids` -- 262 distinct field names force 2-byte field IDs, + while the object under the `nested` field holds a single small value, so its own + field offsets still fit in 1 byte, giving header byte `0x12`. Its enclosing + object has 261 fields, which also makes this the only example with `is_large` + set to 1 (a 4-byte `num_elements`). + ## Regenerating these files The files in this directory were initially generated by running the [`regen.py`](regen.py) diff --git a/variant/data_dictionary.json b/variant/data_dictionary.json index 8a1faae..6a50de6 100644 --- a/variant/data_dictionary.json +++ b/variant/data_dictionary.json @@ -52,6 +52,275 @@ "string_field": "Apache Parquet", "timestamp_field": "2025-04-16T12:34:56.78" }, + "object_wide_field_ids": { + "f000": 0, + "f001": 1, + "f002": 2, + "f003": 3, + "f004": 4, + "f005": 5, + "f006": 6, + "f007": 7, + "f008": 8, + "f009": 9, + "f010": 10, + "f011": 11, + "f012": 12, + "f013": 13, + "f014": 14, + "f015": 15, + "f016": 16, + "f017": 17, + "f018": 18, + "f019": 19, + "f020": 20, + "f021": 21, + "f022": 22, + "f023": 23, + "f024": 24, + "f025": 25, + "f026": 26, + "f027": 27, + "f028": 28, + "f029": 29, + "f030": 30, + "f031": 31, + "f032": 32, + "f033": 33, + "f034": 34, + "f035": 35, + "f036": 36, + "f037": 37, + "f038": 38, + "f039": 39, + "f040": 40, + "f041": 41, + "f042": 42, + "f043": 43, + "f044": 44, + "f045": 45, + "f046": 46, + "f047": 47, + "f048": 48, + "f049": 49, + "f050": 50, + "f051": 51, + "f052": 52, + "f053": 53, + "f054": 54, + "f055": 55, + "f056": 56, + "f057": 57, + "f058": 58, + "f059": 59, + "f060": 60, + "f061": 61, + "f062": 62, + "f063": 63, + "f064": 64, + "f065": 65, + "f066": 66, + "f067": 67, + "f068": 68, + "f069": 69, + "f070": 70, + "f071": 71, + "f072": 72, + "f073": 73, + "f074": 74, + "f075": 75, + "f076": 76, + "f077": 77, + "f078": 78, + "f079": 79, + "f080": 80, + "f081": 81, + "f082": 82, + "f083": 83, + "f084": 84, + "f085": 85, + "f086": 86, + "f087": 87, + "f088": 88, + "f089": 89, + "f090": 90, + "f091": 91, + "f092": 92, + "f093": 93, + "f094": 94, + "f095": 95, + "f096": 96, + "f097": 97, + "f098": 98, + "f099": 99, + "f100": 100, + "f101": 101, + "f102": 102, + "f103": 103, + "f104": 104, + "f105": 105, + "f106": 106, + "f107": 107, + "f108": 108, + "f109": 109, + "f110": 110, + "f111": 111, + "f112": 112, + "f113": 113, + "f114": 114, + "f115": 115, + "f116": 116, + "f117": 117, + "f118": 118, + "f119": 119, + "f120": 120, + "f121": 121, + "f122": 122, + "f123": 123, + "f124": 124, + "f125": 125, + "f126": 126, + "f127": 127, + "f128": 0, + "f129": 1, + "f130": 2, + "f131": 3, + "f132": 4, + "f133": 5, + "f134": 6, + "f135": 7, + "f136": 8, + "f137": 9, + "f138": 10, + "f139": 11, + "f140": 12, + "f141": 13, + "f142": 14, + "f143": 15, + "f144": 16, + "f145": 17, + "f146": 18, + "f147": 19, + "f148": 20, + "f149": 21, + "f150": 22, + "f151": 23, + "f152": 24, + "f153": 25, + "f154": 26, + "f155": 27, + "f156": 28, + "f157": 29, + "f158": 30, + "f159": 31, + "f160": 32, + "f161": 33, + "f162": 34, + "f163": 35, + "f164": 36, + "f165": 37, + "f166": 38, + "f167": 39, + "f168": 40, + "f169": 41, + "f170": 42, + "f171": 43, + "f172": 44, + "f173": 45, + "f174": 46, + "f175": 47, + "f176": 48, + "f177": 49, + "f178": 50, + "f179": 51, + "f180": 52, + "f181": 53, + "f182": 54, + "f183": 55, + "f184": 56, + "f185": 57, + "f186": 58, + "f187": 59, + "f188": 60, + "f189": 61, + "f190": 62, + "f191": 63, + "f192": 64, + "f193": 65, + "f194": 66, + "f195": 67, + "f196": 68, + "f197": 69, + "f198": 70, + "f199": 71, + "f200": 72, + "f201": 73, + "f202": 74, + "f203": 75, + "f204": 76, + "f205": 77, + "f206": 78, + "f207": 79, + "f208": 80, + "f209": 81, + "f210": 82, + "f211": 83, + "f212": 84, + "f213": 85, + "f214": 86, + "f215": 87, + "f216": 88, + "f217": 89, + "f218": 90, + "f219": 91, + "f220": 92, + "f221": 93, + "f222": 94, + "f223": 95, + "f224": 96, + "f225": 97, + "f226": 98, + "f227": 99, + "f228": 100, + "f229": 101, + "f230": 102, + "f231": 103, + "f232": 104, + "f233": 105, + "f234": 106, + "f235": 107, + "f236": 108, + "f237": 109, + "f238": 110, + "f239": 111, + "f240": 112, + "f241": 113, + "f242": 114, + "f243": 115, + "f244": 116, + "f245": 117, + "f246": 118, + "f247": 119, + "f248": 120, + "f249": 121, + "f250": 122, + "f251": 123, + "f252": 124, + "f253": 125, + "f254": 126, + "f255": 127, + "f256": 0, + "f257": 1, + "f258": 2, + "f259": 3, + "nested": { + "last": 1 + } + }, + "object_wide_offsets": { + "id": 1, + "padding": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + }, "primitive_binary": "AxM33q2+78r+", "primitive_boolean_false": false, "primitive_boolean_true": true, diff --git a/variant/object_wide_field_ids.metadata b/variant/object_wide_field_ids.metadata new file mode 100644 index 0000000000000000000000000000000000000000..a22a296bf1477175045449447ad02dcf8862b26e GIT binary patch literal 1579 zcmWO3Ei5()7>Duuj0r5ClOGoXz`Auru2c1VIo4K@bE% z5ClOG1VIo4K@bE%5ClOG1mRraYFGRH{fC=1Y{-Uf#71q*#%;nTZOW#tX>IG?c0GJ+L0aGiJjV+o!f<7+Lc|~josRv-P?mb z+LJxoi@n;Lz1xR<+LwLXkC``o$cKHzM}5r4eZnVw%BQ{QZSVVx&-$Ft`+_g}k}vy; zulky=`-X4&mT&ux@A{ta`+*<&kstespZb}f`-NZnm0$af-};^3`-4CFlRx{5zxtcM z`-gw}mw)?@TQp)QhGQf~V=TsFA|_)hrlT3{=*LXV#$3$DLM+BoEXPW$##*e$Mr_7b zY{yRQ#$N2lK^(?W9LGtV##x-lMO?;JT*pn^#$DXULp;V)JjYAC##_9{M|{Rte8*2% zHgYJ3b0kM|EXQ*qCvz&NvzhJe=S+p5>I(!|z4qu0_ z!`J2O@^$&Td|kdSUze}T*X8T-_4s;xJ-!}akFUqq+|*b z`h0!9KHq?Ez&GF<@D2C|d;`7#-+*s0_$@qyhwu;{!b5ln58)v^gop4D9>PO-2oK>Q zJcNhv5FWxqcnA;SAv}bK@DLutLwE=e;UPSPhwu;{!b5ln58)v^gop4D9>PO-2oK>Q zJcNhv5FWxqcnA;SAv}bK@DLutLwE=e;UPSPhwu;{!b5ln58)v^gop4D9>PO-2oK>Q zJcNhv5FWxqcnA;SAv}bK@DLutLwE=e;UPSPhwu;{g-790coZImN8wR;6dr{~;Zb-L z9)(BYQFs&{g-790coZImN8wR;6dr{~;Zb-L9)(BYQFs&{g-790coZImN8wR;6dr{~ x;Zb-L9)(BYQFs&{g-790coZImN8wR;6dr{~;Zb;0XYlW@|F(XuZT$1!*8c~JhvNVM literal 0 HcmV?d00001 diff --git a/variant/object_wide_field_ids.value b/variant/object_wide_field_ids.value new file mode 100644 index 0000000000000000000000000000000000000000..a83dff1d7324ae3b46d0650ae2503634a28af1bc GIT binary patch literal 1579 zcmeIz37Zd87>4ot`C~9WwjujYL$Z^7Cwq27ma)seH})k-Qc02|Ns>yERFWh~k|aqg zNs=U$BuOguURPhE^9|1Zd#-cRvWPe$K_)Vjg(R|)jqD_I4JHRU$whARkV0PaaV_~N zKtT#om?9LV7{w_;NlHyQYlAyuBQSwP?1VhrV3T5Ms;dXlN+f;ZR${$deo-@ z4QWJUn$VPHG^YhO(UMlQrVVXrM|*Cj10Cr^XS&dpZgeM&9`xiEdeNIc^d+5s^k)DA z8N^_QFqB&v#&AY3l2MFi3}YF`cqTBBN!&&TlbOO)rZJrv%w!g`nZxbODU-?%`hUV-feWmpw(&aKd4nCi$xhy47jLthci6+b?BzZ7@jmlIW>eU zgl^@KtsbhiLbP6JHVVmRq1ZA6+lF3;kn0j^X(84tw9-RrU?>d@p%I}oCS)drN=ArG z3yoPJF*g+Ehrq(9zbMKtiR#Ou_=>2#DoU@3%Il)=`l!1x%5ILTTcYT;sJSCb?uv?g zqTs%$cOc3gifTuq*wLtUJW8F6N~fdH*{E|a%3O#lm!im(sBtw)B%(r+vME_sPUTjL z@+rRxs<4WxxJs(D$|_amRY4V1SyfeCHC0P>R8I}mNKMpCE!0x2)mH7*L7mh^-IS)D z>ZLwPSAPxEU=7tUjnF8K(Kt=eBxPudrfG&|X^!S0em$gM(wN2Z#Lp!xgyR}DqwNLwXKnHb5hjm1s>8OtBxK8M# zPU*DH=&XL!Ii1%9UDPFA))oDwtMLpZ5}6Z8iEN4731d0oOwQ~qPI6Xfb9N{D8rwOX z)4816d7R?B&gW~L-vwOIg" to "VARIANT"" is not yet supported -- INSERT INTO T VALUES ('object_primitive', struct(1234.56::Double as double_field, true as boolean_true_field, false as boolean_false_field, '2025-04-16T12:34:56.78'::Timestamp as timestamp_field, 'Apache Parquet' as string_field, null as null_field)::Variant); ---TODO objects with more than 2**8 distinct fields (that require using more than one byte for field offset) ---TODO objects with more than 2**16 distinct fields (that require using more than 2 bytes for field offset) ---TODO objects with more than 2**24 distinct fields (that require using more than 3 bytes for field offset) + +-- https://github.com/apache/parquet-testing/issues/78 +--TODO objects with more than 2**16 distinct fields (that require using more than 2 bytes for field IDs) +--TODO objects with more than 2**24 distinct fields (that require using more than 3 bytes for field IDs) ------------------------------- -- Array (basic_type=3) @@ -115,6 +139,9 @@ DROP TABLE IF EXISTS output; CREATE TABLE output AS SELECT name, variant_col, to_json(variant_col) as json_col FROM T; """ +sql = sql.replace("__OBJECT_WIDE_FIELD_IDS_JSON__", object_wide_field_ids_json) +sql = sql.replace("__OBJECT_WIDE_OFFSETS_JSON__", object_wide_offsets_json) + for statement in sql.split("\n"): statement = statement.strip() if not statement or statement.startswith("--"):