feat: noncanonical fixtures (PBNode field order)

.gitattributes

@@ -0,0 +1,6 @@
+# Encoded IPLD blocks and CARs are byte-exact payloads: never diff, merge,
+# or normalize line endings, or the bytes stop matching their CIDs.
+*.dag-pb binary
+*.dag-cbor binary
+*.dag-json binary
+*.car binary

.gitignore

@@ -6,3 +6,4 @@ __pycache__/
 *$py.class
 .venv
 venv/
+js/package-lock.json

README.md

@@ -40,6 +40,18 @@ Negative fixtures for an encode phase involve defining a data model form that sh
 
 Negative fixtures for a decode phase involve loading a block's bytes from a hex form from the fixture data and passing those bytes through a `Decode()` for that codec and inspecting the error. Error messages may or may not be matched in some way, depending on the complexity of the implementation—it is more important that a failure occur than the error is exact.
 
+## Non-canonical Fixtures
+
+The [noncanonical-fixtures](./noncanonical-fixtures/) directory contains blocks that decode successfully but are not in the canonical form for their codec, so they can never round-trip byte-for-byte through the positive fixture flow above. Like negative fixtures, they are grouped per codec as `noncanonical-fixtures/<codec-name>/decode/*.json`. Each entry has a `name`, the `hex` of the non-canonical block, the `canonicalHex` of the same logical node in canonical form, and the `canonicalCid` of that canonical block.
+
+Codec implementations are expected to:
+
+1. Decode the non-canonical block successfully
+2. Confirm it decodes to the same logical node as the canonical bytes
+3. Re-encode the node canonically and compare the resulting CID to `canonicalCid`
+
+The current cases cover the opt-in dag-pb `Data`-first field order proposed by [IPIP-550](https://github.com/ipfs/specs/pull/550): decoders accept both field orders, while `Links`-first remains the canonical encode order. The related negative decode fixture "data between links" stays invalid: opt-in ordering does not loosen the rule against interleaved fields.
+
 ## Implementations & Codecs
 
 ### Go

_fixtures_src/dagpb_1namedlink+data.dag-pb

@@ -0,0 +1,3 @@
+3
+$
U X‘µµ"Õßmð±ûÙÒ´üqc¯4Ђ†¢èFö¾	hello.txt
+

fixtures/dagpb_1namedlink+data/bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24.dag-pb

@@ -0,0 +1,3 @@
+3
+$
U X‘µµ"Õßmð±ûÙÒ´üqc¯4Ђ†¢èFö¾	hello.txt
+

fixtures/dagpb_1namedlink+data/baguqeera33s42p2sjw7hz6rsawracsab643ixxch3nx7eovttooneklcim5q.dag-json

@@ -0,0 +1 @@
+{"Data":{"/":{"bytes":"CAE"}},"Links":[{"Hash":{"/":"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am"},"Name":"hello.txt","Tsize":6}]}

go/codecs_test.go

@@ -200,3 +200,79 @@ func testNegativeFixtureDecode(codecName string, fixture negativeFixtureDecode)
 		}
 	}
 }
+
+// TestNoncanonicalFixtures verifies blocks that decode successfully but are
+// not in canonical form, so they can never round-trip byte-for-byte: decoding
+// must succeed, the decoded node must equal the node decoded from the
+// canonical bytes, and canonical re-encoding must produce the canonical CID.
+//
+// TODO: once codec implementations expose an opt-in encoder for alternate
+// forms (e.g. the dag-pb Data-first field order proposed by IPIP-550,
+// https://github.com/ipfs/specs/pull/550), promote these to full round-trip
+// fixtures with a per-fixture encode hint.
+func TestNoncanonicalFixtures(t *testing.T) {
+	dirs, err := os.ReadDir("../noncanonical-fixtures/")
+	if err != nil {
+		t.Fatalf("failed to open noncanonical fixtures dir: %v", err)
+	}
+	for _, dir := range dirs {
+		if !dir.IsDir() {
+			continue
+		}
+		codecName := codecName(dir.Name())
+		t.Run(string(codecName), func(t *testing.T) {
+			files, err := os.ReadDir(filepath.Join("../noncanonical-fixtures/", string(codecName), "decode"))
+			if err != nil {
+				if errors.Is(err, os.ErrNotExist) {
+					return
+				}
+				t.Fatalf("failed to open noncanonical fixtures dir: %v", err)
+			}
+			for _, file := range files {
+				fixtureData, err := os.ReadFile(filepath.Join("../noncanonical-fixtures/", string(codecName), "decode", file.Name()))
+				if err != nil {
+					t.Fatalf("failed to read noncanonical fixture file: %v", err)
+				}
+				var fixtures []noncanonicalFixtureDecode
+				if err := json.Unmarshal(fixtureData, &fixtures); err != nil {
+					t.Fatalf("failed to parse noncanonical fixture file: %v", err)
+				}
+				for _, fixture := range fixtures {
+					t.Run(fixture.Name, testNoncanonicalFixtureDecode(codecName, fixture))
+				}
+			}
+		})
+	}
+}
+
+func testNoncanonicalFixtureDecode(codecName codecName, fixture noncanonicalFixtureDecode) func(t *testing.T) {
+	return func(t *testing.T) {
+		byts, err := hex.DecodeString(fixture.Hex)
+		if err != nil {
+			t.Fatalf("failed to parse fixture hex: %v", err)
+		}
+		canonicalByts, err := hex.DecodeString(fixture.CanonicalHex)
+		if err != nil {
+			t.Fatalf("failed to parse fixture canonicalHex: %v", err)
+		}
+		expectedCid, err := cid.Decode(fixture.CanonicalCid)
+		if err != nil {
+			t.Fatalf("failed to parse fixture canonicalCid: %v", err)
+		}
+
+		node, err := decodeBytes(codecName, byts)
+		if err != nil {
+			t.Fatalf("failed to decode noncanonical block: %v", err)
+		}
+		canonicalNode, err := decodeBytes(codecName, canonicalByts)
+		if err != nil {
+			t.Fatalf("failed to decode canonical block: %v", err)
+		}
+		// Logical equality is verified through canonical form: both decodes
+		// must re-encode to the same canonical CID. ipld.DeepEqual is not
+		// used here because the two wire forms decode to maps whose entry
+		// order differs, which DeepEqual treats as unequal.
+		verifyCid(t, "reencode(noncanonical)", node, codecs[codecName], expectedCid)
+		verifyCid(t, "reencode(canonical)", canonicalNode, codecs[codecName], expectedCid)
+	}
+}

go/fixtures.go

@@ -126,3 +126,26 @@ type negativeFixtureDecode struct {
 	Hex   string `json:"hex"`
 	Error string `json:"error"`
 }
+
+type noncanonicalFixtureDecode struct {
+	Name         string `json:"name"`
+	Hex          string `json:"hex"`
+	CanonicalHex string `json:"canonicalHex"`
+	CanonicalCid string `json:"canonicalCid"`
+}
+
+func decodeBytes(codecName codecName, byts []byte) (ipld.Node, error) {
+	lp, ok := codecs[codecName]
+	if !ok {
+		return nil, fmt.Errorf("unknown codec '%v'", codecName)
+	}
+	decoder, err := linkSystem.DecoderChooser(lp.BuildLink(make([]byte, 32)))
+	if err != nil {
+		return nil, err
+	}
+	na := basicnode.Prototype.Any.NewBuilder()
+	if err := decoder(na, bytes.NewReader(byts)); err != nil {
+		return nil, err
+	}
+	return na.Build(), nil
+}

js/test.js

@@ -9,6 +9,8 @@ import {
   negativeFixtureCodecs,
   negativeFixturesEncode,
   negativeFixturesDecode,
+  noncanonicalFixtureCodecs,
+  noncanonicalFixturesDecode,
   loadFixture
 } from './util.js'
 import { bytes } from 'multiformats'
@@ -31,7 +33,7 @@ describe('Codec fixtures', () => {
   }
 })
 
-describe.only('Codec negative fixtures', () => {
+describe('Codec negative fixtures', () => {
   for (const codec of negativeFixtureCodecs()) {
     describe(codec, () => {
       const { encode, decode } = codecs[codec].codec
@@ -71,3 +73,32 @@ describe.only('Codec negative fixtures', () => {
     })
   }
 })
+
+// Non-canonical fixtures: blocks that decode successfully but are not in
+// canonical form, so they can never round-trip byte-for-byte. The contract:
+// decoding must succeed, the decoded value must equal the value decoded from
+// the canonical bytes, and canonical re-encoding must produce the canonical
+// CID.
+//
+// TODO: once codec implementations expose an opt-in encoder for alternate
+// forms (e.g. the dag-pb Data-first field order proposed by IPIP-550,
+// https://github.com/ipfs/specs/pull/550), promote these to full round-trip
+// fixtures with a per-fixture encode hint.
+describe('Codec noncanonical fixtures', () => {
+  for (const codec of noncanonicalFixtureCodecs()) {
+    describe(codec, () => {
+      const { decode } = codecs[codec].codec
+      for (const fixtures of noncanonicalFixturesDecode(codec)) {
+        for (const { name, hex, canonicalHex, canonicalCid } of fixtures) {
+          it(name, async () => {
+            const value = decode(bytes.fromHex(hex))
+            const canonicalValue = decode(bytes.fromHex(canonicalHex))
+            assert.deepEqual(value, canonicalValue, 'noncanonical and canonical bytes decode to the same value')
+            const block = await Block.encode({ value, codec: codecs[codec].codec, hasher: sha256 })
+            assert.equal(block.cid.toString(), canonicalCid, 'canonical re-encode produces the canonical CID')
+          })
+        }
+      }
+    })
+  }
+})

js/util.js

@@ -3,6 +3,7 @@ import path from 'path'
 
 export const fixturesDir = new URL('../fixtures/', import.meta.url)
 export const negativeFixturesDir = new URL('../negative-fixtures/', import.meta.url)
+export const noncanonicalFixturesDir = new URL('../noncanonical-fixtures/', import.meta.url)
 
 export async function loadFixture (dir) {
   const data = {}
@@ -71,3 +72,15 @@ export function * negativeFixturesEncode (codec) {
 export function * negativeFixturesDecode (codec) {
   yield * negativeFixtures('decode', codec)
 }
+
+export function * noncanonicalFixtureCodecs () {
+  for (const { name } of iterate('dir', noncanonicalFixturesDir)) {
+    yield name
+  }
+}
+
+export function * noncanonicalFixturesDecode (codec) {
+  for (const { url } of iterate('file', noncanonicalFixturesDir, codec, 'decode')) {
+    yield JSON.parse(fs.readFileSync(url, 'utf8'))
+  }
+}

noncanonical-fixtures/dag-pb/decode/field-order.json

@@ -0,0 +1,8 @@
+[
+  {
+    "name": "Data field before Links",
+    "hex": "0a02080112330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e7478741806",
+    "canonicalHex": "12330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e74787418060a020801",
+    "canonicalCid": "bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24"
+  }
+]

python/tests/test_noncanonical_fixtures.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+import json
+
+import pytest
+from ipld_dag_pb import encode, decode, code
+from multiformats import CID, multihash
+
+
+NONCANONICAL_FIXTURES_DIR = Path(__file__).parents[2] / "noncanonical-fixtures/dag-pb"
+
+# TODO: once codec implementations expose an opt-in encoder for alternate
+# forms (e.g. the dag-pb Data-first field order proposed by IPIP-550,
+# https://github.com/ipfs/specs/pull/550), promote these to full round-trip
+# fixtures with a per-fixture encode hint.
+
+
+def bytes_to_cid(data: bytes) -> str:
+    """Convert bytes to a dag-pb CIDv1 using the sha2-256 hash function"""
+    mh = multihash.digest(data, "sha2-256")
+    return str(CID(base="base32", version=1, codec=code, digest=mh))
+
+
+def load_noncanonical_decode_fixtures():
+    """Load non-canonical decode fixtures for dag-pb"""
+    fixtures = []
+    decode_dir = NONCANONICAL_FIXTURES_DIR / "decode"
+    if not decode_dir.is_dir():
+        return fixtures
+
+    for file in decode_dir.iterdir():
+        with open(file, "r") as f:
+            for fixture in json.load(f):
+                fixtures.append((fixture["name"], fixture))
+
+    return fixtures
+
+
+@pytest.mark.parametrize("name, fixture", load_noncanonical_decode_fixtures())
+def test_noncanonical_decode(name, fixture):
+    """Non-canonical blocks decode successfully, equal the canonical decode,
+    and canonical re-encoding produces the canonical CID"""
+    value = decode(bytes.fromhex(fixture["hex"]))
+    canonical_value = decode(bytes.fromhex(fixture["canonicalHex"]))
+    assert value == canonical_value
+
+    reencoded = bytes(encode(value))
+    assert bytes_to_cid(reencoded) == fixture["canonicalCid"]

rust/tests/serde.rs

@@ -144,3 +144,48 @@ fn negative_fixtures() {
         }
     }
 }
+
+/// Non-canonical fixtures: blocks that decode successfully but are not in
+/// canonical form, so they can never round-trip byte-for-byte. The contract:
+/// decoding must succeed, the decoded value must equal the value decoded from
+/// the canonical bytes, and canonical re-encoding must produce the canonical
+/// CID.
+///
+/// TODO: once codec implementations expose an opt-in encoder for alternate
+/// forms (e.g. the dag-pb Data-first field order proposed by IPIP-550,
+/// https://github.com/ipfs/specs/pull/550), promote these to full round-trip
+/// fixtures with a per-fixture encode hint.
+#[test]
+fn noncanonical_fixtures() {
+    for codec_dir in utils::fixture_directories("noncanonical-fixtures") {
+        let codec_name = codec_dir
+            .file_name()
+            .to_str()
+            .expect("Codec names are valid UTF-8")
+            .to_string();
+        let codec = IpldCodec::new(&codec_name);
+
+        for fixture in utils::load_noncanonical_fixtures(codec_dir.path()) {
+            println!(
+                "Testing noncanonical decode fixture for {}: {}",
+                codec_name, fixture.name
+            );
+            let decoded = codec.decode(&fixture.bytes).expect("Decoding must work");
+            let canonical_decoded = codec
+                .decode(&fixture.canonical_bytes)
+                .expect("Decoding canonical bytes must work");
+            assert_eq!(
+                decoded, canonical_decoded,
+                "noncanonical and canonical bytes decode to the same value"
+            );
+
+            let data = codec.encode(&decoded).expect("Encoding must work");
+            let digest = Code::Sha2_256.digest(&data);
+            let cid = Cid::new_v1(u64::from(IpldCodec::new(&codec_name)), digest);
+            assert_eq!(
+                cid, fixture.canonical_cid,
+                "canonical re-encode produces the canonical CID"
+            );
+        }
+    }
+}

rust/tests/utils.rs

@@ -147,3 +147,67 @@ pub fn load_negative_fixtures(mut dir: PathBuf, en_or_decode: &str) -> Vec<Negat
         Vec::new()
     }
 }
+
+/// Contents of a single non-canonical fixture.
+#[derive(Debug)]
+pub struct NoncanonicalFixture {
+    pub name: String,
+    pub bytes: Vec<u8>,
+    pub canonical_bytes: Vec<u8>,
+    pub canonical_cid: Cid,
+}
+
+/// Returns all non-canonical decode fixtures from the given codec directory.
+pub fn load_noncanonical_fixtures(mut dir: PathBuf) -> Vec<NoncanonicalFixture> {
+    dir.push("decode");
+    if let Ok(read_dir) = fs::read_dir(&dir) {
+        read_dir
+            .filter_map(|file| {
+                // Filter out invalid files.
+                let file = file.ok()?;
+
+                let bytes = fs::read(file.path()).expect("File must be able to be read");
+                // Use DAG-JSON for parsing, so we don't need an extra JSON parser.
+                let ipld: Ipld = serde_ipld_dagjson::from_slice(&bytes).expect("It's valid JSON");
+
+                if let Ipld::List(list) = ipld {
+                    let fixtures: Vec<_> = list
+                        .iter()
+                        .map(|fixture| {
+                            let name = match fixture.get("name") {
+                                Ok(Some(Ipld::String(name))) => name.to_string(),
+                                _ => panic!("Noncanonical fixture has no name"),
+                            };
+                            let hex_field = |key: &str| -> Vec<u8> {
+                                if let Ok(Some(Ipld::String(data))) = fixture.get(key) {
+                                    hex::decode(data).unwrap()
+                                } else {
+                                    panic!("Noncanonical fixture is missing '{}'", key)
+                                }
+                            };
+                            let canonical_cid =
+                                if let Ok(Some(Ipld::String(data))) = fixture.get("canonicalCid") {
+                                    Cid::try_from(data.as_str())
+                                        .expect("canonicalCid must be a valid CID")
+                                } else {
+                                    panic!("Noncanonical fixture is missing 'canonicalCid'")
+                                };
+                            NoncanonicalFixture {
+                                name,
+                                bytes: hex_field("hex"),
+                                canonical_bytes: hex_field("canonicalHex"),
+                                canonical_cid,
+                            }
+                        })
+                        .collect();
+                    Some(fixtures)
+                } else {
+                    None
+                }
+            })
+            .flatten()
+            .collect()
+    } else {
+        Vec::new()
+    }
+}