diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7262042 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Encoded IPLD blocks and CARs are byte-exact payloads: never diff, merge, +# or normalize line endings, or the bytes stop matching their CIDs. +*.dag-pb binary +*.dag-cbor binary +*.dag-json binary +*.car binary diff --git a/.gitignore b/.gitignore index 4491de5..acf8b79 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ __pycache__/ *$py.class .venv venv/ +js/package-lock.json diff --git a/README.md b/README.md index 28b437d..054cdc3 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,18 @@ Negative fixtures for an encode phase involve defining a data model form that sh Negative fixtures for a decode phase involve loading a block's bytes from a hex form from the fixture data and passing those bytes through a `Decode()` for that codec and inspecting the error. Error messages may or may not be matched in some way, depending on the complexity of the implementation—it is more important that a failure occur than the error is exact. +## Non-canonical Fixtures + +The [noncanonical-fixtures](./noncanonical-fixtures/) directory contains blocks that decode successfully but are not in the canonical form for their codec, so they can never round-trip byte-for-byte through the positive fixture flow above. Like negative fixtures, they are grouped per codec as `noncanonical-fixtures//decode/*.json`. Each entry has a `name`, the `hex` of the non-canonical block, the `canonicalHex` of the same logical node in canonical form, and the `canonicalCid` of that canonical block. + +Codec implementations are expected to: + +1. Decode the non-canonical block successfully +2. Confirm it decodes to the same logical node as the canonical bytes +3. Re-encode the node canonically and compare the resulting CID to `canonicalCid` + +The current cases cover the opt-in dag-pb `Data`-first field order proposed by [IPIP-550](https://github.com/ipfs/specs/pull/550): decoders accept both field orders, while `Links`-first remains the canonical encode order. The related negative decode fixture "data between links" stays invalid: opt-in ordering does not loosen the rule against interleaved fields. + ## Implementations & Codecs ### Go diff --git a/_fixtures_src/dagpb_1namedlink+data.dag-pb b/_fixtures_src/dagpb_1namedlink+data.dag-pb new file mode 100644 index 0000000..a5e1cfd --- /dev/null +++ b/_fixtures_src/dagpb_1namedlink+data.dag-pb @@ -0,0 +1,3 @@ +3 +$U X‘µµ"Õßmð±ûÙÒ´üqc¯4Ђ†¢èFö¾ hello.txt + \ No newline at end of file diff --git a/fixtures.car b/fixtures.car index 08e36bc..524299d 100644 Binary files a/fixtures.car and b/fixtures.car differ diff --git a/fixtures/dagpb_1namedlink+data/bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24.dag-pb b/fixtures/dagpb_1namedlink+data/bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24.dag-pb new file mode 100644 index 0000000..a5e1cfd --- /dev/null +++ b/fixtures/dagpb_1namedlink+data/bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24.dag-pb @@ -0,0 +1,3 @@ +3 +$U X‘µµ"Õßmð±ûÙÒ´üqc¯4Ђ†¢èFö¾ hello.txt + \ No newline at end of file diff --git a/fixtures/dagpb_1namedlink+data/bafyreifkosctfs7pi4iwenai4keh7ph5kb47gfoo2c5mpthbp7vm25swr4.dag-cbor b/fixtures/dagpb_1namedlink+data/bafyreifkosctfs7pi4iwenai4keh7ph5kb47gfoo2c5mpthbp7vm25swr4.dag-cbor new file mode 100644 index 0000000..b0b55f6 Binary files /dev/null and b/fixtures/dagpb_1namedlink+data/bafyreifkosctfs7pi4iwenai4keh7ph5kb47gfoo2c5mpthbp7vm25swr4.dag-cbor differ diff --git a/fixtures/dagpb_1namedlink+data/baguqeera33s42p2sjw7hz6rsawracsab643ixxch3nx7eovttooneklcim5q.dag-json b/fixtures/dagpb_1namedlink+data/baguqeera33s42p2sjw7hz6rsawracsab643ixxch3nx7eovttooneklcim5q.dag-json new file mode 100644 index 0000000..aa56920 --- /dev/null +++ b/fixtures/dagpb_1namedlink+data/baguqeera33s42p2sjw7hz6rsawracsab643ixxch3nx7eovttooneklcim5q.dag-json @@ -0,0 +1 @@ +{"Data":{"/":{"bytes":"CAE"}},"Links":[{"Hash":{"/":"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am"},"Name":"hello.txt","Tsize":6}]} \ No newline at end of file diff --git a/go/codecs_test.go b/go/codecs_test.go index 774fa0a..9be2f3a 100644 --- a/go/codecs_test.go +++ b/go/codecs_test.go @@ -200,3 +200,79 @@ func testNegativeFixtureDecode(codecName string, fixture negativeFixtureDecode) } } } + +// TestNoncanonicalFixtures verifies blocks that decode successfully but are +// not in canonical form, so they can never round-trip byte-for-byte: decoding +// must succeed, the decoded node must equal the node decoded from the +// canonical bytes, and canonical re-encoding must produce the canonical CID. +// +// TODO: once codec implementations expose an opt-in encoder for alternate +// forms (e.g. the dag-pb Data-first field order proposed by IPIP-550, +// https://github.com/ipfs/specs/pull/550), promote these to full round-trip +// fixtures with a per-fixture encode hint. +func TestNoncanonicalFixtures(t *testing.T) { + dirs, err := os.ReadDir("../noncanonical-fixtures/") + if err != nil { + t.Fatalf("failed to open noncanonical fixtures dir: %v", err) + } + for _, dir := range dirs { + if !dir.IsDir() { + continue + } + codecName := codecName(dir.Name()) + t.Run(string(codecName), func(t *testing.T) { + files, err := os.ReadDir(filepath.Join("../noncanonical-fixtures/", string(codecName), "decode")) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return + } + t.Fatalf("failed to open noncanonical fixtures dir: %v", err) + } + for _, file := range files { + fixtureData, err := os.ReadFile(filepath.Join("../noncanonical-fixtures/", string(codecName), "decode", file.Name())) + if err != nil { + t.Fatalf("failed to read noncanonical fixture file: %v", err) + } + var fixtures []noncanonicalFixtureDecode + if err := json.Unmarshal(fixtureData, &fixtures); err != nil { + t.Fatalf("failed to parse noncanonical fixture file: %v", err) + } + for _, fixture := range fixtures { + t.Run(fixture.Name, testNoncanonicalFixtureDecode(codecName, fixture)) + } + } + }) + } +} + +func testNoncanonicalFixtureDecode(codecName codecName, fixture noncanonicalFixtureDecode) func(t *testing.T) { + return func(t *testing.T) { + byts, err := hex.DecodeString(fixture.Hex) + if err != nil { + t.Fatalf("failed to parse fixture hex: %v", err) + } + canonicalByts, err := hex.DecodeString(fixture.CanonicalHex) + if err != nil { + t.Fatalf("failed to parse fixture canonicalHex: %v", err) + } + expectedCid, err := cid.Decode(fixture.CanonicalCid) + if err != nil { + t.Fatalf("failed to parse fixture canonicalCid: %v", err) + } + + node, err := decodeBytes(codecName, byts) + if err != nil { + t.Fatalf("failed to decode noncanonical block: %v", err) + } + canonicalNode, err := decodeBytes(codecName, canonicalByts) + if err != nil { + t.Fatalf("failed to decode canonical block: %v", err) + } + // Logical equality is verified through canonical form: both decodes + // must re-encode to the same canonical CID. ipld.DeepEqual is not + // used here because the two wire forms decode to maps whose entry + // order differs, which DeepEqual treats as unequal. + verifyCid(t, "reencode(noncanonical)", node, codecs[codecName], expectedCid) + verifyCid(t, "reencode(canonical)", canonicalNode, codecs[codecName], expectedCid) + } +} diff --git a/go/fixtures.go b/go/fixtures.go index 2e733db..6a88a05 100644 --- a/go/fixtures.go +++ b/go/fixtures.go @@ -126,3 +126,26 @@ type negativeFixtureDecode struct { Hex string `json:"hex"` Error string `json:"error"` } + +type noncanonicalFixtureDecode struct { + Name string `json:"name"` + Hex string `json:"hex"` + CanonicalHex string `json:"canonicalHex"` + CanonicalCid string `json:"canonicalCid"` +} + +func decodeBytes(codecName codecName, byts []byte) (ipld.Node, error) { + lp, ok := codecs[codecName] + if !ok { + return nil, fmt.Errorf("unknown codec '%v'", codecName) + } + decoder, err := linkSystem.DecoderChooser(lp.BuildLink(make([]byte, 32))) + if err != nil { + return nil, err + } + na := basicnode.Prototype.Any.NewBuilder() + if err := decoder(na, bytes.NewReader(byts)); err != nil { + return nil, err + } + return na.Build(), nil +} diff --git a/js/test.js b/js/test.js index e6cc7b7..24721c1 100644 --- a/js/test.js +++ b/js/test.js @@ -9,6 +9,8 @@ import { negativeFixtureCodecs, negativeFixturesEncode, negativeFixturesDecode, + noncanonicalFixtureCodecs, + noncanonicalFixturesDecode, loadFixture } from './util.js' import { bytes } from 'multiformats' @@ -31,7 +33,7 @@ describe('Codec fixtures', () => { } }) -describe.only('Codec negative fixtures', () => { +describe('Codec negative fixtures', () => { for (const codec of negativeFixtureCodecs()) { describe(codec, () => { const { encode, decode } = codecs[codec].codec @@ -71,3 +73,32 @@ describe.only('Codec negative fixtures', () => { }) } }) + +// Non-canonical fixtures: blocks that decode successfully but are not in +// canonical form, so they can never round-trip byte-for-byte. The contract: +// decoding must succeed, the decoded value must equal the value decoded from +// the canonical bytes, and canonical re-encoding must produce the canonical +// CID. +// +// TODO: once codec implementations expose an opt-in encoder for alternate +// forms (e.g. the dag-pb Data-first field order proposed by IPIP-550, +// https://github.com/ipfs/specs/pull/550), promote these to full round-trip +// fixtures with a per-fixture encode hint. +describe('Codec noncanonical fixtures', () => { + for (const codec of noncanonicalFixtureCodecs()) { + describe(codec, () => { + const { decode } = codecs[codec].codec + for (const fixtures of noncanonicalFixturesDecode(codec)) { + for (const { name, hex, canonicalHex, canonicalCid } of fixtures) { + it(name, async () => { + const value = decode(bytes.fromHex(hex)) + const canonicalValue = decode(bytes.fromHex(canonicalHex)) + assert.deepEqual(value, canonicalValue, 'noncanonical and canonical bytes decode to the same value') + const block = await Block.encode({ value, codec: codecs[codec].codec, hasher: sha256 }) + assert.equal(block.cid.toString(), canonicalCid, 'canonical re-encode produces the canonical CID') + }) + } + } + }) + } +}) diff --git a/js/util.js b/js/util.js index fadc7bf..97dc677 100644 --- a/js/util.js +++ b/js/util.js @@ -3,6 +3,7 @@ import path from 'path' export const fixturesDir = new URL('../fixtures/', import.meta.url) export const negativeFixturesDir = new URL('../negative-fixtures/', import.meta.url) +export const noncanonicalFixturesDir = new URL('../noncanonical-fixtures/', import.meta.url) export async function loadFixture (dir) { const data = {} @@ -71,3 +72,15 @@ export function * negativeFixturesEncode (codec) { export function * negativeFixturesDecode (codec) { yield * negativeFixtures('decode', codec) } + +export function * noncanonicalFixtureCodecs () { + for (const { name } of iterate('dir', noncanonicalFixturesDir)) { + yield name + } +} + +export function * noncanonicalFixturesDecode (codec) { + for (const { url } of iterate('file', noncanonicalFixturesDir, codec, 'decode')) { + yield JSON.parse(fs.readFileSync(url, 'utf8')) + } +} diff --git a/noncanonical-fixtures/dag-pb/decode/field-order.json b/noncanonical-fixtures/dag-pb/decode/field-order.json new file mode 100644 index 0000000..e52831c --- /dev/null +++ b/noncanonical-fixtures/dag-pb/decode/field-order.json @@ -0,0 +1,8 @@ +[ + { + "name": "Data field before Links", + "hex": "0a02080112330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e7478741806", + "canonicalHex": "12330a24015512205891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03120968656c6c6f2e74787418060a020801", + "canonicalCid": "bafybeigdcg7pksx2zk5336vrfsktjodlr4rbfz37qr3koc5xboxe5ekv24" + } +] diff --git a/python/tests/test_noncanonical_fixtures.py b/python/tests/test_noncanonical_fixtures.py new file mode 100644 index 0000000..707f96a --- /dev/null +++ b/python/tests/test_noncanonical_fixtures.py @@ -0,0 +1,47 @@ +from pathlib import Path +import json + +import pytest +from ipld_dag_pb import encode, decode, code +from multiformats import CID, multihash + + +NONCANONICAL_FIXTURES_DIR = Path(__file__).parents[2] / "noncanonical-fixtures/dag-pb" + +# TODO: once codec implementations expose an opt-in encoder for alternate +# forms (e.g. the dag-pb Data-first field order proposed by IPIP-550, +# https://github.com/ipfs/specs/pull/550), promote these to full round-trip +# fixtures with a per-fixture encode hint. + + +def bytes_to_cid(data: bytes) -> str: + """Convert bytes to a dag-pb CIDv1 using the sha2-256 hash function""" + mh = multihash.digest(data, "sha2-256") + return str(CID(base="base32", version=1, codec=code, digest=mh)) + + +def load_noncanonical_decode_fixtures(): + """Load non-canonical decode fixtures for dag-pb""" + fixtures = [] + decode_dir = NONCANONICAL_FIXTURES_DIR / "decode" + if not decode_dir.is_dir(): + return fixtures + + for file in decode_dir.iterdir(): + with open(file, "r") as f: + for fixture in json.load(f): + fixtures.append((fixture["name"], fixture)) + + return fixtures + + +@pytest.mark.parametrize("name, fixture", load_noncanonical_decode_fixtures()) +def test_noncanonical_decode(name, fixture): + """Non-canonical blocks decode successfully, equal the canonical decode, + and canonical re-encoding produces the canonical CID""" + value = decode(bytes.fromhex(fixture["hex"])) + canonical_value = decode(bytes.fromhex(fixture["canonicalHex"])) + assert value == canonical_value + + reencoded = bytes(encode(value)) + assert bytes_to_cid(reencoded) == fixture["canonicalCid"] diff --git a/rust/tests/serde.rs b/rust/tests/serde.rs index 8227582..76e6206 100644 --- a/rust/tests/serde.rs +++ b/rust/tests/serde.rs @@ -144,3 +144,48 @@ fn negative_fixtures() { } } } + +/// Non-canonical fixtures: blocks that decode successfully but are not in +/// canonical form, so they can never round-trip byte-for-byte. The contract: +/// decoding must succeed, the decoded value must equal the value decoded from +/// the canonical bytes, and canonical re-encoding must produce the canonical +/// CID. +/// +/// TODO: once codec implementations expose an opt-in encoder for alternate +/// forms (e.g. the dag-pb Data-first field order proposed by IPIP-550, +/// https://github.com/ipfs/specs/pull/550), promote these to full round-trip +/// fixtures with a per-fixture encode hint. +#[test] +fn noncanonical_fixtures() { + for codec_dir in utils::fixture_directories("noncanonical-fixtures") { + let codec_name = codec_dir + .file_name() + .to_str() + .expect("Codec names are valid UTF-8") + .to_string(); + let codec = IpldCodec::new(&codec_name); + + for fixture in utils::load_noncanonical_fixtures(codec_dir.path()) { + println!( + "Testing noncanonical decode fixture for {}: {}", + codec_name, fixture.name + ); + let decoded = codec.decode(&fixture.bytes).expect("Decoding must work"); + let canonical_decoded = codec + .decode(&fixture.canonical_bytes) + .expect("Decoding canonical bytes must work"); + assert_eq!( + decoded, canonical_decoded, + "noncanonical and canonical bytes decode to the same value" + ); + + let data = codec.encode(&decoded).expect("Encoding must work"); + let digest = Code::Sha2_256.digest(&data); + let cid = Cid::new_v1(u64::from(IpldCodec::new(&codec_name)), digest); + assert_eq!( + cid, fixture.canonical_cid, + "canonical re-encode produces the canonical CID" + ); + } + } +} diff --git a/rust/tests/utils.rs b/rust/tests/utils.rs index 2fbed12..4402b15 100644 --- a/rust/tests/utils.rs +++ b/rust/tests/utils.rs @@ -147,3 +147,67 @@ pub fn load_negative_fixtures(mut dir: PathBuf, en_or_decode: &str) -> Vec, + pub canonical_bytes: Vec, + pub canonical_cid: Cid, +} + +/// Returns all non-canonical decode fixtures from the given codec directory. +pub fn load_noncanonical_fixtures(mut dir: PathBuf) -> Vec { + dir.push("decode"); + if let Ok(read_dir) = fs::read_dir(&dir) { + read_dir + .filter_map(|file| { + // Filter out invalid files. + let file = file.ok()?; + + let bytes = fs::read(file.path()).expect("File must be able to be read"); + // Use DAG-JSON for parsing, so we don't need an extra JSON parser. + let ipld: Ipld = serde_ipld_dagjson::from_slice(&bytes).expect("It's valid JSON"); + + if let Ipld::List(list) = ipld { + let fixtures: Vec<_> = list + .iter() + .map(|fixture| { + let name = match fixture.get("name") { + Ok(Some(Ipld::String(name))) => name.to_string(), + _ => panic!("Noncanonical fixture has no name"), + }; + let hex_field = |key: &str| -> Vec { + if let Ok(Some(Ipld::String(data))) = fixture.get(key) { + hex::decode(data).unwrap() + } else { + panic!("Noncanonical fixture is missing '{}'", key) + } + }; + let canonical_cid = + if let Ok(Some(Ipld::String(data))) = fixture.get("canonicalCid") { + Cid::try_from(data.as_str()) + .expect("canonicalCid must be a valid CID") + } else { + panic!("Noncanonical fixture is missing 'canonicalCid'") + }; + NoncanonicalFixture { + name, + bytes: hex_field("hex"), + canonical_bytes: hex_field("canonicalHex"), + canonical_cid, + } + }) + .collect(); + Some(fixtures) + } else { + None + } + }) + .flatten() + .collect() + } else { + Vec::new() + } +}