From 89b5a3f4c0e15af22e7db9ce736722a87e93b9c5 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Mon, 7 Sep 2026 20:22:49 +0200 Subject: [PATCH 1/2] format: text and Markdown files can be written in UTF-16 TXT and MD gain two settings, shared in internal/format/textenc so the two cannot describe the same thing differently: encoding (utf-8, utf-16le, utf-16be) and bom. Both default to what these formats already wrote, and five new golden values pin the encoded paths. An odd number of bytes is refused in UTF-16 rather than rounded. A UTF-16 file cut to an odd length is rejected by Python, by V8 and by .NET when each is asked strictly, and repaired in silence by all three when it is not - so the file this tool must never write is the one nobody could tell apart by looking. Half of all sizes become unreachable and the refusal names the reachable size either side of the one asked for. The registry mechanism for that already existed: SmallestAccepted walks to a fixed point and already handled PNG refusing a band above a size it accepts. Single byte encodings are deliberately not offered. The filler vocabulary is ASCII, so latin-1 output is byte for byte identical to UTF-8 - a setting that would change nothing. They belong with content that is not English. TXT and MD also gain the structural check they never had. The comment saying they had nothing to be checked against stopped being true rather than being wrong: a file claiming to be UTF-16LE is a claim somebody else's decoder can settle. The checker is told the encoding rather than sniffing it, for the reason the CSV dialect is told. Two things this found that were not the job: - TestEveryFormatSurvivesItsReferenceTool skipped the structural check for any format with no reference tool, so the new checkers would have been dead on arrival with the guard green and reporting them as covered. It was invisible while the only formats without a reader were also the only ones without a checker. - a strict decode alone blesses a file with its byte order reversed, because 0x74 0x00 read the other way is a valid CJK character. What catches it is asking what the characters are, and only TXT was asking. Nine mutations, all caught. TestEveryTextFormatIsValidUTF8 keeps its name and says out loud that its claim is the default path. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 27 ++ README.md | 3 +- internal/format/md/md.go | 49 ++- internal/format/textenc/textenc.go | 271 +++++++++++++ internal/format/txt/txt.go | 54 ++- internal/guard/generatorbytes_test.go | 15 + internal/guard/layers_test.go | 6 +- internal/guard/oracle_test.go | 76 ++-- internal/guard/parity_test.go | 4 + internal/guard/testdata/generator-golden.json | 20 + internal/guard/textencoding_test.go | 361 ++++++++++++++++++ internal/guard/textformats_test.go | 11 + internal/oracle/oracle.go | 7 +- internal/oracle/strict.py | 91 ++++- web/public/formats/index.html | 20 + web/public/pl/formaty/index.html | 20 + 16 files changed, 976 insertions(+), 59 deletions(-) create mode 100644 internal/format/textenc/textenc.go create mode 100644 internal/guard/textencoding_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a4e4e2..c598c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,33 @@ because it turns other people's test suites red. same new code and were checked against their recorded hashes and across every format at five sizes and two seeds. +### Added + +- **Text and Markdown files can be written in UTF-16, with or without a byte + order mark.** Two new settings on `txt` and `md`: `encoding`, which takes + `utf-8`, `utf-16le` or `utf-16be`, and `bom`, which is `true` or `false`. + Both default to what these formats have always produced, so a recipe that + says nothing gets the same bytes it got before. + + ``` + tfg generate --format txt --size 4kb --set encoding=utf-16le --set bom=true + ``` + + **In UTF-16 an odd number of bytes is refused.** Every character takes two + bytes, so only an even size can be a whole file, and asking for 4001 B gets + an error naming 4000 B and 4002 B rather than a file that is one byte out. + Three readers were measured on a UTF-16 file cut to an odd length: Python, + Node and .NET all reject it when asked strictly, and all three repair it in + silence otherwise, which is why this is an error and not a rounded size. + + The self describing label costs twice as much in UTF-16, so it needs a file + of at least 66 B rather than 33 B to fit. Below that the file is still + produced and the manifest says the label was left out. + + Single byte encodings such as Windows-1252 are deliberately not offered. The + generated text is English, so a file written in one would be byte for byte + the same file as UTF-8 - a setting that changes nothing. + ### Security - **On Windows, the desktop window loads the library it uses for dark menus from diff --git a/README.md b/README.md index fce881d..7f89444 100644 --- a/README.md +++ b/README.md @@ -476,7 +476,8 @@ recipe. `tfg formats ` prints the allowed range or list for each: | `pptx` | `slides` | | `csv` | `delimiter`, `line_ending`, `header`, `quote_style`, `columns` | | `log` | `entry_format`, `timestamps`, `rate`, `methods`, `status_mix`, `level_mix`, `ip_version`, `line_ending` | -| `json`, `xml`, `html`, `md`, `txt`, `svg` | none | +| `txt`, `md` | `encoding`, `bom` | +| `json`, `xml`, `html`, `svg` | none | ``` tfg generate --format jpg --size 500kb --set width=1920 --set height=1080 --set quality=85 diff --git a/internal/format/md/md.go b/internal/format/md/md.go index 5960480..25a328a 100644 --- a/internal/format/md/md.go +++ b/internal/format/md/md.go @@ -21,6 +21,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" ) // The padding channel is the content, measured 2026-08-01 like the rest of the @@ -55,12 +56,15 @@ func init() { Where: format.PlacementEnd, Capacity: 0, }, - Label: format.LabelVisible, + Label: format.LabelVisible, + // No reference tool, for the reason TXT gives: a reader is handed a + // path and would have to guess the encoding. The structural check is + // the layer that can be told. Oracle: format.OracleNone, // Heading depth, table size and which elements appear come later. - // Declaring none now is what makes a recipe asking for them fail - // loudly rather than quietly producing something else. - Properties: nil, + // Declaring only what is here is what makes a recipe asking for them + // fail loudly rather than quietly producing something else. + Properties: textenc.Properties(), GeneratorVersion: generatorVersion, Generator: generator{}, }) @@ -71,6 +75,11 @@ type generator struct{} type memo struct { labelLine string // includes the trailing blank line, empty when absent seed uint64 + codec textenc.Codec + // source is the ASCII content this file holds, counted in characters + // rather than in file bytes - see the same field on TXT. It is what lets + // the block loop below stay the loop it was. + source int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -84,30 +93,41 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } } + codec, err := textenc.Parse("md", r.Properties) + if err != nil { + return format.Plan{}, err + } + if err := codec.Check("MD", r.Bytes); err != nil { + return format.Plan{}, err + } + p := format.Plan{ Bytes: r.Bytes, Exact: true, Determinism: format.DeterminismByte, Properties: map[string]any{ - "encoding": "utf-8", - "line_ending": "lf", - "flavour": "commonmark", + textenc.Setting: codec.Name(), + textenc.SettingBOM: codec.HasBOM(), + "line_ending": "lf", + "flavour": "commonmark", }, } - m := memo{seed: r.Seed} + m := memo{seed: r.Seed, codec: codec, source: codec.Source(r.Bytes)} if r.Label { // A plain line followed by a blank one is a paragraph, which renders // visibly and cannot break the document wherever it sits. line := core.Label("md", r.Bytes, r.Seed) + "\n\n" - if int64(len(line)) <= r.Bytes { + if int64(len(line)) <= m.source { m.labelLine = line } else { + // What the label COSTS here, not how long it reads - the two + // differ by a factor of two in UTF-16. p.Notes = append(p.Notes, format.Note{ Code: "label_omitted", Detail: fmt.Sprintf( "The label needs %d B and the file is %d B, so this file carries no label. Its name and the manifest still identify it.", - len(line), r.Bytes), + codec.Cost(int64(len(line))), r.Bytes), }) } } @@ -123,7 +143,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return fmt.Errorf("md: the plan was not produced by this generator") } - remaining := p.Bytes + // The mark is bytes rather than text, so it goes out as itself, and + // everything after it goes through the encoder. + if err := writeAll(w, m.codec.Preamble()); err != nil { + return err + } + w = m.codec.Writer(w) + + remaining := m.source if m.labelLine != "" { if err := writeAll(w, []byte(m.labelLine)); err != nil { return err diff --git a/internal/format/textenc/textenc.go b/internal/format/textenc/textenc.go new file mode 100644 index 0000000..0520d8f --- /dev/null +++ b/internal/format/textenc/textenc.go @@ -0,0 +1,271 @@ +// Package textenc is the character encoding the text formats share. +// +// It lives here rather than inside one of them because TXT and MD ask the same +// question and have to answer it in the same words. A copy in each is how +// thirteen packages ended up carrying four different versions of one filler +// loop, and the same rule applies to a setting a person reads. +// +// What it holds is arithmetic as much as bytes. A file written in UTF-16 is a +// whole number of sixteen bit units, so its length is always even - which +// means half of all sizes stop being reachable, and the exact size promise +// turns them into refusals rather than into files of the wrong size. +// Measured 2026-09-07 on three independent readers: a UTF-16 file cut to an +// odd length is REJECTED by Python, by V8 and by .NET when each is asked +// strictly, and quietly repaired by all three when it is not. A file whose +// corruption only a strict reader can see is the silence this project bans. +package textenc + +import ( + "fmt" + "io" + "unicode/utf16" + "unicode/utf8" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// Setting names. Public names, so they are spelled once. +const ( + Setting = "encoding" + SettingBOM = "bom" +) + +// The encodings on offer. +// +// Single byte encodings are deliberately absent, and that is a measurement +// rather than an omission. The filler vocabulary of every text format here is +// ASCII, so a file written as latin-1 comes out byte for byte identical to the +// same file written as UTF-8 - compared with cmp on 2026-09-07. Offering one +// would be offering a setting that changes nothing, which this project treats +// as a refusal rather than as a choice. They become real the day the content +// stops being English, and that is a different change with its own golden +// bytes. +const ( + UTF8 = "utf-8" + UTF16LE = "utf-16le" + UTF16BE = "utf-16be" +) + +// Codec is one encoding, with or without a byte order mark. +type Codec struct { + name string + // wide is two bytes per character rather than one, which is what makes + // odd sizes unreachable. + wide bool + // big is the byte order of a wide encoding, and means nothing without it. + big bool + bom bool +} + +// Default is what a recipe that says nothing gets, and it has to stay the +// bytes these formats have always written: UTF-8 with no mark in front. +func Default() Codec { return Codec{name: UTF8} } + +// Parse reads the two settings. +// +// A value outside the declared set has already been refused by the registry, +// which checks every format against its declaration in one place. These +// branches stay for the same reason the CSV dialect keeps its own: this +// function is callable directly, a guard is such a caller, and a generator +// that trusts its input is one registry change away from writing a file +// nobody ordered. +func Parse(formatID string, props map[string]string) (Codec, error) { + c := Default() + + if v, ok := props[Setting]; ok && v != "" { + switch v { + case UTF8: + c.name, c.wide, c.big = UTF8, false, false + case UTF16LE: + c.name, c.wide, c.big = UTF16LE, true, false + case UTF16BE: + c.name, c.wide, c.big = UTF16BE, true, true + default: + return Codec{}, &format.PropertyValueError{ + Format: formatID, Key: Setting, Value: v, + Reason: "it has to be " + UTF8 + ", " + UTF16LE + " or " + UTF16BE, + } + } + } + + if v, ok := props[SettingBOM]; ok && v != "" { + switch v { + case "true": + c.bom = true + case "false": + c.bom = false + default: + return Codec{}, &format.PropertyValueError{ + Format: formatID, Key: SettingBOM, Value: v, + Reason: "it has to be true or false", + } + } + } + + return c, nil +} + +// Name is the encoding as the manifest records it. +func (c Codec) Name() string { return c.name } + +// HasBOM says whether a mark is written, for the manifest. +func (c Codec) HasBOM() bool { return c.bom } + +// Preamble is the bytes in front of the content, empty when there is no mark. +func (c Codec) Preamble() []byte { + if !c.bom { + return nil + } + switch { + case !c.wide: + return []byte{0xEF, 0xBB, 0xBF} + case c.big: + return []byte{0xFE, 0xFF} + default: + return []byte{0xFF, 0xFE} + } +} + +// Width is how many bytes one ASCII character costs in this encoding. +// +// The generators build their content as ASCII and this is what turns that +// into a byte count. It holds because the vocabulary and the label are ASCII, +// which a guard checks rather than this file assuming. +func (c Codec) Width() int64 { + if c.wide { + return 2 + } + return 1 +} + +// Source is how many ASCII bytes of content fit in a file of this size. +func (c Codec) Source(fileBytes int64) int64 { + return (fileBytes - int64(len(c.Preamble()))) / c.Width() +} + +// Cost is what that many ASCII bytes take up once encoded. +func (c Codec) Cost(sourceBytes int64) int64 { return sourceBytes * c.Width() } + +// Check refuses a size this encoding cannot write exactly, in the four parts +// every refusal in this tool carries. +// +// Two things can be wrong and they are different sentences. A file smaller +// than its own mark cannot exist at all. A file of an odd length in a wide +// encoding cannot exist either, but the size ASKED FOR is not too small - the +// one below it is fine - so the reason says which sizes are reachable rather +// than pretending there is a floor. +func (c Codec) Check(formatName string, requested int64) error { + mark := int64(len(c.Preamble())) + if requested < mark { + return &format.BelowMinimumError{ + Format: formatName, Requested: requested, Minimum: mark, + Reason: fmt.Sprintf( + "a byte order mark is %d B and the file has to hold it", mark), + Hint: fmt.Sprintf("Ask for %d B or more, or turn the %s setting off.", mark, SettingBOM), + } + } + if next, ok := c.fits(requested); !ok { + return &format.BelowMinimumError{ + Format: formatName, Requested: requested, Minimum: next, + Reason: fmt.Sprintf( + "%s stores two bytes for every character, so a whole file always has an even number of them", + c.name), + Hint: fmt.Sprintf("Ask for %d B or %d B.", requested-1, next), + } + } + return nil +} + +// fits says whether a file of exactly this many bytes can be written, and +// names the next size that can when it cannot. +func (c Codec) fits(fileBytes int64) (next int64, ok bool) { + if (fileBytes-int64(len(c.Preamble())))%c.Width() == 0 { + return fileBytes, true + } + return fileBytes + 1, false +} + +// Writer wraps a writer so that UTF-8 written to it comes out in this +// encoding. +// +// UTF-8 gets the writer straight back rather than a wrapper that copies. That +// is not a micro optimisation: it is what keeps the default path producing the +// same bytes through the same calls it always did, so the pinned hashes and +// the allocation ceiling both stay where they were. +func (c Codec) Writer(w io.Writer) io.Writer { + if !c.wide { + return w + } + return &wideWriter{out: w, big: c.big} +} + +// wideWriter turns UTF-8 into UTF-16 as it goes, holding no more than one +// incomplete character between calls. +type wideWriter struct { + out io.Writer + big bool + // part is the tail of a character split across two writes. Our own + // generators write whole words, so it stays empty for them - it is here + // because a writer that only works when its caller is careful is a trap + // for the next caller. + part []byte + buf []byte +} + +func (w *wideWriter) Write(p []byte) (int, error) { + n := len(p) + if len(w.part) > 0 { + p = append(w.part, p...) + w.part = w.part[:0] + } + + w.buf = w.buf[:0] + for len(p) > 0 { + r, size := utf8.DecodeRune(p) + if r == utf8.RuneError && size == 1 && !utf8.FullRune(p) { + // An incomplete character at the end. Keep it for the next call + // rather than encoding a replacement nobody asked for. + w.part = append(w.part[:0], p...) + break + } + w.buf = appendRune(w.buf, r, w.big) + p = p[size:] + } + + if _, err := w.out.Write(w.buf); err != nil { + return 0, err + } + return n, nil +} + +// appendRune writes one character as one or two sixteen bit units. +func appendRune(dst []byte, r rune, big bool) []byte { + if r1, r2 := utf16.EncodeRune(r); r1 != utf8.RuneError { + return appendUnit(appendUnit(dst, uint16(r1), big), uint16(r2), big) + } + return appendUnit(dst, uint16(r), big) +} + +func appendUnit(dst []byte, u uint16, big bool) []byte { + if big { + return append(dst, byte(u>>8), byte(u)) + } + return append(dst, byte(u), byte(u>>8)) +} + +// Properties is the declaration both text formats hand to the registry, so +// the two cannot describe the same setting differently. +func Properties() []format.Property { + return []format.Property{ + { + Name: Setting, Kind: format.PropertyChoice, + Choices: []string{UTF16BE, UTF16LE, UTF8}, Default: UTF8, + Detail: "Which encoding the characters are written in. UTF-16 stores two bytes per character, so a file in it always has an even number of bytes and an odd size is refused.", + }, + { + Name: SettingBOM, Kind: format.PropertyBool, + Default: "false", + Detail: "Whether the file opens with a byte order mark. A reader that has to guess the encoding needs one, and a reader that does not expect it shows it as stray characters at the start of the file.", + }, + } +} diff --git a/internal/format/txt/txt.go b/internal/format/txt/txt.go index 8e2b3d1..816bcf4 100644 --- a/internal/format/txt/txt.go +++ b/internal/format/txt/txt.go @@ -11,6 +11,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" ) // The padding channel is the content itself, and it has no limit. A text file @@ -46,12 +47,17 @@ func init() { Where: format.PlacementEnd, Capacity: 0, }, - Label: format.LabelVisible, + Label: format.LabelVisible, + // No reference tool: a reader is handed a path and nothing else, so it + // would have to GUESS which encoding the file claims - and a checker + // that guesses agrees with a file written in the wrong one. The layer + // that can be TOLD is the structural check, and that is where this + // format's encoding is verified. Oracle: format.OracleNone, - // Encoding, line endings and line length come later. Until they do, - // declaring none is what makes a recipe asking for them fail loudly + // Line endings and line length come later. Until they do, declaring + // only what is here is what makes a recipe asking for them fail loudly // instead of quietly producing something else. - Properties: nil, + Properties: textenc.Properties(), GeneratorVersion: generatorVersion, Generator: generator{}, }) @@ -63,6 +69,12 @@ type generator struct{} type memo struct { labelLine string // includes the trailing newline, empty when absent seed uint64 + codec textenc.Codec + // source is how many bytes of ASCII content the file holds, which is the + // ordered size less the mark and divided by the width of a character. + // Everything below counts in these rather than in file bytes, so the + // filling loop is the same loop it always was. + source int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -76,31 +88,44 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } } + codec, err := textenc.Parse("txt", r.Properties) + if err != nil { + return format.Plan{}, err + } + if err := codec.Check("TXT", r.Bytes); err != nil { + return format.Plan{}, err + } + p := format.Plan{ Bytes: r.Bytes, Exact: true, Determinism: format.DeterminismByte, Properties: map[string]any{ - "encoding": "utf-8", - "line_ending": "lf", - "content": "english", + textenc.Setting: codec.Name(), + textenc.SettingBOM: codec.HasBOM(), + "line_ending": "lf", + "content": "english", }, } - m := memo{seed: r.Seed} + m := memo{seed: r.Seed, codec: codec, source: codec.Source(r.Bytes)} if r.Label { line := core.Label("txt", r.Bytes, r.Seed) + "\n" - if int64(len(line)) <= r.Bytes { + if int64(len(line)) <= m.source { m.labelLine = line } else { // Silence is banned. The label did not fit, so that has to be // visible rather than quietly absent from a file the user // believes carries one. + // + // The number is what the label COSTS in this encoding, not how + // long it is to read. In UTF-16 those differ by a factor of two, + // and a note off by half is worse than no note. p.Notes = append(p.Notes, format.Note{ Code: "label_omitted", Detail: fmt.Sprintf( "The label needs %d B and the file is %d B, so this file carries no label. Its name and the manifest still identify it.", - len(line), r.Bytes), + codec.Cost(int64(len(line))), r.Bytes), }) } } @@ -116,7 +141,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return fmt.Errorf("txt: the plan was not produced by this generator") } - remaining := p.Bytes + // The mark is bytes rather than text, so it goes out as itself. Everything + // after it is characters, so it goes through the encoder. + if err := writeAll(w, m.codec.Preamble()); err != nil { + return err + } + w = m.codec.Writer(w) + + remaining := m.source if m.labelLine != "" { if err := writeAll(w, []byte(m.labelLine)); err != nil { diff --git a/internal/guard/generatorbytes_test.go b/internal/guard/generatorbytes_test.go index 42540aa..2ee7104 100644 --- a/internal/guard/generatorbytes_test.go +++ b/internal/guard/generatorbytes_test.go @@ -286,6 +286,21 @@ func goldenCases() map[string]engine.Target { // pinned in both positions. "txt_4kib_no_label": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: false}, + // The encodings, one case per path rather than one per format: both + // byte orders, a mark present and a mark absent, in both formats. The + // default path is pinned by txt_4kib and md_8kib above, and it is the + // pin that says the setting arrived without moving anybody's hashes. + "txt_4kib_utf8_bom": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: true, + Properties: map[string]string{"encoding": "utf-8", "bom": "true"}}, + "txt_4kib_utf16le_bom": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: true, + Properties: map[string]string{"encoding": "utf-16le", "bom": "true"}}, + "txt_4kib_utf16be": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: true, + Properties: map[string]string{"encoding": "utf-16be"}}, + "md_8kib_utf16le_bom": {ID: "g", Format: "md", Sizes: engine.Uniform(1, 8192), Label: true, + Properties: map[string]string{"encoding": "utf-16le", "bom": "true"}}, + "md_8kib_utf16be": {ID: "g", Format: "md", Sizes: engine.Uniform(1, 8192), Label: true, + Properties: map[string]string{"encoding": "utf-16be"}}, + // An archive holding real files of another format. This is the path // "contains" rewrites, and the one case where a refactor could change // the bytes of every archive anybody has generated. diff --git a/internal/guard/layers_test.go b/internal/guard/layers_test.go index 4a28b06..884f711 100644 --- a/internal/guard/layers_test.go +++ b/internal/guard/layers_test.go @@ -36,6 +36,7 @@ var layer = map[string]int{ "internal/format": 1, "internal/format/all": 1, "internal/format/imagelabel": 1, + "internal/format/textenc": 1, "internal/format/archive": 1, "internal/format/txt": 1, "internal/format/md": 1, @@ -114,9 +115,10 @@ var sameLayerAllowed = map[string][]string{ "internal/format/wav", }, "internal/format/imagelabel": {"internal/format"}, + "internal/format/textenc": {"internal/format"}, "internal/format/archive": {"internal/format"}, - "internal/format/txt": {"internal/format", "internal/format/imagelabel"}, - "internal/format/md": {"internal/format"}, + "internal/format/txt": {"internal/format", "internal/format/imagelabel", "internal/format/textenc"}, + "internal/format/md": {"internal/format", "internal/format/textenc"}, "internal/format/logfile": {"internal/format"}, "internal/format/csvfile": {"internal/format"}, "internal/format/jsonfile": {"internal/format"}, diff --git a/internal/guard/oracle_test.go b/internal/guard/oracle_test.go index 4bb0a17..6301dfd 100644 --- a/internal/guard/oracle_test.go +++ b/internal/guard/oracle_test.go @@ -33,18 +33,6 @@ func TestEveryFormatSurvivesItsReferenceTool(t *testing.T) { ) for _, d := range format.All() { - if d.Oracle == format.OracleNone { - noTool = append(noTool, d.ID) - continue - } - - checker, known := oracle.For(d.Oracle) - if !known { - t.Errorf("%s declares the oracle %q and nothing implements it - a declaration nobody honours is worse than none", - d.ID, d.Oracle) - continue - } - // A size big enough to be a realistic file rather than a corner case. size := d.MinBytes + 300*1024 plan, err := d.Generator.Plan(format.Request{Bytes: size, Seed: 7741, Label: true}) @@ -66,17 +54,34 @@ func TestEveryFormatSurvivesItsReferenceTool(t *testing.T) { t.Fatalf("%s: closing failed: %v", d.ID, closeErr) } - res := checker.Check(path) - switch { - case !res.Available: - skipped = append(skipped, d.ID+" ("+checker.Name+" is not installed)") - case res.Err != nil: - checked++ - t.Errorf("%s: %v\n the file is the right size and repeatable, and %s still rejects it", - d.ID, res.Err, checker.Name) + // The first layer, for the formats that name a reader. + // + // A format naming none used to skip the REST of this loop along with + // it, so the structural check below never ran for it. That sat unseen + // while the only two formats without a reader were also the only two + // without a checker - and it would have made the checkers TXT and MD + // gained on 2026-09-07 dead on arrival, silently, with this guard + // green and reporting them as covered. Found by running it and reading + // the log, not by reading the code. + switch checker, known := oracle.For(d.Oracle); { + case d.Oracle == format.OracleNone: + noTool = append(noTool, d.ID) + case !known: + t.Errorf("%s declares the oracle %q and nothing implements it - a declaration nobody honours is worse than none", + d.ID, d.Oracle) default: - checked++ - t.Logf("%s: %s accepted it - %s", d.ID, checker.Name, firstLine(res.Output)) + res := checker.Check(path) + switch { + case !res.Available: + skipped = append(skipped, d.ID+" ("+checker.Name+" is not installed)") + case res.Err != nil: + checked++ + t.Errorf("%s: %v\n the file is the right size and repeatable, and %s still rejects it", + d.ID, res.Err, checker.Name) + default: + checked++ + t.Logf("%s: %s accepted it - %s", d.ID, checker.Name, firstLine(res.Output)) + } } // The tolerant readers answer "would a viewer accept this". The @@ -117,17 +122,26 @@ func TestEveryFormatSurvivesItsReferenceTool(t *testing.T) { // structurallyChecked is the formats the second layer covers, written down. // -// TXT and MD are absent on purpose and that is the whole reason this list -// exists rather than being derived: for those two there is no specification to -// check against beyond "these are the bytes we meant", so they have one layer -// and it is honest to say so. +// TXT and MD were absent on purpose until 2026-09-07, and the reason they gave +// is worth keeping because it stopped being true rather than being wrong: for +// those two there was no specification to check against beyond "these are the +// bytes we meant", so they carried one layer and said so. +// +// Declaring an encoding is what changed it. A file that says it is UTF-16LE +// with a mark in front of it makes a claim somebody else's decoder can settle, +// and three of them settled it - Python, V8 and .NET all reject a UTF-16 file +// cut to an odd length, and all three repair it in silence when asked +// leniently. So the list still exists rather than being derived, and it is now +// every format answering true. // // Without this, dropping a format from oracle.StrictKnows removes its // structural check and every test stays green - the loop above simply skips it. // A guard that can be switched off in silence is the failure this project keeps // finding, so the list is stated and compared rather than trusted. -// Every registered format is named here, and the two that answer false are the -// point of that rule rather than an exception to it. +// Every registered format is named here. The map keeps its shape rather than +// becoming a list, because a format that answers false is a state this project +// has been in and can be in again - a new format arrives before its checker +// does, and saying so out loud is the whole job of this list. // // It held only the trues until 2026-08-25, and an outside review found what // that let through: a format added to neither this list nor oracle.StrictKnows @@ -146,9 +160,9 @@ var structurallyChecked = map[string]bool{ "bmp": true, "gif": true, "ico": true, "jpg": true, "tiff": true, "webp": true, "avif": true, "jxl": true, "docx": true, "xlsx": true, "pptx": true, - // Nothing to check against beyond "these are the bytes we meant", so they - // have one layer and it is honest to say so out loud. - "txt": false, "md": false, + // Since 2026-09-07, when a text file gained something to be checked + // against: the encoding it declares. + "txt": true, "md": true, } func TestTheStructuralCheckerCoversEveryFormatItShould(t *testing.T) { diff --git a/internal/guard/parity_test.go b/internal/guard/parity_test.go index ce4fa30..260df6a 100644 --- a/internal/guard/parity_test.go +++ b/internal/guard/parity_test.go @@ -126,6 +126,8 @@ var reachableFromTheWindow = []string{ "property:log.rate", "property:log.status_mix", "property:log.timestamps", + "property:md.bom", + "property:md.encoding", "property:pdf.page_size", "property:pdf.pages", "property:png.height", @@ -140,6 +142,8 @@ var reachableFromTheWindow = []string{ "property:targz.entry_size", "property:tiff.height", "property:tiff.width", + "property:txt.bom", + "property:txt.encoding", "property:webp.height", "property:webp.width", "property:avif.height", diff --git a/internal/guard/testdata/generator-golden.json b/internal/guard/testdata/generator-golden.json index 2b65f80..cd39848 100644 --- a/internal/guard/testdata/generator-golden.json +++ b/internal/guard/testdata/generator-golden.json @@ -273,6 +273,26 @@ "csv_8kib_seventeen_columns": { "bytes": 8192, "sha256": "23d2fc7e93456ed828159d7d5199faaed2129d164640e0743c5aaf37bb9b6143" + }, + "txt_4kib_utf8_bom": { + "bytes": 4096, + "sha256": "6c131f93605a96b665fc20516e780db972257ed891732f62bec5b8b338c31f46" + }, + "txt_4kib_utf16le_bom": { + "bytes": 4096, + "sha256": "758a5729627bbf17d4cf6157171e495290e023bb23d0b902cbeb40ff44e4f173" + }, + "txt_4kib_utf16be": { + "bytes": 4096, + "sha256": "2fd51842c2457a1464a21c4ed62c90b58904cf04711e9069e528dfd1774b4211" + }, + "md_8kib_utf16le_bom": { + "bytes": 8192, + "sha256": "60156c7f1d936b102cdf8ae100479fcf1a1a6f0fc8b585888d3bdb8ee27eec8a" + }, + "md_8kib_utf16be": { + "bytes": 8192, + "sha256": "a2d0cc69cb609277b97ae7bf7e11505fa26f3eb3214e8e62c1eba07e6de7bc24" } }, "remeasured": [ diff --git a/internal/guard/textencoding_test.go b/internal/guard/textencoding_test.go new file mode 100644 index 0000000..2aa3ef4 --- /dev/null +++ b/internal/guard/textencoding_test.go @@ -0,0 +1,361 @@ +package guard + +// What a text file claims about itself, and whether it is that. +// +// TXT and MD gained an encoding on 2026-09-07, and it is the first setting in +// this project that changes which SIZES exist rather than what fills them. A +// UTF-16 file is a whole number of sixteen bit units, so half of all sizes stop +// being reachable - and the exact size promise turns those into refusals rather +// than into files that are one byte out. +// +// The measurement that shaped all of this, on three readers in three languages: +// a UTF-16 file cut to an odd length is rejected by Python, by V8 and by .NET +// when each is asked strictly, and repaired in silence by all three when it is +// not. Get-Content shows the cut file and the whole one identically. So the +// file this tool must never write is exactly the one a person could not tell +// apart by looking. + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" +) + +// encodedFormats is the formats that take an encoding, named rather than +// derived - so a third one arriving without being added here is a gap somebody +// has to notice rather than a loop that quietly gets shorter. +var encodedFormats = []string{"txt", "md"} + +type encodingCase struct { + encoding string + bom bool + mark []byte + width int64 +} + +func encodingCases() []encodingCase { + return []encodingCase{ + {textenc.UTF8, false, nil, 1}, + {textenc.UTF8, true, []byte{0xEF, 0xBB, 0xBF}, 1}, + {textenc.UTF16LE, false, nil, 2}, + {textenc.UTF16LE, true, []byte{0xFF, 0xFE}, 2}, + {textenc.UTF16BE, false, nil, 2}, + {textenc.UTF16BE, true, []byte{0xFE, 0xFF}, 2}, + } +} + +func (c encodingCase) props() map[string]string { + return map[string]string{ + textenc.Setting: c.encoding, + textenc.SettingBOM: fmt.Sprintf("%t", c.bom), + } +} + +func (c encodingCase) settings() []string { + return []string{ + textenc.Setting + "=" + c.encoding, + textenc.SettingBOM + "=" + fmt.Sprintf("%t", c.bom), + } +} + +func (c encodingCase) String() string { + return fmt.Sprintf("%s/bom=%t", c.encoding, c.bom) +} + +// writeEncoded produces one file and hands back its bytes, insisting on the +// ordered size before anything else looks at it. +func writeEncoded(t *testing.T, id string, size int64, props map[string]string) []byte { + t.Helper() + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + p, err := d.Generator.Plan(format.Request{Bytes: size, Seed: 7741, Label: true, Properties: props}) + if err != nil { + t.Fatalf("%s: planning %d B with %v: %v", id, size, props, err) + } + var buf bytes.Buffer + if err := d.Generator.Write(context.Background(), &buf, p); err != nil { + t.Fatalf("%s: writing %d B with %v: %v", id, size, props, err) + } + if int64(buf.Len()) != size { + t.Fatalf("%s %v: ordered %d B and produced %d - the size is exact or it is an error", + id, props, size, buf.Len()) + } + return buf.Bytes() +} + +// TestATextFileIsTheEncodingItDeclares is the whole claim in one place: the +// size is exact, the mark is there when it was ordered and absent when it was +// not, and somebody else's decoder agrees the bytes are what they say. +// +// The decoder is TOLD which encoding to expect rather than left to work it out. +// A checker that sniffed would decode a UTF-16 file as UTF-16 whatever was +// ordered and call a file written in the wrong encoding correct, which is the +// one question this has to answer. +func TestATextFileIsTheEncodingItDeclares(t *testing.T) { + dir := t.TempDir() + checked, skipped := 0, 0 + + for _, id := range encodedFormats { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + for _, c := range encodingCases() { + smallest := d.SmallestAccepted(format.Request{Seed: 7741, Label: true, Properties: c.props()}) + if c.width == 2 && smallest%2 != 0 { + t.Errorf("%s %v: the smallest size it accepts is %d, which a two byte encoding cannot write", + id, c, smallest) + } + + for _, size := range []int64{smallest, smallest + c.width, 4096, 40960} { + name := fmt.Sprintf("%s/%s/%d", id, c, size) + t.Run(name, func(t *testing.T) { + body := writeEncoded(t, id, size, c.props()) + + if !bytes.HasPrefix(body, c.mark) { + t.Fatalf("a %s mark was ordered and the file opens with % x", c.encoding, first(body, 4)) + } + if !c.bom && startsWithAnyMark(body) { + t.Fatalf("no mark was ordered and the file opens with % x", first(body, 4)) + } + + path := filepath.Join(dir, fmt.Sprintf("s%d.%s", size, id)) + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + res := oracle.Strict(id, path, c.settings()...) + if !res.Available { + skipped++ + t.Skip("the structural check needs python") + } + if res.Err != nil { + t.Fatalf("%s is not %s: %v", id, c.encoding, res.Err) + } + checked++ + }) + } + } + } + + if checked == 0 { + t.Errorf("nothing was decoded by anything outside this package - %d case(s) skipped", skipped) + } + t.Logf("%d file(s) decoded strictly by Python, %d skipped", checked, skipped) +} + +// TestAWideEncodingRefusesAnOddSizeAndNamesOneItCanWrite is the refusal, and +// the control beside it is what makes it mean anything. +// +// The same odd size is accepted under UTF-8, so the refusal is about the +// ENCODING rather than about the number. Without that half, a generator that +// refused every odd size in every encoding would pass this. +func TestAWideEncodingRefusesAnOddSizeAndNamesOneItCanWrite(t *testing.T) { + odd := []int64{4001, 65, 1235} + + for _, id := range encodedFormats { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + for _, c := range encodingCases() { + for _, size := range odd { + _, err := d.Generator.Plan(format.Request{ + Bytes: size, Seed: 7741, Label: true, Properties: c.props()}) + + if c.width == 1 { + if err != nil { + t.Errorf("%s %v: %d B is a size a one byte encoding can write, and it was refused: %v", + id, c, size, err) + } + continue + } + + var below *format.BelowMinimumError + if !errors.As(err, &below) { + t.Errorf("%s %v: %d B cannot be written and the answer was %v, not a BelowMinimumError", + id, c, size, err) + continue + } + if below.Minimum != size+1 { + t.Errorf("%s %v: refusing %d B named %d as the next size it can write", + id, c, size, below.Minimum) + } + // A refusal naming a size it also cannot write would be worse + // than no refusal, so the number it gives is taken up. + if _, err := d.Generator.Plan(format.Request{ + Bytes: below.Minimum, Seed: 7741, Label: true, Properties: c.props()}); err != nil { + t.Errorf("%s %v: refusing %d B pointed at %d B, and that is refused too: %v", + id, c, size, below.Minimum, err) + } + // The four parts every refusal in this tool carries. + if !strings.Contains(below.Reason, "two bytes") { + t.Errorf("%s %v: the reason does not say why an odd size cannot exist: %q", id, c, below.Reason) + } + if !strings.Contains(below.Hint, fmt.Sprintf("%d B", size-1)) { + t.Errorf("%s %v: the hint does not offer the size below: %q", id, c, below.Hint) + } + } + } + } +} + +// TestTheDefaultEncodingIsTheBytesTheseFormatsAlwaysWrote is the way back. +// +// A setting whose default changes the file is a breaking change wearing the +// clothes of a feature. This is the same pin the animated formats got when +// frames arrived: saying nothing and saying the default out loud have to be +// the same bytes, and the golden file beside it holds them to what they were +// before the setting existed. +func TestTheDefaultEncodingIsTheBytesTheseFormatsAlwaysWrote(t *testing.T) { + for _, id := range encodedFormats { + for _, size := range []int64{0, 33, 4096} { + silent := writeEncoded(t, id, size, nil) + spoken := writeEncoded(t, id, size, map[string]string{ + textenc.Setting: textenc.UTF8, textenc.SettingBOM: "false"}) + if !bytes.Equal(silent, spoken) { + t.Errorf("%s at %d B: saying nothing and saying utf-8 produce different bytes", id, size) + } + if size > 0 && startsWithAnyMark(silent) { + t.Errorf("%s at %d B: the default file opens with a byte order mark", id, size) + } + } + } +} + +// TestALabelThatWillNotFitSaysWhatItWouldCost holds the note to the encoding. +// +// The note used to be built from the length of the label as a string, which is +// what it costs in UTF-8 and half of what it costs in UTF-16. A note that is +// out by a factor of two is worse than no note: it tells somebody to ask for +// 66 B when the file needs 132. +func TestALabelThatWillNotFitSaysWhatItWouldCost(t *testing.T) { + const size = int64(64) // below the label's cost in a wide encoding, even + tails := map[string]string{"txt": "\n", "md": "\n\n"} + + for _, id := range encodedFormats { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + props := map[string]string{textenc.Setting: textenc.UTF16LE} + p, err := d.Generator.Plan(format.Request{ + Bytes: size, Seed: 7741, Label: true, Properties: props}) + if err != nil { + t.Fatalf("%s: planning %d B in utf-16le: %v", id, size, err) + } + + line := core.Label(id, size, 7741) + tails[id] + wide, narrow := int64(len(line))*2, int64(len(line)) + if wide <= size { + t.Fatalf("%s: the label costs %d B at %d B, so this case no longer sits below the threshold", + id, wide, size) + } + + note := noteWithCode(p.Notes, "label_omitted") + if note == nil { + t.Fatalf("%s: the label does not fit and nothing said so - silence is banned", id) + } + if !strings.Contains(note.Detail, fmt.Sprintf("needs %d B", wide)) { + t.Errorf("%s: the note should say the label needs %d B in this encoding: %q", id, wide, note.Detail) + } + if strings.Contains(note.Detail, fmt.Sprintf("needs %d B", narrow)) { + t.Errorf("%s: the note reports what the label costs in UTF-8, not in the encoding asked for: %q", + id, note.Detail) + } + } +} + +// TestACharacterSplitAcrossTwoWritesSurvives is the one defence here that our +// own generators cannot redden, so it is reddened on purpose. +// +// They write whole words, so a character never straddles two writes. The +// encoder holds the tail anyway, because a writer that only works when its +// caller is careful is a trap for the next caller - and a defence nothing can +// redden is not a defence, which is why this exists rather than a comment +// saying it was thought about. +func TestACharacterSplitAcrossTwoWritesSurvives(t *testing.T) { + const text = "zażółć" // two byte characters, so a split lands mid character + + for _, name := range []string{textenc.UTF16LE, textenc.UTF16BE} { + codec, err := textenc.Parse("txt", map[string]string{textenc.Setting: name}) + if err != nil { + t.Fatal(err) + } + + var whole bytes.Buffer + if _, err := codec.Writer(&whole).Write([]byte(text)); err != nil { + t.Fatal(err) + } + + for cut := 1; cut < len(text); cut++ { + var split bytes.Buffer + w := codec.Writer(&split) + if _, err := w.Write([]byte(text)[:cut]); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(text)[cut:]); err != nil { + t.Fatal(err) + } + if !bytes.Equal(whole.Bytes(), split.Bytes()) { + t.Fatalf("%s: writing in two goes at byte %d gave % x, and in one go it is % x", + name, cut, split.Bytes(), whole.Bytes()) + } + } + } +} + +// TestTheTextGeneratorsWriteOnlyASCII names the assumption the arithmetic +// stands on. +// +// The budget is worked out as "file bytes divided by the width of a +// character", which is only the same thing as "how much prose fits" while +// every character costs one byte in UTF-8. The day the filler stops being +// English that stops being true, and it should stop here rather than in a file +// that is one byte short. +func TestTheTextGeneratorsWriteOnlyASCII(t *testing.T) { + for _, id := range encodedFormats { + body := writeEncoded(t, id, 40960, nil) + for i, b := range body { + if b > 0x7f { + t.Fatalf("%s: byte %d is %#x, and the size arithmetic assumes one byte per character", id, i, b) + } + } + } +} + +func noteWithCode(notes []format.Note, code string) *format.Note { + for i := range notes { + if notes[i].Code == code { + return ¬es[i] + } + } + return nil +} + +func startsWithAnyMark(body []byte) bool { + for _, m := range [][]byte{{0xEF, 0xBB, 0xBF}, {0xFF, 0xFE}, {0xFE, 0xFF}} { + if bytes.HasPrefix(body, m) { + return true + } + } + return false +} + +func first(body []byte, n int) []byte { + if len(body) < n { + return body + } + return body[:n] +} diff --git a/internal/guard/textformats_test.go b/internal/guard/textformats_test.go index 97cd86f..e14152c 100644 --- a/internal/guard/textformats_test.go +++ b/internal/guard/textformats_test.go @@ -506,6 +506,17 @@ func TestEveryFormatIsClassifiedAsTextOrBinary(t *testing.T) { // refuses with "generated content is English only so far" - and which M5 // describes. The guard goes in before the code that needs it, which is how the // first four guards in this project were built. +// +// AT DEFAULT SETTINGS, and since 2026-09-07 that qualifier is load bearing +// rather than pedantic. TXT and MD can be asked for UTF-16, and a UTF-16 file +// is NOT valid UTF-8 - so the sentence this guard's name makes stopped being +// true of the tool on the day the encoding setting landed. It stayed true of +// what this guard actually does, because generateBytes plans with no +// properties, and a guard that quietly narrows to the case it can still pass +// is the failure this project keeps finding. Said out loud instead: the claim +// here is the DEFAULT path. Every other encoding is held to the encoding it +// declares, by TestATextFileIsTheEncodingItDeclares and by a structural +// checker that is told which one to expect. func TestEveryTextFormatIsValidUTF8(t *testing.T) { for _, id := range textFormats { d, err := format.Get(id) diff --git a/internal/oracle/oracle.go b/internal/oracle/oracle.go index eb0de28..e9895f7 100644 --- a/internal/oracle/oracle.go +++ b/internal/oracle/oracle.go @@ -393,7 +393,12 @@ func Strict(formatID, path string, settings ...string) Result { func StrictKnows(formatID string) bool { switch formatID { case "png", "wav", "pdf", "zip", "targz", "log", "csv", "json", "xml", "svg", "html", - "bmp", "gif", "ico", "jpg", "tiff", "webp", "avif", "jxl", "docx", "xlsx", "pptx": + "bmp", "gif", "ico", "jpg", "tiff", "webp", "avif", "jxl", "docx", "xlsx", "pptx", + // The two text formats joined on 2026-09-07, when they gained an + // encoding. Before that there was nothing here to check against + // beyond "these are the bytes we meant" - a claim about UTF-16 is a + // claim somebody else's decoder can settle. + "txt", "md": return true } return false diff --git a/internal/oracle/strict.py b/internal/oracle/strict.py index b6376fd..4796a45 100644 --- a/internal/oracle/strict.py +++ b/internal/oracle/strict.py @@ -1558,17 +1558,104 @@ def check_jxl(data): } +def decode_declared(kind, data, settings): + """Decode the file in the encoding it was TOLD the file claims. + + Told rather than sniffed, for the reason the CSV dialect is told: a checker + that guessed would decode a UTF-16 file as UTF-16 whatever was ordered and + agree with a file written in the wrong one. Sniffing is genuinely ambiguous + here as well - a UTF-16 file with no mark in front of it is a legal file + that looks like nothing in particular. + + Python's own codecs do the decoding, and that is the point: the bytes are + judged by an implementation that is neither our code nor our language. + Strict on purpose. Measured on three readers on 2026-09-07, every lenient + path replaces a broken character with U+FFFD and reports success, so a + lenient decode here would bless the one defect worth catching. + """ + settings = settings or {} + name = settings.get("encoding", "utf-8") + marks = {"utf-8": b"\xef\xbb\xbf", "utf-16le": b"\xff\xfe", "utf-16be": b"\xfe\xff"} + names = {"utf-8": "utf-8", "utf-16le": "utf-16-le", "utf-16be": "utf-16-be"} + if name not in names: + fail(f"the {kind} check was told encoding={name!r}, which is not one this tool writes") + + wants_mark = settings.get("bom", "false") == "true" + mark = marks[name] + has_mark = data.startswith(mark) + if wants_mark and not has_mark: + fail(f"a {name} byte order mark was ordered and the file does not open with one") + if not wants_mark and has_mark: + fail(f"no byte order mark was ordered and the file opens with one") + body = data[len(mark):] if wants_mark else data + + if name != "utf-8" and len(body) % 2: + fail(f"{name} stores two bytes for every character and the file holds " + f"{len(body)} of them, so the last character is cut in half") + try: + text = body.decode(names[name]) + except UnicodeDecodeError as exc: + fail(f"the file does not decode as {name}: {exc}") + + # Decoding is not enough, and this is the half that says why. Read a + # UTF-16BE file as little endian and every pair of bytes is still a valid + # character - 0x74 0x00 becomes U+7400 rather than "t" - so a strict decode + # blesses a file with its byte order the wrong way round. The prose these + # formats write is words, spaces and newlines, so anything outside that + # means the bytes were read in an order nobody ordered, or landed somewhere + # they were never meant to. + stray = sorted({c for c in text if c != "\n" and not (" " <= c <= "~")}) + if stray: + shown = [hex(ord(c)) for c in stray[:6]] + fail(f"the text holds {len(stray)} character(s) that are not printable ASCII: {shown}") + return text + + +def check_txt(data, settings=None): + """Text, in the encoding it claims, and nothing in it that is not text. + + There was nothing here to check against until a text file could declare an + encoding, which is why TXT and MD carried one layer where every other + format carries two. A file saying it is UTF-16LE with a mark in front is a + claim somebody else's decoder can settle, so from 2026-09-07 they carry two. + + The character check is the second half and it is not about encoding at all: + this prose is words, spaces and newlines, so a control character in the + decoded text means bytes landed somewhere they were never meant to - and + both the size check and the determinism check would call that file correct. + """ + text = decode_declared("txt", data, settings) + ok(f"{len(text)} characters, decoded strictly") + + +def check_md(data, settings=None): + """The document decodes, and every fenced block is closed. + + The fence count is the structural half. This generator writes code fences + in pairs and takes a block whole or not at all, so an odd number of them + means a block was cut in half - which renders as one enormous code block + swallowing the rest of the document, at exactly the size that was ordered + and with a stable hash. Nothing else here would see it. + """ + text = decode_declared("md", data, settings) + fences = text.count("```") + if fences % 2: + fail(f"the document holds {fences} code fences, so one block is never closed") + ok(f"{len(text)} characters, {fences // 2} fenced block(s), decoded strictly") + + CHECKS = {"png": check_png, "wav": check_wav, "pdf": check_pdf, "zip": check_zip, "log": check_log, "csv": check_csv, "json": check_json, "xml": check_xml, "svg": check_svg, "html": check_html, "targz": check_targz, "bmp": check_bmp, "gif": check_gif, "ico": check_ico, "jpg": check_jpg, "tiff": check_tiff, "webp": check_webp, "avif": check_avif, "jxl": check_jxl, - "docx": check_docx, "xlsx": check_xlsx, "pptx": check_pptx} + "docx": check_docx, "xlsx": check_xlsx, "pptx": check_pptx, + "txt": check_txt, "md": check_md} # Checks that take the shape of the file as well as its bytes. Everything else # is handed the bytes alone, so adding a setting to one check cannot change how # any other one is called. -TAKES_SETTINGS = {"csv"} +TAKES_SETTINGS = {"csv", "txt", "md"} if __name__ == "__main__": if len(sys.argv) < 3 or sys.argv[1] not in CHECKS: diff --git a/web/public/formats/index.html b/web/public/formats/index.html index 8db3db1..30cceb5 100644 --- a/web/public/formats/index.html +++ b/web/public/formats/index.html @@ -470,6 +470,16 @@

Settings each format accepts

line_ending crlf, lf + + md + encoding + utf-16be, utf-16le, utf-8 + + + + bom + true or false + pdf pages @@ -545,6 +555,16 @@

Settings each format accepts

height 1 - 20000 pixels + + txt + encoding + utf-16be, utf-16le, utf-8 + + + + bom + true or false + wav sample_rate diff --git a/web/public/pl/formaty/index.html b/web/public/pl/formaty/index.html index cf3178e..8a9f235 100644 --- a/web/public/pl/formaty/index.html +++ b/web/public/pl/formaty/index.html @@ -470,6 +470,16 @@

Ustawienia, które przyjmuje każdy format

line_ending crlf, lf + + md + encoding + utf-16be, utf-16le, utf-8 + + + + bom + prawda albo fałsz + pdf pages @@ -545,6 +555,16 @@

Ustawienia, które przyjmuje każdy format

height 1 - 20000 pikseli + + txt + encoding + utf-16be, utf-16le, utf-8 + + + + bom + prawda albo fałsz + wav sample_rate From 2b6a48444bc559307a35a103e8340c97d0d919b0 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Mon, 7 Sep 2026 20:28:25 +0200 Subject: [PATCH 2/2] textenc: the character width is arithmetic inside the package, not API Width had no caller outside textenc. A fresh export with nobody asking for it adds to a category this tree already counts, so it goes in as unexported and the mutation patterns naming it follow the rename. Co-Authored-By: Claude Opus 5 --- internal/format/textenc/textenc.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/format/textenc/textenc.go b/internal/format/textenc/textenc.go index 0520d8f..c7a6b7f 100644 --- a/internal/format/textenc/textenc.go +++ b/internal/format/textenc/textenc.go @@ -126,12 +126,12 @@ func (c Codec) Preamble() []byte { } } -// Width is how many bytes one ASCII character costs in this encoding. +// width is how many bytes one ASCII character costs in this encoding. // // The generators build their content as ASCII and this is what turns that // into a byte count. It holds because the vocabulary and the label are ASCII, // which a guard checks rather than this file assuming. -func (c Codec) Width() int64 { +func (c Codec) width() int64 { if c.wide { return 2 } @@ -140,11 +140,11 @@ func (c Codec) Width() int64 { // Source is how many ASCII bytes of content fit in a file of this size. func (c Codec) Source(fileBytes int64) int64 { - return (fileBytes - int64(len(c.Preamble()))) / c.Width() + return (fileBytes - int64(len(c.Preamble()))) / c.width() } // Cost is what that many ASCII bytes take up once encoded. -func (c Codec) Cost(sourceBytes int64) int64 { return sourceBytes * c.Width() } +func (c Codec) Cost(sourceBytes int64) int64 { return sourceBytes * c.width() } // Check refuses a size this encoding cannot write exactly, in the four parts // every refusal in this tool carries. @@ -179,7 +179,7 @@ func (c Codec) Check(formatName string, requested int64) error { // fits says whether a file of exactly this many bytes can be written, and // names the next size that can when it cannot. func (c Codec) fits(fileBytes int64) (next int64, ok bool) { - if (fileBytes-int64(len(c.Preamble())))%c.Width() == 0 { + if (fileBytes-int64(len(c.Preamble())))%c.width() == 0 { return fileBytes, true } return fileBytes + 1, false