diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a4e4e2..c598c44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,33 @@ because it turns other people's test suites red. same new code and were checked against their recorded hashes and across every format at five sizes and two seeds. +### Added + +- **Text and Markdown files can be written in UTF-16, with or without a byte + order mark.** Two new settings on `txt` and `md`: `encoding`, which takes + `utf-8`, `utf-16le` or `utf-16be`, and `bom`, which is `true` or `false`. + Both default to what these formats have always produced, so a recipe that + says nothing gets the same bytes it got before. + + ``` + tfg generate --format txt --size 4kb --set encoding=utf-16le --set bom=true + ``` + + **In UTF-16 an odd number of bytes is refused.** Every character takes two + bytes, so only an even size can be a whole file, and asking for 4001 B gets + an error naming 4000 B and 4002 B rather than a file that is one byte out. + Three readers were measured on a UTF-16 file cut to an odd length: Python, + Node and .NET all reject it when asked strictly, and all three repair it in + silence otherwise, which is why this is an error and not a rounded size. + + The self describing label costs twice as much in UTF-16, so it needs a file + of at least 66 B rather than 33 B to fit. Below that the file is still + produced and the manifest says the label was left out. + + Single byte encodings such as Windows-1252 are deliberately not offered. The + generated text is English, so a file written in one would be byte for byte + the same file as UTF-8 - a setting that changes nothing. + ### Security - **On Windows, the desktop window loads the library it uses for dark menus from diff --git a/README.md b/README.md index fce881d..7f89444 100644 --- a/README.md +++ b/README.md @@ -476,7 +476,8 @@ recipe. `tfg formats ` prints the allowed range or list for each: | `pptx` | `slides` | | `csv` | `delimiter`, `line_ending`, `header`, `quote_style`, `columns` | | `log` | `entry_format`, `timestamps`, `rate`, `methods`, `status_mix`, `level_mix`, `ip_version`, `line_ending` | -| `json`, `xml`, `html`, `md`, `txt`, `svg` | none | +| `txt`, `md` | `encoding`, `bom` | +| `json`, `xml`, `html`, `svg` | none | ``` tfg generate --format jpg --size 500kb --set width=1920 --set height=1080 --set quality=85 diff --git a/internal/format/md/md.go b/internal/format/md/md.go index 5960480..25a328a 100644 --- a/internal/format/md/md.go +++ b/internal/format/md/md.go @@ -21,6 +21,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" ) // The padding channel is the content, measured 2026-08-01 like the rest of the @@ -55,12 +56,15 @@ func init() { Where: format.PlacementEnd, Capacity: 0, }, - Label: format.LabelVisible, + Label: format.LabelVisible, + // No reference tool, for the reason TXT gives: a reader is handed a + // path and would have to guess the encoding. The structural check is + // the layer that can be told. Oracle: format.OracleNone, // Heading depth, table size and which elements appear come later. - // Declaring none now is what makes a recipe asking for them fail - // loudly rather than quietly producing something else. - Properties: nil, + // Declaring only what is here is what makes a recipe asking for them + // fail loudly rather than quietly producing something else. + Properties: textenc.Properties(), GeneratorVersion: generatorVersion, Generator: generator{}, }) @@ -71,6 +75,11 @@ type generator struct{} type memo struct { labelLine string // includes the trailing blank line, empty when absent seed uint64 + codec textenc.Codec + // source is the ASCII content this file holds, counted in characters + // rather than in file bytes - see the same field on TXT. It is what lets + // the block loop below stay the loop it was. + source int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -84,30 +93,41 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } } + codec, err := textenc.Parse("md", r.Properties) + if err != nil { + return format.Plan{}, err + } + if err := codec.Check("MD", r.Bytes); err != nil { + return format.Plan{}, err + } + p := format.Plan{ Bytes: r.Bytes, Exact: true, Determinism: format.DeterminismByte, Properties: map[string]any{ - "encoding": "utf-8", - "line_ending": "lf", - "flavour": "commonmark", + textenc.Setting: codec.Name(), + textenc.SettingBOM: codec.HasBOM(), + "line_ending": "lf", + "flavour": "commonmark", }, } - m := memo{seed: r.Seed} + m := memo{seed: r.Seed, codec: codec, source: codec.Source(r.Bytes)} if r.Label { // A plain line followed by a blank one is a paragraph, which renders // visibly and cannot break the document wherever it sits. line := core.Label("md", r.Bytes, r.Seed) + "\n\n" - if int64(len(line)) <= r.Bytes { + if int64(len(line)) <= m.source { m.labelLine = line } else { + // What the label COSTS here, not how long it reads - the two + // differ by a factor of two in UTF-16. p.Notes = append(p.Notes, format.Note{ Code: "label_omitted", Detail: fmt.Sprintf( "The label needs %d B and the file is %d B, so this file carries no label. Its name and the manifest still identify it.", - len(line), r.Bytes), + codec.Cost(int64(len(line))), r.Bytes), }) } } @@ -123,7 +143,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return fmt.Errorf("md: the plan was not produced by this generator") } - remaining := p.Bytes + // The mark is bytes rather than text, so it goes out as itself, and + // everything after it goes through the encoder. + if err := writeAll(w, m.codec.Preamble()); err != nil { + return err + } + w = m.codec.Writer(w) + + remaining := m.source if m.labelLine != "" { if err := writeAll(w, []byte(m.labelLine)); err != nil { return err diff --git a/internal/format/textenc/textenc.go b/internal/format/textenc/textenc.go new file mode 100644 index 0000000..c7a6b7f --- /dev/null +++ b/internal/format/textenc/textenc.go @@ -0,0 +1,271 @@ +// Package textenc is the character encoding the text formats share. +// +// It lives here rather than inside one of them because TXT and MD ask the same +// question and have to answer it in the same words. A copy in each is how +// thirteen packages ended up carrying four different versions of one filler +// loop, and the same rule applies to a setting a person reads. +// +// What it holds is arithmetic as much as bytes. A file written in UTF-16 is a +// whole number of sixteen bit units, so its length is always even - which +// means half of all sizes stop being reachable, and the exact size promise +// turns them into refusals rather than into files of the wrong size. +// Measured 2026-09-07 on three independent readers: a UTF-16 file cut to an +// odd length is REJECTED by Python, by V8 and by .NET when each is asked +// strictly, and quietly repaired by all three when it is not. A file whose +// corruption only a strict reader can see is the silence this project bans. +package textenc + +import ( + "fmt" + "io" + "unicode/utf16" + "unicode/utf8" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// Setting names. Public names, so they are spelled once. +const ( + Setting = "encoding" + SettingBOM = "bom" +) + +// The encodings on offer. +// +// Single byte encodings are deliberately absent, and that is a measurement +// rather than an omission. The filler vocabulary of every text format here is +// ASCII, so a file written as latin-1 comes out byte for byte identical to the +// same file written as UTF-8 - compared with cmp on 2026-09-07. Offering one +// would be offering a setting that changes nothing, which this project treats +// as a refusal rather than as a choice. They become real the day the content +// stops being English, and that is a different change with its own golden +// bytes. +const ( + UTF8 = "utf-8" + UTF16LE = "utf-16le" + UTF16BE = "utf-16be" +) + +// Codec is one encoding, with or without a byte order mark. +type Codec struct { + name string + // wide is two bytes per character rather than one, which is what makes + // odd sizes unreachable. + wide bool + // big is the byte order of a wide encoding, and means nothing without it. + big bool + bom bool +} + +// Default is what a recipe that says nothing gets, and it has to stay the +// bytes these formats have always written: UTF-8 with no mark in front. +func Default() Codec { return Codec{name: UTF8} } + +// Parse reads the two settings. +// +// A value outside the declared set has already been refused by the registry, +// which checks every format against its declaration in one place. These +// branches stay for the same reason the CSV dialect keeps its own: this +// function is callable directly, a guard is such a caller, and a generator +// that trusts its input is one registry change away from writing a file +// nobody ordered. +func Parse(formatID string, props map[string]string) (Codec, error) { + c := Default() + + if v, ok := props[Setting]; ok && v != "" { + switch v { + case UTF8: + c.name, c.wide, c.big = UTF8, false, false + case UTF16LE: + c.name, c.wide, c.big = UTF16LE, true, false + case UTF16BE: + c.name, c.wide, c.big = UTF16BE, true, true + default: + return Codec{}, &format.PropertyValueError{ + Format: formatID, Key: Setting, Value: v, + Reason: "it has to be " + UTF8 + ", " + UTF16LE + " or " + UTF16BE, + } + } + } + + if v, ok := props[SettingBOM]; ok && v != "" { + switch v { + case "true": + c.bom = true + case "false": + c.bom = false + default: + return Codec{}, &format.PropertyValueError{ + Format: formatID, Key: SettingBOM, Value: v, + Reason: "it has to be true or false", + } + } + } + + return c, nil +} + +// Name is the encoding as the manifest records it. +func (c Codec) Name() string { return c.name } + +// HasBOM says whether a mark is written, for the manifest. +func (c Codec) HasBOM() bool { return c.bom } + +// Preamble is the bytes in front of the content, empty when there is no mark. +func (c Codec) Preamble() []byte { + if !c.bom { + return nil + } + switch { + case !c.wide: + return []byte{0xEF, 0xBB, 0xBF} + case c.big: + return []byte{0xFE, 0xFF} + default: + return []byte{0xFF, 0xFE} + } +} + +// width is how many bytes one ASCII character costs in this encoding. +// +// The generators build their content as ASCII and this is what turns that +// into a byte count. It holds because the vocabulary and the label are ASCII, +// which a guard checks rather than this file assuming. +func (c Codec) width() int64 { + if c.wide { + return 2 + } + return 1 +} + +// Source is how many ASCII bytes of content fit in a file of this size. +func (c Codec) Source(fileBytes int64) int64 { + return (fileBytes - int64(len(c.Preamble()))) / c.width() +} + +// Cost is what that many ASCII bytes take up once encoded. +func (c Codec) Cost(sourceBytes int64) int64 { return sourceBytes * c.width() } + +// Check refuses a size this encoding cannot write exactly, in the four parts +// every refusal in this tool carries. +// +// Two things can be wrong and they are different sentences. A file smaller +// than its own mark cannot exist at all. A file of an odd length in a wide +// encoding cannot exist either, but the size ASKED FOR is not too small - the +// one below it is fine - so the reason says which sizes are reachable rather +// than pretending there is a floor. +func (c Codec) Check(formatName string, requested int64) error { + mark := int64(len(c.Preamble())) + if requested < mark { + return &format.BelowMinimumError{ + Format: formatName, Requested: requested, Minimum: mark, + Reason: fmt.Sprintf( + "a byte order mark is %d B and the file has to hold it", mark), + Hint: fmt.Sprintf("Ask for %d B or more, or turn the %s setting off.", mark, SettingBOM), + } + } + if next, ok := c.fits(requested); !ok { + return &format.BelowMinimumError{ + Format: formatName, Requested: requested, Minimum: next, + Reason: fmt.Sprintf( + "%s stores two bytes for every character, so a whole file always has an even number of them", + c.name), + Hint: fmt.Sprintf("Ask for %d B or %d B.", requested-1, next), + } + } + return nil +} + +// fits says whether a file of exactly this many bytes can be written, and +// names the next size that can when it cannot. +func (c Codec) fits(fileBytes int64) (next int64, ok bool) { + if (fileBytes-int64(len(c.Preamble())))%c.width() == 0 { + return fileBytes, true + } + return fileBytes + 1, false +} + +// Writer wraps a writer so that UTF-8 written to it comes out in this +// encoding. +// +// UTF-8 gets the writer straight back rather than a wrapper that copies. That +// is not a micro optimisation: it is what keeps the default path producing the +// same bytes through the same calls it always did, so the pinned hashes and +// the allocation ceiling both stay where they were. +func (c Codec) Writer(w io.Writer) io.Writer { + if !c.wide { + return w + } + return &wideWriter{out: w, big: c.big} +} + +// wideWriter turns UTF-8 into UTF-16 as it goes, holding no more than one +// incomplete character between calls. +type wideWriter struct { + out io.Writer + big bool + // part is the tail of a character split across two writes. Our own + // generators write whole words, so it stays empty for them - it is here + // because a writer that only works when its caller is careful is a trap + // for the next caller. + part []byte + buf []byte +} + +func (w *wideWriter) Write(p []byte) (int, error) { + n := len(p) + if len(w.part) > 0 { + p = append(w.part, p...) + w.part = w.part[:0] + } + + w.buf = w.buf[:0] + for len(p) > 0 { + r, size := utf8.DecodeRune(p) + if r == utf8.RuneError && size == 1 && !utf8.FullRune(p) { + // An incomplete character at the end. Keep it for the next call + // rather than encoding a replacement nobody asked for. + w.part = append(w.part[:0], p...) + break + } + w.buf = appendRune(w.buf, r, w.big) + p = p[size:] + } + + if _, err := w.out.Write(w.buf); err != nil { + return 0, err + } + return n, nil +} + +// appendRune writes one character as one or two sixteen bit units. +func appendRune(dst []byte, r rune, big bool) []byte { + if r1, r2 := utf16.EncodeRune(r); r1 != utf8.RuneError { + return appendUnit(appendUnit(dst, uint16(r1), big), uint16(r2), big) + } + return appendUnit(dst, uint16(r), big) +} + +func appendUnit(dst []byte, u uint16, big bool) []byte { + if big { + return append(dst, byte(u>>8), byte(u)) + } + return append(dst, byte(u), byte(u>>8)) +} + +// Properties is the declaration both text formats hand to the registry, so +// the two cannot describe the same setting differently. +func Properties() []format.Property { + return []format.Property{ + { + Name: Setting, Kind: format.PropertyChoice, + Choices: []string{UTF16BE, UTF16LE, UTF8}, Default: UTF8, + Detail: "Which encoding the characters are written in. UTF-16 stores two bytes per character, so a file in it always has an even number of bytes and an odd size is refused.", + }, + { + Name: SettingBOM, Kind: format.PropertyBool, + Default: "false", + Detail: "Whether the file opens with a byte order mark. A reader that has to guess the encoding needs one, and a reader that does not expect it shows it as stray characters at the start of the file.", + }, + } +} diff --git a/internal/format/txt/txt.go b/internal/format/txt/txt.go index 8e2b3d1..816bcf4 100644 --- a/internal/format/txt/txt.go +++ b/internal/format/txt/txt.go @@ -11,6 +11,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" ) // The padding channel is the content itself, and it has no limit. A text file @@ -46,12 +47,17 @@ func init() { Where: format.PlacementEnd, Capacity: 0, }, - Label: format.LabelVisible, + Label: format.LabelVisible, + // No reference tool: a reader is handed a path and nothing else, so it + // would have to GUESS which encoding the file claims - and a checker + // that guesses agrees with a file written in the wrong one. The layer + // that can be TOLD is the structural check, and that is where this + // format's encoding is verified. Oracle: format.OracleNone, - // Encoding, line endings and line length come later. Until they do, - // declaring none is what makes a recipe asking for them fail loudly + // Line endings and line length come later. Until they do, declaring + // only what is here is what makes a recipe asking for them fail loudly // instead of quietly producing something else. - Properties: nil, + Properties: textenc.Properties(), GeneratorVersion: generatorVersion, Generator: generator{}, }) @@ -63,6 +69,12 @@ type generator struct{} type memo struct { labelLine string // includes the trailing newline, empty when absent seed uint64 + codec textenc.Codec + // source is how many bytes of ASCII content the file holds, which is the + // ordered size less the mark and divided by the width of a character. + // Everything below counts in these rather than in file bytes, so the + // filling loop is the same loop it always was. + source int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -76,31 +88,44 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } } + codec, err := textenc.Parse("txt", r.Properties) + if err != nil { + return format.Plan{}, err + } + if err := codec.Check("TXT", r.Bytes); err != nil { + return format.Plan{}, err + } + p := format.Plan{ Bytes: r.Bytes, Exact: true, Determinism: format.DeterminismByte, Properties: map[string]any{ - "encoding": "utf-8", - "line_ending": "lf", - "content": "english", + textenc.Setting: codec.Name(), + textenc.SettingBOM: codec.HasBOM(), + "line_ending": "lf", + "content": "english", }, } - m := memo{seed: r.Seed} + m := memo{seed: r.Seed, codec: codec, source: codec.Source(r.Bytes)} if r.Label { line := core.Label("txt", r.Bytes, r.Seed) + "\n" - if int64(len(line)) <= r.Bytes { + if int64(len(line)) <= m.source { m.labelLine = line } else { // Silence is banned. The label did not fit, so that has to be // visible rather than quietly absent from a file the user // believes carries one. + // + // The number is what the label COSTS in this encoding, not how + // long it is to read. In UTF-16 those differ by a factor of two, + // and a note off by half is worse than no note. p.Notes = append(p.Notes, format.Note{ Code: "label_omitted", Detail: fmt.Sprintf( "The label needs %d B and the file is %d B, so this file carries no label. Its name and the manifest still identify it.", - len(line), r.Bytes), + codec.Cost(int64(len(line))), r.Bytes), }) } } @@ -116,7 +141,14 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { return fmt.Errorf("txt: the plan was not produced by this generator") } - remaining := p.Bytes + // The mark is bytes rather than text, so it goes out as itself. Everything + // after it is characters, so it goes through the encoder. + if err := writeAll(w, m.codec.Preamble()); err != nil { + return err + } + w = m.codec.Writer(w) + + remaining := m.source if m.labelLine != "" { if err := writeAll(w, []byte(m.labelLine)); err != nil { diff --git a/internal/guard/generatorbytes_test.go b/internal/guard/generatorbytes_test.go index 42540aa..2ee7104 100644 --- a/internal/guard/generatorbytes_test.go +++ b/internal/guard/generatorbytes_test.go @@ -286,6 +286,21 @@ func goldenCases() map[string]engine.Target { // pinned in both positions. "txt_4kib_no_label": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: false}, + // The encodings, one case per path rather than one per format: both + // byte orders, a mark present and a mark absent, in both formats. The + // default path is pinned by txt_4kib and md_8kib above, and it is the + // pin that says the setting arrived without moving anybody's hashes. + "txt_4kib_utf8_bom": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: true, + Properties: map[string]string{"encoding": "utf-8", "bom": "true"}}, + "txt_4kib_utf16le_bom": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: true, + Properties: map[string]string{"encoding": "utf-16le", "bom": "true"}}, + "txt_4kib_utf16be": {ID: "g", Format: "txt", Sizes: engine.Uniform(1, 4096), Label: true, + Properties: map[string]string{"encoding": "utf-16be"}}, + "md_8kib_utf16le_bom": {ID: "g", Format: "md", Sizes: engine.Uniform(1, 8192), Label: true, + Properties: map[string]string{"encoding": "utf-16le", "bom": "true"}}, + "md_8kib_utf16be": {ID: "g", Format: "md", Sizes: engine.Uniform(1, 8192), Label: true, + Properties: map[string]string{"encoding": "utf-16be"}}, + // An archive holding real files of another format. This is the path // "contains" rewrites, and the one case where a refactor could change // the bytes of every archive anybody has generated. diff --git a/internal/guard/layers_test.go b/internal/guard/layers_test.go index 4a28b06..884f711 100644 --- a/internal/guard/layers_test.go +++ b/internal/guard/layers_test.go @@ -36,6 +36,7 @@ var layer = map[string]int{ "internal/format": 1, "internal/format/all": 1, "internal/format/imagelabel": 1, + "internal/format/textenc": 1, "internal/format/archive": 1, "internal/format/txt": 1, "internal/format/md": 1, @@ -114,9 +115,10 @@ var sameLayerAllowed = map[string][]string{ "internal/format/wav", }, "internal/format/imagelabel": {"internal/format"}, + "internal/format/textenc": {"internal/format"}, "internal/format/archive": {"internal/format"}, - "internal/format/txt": {"internal/format", "internal/format/imagelabel"}, - "internal/format/md": {"internal/format"}, + "internal/format/txt": {"internal/format", "internal/format/imagelabel", "internal/format/textenc"}, + "internal/format/md": {"internal/format", "internal/format/textenc"}, "internal/format/logfile": {"internal/format"}, "internal/format/csvfile": {"internal/format"}, "internal/format/jsonfile": {"internal/format"}, diff --git a/internal/guard/oracle_test.go b/internal/guard/oracle_test.go index 4bb0a17..6301dfd 100644 --- a/internal/guard/oracle_test.go +++ b/internal/guard/oracle_test.go @@ -33,18 +33,6 @@ func TestEveryFormatSurvivesItsReferenceTool(t *testing.T) { ) for _, d := range format.All() { - if d.Oracle == format.OracleNone { - noTool = append(noTool, d.ID) - continue - } - - checker, known := oracle.For(d.Oracle) - if !known { - t.Errorf("%s declares the oracle %q and nothing implements it - a declaration nobody honours is worse than none", - d.ID, d.Oracle) - continue - } - // A size big enough to be a realistic file rather than a corner case. size := d.MinBytes + 300*1024 plan, err := d.Generator.Plan(format.Request{Bytes: size, Seed: 7741, Label: true}) @@ -66,17 +54,34 @@ func TestEveryFormatSurvivesItsReferenceTool(t *testing.T) { t.Fatalf("%s: closing failed: %v", d.ID, closeErr) } - res := checker.Check(path) - switch { - case !res.Available: - skipped = append(skipped, d.ID+" ("+checker.Name+" is not installed)") - case res.Err != nil: - checked++ - t.Errorf("%s: %v\n the file is the right size and repeatable, and %s still rejects it", - d.ID, res.Err, checker.Name) + // The first layer, for the formats that name a reader. + // + // A format naming none used to skip the REST of this loop along with + // it, so the structural check below never ran for it. That sat unseen + // while the only two formats without a reader were also the only two + // without a checker - and it would have made the checkers TXT and MD + // gained on 2026-09-07 dead on arrival, silently, with this guard + // green and reporting them as covered. Found by running it and reading + // the log, not by reading the code. + switch checker, known := oracle.For(d.Oracle); { + case d.Oracle == format.OracleNone: + noTool = append(noTool, d.ID) + case !known: + t.Errorf("%s declares the oracle %q and nothing implements it - a declaration nobody honours is worse than none", + d.ID, d.Oracle) default: - checked++ - t.Logf("%s: %s accepted it - %s", d.ID, checker.Name, firstLine(res.Output)) + res := checker.Check(path) + switch { + case !res.Available: + skipped = append(skipped, d.ID+" ("+checker.Name+" is not installed)") + case res.Err != nil: + checked++ + t.Errorf("%s: %v\n the file is the right size and repeatable, and %s still rejects it", + d.ID, res.Err, checker.Name) + default: + checked++ + t.Logf("%s: %s accepted it - %s", d.ID, checker.Name, firstLine(res.Output)) + } } // The tolerant readers answer "would a viewer accept this". The @@ -117,17 +122,26 @@ func TestEveryFormatSurvivesItsReferenceTool(t *testing.T) { // structurallyChecked is the formats the second layer covers, written down. // -// TXT and MD are absent on purpose and that is the whole reason this list -// exists rather than being derived: for those two there is no specification to -// check against beyond "these are the bytes we meant", so they have one layer -// and it is honest to say so. +// TXT and MD were absent on purpose until 2026-09-07, and the reason they gave +// is worth keeping because it stopped being true rather than being wrong: for +// those two there was no specification to check against beyond "these are the +// bytes we meant", so they carried one layer and said so. +// +// Declaring an encoding is what changed it. A file that says it is UTF-16LE +// with a mark in front of it makes a claim somebody else's decoder can settle, +// and three of them settled it - Python, V8 and .NET all reject a UTF-16 file +// cut to an odd length, and all three repair it in silence when asked +// leniently. So the list still exists rather than being derived, and it is now +// every format answering true. // // Without this, dropping a format from oracle.StrictKnows removes its // structural check and every test stays green - the loop above simply skips it. // A guard that can be switched off in silence is the failure this project keeps // finding, so the list is stated and compared rather than trusted. -// Every registered format is named here, and the two that answer false are the -// point of that rule rather than an exception to it. +// Every registered format is named here. The map keeps its shape rather than +// becoming a list, because a format that answers false is a state this project +// has been in and can be in again - a new format arrives before its checker +// does, and saying so out loud is the whole job of this list. // // It held only the trues until 2026-08-25, and an outside review found what // that let through: a format added to neither this list nor oracle.StrictKnows @@ -146,9 +160,9 @@ var structurallyChecked = map[string]bool{ "bmp": true, "gif": true, "ico": true, "jpg": true, "tiff": true, "webp": true, "avif": true, "jxl": true, "docx": true, "xlsx": true, "pptx": true, - // Nothing to check against beyond "these are the bytes we meant", so they - // have one layer and it is honest to say so out loud. - "txt": false, "md": false, + // Since 2026-09-07, when a text file gained something to be checked + // against: the encoding it declares. + "txt": true, "md": true, } func TestTheStructuralCheckerCoversEveryFormatItShould(t *testing.T) { diff --git a/internal/guard/parity_test.go b/internal/guard/parity_test.go index ce4fa30..260df6a 100644 --- a/internal/guard/parity_test.go +++ b/internal/guard/parity_test.go @@ -126,6 +126,8 @@ var reachableFromTheWindow = []string{ "property:log.rate", "property:log.status_mix", "property:log.timestamps", + "property:md.bom", + "property:md.encoding", "property:pdf.page_size", "property:pdf.pages", "property:png.height", @@ -140,6 +142,8 @@ var reachableFromTheWindow = []string{ "property:targz.entry_size", "property:tiff.height", "property:tiff.width", + "property:txt.bom", + "property:txt.encoding", "property:webp.height", "property:webp.width", "property:avif.height", diff --git a/internal/guard/testdata/generator-golden.json b/internal/guard/testdata/generator-golden.json index 2b65f80..cd39848 100644 --- a/internal/guard/testdata/generator-golden.json +++ b/internal/guard/testdata/generator-golden.json @@ -273,6 +273,26 @@ "csv_8kib_seventeen_columns": { "bytes": 8192, "sha256": "23d2fc7e93456ed828159d7d5199faaed2129d164640e0743c5aaf37bb9b6143" + }, + "txt_4kib_utf8_bom": { + "bytes": 4096, + "sha256": "6c131f93605a96b665fc20516e780db972257ed891732f62bec5b8b338c31f46" + }, + "txt_4kib_utf16le_bom": { + "bytes": 4096, + "sha256": "758a5729627bbf17d4cf6157171e495290e023bb23d0b902cbeb40ff44e4f173" + }, + "txt_4kib_utf16be": { + "bytes": 4096, + "sha256": "2fd51842c2457a1464a21c4ed62c90b58904cf04711e9069e528dfd1774b4211" + }, + "md_8kib_utf16le_bom": { + "bytes": 8192, + "sha256": "60156c7f1d936b102cdf8ae100479fcf1a1a6f0fc8b585888d3bdb8ee27eec8a" + }, + "md_8kib_utf16be": { + "bytes": 8192, + "sha256": "a2d0cc69cb609277b97ae7bf7e11505fa26f3eb3214e8e62c1eba07e6de7bc24" } }, "remeasured": [ diff --git a/internal/guard/textencoding_test.go b/internal/guard/textencoding_test.go new file mode 100644 index 0000000..2aa3ef4 --- /dev/null +++ b/internal/guard/textencoding_test.go @@ -0,0 +1,361 @@ +package guard + +// What a text file claims about itself, and whether it is that. +// +// TXT and MD gained an encoding on 2026-09-07, and it is the first setting in +// this project that changes which SIZES exist rather than what fills them. A +// UTF-16 file is a whole number of sixteen bit units, so half of all sizes stop +// being reachable - and the exact size promise turns those into refusals rather +// than into files that are one byte out. +// +// The measurement that shaped all of this, on three readers in three languages: +// a UTF-16 file cut to an odd length is rejected by Python, by V8 and by .NET +// when each is asked strictly, and repaired in silence by all three when it is +// not. Get-Content shows the cut file and the whole one identically. So the +// file this tool must never write is exactly the one a person could not tell +// apart by looking. + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/textenc" + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" +) + +// encodedFormats is the formats that take an encoding, named rather than +// derived - so a third one arriving without being added here is a gap somebody +// has to notice rather than a loop that quietly gets shorter. +var encodedFormats = []string{"txt", "md"} + +type encodingCase struct { + encoding string + bom bool + mark []byte + width int64 +} + +func encodingCases() []encodingCase { + return []encodingCase{ + {textenc.UTF8, false, nil, 1}, + {textenc.UTF8, true, []byte{0xEF, 0xBB, 0xBF}, 1}, + {textenc.UTF16LE, false, nil, 2}, + {textenc.UTF16LE, true, []byte{0xFF, 0xFE}, 2}, + {textenc.UTF16BE, false, nil, 2}, + {textenc.UTF16BE, true, []byte{0xFE, 0xFF}, 2}, + } +} + +func (c encodingCase) props() map[string]string { + return map[string]string{ + textenc.Setting: c.encoding, + textenc.SettingBOM: fmt.Sprintf("%t", c.bom), + } +} + +func (c encodingCase) settings() []string { + return []string{ + textenc.Setting + "=" + c.encoding, + textenc.SettingBOM + "=" + fmt.Sprintf("%t", c.bom), + } +} + +func (c encodingCase) String() string { + return fmt.Sprintf("%s/bom=%t", c.encoding, c.bom) +} + +// writeEncoded produces one file and hands back its bytes, insisting on the +// ordered size before anything else looks at it. +func writeEncoded(t *testing.T, id string, size int64, props map[string]string) []byte { + t.Helper() + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + p, err := d.Generator.Plan(format.Request{Bytes: size, Seed: 7741, Label: true, Properties: props}) + if err != nil { + t.Fatalf("%s: planning %d B with %v: %v", id, size, props, err) + } + var buf bytes.Buffer + if err := d.Generator.Write(context.Background(), &buf, p); err != nil { + t.Fatalf("%s: writing %d B with %v: %v", id, size, props, err) + } + if int64(buf.Len()) != size { + t.Fatalf("%s %v: ordered %d B and produced %d - the size is exact or it is an error", + id, props, size, buf.Len()) + } + return buf.Bytes() +} + +// TestATextFileIsTheEncodingItDeclares is the whole claim in one place: the +// size is exact, the mark is there when it was ordered and absent when it was +// not, and somebody else's decoder agrees the bytes are what they say. +// +// The decoder is TOLD which encoding to expect rather than left to work it out. +// A checker that sniffed would decode a UTF-16 file as UTF-16 whatever was +// ordered and call a file written in the wrong encoding correct, which is the +// one question this has to answer. +func TestATextFileIsTheEncodingItDeclares(t *testing.T) { + dir := t.TempDir() + checked, skipped := 0, 0 + + for _, id := range encodedFormats { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + for _, c := range encodingCases() { + smallest := d.SmallestAccepted(format.Request{Seed: 7741, Label: true, Properties: c.props()}) + if c.width == 2 && smallest%2 != 0 { + t.Errorf("%s %v: the smallest size it accepts is %d, which a two byte encoding cannot write", + id, c, smallest) + } + + for _, size := range []int64{smallest, smallest + c.width, 4096, 40960} { + name := fmt.Sprintf("%s/%s/%d", id, c, size) + t.Run(name, func(t *testing.T) { + body := writeEncoded(t, id, size, c.props()) + + if !bytes.HasPrefix(body, c.mark) { + t.Fatalf("a %s mark was ordered and the file opens with % x", c.encoding, first(body, 4)) + } + if !c.bom && startsWithAnyMark(body) { + t.Fatalf("no mark was ordered and the file opens with % x", first(body, 4)) + } + + path := filepath.Join(dir, fmt.Sprintf("s%d.%s", size, id)) + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + res := oracle.Strict(id, path, c.settings()...) + if !res.Available { + skipped++ + t.Skip("the structural check needs python") + } + if res.Err != nil { + t.Fatalf("%s is not %s: %v", id, c.encoding, res.Err) + } + checked++ + }) + } + } + } + + if checked == 0 { + t.Errorf("nothing was decoded by anything outside this package - %d case(s) skipped", skipped) + } + t.Logf("%d file(s) decoded strictly by Python, %d skipped", checked, skipped) +} + +// TestAWideEncodingRefusesAnOddSizeAndNamesOneItCanWrite is the refusal, and +// the control beside it is what makes it mean anything. +// +// The same odd size is accepted under UTF-8, so the refusal is about the +// ENCODING rather than about the number. Without that half, a generator that +// refused every odd size in every encoding would pass this. +func TestAWideEncodingRefusesAnOddSizeAndNamesOneItCanWrite(t *testing.T) { + odd := []int64{4001, 65, 1235} + + for _, id := range encodedFormats { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + for _, c := range encodingCases() { + for _, size := range odd { + _, err := d.Generator.Plan(format.Request{ + Bytes: size, Seed: 7741, Label: true, Properties: c.props()}) + + if c.width == 1 { + if err != nil { + t.Errorf("%s %v: %d B is a size a one byte encoding can write, and it was refused: %v", + id, c, size, err) + } + continue + } + + var below *format.BelowMinimumError + if !errors.As(err, &below) { + t.Errorf("%s %v: %d B cannot be written and the answer was %v, not a BelowMinimumError", + id, c, size, err) + continue + } + if below.Minimum != size+1 { + t.Errorf("%s %v: refusing %d B named %d as the next size it can write", + id, c, size, below.Minimum) + } + // A refusal naming a size it also cannot write would be worse + // than no refusal, so the number it gives is taken up. + if _, err := d.Generator.Plan(format.Request{ + Bytes: below.Minimum, Seed: 7741, Label: true, Properties: c.props()}); err != nil { + t.Errorf("%s %v: refusing %d B pointed at %d B, and that is refused too: %v", + id, c, size, below.Minimum, err) + } + // The four parts every refusal in this tool carries. + if !strings.Contains(below.Reason, "two bytes") { + t.Errorf("%s %v: the reason does not say why an odd size cannot exist: %q", id, c, below.Reason) + } + if !strings.Contains(below.Hint, fmt.Sprintf("%d B", size-1)) { + t.Errorf("%s %v: the hint does not offer the size below: %q", id, c, below.Hint) + } + } + } + } +} + +// TestTheDefaultEncodingIsTheBytesTheseFormatsAlwaysWrote is the way back. +// +// A setting whose default changes the file is a breaking change wearing the +// clothes of a feature. This is the same pin the animated formats got when +// frames arrived: saying nothing and saying the default out loud have to be +// the same bytes, and the golden file beside it holds them to what they were +// before the setting existed. +func TestTheDefaultEncodingIsTheBytesTheseFormatsAlwaysWrote(t *testing.T) { + for _, id := range encodedFormats { + for _, size := range []int64{0, 33, 4096} { + silent := writeEncoded(t, id, size, nil) + spoken := writeEncoded(t, id, size, map[string]string{ + textenc.Setting: textenc.UTF8, textenc.SettingBOM: "false"}) + if !bytes.Equal(silent, spoken) { + t.Errorf("%s at %d B: saying nothing and saying utf-8 produce different bytes", id, size) + } + if size > 0 && startsWithAnyMark(silent) { + t.Errorf("%s at %d B: the default file opens with a byte order mark", id, size) + } + } + } +} + +// TestALabelThatWillNotFitSaysWhatItWouldCost holds the note to the encoding. +// +// The note used to be built from the length of the label as a string, which is +// what it costs in UTF-8 and half of what it costs in UTF-16. A note that is +// out by a factor of two is worse than no note: it tells somebody to ask for +// 66 B when the file needs 132. +func TestALabelThatWillNotFitSaysWhatItWouldCost(t *testing.T) { + const size = int64(64) // below the label's cost in a wide encoding, even + tails := map[string]string{"txt": "\n", "md": "\n\n"} + + for _, id := range encodedFormats { + d, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + props := map[string]string{textenc.Setting: textenc.UTF16LE} + p, err := d.Generator.Plan(format.Request{ + Bytes: size, Seed: 7741, Label: true, Properties: props}) + if err != nil { + t.Fatalf("%s: planning %d B in utf-16le: %v", id, size, err) + } + + line := core.Label(id, size, 7741) + tails[id] + wide, narrow := int64(len(line))*2, int64(len(line)) + if wide <= size { + t.Fatalf("%s: the label costs %d B at %d B, so this case no longer sits below the threshold", + id, wide, size) + } + + note := noteWithCode(p.Notes, "label_omitted") + if note == nil { + t.Fatalf("%s: the label does not fit and nothing said so - silence is banned", id) + } + if !strings.Contains(note.Detail, fmt.Sprintf("needs %d B", wide)) { + t.Errorf("%s: the note should say the label needs %d B in this encoding: %q", id, wide, note.Detail) + } + if strings.Contains(note.Detail, fmt.Sprintf("needs %d B", narrow)) { + t.Errorf("%s: the note reports what the label costs in UTF-8, not in the encoding asked for: %q", + id, note.Detail) + } + } +} + +// TestACharacterSplitAcrossTwoWritesSurvives is the one defence here that our +// own generators cannot redden, so it is reddened on purpose. +// +// They write whole words, so a character never straddles two writes. The +// encoder holds the tail anyway, because a writer that only works when its +// caller is careful is a trap for the next caller - and a defence nothing can +// redden is not a defence, which is why this exists rather than a comment +// saying it was thought about. +func TestACharacterSplitAcrossTwoWritesSurvives(t *testing.T) { + const text = "zażółć" // two byte characters, so a split lands mid character + + for _, name := range []string{textenc.UTF16LE, textenc.UTF16BE} { + codec, err := textenc.Parse("txt", map[string]string{textenc.Setting: name}) + if err != nil { + t.Fatal(err) + } + + var whole bytes.Buffer + if _, err := codec.Writer(&whole).Write([]byte(text)); err != nil { + t.Fatal(err) + } + + for cut := 1; cut < len(text); cut++ { + var split bytes.Buffer + w := codec.Writer(&split) + if _, err := w.Write([]byte(text)[:cut]); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(text)[cut:]); err != nil { + t.Fatal(err) + } + if !bytes.Equal(whole.Bytes(), split.Bytes()) { + t.Fatalf("%s: writing in two goes at byte %d gave % x, and in one go it is % x", + name, cut, split.Bytes(), whole.Bytes()) + } + } + } +} + +// TestTheTextGeneratorsWriteOnlyASCII names the assumption the arithmetic +// stands on. +// +// The budget is worked out as "file bytes divided by the width of a +// character", which is only the same thing as "how much prose fits" while +// every character costs one byte in UTF-8. The day the filler stops being +// English that stops being true, and it should stop here rather than in a file +// that is one byte short. +func TestTheTextGeneratorsWriteOnlyASCII(t *testing.T) { + for _, id := range encodedFormats { + body := writeEncoded(t, id, 40960, nil) + for i, b := range body { + if b > 0x7f { + t.Fatalf("%s: byte %d is %#x, and the size arithmetic assumes one byte per character", id, i, b) + } + } + } +} + +func noteWithCode(notes []format.Note, code string) *format.Note { + for i := range notes { + if notes[i].Code == code { + return ¬es[i] + } + } + return nil +} + +func startsWithAnyMark(body []byte) bool { + for _, m := range [][]byte{{0xEF, 0xBB, 0xBF}, {0xFF, 0xFE}, {0xFE, 0xFF}} { + if bytes.HasPrefix(body, m) { + return true + } + } + return false +} + +func first(body []byte, n int) []byte { + if len(body) < n { + return body + } + return body[:n] +} diff --git a/internal/guard/textformats_test.go b/internal/guard/textformats_test.go index 97cd86f..e14152c 100644 --- a/internal/guard/textformats_test.go +++ b/internal/guard/textformats_test.go @@ -506,6 +506,17 @@ func TestEveryFormatIsClassifiedAsTextOrBinary(t *testing.T) { // refuses with "generated content is English only so far" - and which M5 // describes. The guard goes in before the code that needs it, which is how the // first four guards in this project were built. +// +// AT DEFAULT SETTINGS, and since 2026-09-07 that qualifier is load bearing +// rather than pedantic. TXT and MD can be asked for UTF-16, and a UTF-16 file +// is NOT valid UTF-8 - so the sentence this guard's name makes stopped being +// true of the tool on the day the encoding setting landed. It stayed true of +// what this guard actually does, because generateBytes plans with no +// properties, and a guard that quietly narrows to the case it can still pass +// is the failure this project keeps finding. Said out loud instead: the claim +// here is the DEFAULT path. Every other encoding is held to the encoding it +// declares, by TestATextFileIsTheEncodingItDeclares and by a structural +// checker that is told which one to expect. func TestEveryTextFormatIsValidUTF8(t *testing.T) { for _, id := range textFormats { d, err := format.Get(id) diff --git a/internal/oracle/oracle.go b/internal/oracle/oracle.go index eb0de28..e9895f7 100644 --- a/internal/oracle/oracle.go +++ b/internal/oracle/oracle.go @@ -393,7 +393,12 @@ func Strict(formatID, path string, settings ...string) Result { func StrictKnows(formatID string) bool { switch formatID { case "png", "wav", "pdf", "zip", "targz", "log", "csv", "json", "xml", "svg", "html", - "bmp", "gif", "ico", "jpg", "tiff", "webp", "avif", "jxl", "docx", "xlsx", "pptx": + "bmp", "gif", "ico", "jpg", "tiff", "webp", "avif", "jxl", "docx", "xlsx", "pptx", + // The two text formats joined on 2026-09-07, when they gained an + // encoding. Before that there was nothing here to check against + // beyond "these are the bytes we meant" - a claim about UTF-16 is a + // claim somebody else's decoder can settle. + "txt", "md": return true } return false diff --git a/internal/oracle/strict.py b/internal/oracle/strict.py index b6376fd..4796a45 100644 --- a/internal/oracle/strict.py +++ b/internal/oracle/strict.py @@ -1558,17 +1558,104 @@ def check_jxl(data): } +def decode_declared(kind, data, settings): + """Decode the file in the encoding it was TOLD the file claims. + + Told rather than sniffed, for the reason the CSV dialect is told: a checker + that guessed would decode a UTF-16 file as UTF-16 whatever was ordered and + agree with a file written in the wrong one. Sniffing is genuinely ambiguous + here as well - a UTF-16 file with no mark in front of it is a legal file + that looks like nothing in particular. + + Python's own codecs do the decoding, and that is the point: the bytes are + judged by an implementation that is neither our code nor our language. + Strict on purpose. Measured on three readers on 2026-09-07, every lenient + path replaces a broken character with U+FFFD and reports success, so a + lenient decode here would bless the one defect worth catching. + """ + settings = settings or {} + name = settings.get("encoding", "utf-8") + marks = {"utf-8": b"\xef\xbb\xbf", "utf-16le": b"\xff\xfe", "utf-16be": b"\xfe\xff"} + names = {"utf-8": "utf-8", "utf-16le": "utf-16-le", "utf-16be": "utf-16-be"} + if name not in names: + fail(f"the {kind} check was told encoding={name!r}, which is not one this tool writes") + + wants_mark = settings.get("bom", "false") == "true" + mark = marks[name] + has_mark = data.startswith(mark) + if wants_mark and not has_mark: + fail(f"a {name} byte order mark was ordered and the file does not open with one") + if not wants_mark and has_mark: + fail(f"no byte order mark was ordered and the file opens with one") + body = data[len(mark):] if wants_mark else data + + if name != "utf-8" and len(body) % 2: + fail(f"{name} stores two bytes for every character and the file holds " + f"{len(body)} of them, so the last character is cut in half") + try: + text = body.decode(names[name]) + except UnicodeDecodeError as exc: + fail(f"the file does not decode as {name}: {exc}") + + # Decoding is not enough, and this is the half that says why. Read a + # UTF-16BE file as little endian and every pair of bytes is still a valid + # character - 0x74 0x00 becomes U+7400 rather than "t" - so a strict decode + # blesses a file with its byte order the wrong way round. The prose these + # formats write is words, spaces and newlines, so anything outside that + # means the bytes were read in an order nobody ordered, or landed somewhere + # they were never meant to. + stray = sorted({c for c in text if c != "\n" and not (" " <= c <= "~")}) + if stray: + shown = [hex(ord(c)) for c in stray[:6]] + fail(f"the text holds {len(stray)} character(s) that are not printable ASCII: {shown}") + return text + + +def check_txt(data, settings=None): + """Text, in the encoding it claims, and nothing in it that is not text. + + There was nothing here to check against until a text file could declare an + encoding, which is why TXT and MD carried one layer where every other + format carries two. A file saying it is UTF-16LE with a mark in front is a + claim somebody else's decoder can settle, so from 2026-09-07 they carry two. + + The character check is the second half and it is not about encoding at all: + this prose is words, spaces and newlines, so a control character in the + decoded text means bytes landed somewhere they were never meant to - and + both the size check and the determinism check would call that file correct. + """ + text = decode_declared("txt", data, settings) + ok(f"{len(text)} characters, decoded strictly") + + +def check_md(data, settings=None): + """The document decodes, and every fenced block is closed. + + The fence count is the structural half. This generator writes code fences + in pairs and takes a block whole or not at all, so an odd number of them + means a block was cut in half - which renders as one enormous code block + swallowing the rest of the document, at exactly the size that was ordered + and with a stable hash. Nothing else here would see it. + """ + text = decode_declared("md", data, settings) + fences = text.count("```") + if fences % 2: + fail(f"the document holds {fences} code fences, so one block is never closed") + ok(f"{len(text)} characters, {fences // 2} fenced block(s), decoded strictly") + + CHECKS = {"png": check_png, "wav": check_wav, "pdf": check_pdf, "zip": check_zip, "log": check_log, "csv": check_csv, "json": check_json, "xml": check_xml, "svg": check_svg, "html": check_html, "targz": check_targz, "bmp": check_bmp, "gif": check_gif, "ico": check_ico, "jpg": check_jpg, "tiff": check_tiff, "webp": check_webp, "avif": check_avif, "jxl": check_jxl, - "docx": check_docx, "xlsx": check_xlsx, "pptx": check_pptx} + "docx": check_docx, "xlsx": check_xlsx, "pptx": check_pptx, + "txt": check_txt, "md": check_md} # Checks that take the shape of the file as well as its bytes. Everything else # is handed the bytes alone, so adding a setting to one check cannot change how # any other one is called. -TAKES_SETTINGS = {"csv"} +TAKES_SETTINGS = {"csv", "txt", "md"} if __name__ == "__main__": if len(sys.argv) < 3 or sys.argv[1] not in CHECKS: diff --git a/web/public/formats/index.html b/web/public/formats/index.html index 8db3db1..30cceb5 100644 --- a/web/public/formats/index.html +++ b/web/public/formats/index.html @@ -470,6 +470,16 @@

Settings each format accepts

line_ending crlf, lf + + md + encoding + utf-16be, utf-16le, utf-8 + + + + bom + true or false + pdf pages @@ -545,6 +555,16 @@

Settings each format accepts

height 1 - 20000 pixels + + txt + encoding + utf-16be, utf-16le, utf-8 + + + + bom + true or false + wav sample_rate diff --git a/web/public/pl/formaty/index.html b/web/public/pl/formaty/index.html index cf3178e..8a9f235 100644 --- a/web/public/pl/formaty/index.html +++ b/web/public/pl/formaty/index.html @@ -470,6 +470,16 @@

Ustawienia, które przyjmuje każdy format

line_ending crlf, lf + + md + encoding + utf-16be, utf-16le, utf-8 + + + + bom + prawda albo fałsz + pdf pages @@ -545,6 +555,16 @@

Ustawienia, które przyjmuje każdy format

height 1 - 20000 pikseli + + txt + encoding + utf-16be, utf-16le, utf-8 + + + + bom + prawda albo fałsz + wav sample_rate