From 7d08219d9bb10585841fa126ba66a86994570571 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Wed, 9 Sep 2026 17:23:37 +0200 Subject: [PATCH] fix: the structural checker is found by a binary built somewhere else oracle.Strict looked for its script beside its own source file, through runtime.Caller, and that path is compiled in. A test binary cross compiled on Windows and run in a Linux container therefore looked for a Windows directory, found nothing, and answered "not available" for every structural check in the suite. Nothing went red for it. An unavailable checker is a skip, and a skip reads like a check that ran, so the container runs had been reporting a green suite in which not one file was ever checked against the specification. It surfaced only when two guards started refusing to pass on zero checks rather than reporting a skip, and the blind spot was older than both of them. There is a second way in now: walk up from the working directory looking for the script at its place in the tree. A test binary runs from inside the tree it tests, which is the case the compiled in path cannot cover. That path is still tried first, so nothing changes on a machine that built its own binary. The first diagnosis was wrong and it is worth recording why. The failure said "0 file(s) decoded strictly by Python", so the container was given an image carrying Python - and nothing changed, because there were two causes and both were needed. The message named the outcome. A probe printing what the function returned named the cause: strictScriptPath answering nothing while LookPath answered /usr/local/bin/python. Three mutations, all caught. The guard asks the second way directly, because the compiled in path is chosen first on any machine that built the binary, so asking through Strict would prove the first way and never reach the second. The control beside it asks the real entry point and is broken by taking the interpreter away rather than either path, since breaking one path leaves the other answering. internal/oracle exists for tests and is linked into neither binary, so no shipped byte moves. Co-Authored-By: Claude Opus 5 --- internal/guard/oraclescript_test.go | 93 +++++++++++++++++++++++++++++ internal/oracle/oracle.go | 53 ++++++++++++++-- 2 files changed, 140 insertions(+), 6 deletions(-) create mode 100644 internal/guard/oraclescript_test.go diff --git a/internal/guard/oraclescript_test.go b/internal/guard/oraclescript_test.go new file mode 100644 index 0000000..f2e63b5 --- /dev/null +++ b/internal/guard/oraclescript_test.go @@ -0,0 +1,93 @@ +package guard + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/oracle" +) + +// The structural checker is found by a binary that was built somewhere else. +// +// What this defends against was measured on 2026-09-09 and had been true for as +// long as this project has run its suite in a container. oracle.Strict looks for +// its script beside its own source file, and that path is compiled in - so a +// binary cross compiled on Windows and run on Linux looked for a Windows +// directory, found nothing, and answered "not available" for every structural +// check. Nothing went red, because an unavailable checker is a skip and a skip +// reads like a check that ran. +// +// It surfaced only when two guards started refusing to pass on zero checks, and +// the blind spot was older than both of them. The first diagnosis was wrong in +// a way worth recording: the message said "0 file(s) decoded strictly by +// Python", so the container was given an image carrying Python, and nothing +// changed. The message named the outcome. A probe printing what the function +// returned named the cause. +// +// This asks the second way directly, because the compiled in path is chosen +// first on any machine that built the binary - including the one running this +// guard - so asking through Strict would prove the first way and never reach +// the second. +func TestTheStructuralCheckerIsFoundByABinaryBuiltSomewhereElse(t *testing.T) { + root := t.TempDir() + want := filepath.Join(root, "internal", "oracle", "strict.py") + if err := os.MkdirAll(filepath.Dir(want), 0o755); err != nil { + t.Fatalf("laying out a tree to search: %v", err) + } + if err := os.WriteFile(want, []byte("print('OK')\n"), 0o600); err != nil { + t.Fatalf("writing a stand in for the checker: %v", err) + } + + // Where a test binary actually runs from: a package directory, some way + // down the tree from the script. + from := filepath.Join(root, "internal", "guard") + if err := os.MkdirAll(from, 0o755); err != nil { + t.Fatalf("laying out the directory to search from: %v", err) + } + + got, ok := oracle.StrictScriptUnder(from) + if !ok { + t.Fatalf("the checker was not found by walking up from %s.\n"+ + "Reason: a binary built on one machine and run on another cannot use the path\n"+ + "compiled into it, so this is the only way left. Without it every structural\n"+ + "check in a container is skipped and the suite still reports success.", from) + } + if got != want { + t.Errorf("the walk found %q and the checker is at %q", got, want) + } + + // The other half, and it is not decoration: a walk with no stopping + // condition climbs to the root of the disk and answers about a file that + // belongs to something else. + if p, ok := oracle.StrictScriptUnder(t.TempDir()); ok { + t.Errorf("a tree with no checker in it answered %q.\n"+ + "Reason: the walk has to give up at the top rather than keep going and pick up\n"+ + "whatever it finds outside the tree it was asked about.", p) + } +} + +// The way that works on the machine that built the binary still works. +// +// A control, and the reason it is here: the guard above lays out its own tree, +// so it would pass unchanged if the real lookup were broken outright. This one +// asks the real thing, from the real repository. +// +// It steps aside when there is no interpreter to find, because then there is +// nothing for the lookup to be available for and a red result would be a +// sentence about the machine rather than about this code. The name looked for +// is "python" and not "python3", which is the name oracle.Strict itself looks +// for - asking a different question here would make this guard agree with a +// machine the checker cannot actually use. +func TestTheStructuralCheckerIsAvailableWhereItWasBuilt(t *testing.T) { + if _, err := exec.LookPath("python"); err != nil { + t.Skip("no python on this machine, so there is nothing for the checker to be") + } + res := oracle.Strict("txt", filepath.Join(t.TempDir(), "nothing.txt")) + if !res.Available { + t.Errorf("python is on this machine and the structural checker is still not available.\n" + + "Reason: the script sits beside its own source here, so the compiled in path should\n" + + "find it. If this is red, the lookup is broken rather than the machine unusual.") + } +} diff --git a/internal/oracle/oracle.go b/internal/oracle/oracle.go index e9895f7..b069973 100644 --- a/internal/oracle/oracle.go +++ b/internal/oracle/oracle.go @@ -406,14 +406,55 @@ func StrictKnows(formatID string) bool { // strictScriptPath finds the checker next to this source file, so the tests // work wherever the repository is checked out. +// strictScriptPath finds the checker script, and has to look twice. +// +// The first way works on the machine that compiled the binary: the script sits +// beside this file, and runtime.Caller says where that is. +// +// The second way exists because that path is COMPILED IN, so a binary built +// here and carried anywhere else looks for a directory that is not there. +// Measured on 2026-09-09: the guard suite cross compiled on Windows and run in +// a Linux container reported "0 file(s) decoded strictly by Python" with python +// sitting in that container's PATH the whole time. Every structural check had +// been quietly unavailable since the container runs began, and it took two +// guards that refuse to pass on nothing to make it visible - an unavailable +// checker is a skip, and a skip reads like a check that ran. +// +// So the second way asks the working directory instead and walks up looking for +// the script at its place in the tree. A test binary runs from inside the tree +// it tests, which is exactly the case the compiled in path cannot cover. func strictScriptPath() (string, bool) { - _, thisFile, _, ok := runtime.Caller(0) - if !ok { - return "", false + if _, thisFile, _, ok := runtime.Caller(0); ok { + p := filepath.Join(filepath.Dir(thisFile), "strict.py") + if _, err := os.Stat(p); err == nil { + return p, true + } } - p := filepath.Join(filepath.Dir(thisFile), "strict.py") - if _, err := os.Stat(p); err != nil { + dir, err := os.Getwd() + if err != nil { return "", false } - return p, true + return StrictScriptUnder(dir) +} + +// StrictScriptUnder walks up from dir looking for the checker script. +// +// Exported for the guard that holds it. The compiled in path above is chosen +// first on any machine that built the binary, so a guard calling +// strictScriptPath would prove that one and never reach this. +func StrictScriptUnder(dir string) (string, bool) { + for { + p := filepath.Join(dir, "internal", "oracle", "strict.py") + if _, err := os.Stat(p); err == nil { + return p, true + } + up := filepath.Dir(dir) + if up == dir { + // The top of the disk. Whatever sits above this belongs to + // something else, so the answer is no rather than the first + // checker that happens to turn up outside the tree. + return "", false + } + dir = up + } }