From 55d1fa7ccaefb8b076dfd28f33f435040029e7fb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:06:53 +0000 Subject: [PATCH 1/8] Add fold/textlen/textslice/listdir, and contain ambient paths to ctx.dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four builtins a build description needs, and the containment that makes a sandboxed build mean something. `fold { .table, .init, .step }` is the one general traversal primitive - the language had no loops and no way to enumerate a Table's entries, so `map`, `filter` and sequence append are now writable in HashedBuild rather than each needing a builtin. It visits in ascending key order rather than entry order, which buys two properties: a sequence folds in index order, and two Tables that compare equal (§6 ignores entry order) fold to the same answer. `textlen`/`textslice` count codepoints, not bytes, since the type is Utf8 and a byte index could cut a character in half. They are the primitives; `endswith` is a one-liner on top, which is how the C sources get filtered. `listdir ` resolves the open half of §16's TODO: names only, sorted byte-wise for the same reason the directory hash sorts - readdir order is the filesystem's business, and a build whose argument order varied by machine would cache differently on each. `ctx.dir` is a handle to the directory a run is rooted at, and handle-less path resolution becomes three states chosen by permission: `anypath` (as before), `workdir` (contained to ctx.dir - ".." , an absolute path and a symlink are all refused, and "." resolves to ctx.dir rather than through it), or denied. Both spellings of a handle-less path share one resolver, so the guarantee cannot hold for `loadfile "x"` and not for `createfile { .path }`. ctx.dir is its own type rather than a directory File for one specific reason: §15 puts the whole ctx into every `cached` key, and a directory File hashes over its contents, so a File here would make every cache entry depend on every byte of the project tree. Like ctx.cache it hashes as a bare tag; what is read *through* it are ordinary Files that still hash by content. The root context grants io, exec and anypath, so existing behaviour is unchanged - context-permissions.hb's documented value grew accordingly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- examples/context-permissions.hb | 12 +- src/builtins_build.odin | 230 ++++++++++++++++++++++++++++++++ src/builtins_fs.odin | 124 ++++++++++++++++- src/cache_format.odin | 2 + src/examples_test.odin | 2 +- src/hash.odin | 12 ++ src/main.odin | 4 + src/print_value.odin | 8 ++ src/value.odin | 19 +++ 9 files changed, 402 insertions(+), 11 deletions(-) create mode 100644 src/builtins_build.odin diff --git a/examples/context-permissions.hb b/examples/context-permissions.hb index 30f52a5..840763c 100644 --- a/examples/context-permissions.hb +++ b/examples/context-permissions.hb @@ -10,9 +10,15 @@ // as `.still_ambient` shows. Two things to know: there are no `true`/`false` // literals yet, hence `1 == 0` for "false"; and a `loadfile` under a // narrowed context doesn't evaluate to an error value - it fails the whole -// program, since only a false `then` is catchable (§8). Evaluates to -// { ambient: {io: nothing}, io_denied: {}, replaced: {}, -// still_ambient: {io: nothing} }. +// program, since only a false `then` is catchable (§8). +// +// The root grants three: `io` to touch the filesystem at all, `exec` to run a +// program, and `anypath` to resolve a path with no directory handle. Dropping +// `anypath` for `workdir` is what contains a run to `ctx.dir` - see +// workdir-containment.hb. Evaluates to +// { ambient: {io: nothing, exec: nothing, anypath: nothing}, io_denied: +// {exec: nothing, anypath: nothing}, replaced: {}, +// still_ambient: {io: nothing, exec: nothing, anypath: nothing} }. { .ambient = ctx.permissions, .io_denied = ctx.permissions chctx chperm { .name = "io", .enabled = 1 == 0 }, diff --git a/src/builtins_build.odin b/src/builtins_build.odin new file mode 100644 index 0000000..40ee989 --- /dev/null +++ b/src/builtins_build.odin @@ -0,0 +1,230 @@ +package hashedbuild + +// The builtins a build description needs, beyond §16's filesystem set: +// traversing a Table (`fold`), measuring and cutting text (`textlen`, +// `textslice`), listing a directory (`listdir`, SPEC.md §16) and running a +// program (`exec`). Like §16's, none of these are syntax - they are ordinary +// Function values pre-bound in the global scope, so adding one never touches +// the grammar. + +import "core:slice" +import "core:strings" +import "core:unicode/utf8" + +// ---- fold --------------------------------------------------------------------- + +// fold { .table, .init, .step } -> Value +// +// The language has no loops (§8), and recursion cannot get at a Table's +// entries without a way to enumerate them - so this is the one general +// traversal primitive, and `map`/`filter`/`append` are written in HashedBuild +// on top of it rather than each being a builtin of its own. +// +// `.step` is called with { .acc, .key, .value } and returns the next +// accumulator. Not gated: it reads nothing outside the value it was handed. +// +// **Visiting order is ascending key order**, which is a deliberate choice and +// not the order the entries happen to sit in. Two consequences are the reason: +// a sequence (§5 - keys 1..N) folds in index order, which is what makes +// folding a list of filenames mean anything; and two Tables that compare +// equal (§6, which ignores entry order) fold to the same answer, which they +// would not if this walked `entries` as written. The order is numbers first +// by value, then Utf8 keys byte-wise; any other key kind sorts after those, +// keeping its relative position (see fold_key_less). +@(private = "file") +builtin_fold :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, bool) { + t, is_table := arg.(^Table_Value) + if !is_table do return fail(interp, "fold expects a { .table, .init, .step } Table") + + table_val, has_table := table_find(t, "table") + init_val, has_init := table_find(t, "init") + step_val, has_step := table_find(t, "step") + if !has_table do return fail(interp, "fold needs a .table") + if !has_init do return fail(interp, "fold needs an .init") + if !has_step do return fail(interp, "fold needs a .step") + + target, target_is_table := table_val.(^Table_Value) + if !target_is_table do return fail(interp, "fold's .table must be a Table") + step, step_is_fn := step_val.(^Function_Value) + if !step_is_fn do return fail(interp, "fold's .step must be a Function") + + order := fold_order(target) + defer delete(order) + + acc := init_val + for idx in order { + entry := target.entries[idx] + frame := new(Table_Value) + append(&frame.entries, Table_Entry_Value{key = "acc", value = acc}) + append(&frame.entries, Table_Entry_Value{key = "key", value = entry.key}) + append(&frame.entries, Table_Entry_Value{key = "value", value = entry.value}) + next, ok := call_function(interp, step, frame) + if !ok do return nil, false + next, ok = concrete_value(interp, next) + if !ok do return nil, false + acc = next + } + return acc, true +} + +// Indices into `t.entries`, ascending by key. A hand-rolled insertion sort +// rather than slice.sort_by because it has to be *stable*: keys the ordering +// below does not distinguish (a Table used as a key, say) must still come out +// in a fixed order rather than whatever an unstable sort happens to do, or a +// fold over such a Table would not be reproducible run to run. +@(private = "file") +fold_order :: proc(t: ^Table_Value) -> []int { + order := make([]int, len(t.entries)) + for i in 0 ..< len(t.entries) do order[i] = i + for i in 1 ..< len(order) { + j := i + for j > 0 && fold_key_less(t.entries[order[j]].key, t.entries[order[j - 1]].key) { + order[j], order[j - 1] = order[j - 1], order[j] + j -= 1 + } + } + return order +} + +// Numbers before text before everything else; within numbers by value, within +// text byte-wise. Anything else compares equal to anything else, which the +// stable sort above turns into "keeps the order it was written in" - the +// honest answer, since §6 defines no ordering over those kinds and inventing +// one here would be a language decision this builtin has no business making. +@(private = "file") +fold_key_less :: proc(a: Value, b: Value) -> bool { + ra, rb := fold_key_rank(a), fold_key_rank(b) + if ra != rb do return ra < rb + switch ra { + case 0: + return fold_key_number(a) < fold_key_number(b) + case 1: + return a.(string) < b.(string) + } + return false +} + +@(private = "file") +fold_key_rank :: proc(v: Value) -> int { + #partial switch _ in v { + case i64, f64: return 0 + case string: return 1 + } + return 2 +} + +@(private = "file") +fold_key_number :: proc(v: Value) -> f64 { + #partial switch x in v { + case i64: return f64(x) + case f64: return x + } + return 0 +} + +// ---- textlen / textslice ------------------------------------------------------ + +// Both count in *codepoints*, not bytes: the type is Utf8 (§3), and an index +// that could land inside a multi-byte character would make `textslice` able to +// produce something that is not Utf8 at all. Neither is gated by `io` - they +// only look at text already in hand, the same reasoning that leaves `filetext` +// ungated (§16). + +// textlen -> Integer +@(private = "file") +builtin_textlen :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, bool) { + s, is_text := arg.(string) + if !is_text do return fail(interp, "textlen expects a Utf8") + return i64(utf8.rune_count_in_string(s)), true +} + +// textslice { .text, .start, .count } -> Utf8 +// +// `.start` is 1-based, matching `[i]`'s element access (§5). Asking for +// anything outside the text is fatal rather than clamped: a silently short +// answer is how an off-by-one becomes a wrong build instead of a stopped one. +@(private = "file") +builtin_textslice :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, bool) { + t, is_table := arg.(^Table_Value) + if !is_table do return fail(interp, "textslice expects a { .text, .start, .count } Table") + + text_val, has_text := table_find(t, "text") + start_val, has_start := table_find(t, "start") + count_val, has_count := table_find(t, "count") + + s, text_ok := text_val.(string) + start, start_ok := start_val.(i64) + count, count_ok := count_val.(i64) + if !has_text || !text_ok do return fail(interp, "textslice needs a Utf8 .text") + if !has_start || !start_ok do return fail(interp, "textslice needs an Integer .start") + if !has_count || !count_ok do return fail(interp, "textslice needs an Integer .count") + + n := i64(utf8.rune_count_in_string(s)) + if start < 1 do return fail(interp, "textslice's .start is 1-based, so it must be at least 1") + if count < 0 do return fail(interp, "textslice's .count cannot be negative") + if start - 1 + count > n { + return fail(interp, "textslice: .start + .count reaches past the end of the text") + } + return strings.clone(strings.cut(s, int(start - 1), int(count))), true +} + +// ---- listdir ------------------------------------------------------------------ + +// listdir -> a sequence of names (Utf8), sorted +// +// SPEC.md §16 left this open ("what a directory File's listing looks like as a +// value, so you can enumerate its entries, not just address a name you already +// know"); this is that half resolved. The plumbing was already here - the +// directory hasher walks a tree with the same call - it simply was not exposed. +// +// **Names only, not kinds.** What an entry *is* can already be asked by opening +// it (`loadfile { .dir, .path }`), and a bare sequence is the shape `fold` +// traverses and `[i]` indexes; a Table of name -> tag would need unwrapping at +// every use for something most callers do not consult. Adding kinds later is a +// widening, which is the direction that stays compatible. +// +// Sorted by name, byte-wise, for the same reason the directory *hash* sorts +// (§3): readdir order is a filesystem's private business, and a build whose +// argument order changed between machines would hash - and therefore cache - +// differently on each. Gated by `io` like every other read (§16). +@(private = "file") +builtin_listdir :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, bool) { + if !ctx_allows_io(interp) do return fail(interp, "listdir: io permission not granted in the current context") + + fv, is_file := arg.(^File_Value) + if !is_file do return fail(interp, "listdir expects a File") + if fv.kind != .Directory do return fail(interp, "listdir expects a directory, not a regular file") + + entries, err := fs_list_entries_at(fv.dir_fd, context.temp_allocator) + if err != .None do return fail(interp, "listdir: could not read the directory") + slice.sort_by(entries, proc(a, b: Fs_Dir_Entry) -> bool { return a.name < b.name }) + + out := new(Table_Value) + for entry, i in entries { + append(&out.entries, Table_Entry_Value{key = i64(i + 1), value = strings.clone(entry.name)}) + } + return out, true +} + +// ---- registration ------------------------------------------------------------- + +// Called by make_global_env (builtins_fs.odin) alongside §16's six. Kept here +// rather than there so the two sets stay separately readable; the names are +// what each hashes as (hash_function.odin), so they are as load-bearing as +// the procs and must not be renamed casually. +bind_build_builtins :: proc(env: ^Env) { + env_bind(env, "fold", new_build_native("fold", builtin_fold)) + env_bind(env, "listdir", new_build_native("listdir", builtin_listdir)) + env_bind(env, "textlen", new_build_native("textlen", builtin_textlen)) + env_bind(env, "textslice", new_build_native("textslice", builtin_textslice)) +} + +@(private = "file") +new_build_native :: proc(name: string, fn: Native_Fn, closure: Value = nil) -> Value { + f := new(Function_Value) + f.name = name + f.native = fn + f.native_closure = closure + return f +} + diff --git a/src/builtins_fs.odin b/src/builtins_fs.odin index bfcd617..2551d01 100644 --- a/src/builtins_fs.odin +++ b/src/builtins_fs.odin @@ -33,16 +33,46 @@ import "core:unicode/utf8" make_root_context :: proc(cache_dir_override: string = "") -> Value { perms := new(Table_Value) append(&perms.entries, Table_Entry_Value{key = "io", value = Nothing_Value{}}) + append(&perms.entries, Table_Entry_Value{key = "exec", value = Nothing_Value{}}) + // `anypath` at the root is what keeps a handle-less `loadfile "x"` resolving + // the way it always has (§16). Narrowing to `workdir` - or to neither - is + // something a program or a host does on purpose; see ambient_mode below. + append(&perms.entries, Table_Entry_Value{key = "anypath", value = Nothing_Value{}}) cache := new(Cache_Value) cache.dir_path = resolve_cache_dir(cache_dir_override) + // Starts at the process's cwd; a host that knows better replaces it (see + // ctx_set_workdir - eval_source_file points it at the source file's own + // directory, so a script behaves the same wherever it is invoked from). + wd := new(Workdir_Value) + wd.dir_fd = fs_cwd_dir() + if cwd, err := os.get_working_directory(context.allocator); err == nil { + wd.dir_path = to_forward_slashes(cwd) + } + ctx_table := new(Table_Value) append(&ctx_table.entries, Table_Entry_Value{key = "permissions", value = perms}) append(&ctx_table.entries, Table_Entry_Value{key = "cache", value = cache}) + append(&ctx_table.entries, Table_Entry_Value{key = "dir", value = wd}) return ctx_table } +// Points a context's `.dir` (§9) at a specific directory, in place. Called by +// whoever set up the run once it knows what "the current directory" means for +// it: the source file's own directory for `hb file.hb`, the build file's for +// hashmake, the process cwd for `-e` and the REPL (already the default). +ctx_set_workdir :: proc(ctx: Value, dir_fd: Fs_Fd, dir_path: string) { + t, is_table := ctx.(^Table_Value) + if !is_table do return + wd_val, found := table_find(t, "dir") + if !found do return + wd, is_wd := wd_val.(^Workdir_Value) + if !is_wd do return + wd.dir_fd = dir_fd + wd.dir_path = dir_path +} + // XDG Base Directory spec: $XDG_CACHE_HOME/hashedbuild, falling back to // $HOME/.cache/hashedbuild if XDG_CACHE_HOME isn't set (or is empty). // @@ -90,6 +120,7 @@ make_global_env :: proc() -> ^Env { env_bind(env, "readlink", new_native_function("readlink", builtin_readlink)) env_bind(env, "chperm", new_native_function("chperm", builtin_chperm)) env_bind(env, "filetext", new_native_function("filetext", builtin_filetext)) + bind_build_builtins(env) return env } @@ -98,14 +129,55 @@ make_global_env :: proc() -> ^Env { // it (hash.odin), and §15's `cached` reads and writes cache entries // (eval.odin). Both are I/O operations like any other here (SPEC.md §3/§9). ctx_allows_io :: proc(interp: ^Interpreter) -> bool { + return ctx_has_permission(interp, "io") +} + +// §9's permissions are a Table used as a set: granted iff the key is *present*, +// whatever its (always-`nothing`) value. Read live off the call site, never +// captured - that is what lets a wrapping `withctx`/`chctx` restrict a builtin +// from the outside. +ctx_has_permission :: proc(interp: ^Interpreter, name: string) -> bool { t, is_table := interp.current_ctx.(^Table_Value) if !is_table do return false perms_val, found := table_find(t, "permissions") if !found do return false perms, perms_is_table := perms_val.(^Table_Value) if !perms_is_table do return false - _, io_found := table_find(perms, "io") - return io_found + _, name_found := table_find(perms, name) + return name_found +} + +// What a handle-less path (`loadfile "x"`, `createfile { .path = … }`) is +// allowed to resolve to. Three states, and which one is in force is decided +// purely by which permission is present (§9): +// +// anypath - anywhere, as it always has: relative to the source file's own +// directory, or absolute, with no containment beyond `io`. +// workdir - contained to ctx.dir, by the same component-by-component walk +// the { .dir, .path } form uses: ".." and an absolute path are +// refused, a symlink anywhere along the way is refused, and "." +// resolves to ctx.dir itself rather than escaping through it. +// denied - refused outright; only the handle forms work. +// +// `anypath` subsumes `workdir`, so holding both is not an error - it just +// means anypath. A host narrows by dropping anypath, which is what hashmake +// does so a build description cannot read outside its own project. +Ambient_Mode :: enum { Denied, Workdir, Anypath } + +ambient_mode :: proc(interp: ^Interpreter) -> Ambient_Mode { + if ctx_has_permission(interp, "anypath") do return .Anypath + if ctx_has_permission(interp, "workdir") do return .Workdir + return .Denied +} + +// ctx.dir's handle, for the contained ambient mode and for use as a `.dir`. +ctx_workdir :: proc(interp: ^Interpreter) -> (^Workdir_Value, bool) { + t, is_table := interp.current_ctx.(^Table_Value) + if !is_table do return nil, false + val, found := table_find(t, "dir") + if !found do return nil, false + wd, is_wd := val.(^Workdir_Value) + return wd, is_wd } // The directory unsandboxed (no .dir given) loadfile/createfile calls @@ -200,20 +272,51 @@ resolve_target :: proc(interp: ^Interpreter, t: ^Table_Value, path_str: string, dir_val, has_dir := table_find(t, "dir") if !has_dir { if dir_required do return {}, "requires a .dir directory handle", false - return Resolved_Path{fd = unsandboxed_dir_fd(interp), basename = path_str, display_dir = unsandboxed_dir_path(interp)}, "", true + return resolve_ambient(interp, path_str) + } + + // ctx.dir (§9) is accepted wherever a directory handle is, exactly as + // ctx.cache is by createfile - it *is* a directory, just not a File. + if wd, is_wd := dir_val.(^Workdir_Value); is_wd { + return resolve_beneath_handle(wd.dir_fd, wd.dir_path, path_str) } dir_file, dir_ok := dir_val.(^File_Value) if !dir_ok || dir_file.kind != .Directory { return {}, ".dir must be a directory File", false } - parent_fd, basename, rerr := resolve_parent_beneath(dir_file.dir_fd, path_str) + return resolve_beneath_handle(dir_file.dir_fd, dir_file.display_path, path_str) +} + +// A path written with no directory handle, resolved per the ambient mode in +// force (§9 - see Ambient_Mode). Both spellings of a handle-less path go +// through here - `loadfile "x"` and `createfile { .path = "x" }` - so the +// containment cannot hold for one and not the other. +resolve_ambient :: proc(interp: ^Interpreter, path_str: string) -> (r: Resolved_Path, err_msg: string, ok: bool) { + switch ambient_mode(interp) { + case .Anypath: + return Resolved_Path{fd = unsandboxed_dir_fd(interp), basename = path_str, display_dir = unsandboxed_dir_path(interp)}, "", true + case .Workdir: + wd, has_wd := ctx_workdir(interp) + if !has_wd do return {}, "workdir is granted but this context has no .dir to contain to", false + return resolve_beneath_handle(wd.dir_fd, wd.dir_path, path_str) + case .Denied: + return {}, "resolving a path without a .dir handle needs the workdir or anypath permission", false + } + return {}, "unknown ambient path mode", false +} + +// The contained resolution both the handle forms and the `workdir` ambient +// mode share - one walk, so the guarantee cannot drift between them. +@(private = "file") +resolve_beneath_handle :: proc(dir_fd: Fs_Fd, display: string, path_str: string) -> (r: Resolved_Path, err_msg: string, ok: bool) { + parent_fd, basename, rerr := resolve_parent_beneath(dir_fd, path_str) if rerr != .None { return {}, fmt.tprintf("path escapes its directory or doesn't exist (%v)", rerr), false } return Resolved_Path{ fd = parent_fd, basename = basename, - needs_close = parent_fd != dir_file.dir_fd, - display_dir = dir_file.display_path, + needs_close = parent_fd != dir_fd, + display_dir = display, }, "", true } @@ -429,7 +532,14 @@ builtin_loadfile :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, if !ctx_allows_io(interp) do return fail(interp, "loadfile: io permission not granted in the current context") if path, is_str := arg.(string); is_str { - return open_and_load(interp, unsandboxed_dir_fd(interp), path, false, display_join(unsandboxed_dir_path(interp), path)) + contained := ambient_mode(interp) == .Workdir + r, err_msg, ok := resolve_ambient(interp, path) + if !ok do return fail(interp, fmt.tprintf("loadfile: %s", err_msg)) + defer close_resolved(r) + // no_follow only where the walk actually contained the path. Under + // `anypath` this form has always followed symlinks, and quietly changing + // that would break programs that legitimately load through one. + return open_and_load(interp, r.fd, r.basename, contained, display_join(r.display_dir, path)) } t, is_table := arg.(^Table_Value) diff --git a/src/cache_format.odin b/src/cache_format.odin index 86fd953..f479025 100644 --- a/src/cache_format.odin +++ b/src/cache_format.odin @@ -189,6 +189,8 @@ write_value :: proc(b: ^strings.Builder, v: Value, w: ^Write_Ctx) -> string { return "a Function cannot be cached - a closure's meaning is its environment" case ^Cache_Value: return "ctx.cache cannot be cached - it is write-only and has no content" + case ^Workdir_Value: + return "ctx.dir cannot be cached - it is a handle, not content" case ^Async_Handle: return "an un-awaited async handle cannot be cached" diff --git a/src/examples_test.odin b/src/examples_test.odin index 19f9428..f804a15 100644 --- a/src/examples_test.odin +++ b/src/examples_test.odin @@ -36,7 +36,7 @@ EXAMPLE_CASES := []Example_Case{ {"cached.hb", "{answer: 42, asking_again_agrees: true, per_argument: {small: 2, large: 11}, file_survives_the_round_trip: true, a_cycle_survives_too: true}"}, {"check-and-invariants.hb", "100"}, {"comparison-and-logic.hb", "{ordered: true, both: true, either: true, mixed: false}"}, - {"context-permissions.hb", "{ambient: {io: nothing}, io_denied: {}, replaced: {}, still_ambient: {io: nothing}}"}, + {"context-permissions.hb", "{ambient: {io: nothing, exec: nothing, anypath: nothing}, io_denied: {exec: nothing, anypath: nothing}, replaced: {}, still_ambient: {io: nothing, exec: nothing, anypath: nothing}}"}, {"cyclic-data.hb", `{round_trip: "Alice", mutual: true, second_hop: "Carol", reordered: 3, same_shape: true}`}, {"files-symlink.hb", `"optiona.txt"`}, {"functions.hb", "121"}, diff --git a/src/hash.odin b/src/hash.odin index cd86318..c84bb8f 100644 --- a/src/hash.odin +++ b/src/hash.odin @@ -71,6 +71,7 @@ TAG_CYCLIC_NODE :: 0x11 // plus what the expression can read out of the dynamic `#arg`/`#self` stacks. // It lives here so the tag space stays in one place. See hash_implicit.odin. TAG_IMPLICIT_REACH :: 0x12 +TAG_WORKDIR :: 0x13 // Byte-lexicographic order on digests. Written as an explicit loop rather than // slice.cmp because a `proc` parameter isn't addressable, so it can't be sliced. @@ -303,6 +304,17 @@ value_digest_walk :: proc(v: Value, w: ^Hash_Walk) -> (Value_Digest, Hash_Fail) // it to the path would hand that back through a side door. return sha256_tagged(TAG_CACHE, nil), HASH_OK + case ^Workdir_Value: + // §9's ctx.dir, and the same reasoning as ctx.cache above with one more + // reason on top. Hashing it as the directory's *contents* (which is what + // a directory File hashes as, §3) would put the whole project tree into + // every `cached` key, so editing any one file would invalidate every + // entry - the exact opposite of what a content-addressed build wants. + // Hashing it as its *path* would bake the checkout location in. A bare + // tag is what is left, and it loses nothing: what a program reads through + // this handle are ordinary Files, and those still hash by content. + return sha256_tagged(TAG_WORKDIR, nil), HASH_OK + case ^Async_Handle: return {}, fail_kind(.Async) diff --git a/src/main.odin b/src/main.odin index 89b297b..4a0e4b8 100644 --- a/src/main.odin +++ b/src/main.odin @@ -183,6 +183,10 @@ eval_source_file :: proc(path_str: string, show_ast: bool, cache_dir: string) -> interp.base_dir_fd = dir_fd interp.has_base_dir = true interp.base_dir_path = absolute_dir_path(dir_path) + // ctx.dir (§9) means "the directory this run is rooted at", which for a + // script is its own directory - the same one a handle-less path resolves + // against - so the two cannot disagree. + ctx_set_workdir(interp.current_ctx, dir_fd, interp.base_dir_path) } defer if dir_errno == .None do fs_close(dir_fd) diff --git a/src/print_value.odin b/src/print_value.odin index 1317c2e..5c22aff 100644 --- a/src/print_value.odin +++ b/src/print_value.odin @@ -107,6 +107,14 @@ write_value :: proc(b: ^strings.Builder, val: Value, st: ^Print_State) { } case ^Cache_Value: fmt.sbprint(b, "") + case ^Workdir_Value: + // §3's display rule, as for a File: show where it actually is. Nothing in + // the language reads that path back out as a value. + if v.dir_path == "" { + fmt.sbprint(b, "") + } else { + fmt.sbprintf(b, "", v.dir_path) + } case ^Async_Handle: // Every real call site awaits the top-level result before formatting it // (see eval_async.odin/await_value) - reaching here un-awaited would diff --git a/src/value.odin b/src/value.odin index 6a5ee67..75fd3ee 100644 --- a/src/value.odin +++ b/src/value.odin @@ -84,6 +84,21 @@ Cache_Value :: struct { opened: bool, } +// SPEC.md §9's ctx.dir: a handle to the directory a program was started in - +// the running source file's own directory, or the process's cwd. Its own type +// rather than a directory File, for one specific reason: §15 puts the whole +// `ctx` into every `cached` key, and a directory File hashes over its contents +// (§3), so a File here would make every cache entry depend on every byte of +// the project tree - touching any file would invalidate all of them. Like +// ctx.cache it therefore hashes as a bare tag (hash.odin). Reading *through* +// it yields ordinary Files that hash by content as usual, so nothing about +// incremental correctness is given up; what is discarded is only the identity +// of the directory itself, which is the same thing ctx.cache discards. +Workdir_Value :: struct { + dir_path: string, // absolute - display-only, same as File_Value's + dir_fd: Fs_Fd, +} + // SPEC.md §10's forward reference: a stand-in for a `let rec` Table entry that // is still being evaluated. Demand-driven evaluation reorders away every // dependency that has a topological order (see eval.odin's Rec_Build); one of @@ -116,6 +131,7 @@ Value :: union { ^Function_Value, ^File_Value, ^Cache_Value, + ^Workdir_Value, ^Async_Handle, // SPEC.md §2 - a fired-but-not-yet-awaited `async` expression; see eval_async.odin ^Forward_Ref_Value, // SPEC.md §10 - a `let rec` cycle's back-edge, only ever unresolved mid-construction } @@ -315,6 +331,9 @@ values_equal_bisim :: proc(a: Value, b: Value, bs: ^Bisim) -> bool { case ^Cache_Value: y, ok := bv.(^Cache_Value) return ok && x == y // reference equality - there's only ever one per context anyway + case ^Workdir_Value: + y, ok := bv.(^Workdir_Value) + return ok && x == y // as ctx.cache: one per context, and no content to compare case ^Async_Handle: // Every real call site awaits an operand before comparing it (see // eval_async.odin) - an un-awaited handle reaching here would be a bug From d17374e4c99a331dbba20c76c645ed4f03093300 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:10:27 +0000 Subject: [PATCH 2/8] Add `exec`: run a program, with inputs and outputs as values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The language could not start a process at all. This adds the one builtin that can, in the shape the rest of the design needs rather than the thinnest one that would work. exec { .cmd, .args, .inputs, .outputs, .stdin } -> { .status, .stdout, .stderr, .outputs = { name -> File } } The command runs in a fresh scratch directory holding nothing but the inputs it was handed, and what comes back are the declared outputs as File values, never paths. That is what makes `cached exec { … }` correct: §15's key excludes anything an expression reads at run time, so a thinner exec that wrote into a directory and let the caller loadfile the result afterwards would answer with the first run's bytes forever. Here an input is a File, a File is its content (§3), so inputs are in the key and a changed source invalidates exactly the steps that consumed it - verified end to end: a warm run hits, and editing a source produces a new digest and a second entry. A non-zero exit is deliberately not a failure - it comes back as .status so a build can `check` it and show .stderr, which is the useful thing to do with a compiler that rejected its input. A command that cannot be started, an input that cannot be written and a declared output that is not there are all fatal like any other builtin (§16). Gated by a new ctx.permissions.exec rather than riding on `io`: running an arbitrary program is strictly more authority than reading a file. WASI has no process spawn, and core:os's backend answers .Unsupported, so a wasm build says so rather than reporting a missing program - no per-target split needed. Six helpers in cache_store.odin become package-visible: materialising a File into a directory, reading one back out, and removing a tree are exactly what `cached` already does, and sharing them is what keeps a File round-tripping identically through a build step and through the cache. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- src/builtins_build.odin | 199 ++++++++++++++++++++++++++++++++++++++++ src/cache_store.odin | 35 +++---- src/eval.odin | 5 +- 3 files changed, 221 insertions(+), 18 deletions(-) diff --git a/src/builtins_build.odin b/src/builtins_build.odin index 40ee989..2d3c037 100644 --- a/src/builtins_build.odin +++ b/src/builtins_build.odin @@ -7,6 +7,8 @@ package hashedbuild // Function values pre-bound in the global scope, so adding one never touches // the grammar. +import "core:fmt" +import "core:os" import "core:slice" import "core:strings" import "core:unicode/utf8" @@ -206,6 +208,202 @@ builtin_listdir :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, b return out, true } +// ---- exec --------------------------------------------------------------------- + +// exec { .cmd, .args, .inputs, .outputs, .stdin } +// -> { .status, .stdout, .stderr, .outputs = { name -> File } } +// +// Runs a program, and is the first builtin here that is designed *around* the +// cache rather than beside it. The shape is the whole point: the command runs +// in a fresh scratch directory that holds nothing but the `.inputs` it was +// given, and what comes back out are the `.outputs` **as values**, never +// paths. That is what makes `cached exec { … }` correct - §15's key excludes +// anything an expression reads at run time, so a thinner exec that wrote into +// a directory and let the caller `loadfile` the result afterwards would cache +// the first run's answer forever. Here every input is a File, a File is its +// content (§3), so the inputs are *in* the key and a changed source +// invalidates exactly the steps that consumed it. +// +// .cmd Utf8, looked up on PATH. +// .args a sequence of Utf8; default none. +// .inputs either a directory File (its contents become the scratch root) or +// a Table of relative-name -> File. Default: an empty scratch. +// .outputs a sequence of Utf8 relative paths to collect afterwards. +// .stdin optional Utf8 fed to the program. +// +// **A non-zero exit is not a failure here.** It comes back as `.status`, so a +// build can `check(r.status == 0, …)` and show `.stderr` - which is the useful +// thing to do with a compiler that rejected its input. Everything else is +// fatal like any other builtin (§16): a command that could not be started, an +// input that could not be written, a declared output that is not there. +// +// Gated by `ctx.permissions.exec`, checked live at the call site. Running an +// arbitrary program is strictly more authority than reading a file, so it does +// not ride on `io`. +// +// Worth being plain about in the docs: this contains what is *handed to* a +// build step, not what that step then does. A compiler started here is an +// ordinary process and can read whatever the user running it can. +@(private = "file") +builtin_exec :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, bool) { + if !ctx_has_permission(interp, "exec") { + return fail(interp, "exec: exec permission not granted in the current context") + } + if !ctx_allows_io(interp) { + return fail(interp, "exec: io permission not granted in the current context") + } + + t, is_table := arg.(^Table_Value) + if !is_table do return fail(interp, "exec expects a { .cmd, .args, .inputs, .outputs } Table") + + cmd_val, has_cmd := table_find(t, "cmd") + cmd, cmd_ok := cmd_val.(string) + if !has_cmd || !cmd_ok do return fail(interp, "exec needs a Utf8 .cmd") + + // argv[0] is the command itself, as every exec-family call expects. + command := make([dynamic]string, 0, 8, context.temp_allocator) + append(&command, cmd) + if args_val, has_args := table_find(t, "args"); has_args { + args_t, args_is_table := args_val.(^Table_Value) + if !args_is_table do return fail(interp, "exec's .args must be a Table of Utf8") + for idx in fold_order(args_t) { + a, a_ok := args_t.entries[idx].value.(string) + if !a_ok do return fail(interp, "exec's .args must all be Utf8") + append(&command, a) + } + } + + scratch_path, scratch_fd, made := exec_make_scratch(interp) + if !made do return fail(interp, "exec: could not create a scratch directory to run in") + defer fs_close(scratch_fd) + defer exec_remove_scratch(interp, scratch_path) + + if inputs_val, has_inputs := table_find(t, "inputs"); has_inputs { + if msg := exec_materialize(scratch_fd, inputs_val); msg != "" { + return fail(interp, fmt.tprintf("exec: %s", msg)) + } + } + + desc := os.Process_Desc{working_dir = scratch_path, command = command[:]} + + // .stdin is handed over as a real file inside the scratch rather than a + // pipe: the program is waited on to completion anyway, so there is nothing + // a pipe would buy beyond a second failure mode to get wrong. + stdin_file: ^os.File + if stdin_val, has_stdin := table_find(t, "stdin"); has_stdin { + text, text_ok := stdin_val.(string) + if !text_ok do return fail(interp, "exec's .stdin must be a Utf8") + if msg := write_bytes(scratch_fd, EXEC_STDIN_NAME, transmute([]u8)text, false); msg != "" { + return fail(interp, fmt.tprintf("exec: %s", msg)) + } + f, ferr := os.open(strings.concatenate({scratch_path, "/", EXEC_STDIN_NAME}, context.temp_allocator)) + if ferr != nil do return fail(interp, "exec: could not open .stdin for the program to read") + stdin_file = f + desc.stdin = f + } + defer if stdin_file != nil do os.close(stdin_file) + + state, stdout_bytes, stderr_bytes, run_err := os.process_exec(desc, context.allocator) + if run_err != nil { + // WASI has no way to start a process at all - core:os's backend answers + // .Unsupported - so this is also where a wasm build lands, and it says so + // rather than reporting a missing program. + if run_err == .Unsupported { + return fail(interp, "exec: running a program is not available on this target") + } + return fail(interp, fmt.tprintf("exec: could not run %s (%v)", cmd, run_err)) + } + + outputs := new(Table_Value) + if outs_val, has_outs := table_find(t, "outputs"); has_outs { + outs_t, outs_is_table := outs_val.(^Table_Value) + if !outs_is_table do return fail(interp, "exec's .outputs must be a Table of Utf8") + for idx in fold_order(outs_t) { + name, name_ok := outs_t.entries[idx].value.(string) + if !name_ok do return fail(interp, "exec's .outputs must all be Utf8") + is_dir, stat_err := fs_stat_is_dir_at(scratch_fd, name, true) + if stat_err != .None { + return fail(interp, fmt.tprintf("exec: %s declared no output named %s", cmd, name)) + } + if is_dir { + return fail(interp, fmt.tprintf("exec: .outputs names %s, which is a directory - only regular files can be collected today", name)) + } + fv, msg := open_as_file_value(scratch_fd, name, false, display_join(scratch_path, name)) + if msg != "" do return fail(interp, fmt.tprintf("exec: %s", msg)) + append(&outputs.entries, Table_Entry_Value{key = strings.clone(name), value = fv}) + } + } + + result := new(Table_Value) + append(&result.entries, Table_Entry_Value{key = "status", value = i64(state.exit_code)}) + append(&result.entries, Table_Entry_Value{key = "stdout", value = exec_text(stdout_bytes)}) + append(&result.entries, Table_Entry_Value{key = "stderr", value = exec_text(stderr_bytes)}) + append(&result.entries, Table_Entry_Value{key = "outputs", value = outputs}) + return result, true +} + +EXEC_STDIN_NAME :: ".hb-exec-stdin" + +// A program's output is whatever bytes it chose to write, which need not be +// text at all. Utf8 is the only shape the language has for it, so invalid +// bytes are replaced rather than failing the build - a compiler that emitted +// one stray byte on stderr should not take the whole run down, and the +// diagnostic is still what the user needs to read. +@(private = "file") +exec_text :: proc(raw: []u8) -> Value { + if utf8.valid_string(string(raw)) do return strings.clone(string(raw)) + b := strings.builder_make() + for r in string(raw) do strings.write_rune(&b, r == utf8.RUNE_ERROR ? '?' : r) + return strings.to_string(b) +} + +// The scratch lives under the cache directory: it is already the place this +// language keeps working files, `--cache-dir` already points it somewhere +// writable, and putting it there keeps build droppings out of the project. +@(private = "file") +exec_make_scratch :: proc(interp: ^Interpreter) -> (path: string, fd: Fs_Fd, ok: bool) { + cache, has_cache := cache_of_ctx(interp.current_ctx) + if !has_cache do return "", FS_INVALID_FD, false + if ensure_cache_dir_open(cache) != .None do return "", FS_INVALID_FD, false + + name, made := make_temp_dir(cache.dir_fd, "exec") + if !made do return "", FS_INVALID_FD, false + dir_fd, err := fs_open_dir_at(cache.dir_fd, name, true) + if err != .None do return "", FS_INVALID_FD, false + return strings.concatenate({cache.dir_path, "/", name}), dir_fd, true +} + +@(private = "file") +exec_remove_scratch :: proc(interp: ^Interpreter, scratch_path: string) { + cache, has_cache := cache_of_ctx(interp.current_ctx) + if !has_cache do return + idx := strings.last_index_byte(scratch_path, '/') + if idx < 0 do return + remove_tree_at(cache.dir_fd, scratch_path[idx + 1:]) +} + +// `.inputs` is either one directory File - whose contents become the scratch +// root, which is what lets `listdir` names be used as arguments verbatim - or +// a Table placing each File at its own key. +@(private = "file") +exec_materialize :: proc(scratch_fd: Fs_Fd, inputs: Value) -> string { + #partial switch v in inputs { + case ^File_Value: + if v.kind != .Directory do return ".inputs given as a single File must be a directory" + return copy_tree(v.dir_fd, scratch_fd) + case ^Table_Value: + for entry in v.entries { + name, name_ok := entry.key.(string) + if !name_ok do return ".inputs keys must be Utf8 names" + fv, is_file := entry.value.(^File_Value) + if !is_file do return fmt.tprintf(".inputs entry %s is not a File", name) + if msg := write_file_value(scratch_fd, name, fv); msg != "" do return msg + } + return "" + } + return ".inputs must be a directory File or a Table of name -> File" +} + // ---- registration ------------------------------------------------------------- // Called by make_global_env (builtins_fs.odin) alongside §16's six. Kept here @@ -215,6 +413,7 @@ builtin_listdir :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, b bind_build_builtins :: proc(env: ^Env) { env_bind(env, "fold", new_build_native("fold", builtin_fold)) env_bind(env, "listdir", new_build_native("listdir", builtin_listdir)) + env_bind(env, "exec", new_build_native("exec", builtin_exec)) env_bind(env, "textlen", new_build_native("textlen", builtin_textlen)) env_bind(env, "textslice", new_build_native("textslice", builtin_textslice)) } diff --git a/src/cache_store.odin b/src/cache_store.odin index b5201b6..54db84b 100644 --- a/src/cache_store.odin +++ b/src/cache_store.odin @@ -107,7 +107,7 @@ load_file_entry :: proc(cache: ^Cache_Value, name: string, is_dir: bool) -> (Val load_text_entry :: proc(cache: ^Cache_Value, dir_name: string) -> (Value, bool, string) { entry_fd, open_err := fs_open_dir_at(cache.dir_fd, dir_name, true) if open_err != .None { - return nil, false, fmt.tprintf("could not open cache entry %s (%v)", dir_name, open_err) + return nil, false, fmt.tprintf("could not open %s (%v)", dir_name, open_err) } defer fs_close(entry_fd) @@ -162,29 +162,28 @@ resolve_entry_file :: proc(entry_name: string, is_dir: bool, userdata: rawptr) - // One File value for a name inside the store, of the kind the text said it // was. A mismatch is reported rather than followed: it means the entry and // what is on disk have drifted apart. -@(private = "file") open_as_file_value :: proc(dir_fd: Fs_Fd, name: string, is_dir: bool, display: string) -> (^File_Value, string) { actually_dir, stat_err := fs_stat_is_dir_at(dir_fd, name, true) - if stat_err != .None do return nil, fmt.tprintf("could not read cache entry %s (%v)", name, stat_err) + if stat_err != .None do return nil, fmt.tprintf("could not read %s (%v)", name, stat_err) if actually_dir != is_dir { - return nil, fmt.tprintf("cache entry %s is not the kind of File the entry says it is", name) + return nil, fmt.tprintf("%s is not the kind of File it was expected to be", name) } fv := new(File_Value) fv.display_path = display if is_dir { fd, err := fs_open_dir_at(dir_fd, name, true) - if err != .None do return nil, fmt.tprintf("could not open cache entry %s (%v)", name, err) + if err != .None do return nil, fmt.tprintf("could not open %s (%v)", name, err) fv.kind = .Directory fv.dir_fd = fd return fv, "" } fd, err := fs_open_read_at(dir_fd, name, true) - if err != .None do return nil, fmt.tprintf("could not open cache entry %s (%v)", name, err) + if err != .None do return nil, fmt.tprintf("could not open %s (%v)", name, err) defer fs_close(fd) content, read_err := fs_read_all(fd) - if read_err != .None do return nil, fmt.tprintf("could not read cache entry %s (%v)", name, read_err) + if read_err != .None do return nil, fmt.tprintf("could not read %s (%v)", name, read_err) fv.kind = .Regular fv.content = content return fv, "" @@ -264,7 +263,6 @@ cache_store :: proc(cache: ^Cache_Value, key_name: string, v: Value, interp: ^In // source of randomness and no process id - neither of which every target here // has. The cap only has to exceed the number of runs racing on one key at // once; anything near it means something else is wrong. -@(private = "file") make_temp_dir :: proc(dir_fd: Fs_Fd, final_name: string) -> (string, bool) { for i in 0 ..< 64 { name := fmt.tprintf("%s.tmp%d", final_name, i) @@ -328,16 +326,22 @@ collect_files :: proc(v: Value, out: ^[dynamic]^File_Value, seen: ^map[^Table_Va } // ---- writing a File out ----------------------------------------------------- +// +// The four procs below (write_file_value, open_as_file_value, make_temp_dir, +// remove_tree_at, copy_tree and write_bytes) are package-visible rather than file-private because `exec` +// (builtins_build.odin) needs exactly the same four operations: it materialises +// input Files into a scratch directory, reads the declared outputs back out as +// Files, and removes the scratch afterwards. Sharing them is what keeps a File +// round-tripping identically through a build step and through the cache. -@(private = "file") write_file_value :: proc(dir_fd: Fs_Fd, name: string, fv: ^File_Value) -> string { if fv.kind == .Regular do return write_bytes(dir_fd, name, fv.content, false) if err := fs_mkdir_at(dir_fd, name); err != .None { - return fmt.tprintf("could not create %s in the cache (%v)", name, err) + return fmt.tprintf("could not create %s (%v)", name, err) } dst, open_err := fs_open_dir_at(dir_fd, name, true) - if open_err != .None do return fmt.tprintf("could not open %s in the cache (%v)", name, open_err) + if open_err != .None do return fmt.tprintf("could not open %s (%v)", name, open_err) defer fs_close(dst) return copy_tree(fv.dir_fd, dst) } @@ -352,7 +356,6 @@ write_file_value :: proc(dir_fd: Fs_Fd, name: string, fv: ^File_Value) -> string // so copying it changes no digest; it is copied because caching a build output // and getting back something you can no longer run would be a poor trade for a // build system. Nothing else about a directory is copied. -@(private = "file") copy_tree :: proc(src_fd: Fs_Fd, dst_fd: Fs_Fd) -> string { entries, list_err := fs_list_entries_at(src_fd, context.temp_allocator) if list_err != .None do return fmt.tprintf("could not read a directory being cached (%v)", list_err) @@ -369,13 +372,13 @@ copy_tree :: proc(src_fd: Fs_Fd, dst_fd: Fs_Fd) -> string { case .Directory: if err := fs_mkdir_at(dst_fd, entry.name); err != .None { - return fmt.tprintf("could not create %s in the cache (%v)", entry.name, err) + return fmt.tprintf("could not create %s (%v)", entry.name, err) } child_src, src_err := fs_open_dir_at(src_fd, entry.name, true) if src_err != .None do return fmt.tprintf("could not open %s (%v)", entry.name, src_err) defer fs_close(child_src) child_dst, dst_err := fs_open_dir_at(dst_fd, entry.name, true) - if dst_err != .None do return fmt.tprintf("could not open %s in the cache (%v)", entry.name, dst_err) + if dst_err != .None do return fmt.tprintf("could not open %s (%v)", entry.name, dst_err) defer fs_close(child_dst) if msg := copy_tree(child_src, child_dst); msg != "" do return msg @@ -398,10 +401,9 @@ copy_tree :: proc(src_fd: Fs_Fd, dst_fd: Fs_Fd) -> string { return "" } -@(private = "file") write_bytes :: proc(dir_fd: Fs_Fd, name: string, data: []u8, executable: bool) -> string { fd, err := fs_create_exclusive_at(dir_fd, name) - if err != .None do return fmt.tprintf("could not create %s in the cache (%v)", name, err) + if err != .None do return fmt.tprintf("could not create %s (%v)", name, err) werr := fs_write_all(fd, data) fs_close(fd) if werr != .None do return fmt.tprintf("could not write %s in the cache (%v)", name, werr) @@ -415,7 +417,6 @@ write_bytes :: proc(dir_fd: Fs_Fd, name: string, data: []u8, executable: bool) - // the caller is already on its way to reporting something else (or to using // the entry another run published). Leaving a `.tmpN` behind is untidy, not // wrong - the next run picks a different N. -@(private = "file") remove_tree_at :: proc(parent: Fs_Fd, name: string) { if fs_unlink_at(parent, name) == .None do return if fd, err := fs_open_dir_at(parent, name, true); err == .None { diff --git a/src/eval.odin b/src/eval.odin index 92d8c0b..0e3f3f6 100644 --- a/src/eval.odin +++ b/src/eval.odin @@ -1228,7 +1228,10 @@ top_of :: proc(stack: []Value, count: int) -> []Value { // `ctx.cache`, if this context has one. §9 lets a program build a context by // hand, and one built without carrying `.cache` over simply hasn't got a cache // to use - so this is a question, not an assertion. -@(private = "file") +// +// Package-visible because `exec` (builtins_build.odin) asks the same question: +// its scratch directory lives under the cache's, so a run with no cache has +// nowhere to put one. cache_of_ctx :: proc(ctx: Value) -> (^Cache_Value, bool) { t, is_table := ctx.(^Table_Value) if !is_table do return nil, false From 1855fc312386825692b362c75e5efc93bbc953b7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:17:29 +0000 Subject: [PATCH 3/8] Add the hashmake CLI, and the cJSON build it drives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hashmake reads a hashmake.hb, evaluates it to a dependency graph, orders it, refuses cycles, and calls each node with what it asked for. That is all it does: it has no cache of its own, because a node opts in by writing `cached` around its own work and §15 then gives correct incremental rebuilds - an input is a File and a File is its content, so there are no timestamps anywhere in this program. The demo builds DaveGamble/cJSON, vendored as a submodule and pinned. Nothing in hashmake.hb names a C file: the sources are found by listing the checkout and filtering on a suffix with `endswith`, itself written in HashedBuild on top of textlen/textslice. That yields one compile node per source, a link node fanning the three objects in, and a run node that produces no artifact. Measured on this checkout: a cold build compiles three sources; a warm run is 27ms with nothing rebuilt; editing one source adds exactly two cache entries - that object and the link, not the two untouched objects; and *restoring* the file returns to the original entries at 27ms, which a timestamp-based tool could not do. The graph rules are enforced rather than documented: a cycle is reported in full (a -> b -> c -> a) before anything is built, a .needs naming a target that does not exist is a clean error, and nothing may depend on a target that produced no artifact. By default the build file is contained to its own directory (ctx.dir plus `workdir`), so it cannot read the rest of the machine; --allow-any-path opts back out. Three supporting changes in src: - eval_source_file is factored into eval_source_file_run, which hands the value to a callback while the AST is still alive. A Function value points into the AST, so returning one past ast_destroy would return a dangling reference - and calling functions out of the graph is hashmake's whole job. eval_source_file is now a thin wrapper, so every existing caller is unchanged. - chperm's edit is split from its Function wrapper, so a host can narrow a context by the same means a program would. - A regular File carries its executable bit. §3 hashes no permission bit, so no digest changes; it is carried because a linked binary that stopped being executable by passing through the cache could not then be run, which is exactly what the run node does. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- .gitignore | 2 + .gitmodules | 3 + examples/hashmake/hashmake.hb | 107 +++++++++ examples/hashmake/vendor/cJSON | 1 + src/builtins_build.odin | 13 ++ src/builtins_fs.odin | 18 +- src/cache_store.odin | 17 +- src/main.odin | 61 ++++- src/value.odin | 10 + tools/hashmake/hashmake.odin | 391 +++++++++++++++++++++++++++++++++ 10 files changed, 611 insertions(+), 12 deletions(-) create mode 100644 .gitmodules create mode 100644 examples/hashmake/hashmake.hb create mode 160000 examples/hashmake/vendor/cJSON create mode 100644 tools/hashmake/hashmake.odin diff --git a/.gitignore b/.gitignore index 38d2442..e7f2a19 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ __pycache__/ # the real site, and by scripts/ for a local one. Committing them coupled every # tracked file to a build product. docs/repo-files.json +/hashmake +/hashmake.exe diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d6bb01f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "examples/hashmake/vendor/cJSON"] + path = examples/hashmake/vendor/cJSON + url = https://github.com/DaveGamble/cJSON diff --git a/examples/hashmake/hashmake.hb b/examples/hashmake/hashmake.hb new file mode 100644 index 0000000..43d3b67 --- /dev/null +++ b/examples/hashmake/hashmake.hb @@ -0,0 +1,107 @@ +// A hashmake build: the dependency graph for a real, multi-source C program. +// +// `hashmake` evaluates this file and gets back a graph. Every node is an +// ordinary HashedBuild function from its prerequisites to the artifact it +// builds, and `.needs` maps a local alias to the name of the target that +// produces it - so a build function receives exactly `{ alias -> artifact }` +// and never a path. +// +// Nothing here names a C file. The sources are discovered by listing the +// vendored cJSON checkout, so adding one to that tree adds a node to this +// graph with nothing edited below. +// +// The incremental behaviour is not this file's doing and not hashmake's: each +// build wraps its `exec` in `cached` (SPEC.md §15), whose key is the code plus +// the *values* it reads. An input is a File and a File is its content (§3), so +// editing one source changes that object's key and the link's, and nothing +// else. There are no timestamps anywhere in this. + +let endswith (let a; + let t a.text; let s a.suffix; + (textlen t) >= (textlen s) + and (textslice { .text = t, .start = (textlen t) - (textlen s) + 1, .count = textlen s }) == s); + +// The language has no loops; `fold` is the one traversal, so the list helpers +// a build needs are written here rather than being builtins of their own. +let seq_len (let t; fold { .table = t, .init = 0, .step = (let s; s.acc + 1) }); +let append (let a; a.seq concat { [(seq_len a.seq) + 1] = a.item }); +let concat_seq (let a; + fold { .table = a.b, .init = a.a, .step = (let s; append { .seq = s.acc, .item = s.value }) }); + +// Every name in the checkout ending in , as { name -> File }. +let cjson loadfile "vendor/cJSON"; +let files_matching (let suffix; + fold { + .table = listdir cjson, + .init = empty, + .step = (let s; (endswith { .text = s.value, .suffix = suffix }) + then (s.acc concat { [s.value] = loadfile { .dir = cjson, .path = s.value } }) + else s.acc), + }); + +let sources files_matching ".c"; +let headers files_matching ".h"; + +// "cJSON.c" -> "cJSON.o" +let object_name (let n; (textslice { .text = n, .start = 1, .count = (textlen n) - 2 }) concat ".o"); + +// One compile target per discovered source. Each is given only its own source +// and the headers - not the whole tree - so editing one .c rebuilds one .o. +let compile_targets fold { + .table = sources, + .init = empty, + .step = (let s; + let name s.key; + let obj object_name name; + s.acc concat { + [obj] = { + .needs = empty, + .build = (let prereqs; cached ( + let r exec { + .cmd = "clang", + .args = { "-c", "-O2", "-std=c89", "-I.", name, "-o", obj }, + .inputs = headers concat { [name] = s.value }, + .outputs = { obj }, + }; + check(r.status == 0, "clang failed to compile " concat name) r.outputs[obj])), + }, + }), +}; + +// The link needs every object; the alias each arrives under is its own name. +let link_needs fold { + .table = compile_targets, + .init = empty, + .step = (let s; s.acc concat { [s.key] = s.key }), +}; + +{ + .default = "run", + .targets = compile_targets concat { + .link = { + .needs = link_needs, + .build = (let objects; cached ( + let r exec { + .cmd = "clang", + .args = concat_seq { + .a = fold { .table = objects, .init = empty, .step = (let s; append { .seq = s.acc, .item = s.key }) }, + .b = { "-o", "cjson-demo", "-lm" }, + }, + .inputs = objects, + .outputs = { "cjson-demo" }, + }; + check(r.status == 0, "clang failed to link cjson-demo") r.outputs["cjson-demo"])), + }, + + // The run target produces **no artifact**: it answers with the program's + // own output as text, which is there so hashmake can show it, not for + // anything to consume. hashmake enforces the difference - a target may only + // be depended on if it produced a File, so nothing can be built "from" this. + .run = { + .needs = { .app = "link" }, + .build = (let prereqs; + let r exec { .cmd = "./cjson-demo", .inputs = { ["cjson-demo"] = prereqs.app } }; + check(r.status == 0, "cjson-demo exited non-zero") r.stdout), + }, + }, +} diff --git a/examples/hashmake/vendor/cJSON b/examples/hashmake/vendor/cJSON new file mode 160000 index 0000000..fb16e5c --- /dev/null +++ b/examples/hashmake/vendor/cJSON @@ -0,0 +1 @@ +Subproject commit fb16e5cf358798aabb049655975cde8427101056 diff --git a/src/builtins_build.odin b/src/builtins_build.odin index 2d3c037..dba7c1d 100644 --- a/src/builtins_build.odin +++ b/src/builtins_build.odin @@ -284,6 +284,18 @@ builtin_exec :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, bool } } + // A .cmd naming a path ("./cjson-demo") means "in the directory this run + // happens in" - which is the only directory the program can see anyway. It + // has to be made absolute here: the exec-family lookup checks the name + // against the *parent's* working directory, before the child ever changes + // into the scratch, so a relative one would be looked for in the wrong + // place. A bare name ("clang") is left alone and found on PATH as usual. + if strings.index_byte(cmd, '/') >= 0 && !is_absolute_path(cmd) { + trimmed := cmd + if strings.has_prefix(trimmed, "./") do trimmed = trimmed[2:] + command[0] = strings.concatenate({scratch_path, "/", trimmed}, context.temp_allocator) + } + desc := os.Process_Desc{working_dir = scratch_path, command = command[:]} // .stdin is handed over as a real file inside the scratch rather than a @@ -344,6 +356,7 @@ builtin_exec :: proc(interp: ^Interpreter, _: Value, arg: Value) -> (Value, bool EXEC_STDIN_NAME :: ".hb-exec-stdin" + // A program's output is whatever bytes it chose to write, which need not be // text at all. Utf8 is the only shape the language has for it, so invalid // bytes are replaced rather than failing the build - a compiler that emitted diff --git a/src/builtins_fs.odin b/src/builtins_fs.odin index 2551d01..e18b1ea 100644 --- a/src/builtins_fs.odin +++ b/src/builtins_fs.odin @@ -132,6 +132,15 @@ ctx_allows_io :: proc(interp: ^Interpreter) -> bool { return ctx_has_permission(interp, "io") } +// A copy of `ctx` with `anypath` dropped and `workdir` granted - i.e. one +// whose handle-less paths are contained to ctx.dir. Built out of the same +// chperm machinery a program would use, so there is nothing a host can reach +// here that a HashedBuild expression could not do to itself. +ctx_contained_to_workdir :: proc(ctx: Value) -> Value { + narrowed := apply_chperm_pair(ctx, "anypath", false) + return apply_chperm_pair(narrowed, "workdir", true) +} + // §9's permissions are a Table used as a set: granted iff the key is *present*, // whatever its (always-`nothing`) value. Read live off the call site, never // captured - that is what lets a wrapping `withctx`/`chctx` restrict a builtin @@ -765,9 +774,12 @@ apply_chperm :: proc(interp: ^Interpreter, closure: Value, old_ctx: Value) -> (V closure_t, _ := closure.(^Table_Value) name_val, _ := table_find(closure_t, "name") enabled_val, _ := table_find(closure_t, "enabled") - name_str := name_val.(string) - enabled := enabled_val.(bool) + return apply_chperm_pair(old_ctx, name_val.(string), enabled_val.(bool)), true +} +// The edit itself, without the Function wrapper - so a host (main.odin's +// Run_Options) can make the same change directly. +apply_chperm_pair :: proc(old_ctx: Value, name_str: string, enabled: bool) -> Value { old_t, old_is_table := old_ctx.(^Table_Value) old_perms: ^Table_Value if old_is_table { @@ -795,5 +807,5 @@ apply_chperm :: proc(interp: ^Interpreter, closure: Value, old_ctx: Value) -> (V } } append(&new_ctx.entries, Table_Entry_Value{key = "permissions", value = new_perms}) - return new_ctx, true + return new_ctx } diff --git a/src/cache_store.odin b/src/cache_store.odin index 54db84b..c95e944 100644 --- a/src/cache_store.odin +++ b/src/cache_store.odin @@ -186,9 +186,24 @@ open_as_file_value :: proc(dir_fd: Fs_Fd, name: string, is_dir: bool, display: s if read_err != .None do return nil, fmt.tprintf("could not read %s (%v)", name, read_err) fv.kind = .Regular fv.content = content + fv.is_executable = is_executable_at(dir_fd, name) return fv, "" } +// Whether a name in an open directory is executable. Read off the directory +// listing, which is the one place the fs layer reports the bit (Fs_Dir_Entry); +// there is no single-name stat for it. §3 hashes no permission bit, so this +// changes no digest - it is carried so that a program does not stop being one +// by passing through the cache, or through `exec`'s scratch directory. +is_executable_at :: proc(dir_fd: Fs_Fd, name: string) -> bool { + entries, err := fs_list_entries_at(dir_fd, context.temp_allocator) + if err != .None do return false + for entry in entries { + if entry.name == name do return entry.is_executable + } + return false +} + // ---- store ------------------------------------------------------------------ // Writes `v` under `key_name` and returns what a lookup of that key now @@ -335,7 +350,7 @@ collect_files :: proc(v: Value, out: ^[dynamic]^File_Value, seen: ^map[^Table_Va // round-tripping identically through a build step and through the cache. write_file_value :: proc(dir_fd: Fs_Fd, name: string, fv: ^File_Value) -> string { - if fv.kind == .Regular do return write_bytes(dir_fd, name, fv.content, false) + if fv.kind == .Regular do return write_bytes(dir_fd, name, fv.content, fv.is_executable) if err := fs_mkdir_at(dir_fd, name); err != .None { return fmt.tprintf("could not create %s (%v)", name, err) diff --git a/src/main.odin b/src/main.odin index 4a0e4b8..743ffba 100644 --- a/src/main.odin +++ b/src/main.odin @@ -152,10 +152,32 @@ run_file :: proc(path_str: string, show_ast: bool, cache_dir: string) { // grants `io` by default (§9) - the same environment the REPL and the live // editor evaluate under. Split out of run_file so the examples can be run // end to end as tests (examples_test.odin). -eval_source_file :: proc(path_str: string, show_ast: bool, cache_dir: string) -> (formatted: string, err_msg: string, ok: bool) { +// How a run of a source file is set up, beyond the path itself. Split out so +// hashmake (tools/hashmake) can ask for the same evaluation under a narrowed +// context without duplicating any of the plumbing below. +Run_Options :: struct { + show_ast: bool, + cache_dir: string, // "" resolves the XDG default (SPEC.md §9/§16) + // Drop `anypath` and grant `workdir`, so handle-less paths in the program + // are contained to ctx.dir - the source file's own directory. What hashmake + // uses so a build description cannot read outside its project. + contain_to_workdir: bool, +} + +// What to do with the value a source file evaluated to, while the AST and the +// interpreter that produced it are still alive. A callback rather than a +// return, because a Function value points into the AST (Function_Value.body is +// a Node_Idx) - handing one back past ast_destroy would hand back a dangling +// reference, and hashmake's whole job is calling functions out of the graph it +// just evaluated. +Source_Value_Proc :: proc(interp: ^Interpreter, value: Value, userdata: rawptr) -> bool + +// Runs a source file and hands the result to `on_value`. Everything +// eval_source_file used to do inline, so the two cannot drift. +eval_source_file_run :: proc(path_str: string, opts: Run_Options, on_value: Source_Value_Proc, userdata: rawptr) -> (err_msg: string, ok: bool) { source, errno := load_source_file(path_str) if errno != .None { - return "", fmt.tprintf("could not read %s (%v)", path_str, errno), false + return fmt.tprintf("could not read %s (%v)", path_str, errno), false } defer free_source_file(source) @@ -163,15 +185,15 @@ eval_source_file :: proc(path_str: string, show_ast: bool, cache_dir: string) -> ast := parse(source, ast_t{}) defer ast_destroy(&ast) - if show_ast { + if opts.show_ast { print_ast(&ast, src) } if len(ast.errors) > 0 { - if !show_ast do print_ast(&ast, src) // errors always need the tree to make sense of them - return "", "", false + if !opts.show_ast do print_ast(&ast, src) // errors always need the tree to make sense of them + return "", false } - interp := Interpreter{ast = &ast, src = src, current_ctx = make_root_context(cache_dir)} + interp := Interpreter{ast = &ast, src = src, current_ctx = make_root_context(opts.cache_dir)} // Unsandboxed loadfile/createfile calls (no .dir given) resolve relative // paths against the source file's own directory, not the process's cwd - // filepath.dir returns "" (not ".") for a bare filename with no directory @@ -190,14 +212,37 @@ eval_source_file :: proc(path_str: string, show_ast: bool, cache_dir: string) -> } defer if dir_errno == .None do fs_close(dir_fd) + if opts.contain_to_workdir { + interp.current_ctx = ctx_contained_to_workdir(interp.current_ctx) + } + // eval_program, not eval: it also resolves a bare top-level `async` (§2) // and waits for any task nothing awaited, so the program can't exit with // work half-done (eval_async.odin). val, eval_ok := eval_program(&interp, ast.root, make_global_env()) if !eval_ok { - return "", strings.clone(interp.error_message), false + return strings.clone(interp.error_message), false } - return format_value(val), "", true + if !on_value(&interp, val, userdata) do return "", false + return "", true +} + +@(private = "file") +Format_Result :: struct { out: string } + +@(private = "file") +format_the_value :: proc(interp: ^Interpreter, value: Value, userdata: rawptr) -> bool { + (^Format_Result)(userdata).out = format_value(value) + return true +} + +// Runs a source file and returns its value already formatted. Exists so the +// examples can be run end to end as tests. +eval_source_file :: proc(path_str: string, show_ast: bool, cache_dir: string) -> (formatted: string, err_msg: string, ok: bool) { + result := Format_Result{} + err_msg, ok = eval_source_file_run(path_str, Run_Options{show_ast = show_ast, cache_dir = cache_dir}, format_the_value, &result) + if !ok do return "", err_msg, false + return result.out, "", true } // A real read-eval-print loop: accumulates lines into a snippet, evaluates it diff --git a/src/value.odin b/src/value.odin index 75fd3ee..46ee8c2 100644 --- a/src/value.odin +++ b/src/value.odin @@ -71,6 +71,16 @@ File_Value :: struct { // it is then - which is how a build observes a change.) dir_digest: Value_Digest, dir_digest_known: bool, + + // Regular only, and **not part of any hash** - exactly as Fs_Dir_Entry's + // field of the same name isn't (§3 carries no permission bit, and hash.odin + // says why). It is here for the same reason cache_store.odin already puts + // the bit back when it copies a tree: a build step that produced a program + // and got back something that could no longer be run would be a poor trade. + // `exec` sets it on the outputs it collects, and honours it on the inputs it + // materialises, which is what lets one step's linked binary be the next + // step's command. + is_executable: bool, } // SPEC.md §9's ctx.cache: a write-only, content-addressed blob store rooted diff --git a/tools/hashmake/hashmake.odin b/tools/hashmake/hashmake.odin new file mode 100644 index 0000000..df98891 --- /dev/null +++ b/tools/hashmake/hashmake.odin @@ -0,0 +1,391 @@ +package main + +// hashmake - a build tool whose build files are HashedBuild programs. +// +// `hashmake.hb` evaluates to a graph. Every node is an ordinary function from +// its prerequisites to the artifact it builds, and `.needs` maps a local alias +// to the name of the target producing it, so a build function is handed +// exactly `{ alias -> artifact }` and never a path. +// +// What this tool does is deliberately small: find the build file, evaluate it, +// order the graph, refuse cycles, and call each node with what it asked for. +// It does no caching of its own - a node opts in by writing `cached` around +// its own work, and the language's content-addressed cache (SPEC.md §15) then +// gives correct incremental rebuilds, because an input is a File and a File is +// its content (§3). There are no timestamps in this program. +// +// Usage: odin build tools/hashmake -out:hashmake + +import "core:fmt" +import "core:os" +import "core:strings" +import hb "../../src" + +VERSION :: "0.1.0" + +USAGE :: `Usage: hashmake [options] [target...] + +Builds targets from the hashmake.hb in the current directory. +With no target, builds the graph's .default. + +Options: + -C, --directory Run as if started in + -f, --file The build file (default: hashmake.hb) + -n, --dry-run Print the order targets would be built in, and stop + --graph Print the dependency graph, and stop + --allow-any-path Let the build file resolve paths outside its own + directory (by default it is contained to it) + --cache-dir Where cached entries are kept + -h, --help Print this help and exit + --version Print the version and exit` + +Mode :: enum { Build, Graph, Dry_Run, Help, Version } + +Options :: struct { + mode: Mode, + directory: string, + file: string, + cache_dir: string, + allow_any_path: bool, + targets: [dynamic]string, +} + +// A pure function of argv, like hb's own parse_args: no printing and no +// os.exit in here, so main stays a thin dispatch and the whole flag surface is +// testable. +parse_args :: proc(args: []string) -> (opts: Options, err_msg: string, ok: bool) { + opts.file = "hashmake.hb" + want_help := false + want_version := false + graph := false + dry_run := false + + for i := 0; i < len(args); i += 1 { + switch args[i] { + case "-h", "--help": + want_help = true + case "--version": + want_version = true + case "--graph": + graph = true + case "-n", "--dry-run": + dry_run = true + case "--allow-any-path": + opts.allow_any_path = true + case "-C", "--directory": + i += 1 + if i >= len(args) do return opts, "-C/--directory requires a path argument", false + opts.directory = args[i] + case "-f", "--file": + i += 1 + if i >= len(args) do return opts, "-f/--file requires a path argument", false + opts.file = args[i] + case "--cache-dir": + i += 1 + if i >= len(args) do return opts, "--cache-dir requires a path argument", false + opts.cache_dir = args[i] + case: + if strings.has_prefix(args[i], "-") { + return opts, fmt.tprintf("unknown option %s (see --help)", args[i]), false + } + append(&opts.targets, args[i]) + } + } + + switch { + case want_help: opts.mode = .Help + case want_version: opts.mode = .Version + case graph: + if dry_run do return opts, "--graph cannot be combined with -n/--dry-run", false + opts.mode = .Graph + case dry_run: opts.mode = .Dry_Run + case: opts.mode = .Build + } + return opts, "", true +} + +main :: proc() { + opts, err_msg, ok := parse_args(os.args[1:]) + if !ok { + fmt.eprintfln("error: %s", err_msg) + os.exit(1) + } + + switch opts.mode { + case .Help: + fmt.println(USAGE) + return + case .Version: + fmt.println("hashmake", VERSION) + return + case .Build, .Graph, .Dry_Run: + } + + if opts.directory != "" { + if os.set_working_directory(opts.directory) != nil { + fmt.eprintfln("error: could not change to %s", opts.directory) + os.exit(1) + } + } + + if !os.exists(opts.file) { + fmt.eprintfln("error: no %s here (use -f to name one, or -C to run elsewhere)", opts.file) + os.exit(1) + } + + run := Run{opts = opts} + eval_err, eval_ok := hb.eval_source_file_run( + opts.file, + hb.Run_Options{cache_dir = opts.cache_dir, contain_to_workdir = !opts.allow_any_path}, + on_graph, + &run, + ) + if !eval_ok { + // An empty message means the parse errors were already printed as part of + // the AST dump, exactly as hb does it. + if eval_err != "" do fmt.eprintfln("error: %s", eval_err) + os.exit(1) + } + if run.failed do os.exit(1) +} + +// ---- the graph ---------------------------------------------------------------- + +Node :: struct { + name: string, + needs: ^hb.Table_Value, // alias -> target name (Utf8) + build: ^hb.Function_Value, + built: bool, + result: hb.Value, +} + +Run :: struct { + opts: Options, + failed: bool, + nodes: map[string]^Node, + order: [dynamic]string, // targets in the order the graph declared them + // Three-colour DFS: a name absent is white, `false` is grey (on the current + // path, so meeting it again is a cycle) and `true` is black (done). + visited: map[string]bool, + path: [dynamic]string, // the grey stack, so a cycle can be named in full +} + +// Called with the value hashmake.hb evaluated to, while the AST behind its +// functions is still alive (see hb.Source_Value_Proc). +on_graph :: proc(interp: ^hb.Interpreter, value: hb.Value, userdata: rawptr) -> bool { + run := (^Run)(userdata) + + root, is_table := value.(^hb.Table_Value) + if !is_table { + return fail(run, "the build file must evaluate to a Table of targets") + } + + targets_val, has_targets := hb.table_find(root, "targets") + targets, targets_ok := targets_val.(^hb.Table_Value) + if !has_targets || !targets_ok { + // A bare table of targets is accepted too - `.targets`/`.default` is the + // fuller spelling, and this is the same graph with less ceremony. + targets = root + } + + for entry in targets.entries { + name, name_ok := entry.key.(string) + if !name_ok do continue + if name == "default" do continue + + node_t, node_ok := entry.value.(^hb.Table_Value) + if !node_ok { + return fail(run, fmt.tprintf("target %s is not a Table", name)) + } + build_val, has_build := hb.table_find(node_t, "build") + build_fn, build_ok := build_val.(^hb.Function_Value) + if !has_build || !build_ok { + return fail(run, fmt.tprintf("target %s has no .build function", name)) + } + needs_t: ^hb.Table_Value + if needs_val, has_needs := hb.table_find(node_t, "needs"); has_needs { + nt, needs_ok := needs_val.(^hb.Table_Value) + if !needs_ok do return fail(run, fmt.tprintf("target %s has a .needs that is not a Table", name)) + needs_t = nt + } + + node := new(Node) + node.name = name + node.needs = needs_t + node.build = build_fn + run.nodes[name] = node + append(&run.order, name) + } + + if len(run.nodes) == 0 do return fail(run, "the build file declares no targets") + + wanted := run.opts.targets[:] + if len(wanted) == 0 { + def, has_def := hb.table_find(root, "default") + def_name, def_ok := def.(string) + if has_def && def_ok { + wanted = []string{def_name} + } else if len(run.order) == 1 { + wanted = []string{run.order[0]} + } else { + return fail(run, "no target given and the graph has no .default") + } + } + for name in wanted { + if name not_in run.nodes { + return fail(run, fmt.tprintf("no such target: %s", name)) + } + } + + // Cycles are refused before anything is built, so a bad graph costs nothing + // and the error names the whole loop rather than one edge of it. + for name in wanted { + if !check_acyclic(run, name) do return false + } + + switch run.opts.mode { + case .Graph: + print_graph(run) + return true + case .Dry_Run: + for name in wanted { + for step in build_order(run, name) do fmt.println(step) + } + return true + case .Build, .Help, .Version: + } + + for name in wanted { + if _, ok := build(run, interp, name); !ok do return false + } + return true +} + +@(private = "file") +fail :: proc(run: ^Run, msg: string) -> bool { + fmt.eprintfln("error: %s", msg) + run.failed = true + return false +} + +// ---- cycles ------------------------------------------------------------------- + +check_acyclic :: proc(run: ^Run, name: string) -> bool { + done, seen := run.visited[name] + if seen && done do return true + if seen && !done { + // `name` is grey: it is on the path we walked to get here. + start := 0 + for step, i in run.path do if step == name { start = i; break } + loop := make([dynamic]string, context.temp_allocator) + for i in start ..< len(run.path) do append(&loop, run.path[i]) + append(&loop, name) + return fail(run, fmt.tprintf("dependency cycle: %s", strings.join(loop[:], " -> ", context.temp_allocator))) + } + + run.visited[name] = false + append(&run.path, name) + node := run.nodes[name] + if node.needs != nil { + for entry in node.needs.entries { + dep, dep_ok := entry.value.(string) + if !dep_ok { + return fail(run, fmt.tprintf("target %s has a .needs entry that is not a target name", name)) + } + if dep not_in run.nodes { + return fail(run, fmt.tprintf("target %s needs %s, which no target produces", name, dep)) + } + if !check_acyclic(run, dep) do return false + } + } + pop(&run.path) + run.visited[name] = true + return true +} + +// ---- building ----------------------------------------------------------------- + +build_order :: proc(run: ^Run, name: string) -> []string { + out := make([dynamic]string, context.temp_allocator) + seen := make(map[string]bool, context.temp_allocator) + walk_order(run, name, &out, &seen) + return out[:] +} + +@(private = "file") +walk_order :: proc(run: ^Run, name: string, out: ^[dynamic]string, seen: ^map[string]bool) { + if name in seen do return + seen[name] = true + node := run.nodes[name] + if node.needs != nil { + for entry in node.needs.entries { + if dep, ok := entry.value.(string); ok do walk_order(run, dep, out, seen) + } + } + append(out, name) +} + +build :: proc(run: ^Run, interp: ^hb.Interpreter, name: string) -> (hb.Value, bool) { + node := run.nodes[name] + if node.built do return node.result, true + + // Prerequisites first, gathered into the Table the build function receives: + // the alias it asked for, mapped to what that dependency actually produced. + prereqs := new(hb.Table_Value) + if node.needs != nil { + for entry in node.needs.entries { + alias, alias_ok := entry.key.(string) + dep, dep_ok := entry.value.(string) + if !alias_ok || !dep_ok do continue + + artifact, ok := build(run, interp, dep) + if !ok do return nil, false + // "Produces no artifact" is enforced here rather than by convention: a + // target that answered with something other than a File has nothing for + // a dependent to build from, and saying so beats passing it along. + if _, is_file := artifact.(^hb.File_Value); !is_file { + return nil, false_with(run, fmt.tprintf("%s needs %s, but %s produces no artifact", name, dep, dep)) + } + append(&prereqs.entries, hb.Table_Entry_Value{key = alias, value = artifact}) + } + } + + fmt.eprintfln("hashmake: %s", name) + result, ok := hb.call_function(interp, node.build, prereqs) + if !ok { + // The interpreter has already recorded why; §8's failures are fatal, so + // there is nothing to recover and the message is the whole story. + return nil, false_with(run, interp.error_message) + } + node.result = result + node.built = true + + // A target that answers with text has produced no artifact - it ran for its + // effect - so show what it said. That is how the demo's own output reaches + // the terminal. + if text, is_text := result.(string); is_text && text != "" { + fmt.print(text) + } + return result, true +} + +@(private = "file") +false_with :: proc(run: ^Run, msg: string) -> bool { + fail(run, msg) + return false +} + +print_graph :: proc(run: ^Run) { + for name in run.order { + node := run.nodes[name] + if node.needs == nil || len(node.needs.entries) == 0 { + fmt.printfln("%s", name) + continue + } + deps := make([dynamic]string, context.temp_allocator) + for entry in node.needs.entries { + if dep, ok := entry.value.(string); ok do append(&deps, dep) + } + fmt.printfln("%s <- %s", name, strings.join(deps[:], ", ", context.temp_allocator)) + } +} From 3260ba37aa8b9b7669327522abbefc9fe9d7532d Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:22:48 +0000 Subject: [PATCH 4/8] Add examples and tests for the build builtins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five examples, in the house style with their documented values asserted by the suite, plus the unit tests for what an example structurally cannot show: a denied or malformed call is a fatal failure (§8/§16), so an example that tripped one would end rather than evaluate to anything. folding-a-table, text-slicing and listing-a-directory are ordinary EXAMPLE_CASES rows. workdir-containment documents the three ambient path modes and shows the two it can - the refusals are left as a by-hand invitation in its header, for the reason above. running-a-program drives clang, so it gets its own test that skips with a logged reason where clang is absent, exactly as files-symlink.hb does for symlinks. The unit tests cover the promises rather than the happy path: that `fold` visits in ascending key order and so agrees on two Tables that compare equal; that textlen/textslice count codepoints and refuse to run past the end; that `workdir` refuses "..", an absolute path, and - the one that looks like it stays put - a ".." written behind a "."; that neither permission denies a handle-less path outright; that exec has its own permission and requires the outputs it declared; and that ctx.dir does not hash as its directory's contents, which is the property the whole incremental story rests on. examples/listing/ is a small fixture with stable contents, so listdir's example asserts a fixed answer rather than one that changes whenever an example is added. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- examples/folding-a-table.hb | 29 ++++ examples/listing-a-directory.hb | 35 +++++ examples/listing/alpha.txt | 1 + examples/listing/beta.md | 1 + examples/listing/gamma.txt | 1 + examples/running-a-program.hb | 51 ++++++ examples/text-slicing.hb | 30 ++++ examples/workdir-containment.hb | 37 +++++ src/builtins_build_test.odin | 269 ++++++++++++++++++++++++++++++++ src/examples_test.odin | 53 ++++++- 10 files changed, 505 insertions(+), 2 deletions(-) create mode 100644 examples/folding-a-table.hb create mode 100644 examples/listing-a-directory.hb create mode 100644 examples/listing/alpha.txt create mode 100644 examples/listing/beta.md create mode 100644 examples/listing/gamma.txt create mode 100644 examples/running-a-program.hb create mode 100644 examples/text-slicing.hb create mode 100644 examples/workdir-containment.hb create mode 100644 src/builtins_build_test.odin diff --git a/examples/folding-a-table.hb b/examples/folding-a-table.hb new file mode 100644 index 0000000..4215290 --- /dev/null +++ b/examples/folding-a-table.hb @@ -0,0 +1,29 @@ +// `fold` (SPEC.md §16): the one way to traverse a Table. There are no loops in +// this language and recursion cannot reach a Table's entries on its own, so +// this is the primitive `map`, `filter` and appending are written on top of - +// each of them a few lines of HashedBuild rather than a builtin of its own. +// +// `.step` is called with { .acc, .key, .value } and returns the next +// accumulator. +// +// **Entries are visited in ascending key order, not the order they were +// written.** Two things follow, and they are the reason for the choice. A +// sequence (§5 - keys 1..N) folds in index order, which is what makes folding +// a list of filenames mean anything. And two Tables that compare equal (§6 +// ignores entry order) fold to the same answer, which walking them as written +// could not promise. Evaluates to +// { sum: 60, in_index_order: "abc", by_key_not_by_entry: "AMZ", +// equal_tables_agree: true, length: 3 }. + +let sum_step (let s; s.acc + s.value); +let seq_len (let t; fold { .table = t, .init = 0, .step = (let s; s.acc + 1) }); +let digits (let t; fold { .table = t, .init = 0, .step = (let s; s.acc * 10 + s.value) }); + +{ + .sum = fold { .table = {10, 20, 30}, .init = 0, .step = sum_step }, + .in_index_order = fold { .table = {"a", "b", "c"}, .init = "", .step = (let s; s.acc concat s.value) }, + .by_key_not_by_entry = + fold { .table = { .z = "Z", .a = "A", .m = "M" }, .init = "", .step = (let s; s.acc concat s.value) }, + .equal_tables_agree = (digits { .a = 1, .b = 2 }) == (digits { .b = 2, .a = 1 }), + .length = seq_len {"x", "y", "z"}, +} diff --git a/examples/listing-a-directory.hb b/examples/listing-a-directory.hb new file mode 100644 index 0000000..af74826 --- /dev/null +++ b/examples/listing-a-directory.hb @@ -0,0 +1,35 @@ +// `listdir` (SPEC.md §16): a directory's entries, as a value. This is the half +// of §16's open question that is now answered - a program could always address +// a name it already knew, and can now find out what is there. +// +// **Names only, not kinds.** What an entry *is* can be asked by opening it +// (`loadfile { .dir, .path }`), and a bare sequence is the shape `fold` +// traverses and `[i]` indexes. Sorted byte-wise, for the same reason the +// directory *hash* sorts (§3): readdir order is the filesystem's own business, +// and a build whose argument order changed between machines would cache +// differently on each. +// +// Together with `fold` and `textslice` this is how a build finds its sources +// without naming them - see examples/hashmake/hashmake.hb, which does exactly +// the filter below over a checkout of C sources. Gated by `io` like every +// other read. Evaluates to +// { all: {"alpha.txt", "beta.md", "gamma.txt"}, only_txt: {"alpha.txt", "gamma.txt"} }. + +let endswith (let a; + let t a.text; let s a.suffix; + (textlen t) >= (textlen s) + and (textslice { .text = t, .start = (textlen t) - (textlen s) + 1, .count = textlen s }) == s); +let seq_len (let t; fold { .table = t, .init = 0, .step = (let s; s.acc + 1) }); +let append (let a; a.seq concat { [(seq_len a.seq) + 1] = a.item }); + +let names listdir (loadfile "listing"); +{ + .all = names, + .only_txt = fold { + .table = names, + .init = empty, + .step = (let s; (endswith { .text = s.value, .suffix = ".txt" }) + then (append { .seq = s.acc, .item = s.value }) + else s.acc), + }, +} diff --git a/examples/listing/alpha.txt b/examples/listing/alpha.txt new file mode 100644 index 0000000..4a58007 --- /dev/null +++ b/examples/listing/alpha.txt @@ -0,0 +1 @@ +alpha diff --git a/examples/listing/beta.md b/examples/listing/beta.md new file mode 100644 index 0000000..eeebf43 --- /dev/null +++ b/examples/listing/beta.md @@ -0,0 +1 @@ +# beta diff --git a/examples/listing/gamma.txt b/examples/listing/gamma.txt new file mode 100644 index 0000000..af17f6c --- /dev/null +++ b/examples/listing/gamma.txt @@ -0,0 +1 @@ +gamma diff --git a/examples/running-a-program.hb b/examples/running-a-program.hb new file mode 100644 index 0000000..42930c5 --- /dev/null +++ b/examples/running-a-program.hb @@ -0,0 +1,51 @@ +// `exec` (SPEC.md §16): running a program, with its inputs and outputs as +// **values** rather than paths. +// +// The command runs in a fresh scratch directory holding nothing but the +// `.inputs` it was handed, and the `.outputs` it declares come back as `File`s. +// That shape is the point rather than a convenience: §15's cache key excludes +// anything an expression reads at run time, so a thinner exec that wrote into +// a directory and let the caller `loadfile` the result afterwards would answer +// with the first run's bytes forever. Here an input is a File and a File is +// its content (§3), so inputs are *in* the key - which is what makes +// `cached exec { … }` correct, and what examples/hashmake/hashmake.hb is built +// on. +// +// **A non-zero exit is not a failure.** It comes back as `.status`, so a build +// can `check` it and show `.stderr` - the useful thing to do with a compiler +// that rejected its input. A command that cannot be started, an input that +// cannot be written, or a declared output that is not there are all fatal +// (§16), as is calling this without `ctx.permissions.exec`. +// +// Worth being plain about: this contains what is *handed to* a program, not +// what that program then does. A compiler started here can read whatever the +// user running it can. +// +// This one shells out to `clang`, so unlike every other example it needs +// something outside the repository; the test that runs it skips itself, with a +// logged reason, where clang isn't installed. Evaluates to +// { compiled: 0, said_its_version: true, ran_what_it_built: "hello from C\n" }. + +let source createfile { + .dir = ctx.cache, + .content = "#include \nint main(void){printf(\"hello from C\\n\");return 0;}\n", +}; + +let version exec { .cmd = "clang", .args = { "--version" } }; + +let built exec { + .cmd = "clang", + .args = { "greet.c", "-o", "greet" }, + .inputs = { ["greet.c"] = source }, + .outputs = { "greet" }, +}; + +// The program just built is the next step's command - which works because a +// File carries its executable bit across the handover. +let ran exec { .cmd = "./greet", .inputs = { ["greet"] = built.outputs["greet"] } }; + +{ + .compiled = built.status, + .said_its_version = (textlen version.stdout) > 0, + .ran_what_it_built = ran.stdout, +} diff --git a/examples/text-slicing.hb b/examples/text-slicing.hb new file mode 100644 index 0000000..17aa071 --- /dev/null +++ b/examples/text-slicing.hb @@ -0,0 +1,30 @@ +// `textlen` and `textslice` (SPEC.md §16): measuring and cutting `Utf8`. +// +// Both count **codepoints, not bytes**, because the type is Utf8 (§3) - a byte +// index could land inside a multi-byte character and hand back something that +// is not text at all. `.start` is 1-based, matching `[i]`'s element access +// (§5), and asking for anything past the end is a fatal failure rather than a +// silently short answer. +// +// They are deliberately the primitives rather than a set of ready-made +// predicates: `endswith` below is the whole of what a build needs to pick the +// C files out of a directory listing, and it is four lines of ordinary +// HashedBuild. Evaluates to +// { length: 7, bytes_would_say: 5, extension: ".c", stem: "cJSON", +// is_c: true, is_not_h: false, too_short_is_false: false }. + +let endswith (let a; + let t a.text; let s a.suffix; + (textlen t) >= (textlen s) + and (textslice { .text = t, .start = (textlen t) - (textlen s) + 1, .count = textlen s }) == s); + +let name "cJSON.c"; +{ + .length = textlen name, + .bytes_would_say = textlen "héllo", // 5 codepoints, 6 bytes + .extension = textslice { .text = name, .start = (textlen name) - 1, .count = 2 }, + .stem = textslice { .text = name, .start = 1, .count = (textlen name) - 2 }, + .is_c = endswith { .text = name, .suffix = ".c" }, + .is_not_h = endswith { .text = name, .suffix = ".h" }, + .too_short_is_false = endswith { .text = "c", .suffix = ".c" }, +} diff --git a/examples/workdir-containment.hb b/examples/workdir-containment.hb new file mode 100644 index 0000000..4eedc89 --- /dev/null +++ b/examples/workdir-containment.hb @@ -0,0 +1,37 @@ +// `ctx.dir` and the path permissions (SPEC.md §9/§16). A path written without +// a directory handle - `loadfile "notes.txt"` - used to resolve with no +// containment at all. Which of three things it now does is decided by a +// permission, and by nothing else: +// +// anypath anywhere, as before: relative to this source file's own +// directory, or absolute. Granted at the root, so nothing changed +// for programs that were already written. +// workdir contained to `ctx.dir` - the directory this run is rooted at. +// "..", an absolute path, and a symlink pointing outward are all +// refused, and "." resolves to ctx.dir rather than through it. The +// same component-by-component walk the { .dir, .path } form uses, +// so the guarantee cannot differ between the two spellings. +// neither refused outright; only the handle forms work. +// +// `anypath` subsumes `workdir`, so holding both just means anypath. Narrowing +// is what `hashmake` does before evaluating a build file, which is why a +// hashmake.hb cannot read the rest of the machine. +// +// The refusals are not shown here for a reason worth knowing: a denied read is +// a **fatal** failure (§8/§16), so an example that tried to demonstrate one +// would end rather than evaluate to anything. Try it by hand instead: +// +// ./hb -e '(loadfile "..") chctx chperm { .name = "anypath", .enabled = 1 == 0 } +// chctx chperm { .name = "workdir", .enabled = 1 == 1 }' +// +// Evaluates to { root_grants: {io: nothing, exec: nothing, anypath: nothing}, +// contained_grants: {io: nothing, exec: nothing, workdir: nothing}, +// reads_inside: "This is the payload for option A.\n" }. + +let deny_any chperm { .name = "anypath", .enabled = 1 == 0 }; +let grant_wd chperm { .name = "workdir", .enabled = 1 == 1 }; +{ + .root_grants = ctx.permissions, + .contained_grants = ctx.permissions chctx deny_any chctx grant_wd, + .reads_inside = (filetext (loadfile "optiona.txt")) chctx deny_any chctx grant_wd, +} diff --git a/src/builtins_build_test.odin b/src/builtins_build_test.odin new file mode 100644 index 0000000..e171733 --- /dev/null +++ b/src/builtins_build_test.odin @@ -0,0 +1,269 @@ +#+build linux, windows +// core:testing doesn't compile for wasm32, so the suite is native-only - the +// WASI backends are covered by scripts/wasi_smoke.sh instead. + +package hashedbuild + +// The build builtins (builtins_build.odin). The examples cover what each one +// *returns*; these cover the parts an example structurally cannot, because a +// denied or malformed call is a **fatal** failure (§8/§16) - an example that +// tripped one would end rather than evaluate to anything. + +import "core:fmt" +import "core:log" +import "core:os" +import "core:strings" +import "core:testing" + +@(private = "file") +build_parse :: proc(s: string) -> ast_t { + return parse(source_t{name = "test", n_bytes = u64(len(s)), data = raw_data(s)}, ast_t{}) +} + +@(private = "file") +eval_build :: proc(src: string, cache_name := "build_default") -> (val: Value, ok: bool, err: string) { + ast := build_parse(src) + defer ast_destroy(&ast) + cache := fmt.tprintf("%s/.build_test_%s", repo_root(), cache_name) + interp := Interpreter{ast = &ast, src = src, current_ctx = make_root_context(cache)} + val, ok = eval_program(&interp, ast.root, make_global_env()) + return val, ok, interp.error_message +} + +@(private = "file") +expect_fails_with :: proc(t: ^testing.T, src: string, fragment: string, cache_name := "build_default") { + _, ok, err := eval_build(src, cache_name) + if !testing.expect(t, !ok, fmt.tprintf("expected %s to fail, but it succeeded", src)) do return + testing.expect( + t, + strings.contains(err, fragment), + fmt.tprintf("expected the failure for %s to mention %q, got %q", src, fragment, err), + ) +} + +// ---- fold --------------------------------------------------------------------- + +// The ordering promise, on the one case that would expose a hash-order or an +// entry-order walk: keys of two kinds, written out of order. Ascending order +// puts the Integer keys first and the Utf8 keys after, each group sorted. +@(test) +test_fold_visits_in_ascending_key_order :: proc(t: ^testing.T) { + val, ok, err := eval_build(`fold { + .table = { .b = "B", [2] = "2", .a = "A", [1] = "1" }, + .init = "", + .step = (let s; s.acc concat s.value), + }`) + testing.expect(t, ok, err) + if !ok do return + testing.expect_value(t, val.(string), "12AB") +} + +// §6 ignores the order a Table's entries were written in, so a fold over two +// equal Tables has to agree - which walking `entries` as stored would not. +@(test) +test_fold_agrees_on_equal_tables :: proc(t: ^testing.T) { + val, ok, err := eval_build(`let digits (let tbl; + fold { .table = tbl, .init = 0, .step = (let s; s.acc * 10 + s.value) }); + (digits { .a = 1, .b = 2, .c = 3 }) == (digits { .c = 3, .a = 1, .b = 2 })`) + testing.expect(t, ok, err) + if !ok do return + testing.expect_value(t, val.(bool), true) +} + +@(test) +test_fold_rejects_a_non_function_step :: proc(t: ^testing.T) { + expect_fails_with(t, `fold { .table = {1}, .init = 0, .step = 7 }`, "must be a Function") +} + +@(test) +test_fold_propagates_a_failing_step :: proc(t: ^testing.T) { + expect_fails_with( + t, + `fold { .table = {1, 2}, .init = 0, .step = (let s; error "step gave up") }`, + "step gave up", + ) +} + +// ---- textlen / textslice ------------------------------------------------------ + +// Codepoints, not bytes: "héllo" is 5 characters in 6 bytes, and slicing at 2 +// must not cut the é in half. +@(test) +test_text_builtins_count_codepoints_not_bytes :: proc(t: ^testing.T) { + val, ok, err := eval_build(`{ + .len = textlen "héllo", + .cut = textslice { .text = "héllo", .start = 2, .count = 2 }, + }`) + testing.expect(t, ok, err) + if !ok do return + testing.expect_value(t, format_value(val), `{len: 5, cut: "él"}`) +} + +@(test) +test_textslice_refuses_to_run_past_the_end :: proc(t: ^testing.T) { + expect_fails_with(t, `textslice { .text = "abc", .start = 3, .count = 2 }`, "past the end") +} + +@(test) +test_textslice_start_is_one_based :: proc(t: ^testing.T) { + expect_fails_with(t, `textslice { .text = "abc", .start = 0, .count = 1 }`, "1-based") +} + +// ---- listdir ------------------------------------------------------------------ + +@(test) +test_listdir_refuses_a_regular_file :: proc(t: ^testing.T) { + expect_fails_with(t, `listdir (loadfile "examples/optiona.txt")`, "not a regular file") +} + +@(test) +test_listdir_needs_io :: proc(t: ^testing.T) { + expect_fails_with( + t, + `(listdir (loadfile "examples")) chctx chperm { .name = "io", .enabled = 1 == 0 }`, + "io permission not granted", + ) +} + +// ---- ctx.dir and the ambient path modes --------------------------------------- + +// The containment the examples describe but cannot demonstrate, since each of +// these ends the program rather than evaluating to anything. +@(private = "file") +CONTAINED :: + ` chctx chperm { .name = "anypath", .enabled = 1 == 0 }` + + ` chctx chperm { .name = "workdir", .enabled = 1 == 1 }` + +@(test) +test_workdir_refuses_an_absolute_path :: proc(t: ^testing.T) { + expect_fails_with(t, `(loadfile "/etc/hostname")` + CONTAINED, "escapes its directory") +} + +@(test) +test_workdir_refuses_dot_dot :: proc(t: ^testing.T) { + expect_fails_with(t, `(loadfile "../README.md")` + CONTAINED, "escapes its directory") +} + +// The `.` component specifically: it is the one that looks like it stays put +// and can still be written through to get out. +@(test) +test_workdir_refuses_dot_dot_behind_a_dot :: proc(t: ^testing.T) { + expect_fails_with(t, `(loadfile "./../README.md")` + CONTAINED, "escapes its directory") +} + +@(test) +test_workdir_allows_a_path_inside :: proc(t: ^testing.T) { + val, ok, err := eval_build(`(filetext (loadfile "examples/optiona.txt"))` + CONTAINED) + testing.expect(t, ok, err) + if !ok do return + testing.expect_value(t, val.(string), "This is the payload for option A.\n") +} + +// Neither permission: only the handle forms work at all. +@(test) +test_neither_permission_denies_a_handle_less_path :: proc(t: ^testing.T) { + expect_fails_with( + t, + `(loadfile "examples/optiona.txt") chctx chperm { .name = "anypath", .enabled = 1 == 0 }`, + "needs the workdir or anypath permission", + ) +} + +// ctx.dir is accepted wherever a directory handle is, exactly as ctx.cache is. +@(test) +test_ctx_dir_works_as_a_directory_handle :: proc(t: ^testing.T) { + val, ok, err := eval_build(`filetext (loadfile { .dir = ctx.dir, .path = "examples/optiona.txt" })`) + testing.expect(t, ok, err) + if !ok do return + testing.expect_value(t, val.(string), "This is the payload for option A.\n") +} + +// The reason ctx.dir is its own type rather than a directory File: §15 puts +// the whole ctx into every cache key, and a directory File hashes over its +// contents (§3), so a File here would make every entry depend on every byte of +// the tree. Hashing as a bare tag is what keeps a cache key stable while the +// project changes around it. +@(test) +test_ctx_dir_does_not_hash_as_the_directorys_contents :: proc(t: ^testing.T) { + // The same directory, reached two ways: as ctx.dir, and as a directory File. + // The File hashes over every byte of the tree (§3); ctx.dir must not, or the + // whole project would be inside every cache key. + val, ok, err := eval_build(`(sha256 ctx.dir) == (sha256 (loadfile "."))`) + testing.expect(t, ok, err) + if !ok do return + testing.expect( + t, + !val.(bool), + "ctx.dir hashed as its directory's contents - every cached entry would then depend on the whole tree", + ) +} + +// ---- exec --------------------------------------------------------------------- + +@(test) +test_exec_needs_its_own_permission :: proc(t: ^testing.T) { + expect_fails_with( + t, + `(exec { .cmd = "clang" }) chctx chperm { .name = "exec", .enabled = 1 == 0 }`, + "exec permission not granted", + ) +} + +@(test) +test_exec_rejects_a_missing_cmd :: proc(t: ^testing.T) { + expect_fails_with(t, `exec { .args = { "x" } }`, "needs a Utf8 .cmd") +} + +// A non-zero exit is a value, not a failure - the whole reason `.status` is +// returned rather than the call dying on the caller's behalf. +@(test) +test_exec_reports_a_non_zero_exit_as_a_value :: proc(t: ^testing.T) { + if !clang_available() { + log.info("skipping: clang is not on PATH in this environment") + return + } + exec_status_cache := fmt.tprintf("%s/.build_test_exec_status", repo_root()) + remove_dir_and_entries(exec_status_cache) + defer remove_dir_and_entries(exec_status_cache) + val, ok, err := eval_build( + `(exec { .cmd = "clang", .args = { "--no-such-flag-at-all" } }).status == 0`, + "exec_status", + ) + testing.expect(t, ok, err) + if !ok do return + testing.expect_value(t, val.(bool), false) +} + +// A declared output that the program did not produce is fatal: silently +// handing back a graph node with no artifact would turn a broken build into a +// mysterious one further along. +@(test) +test_exec_requires_its_declared_outputs :: proc(t: ^testing.T) { + if !clang_available() { + log.info("skipping: clang is not on PATH in this environment") + return + } + exec_outputs_cache := fmt.tprintf("%s/.build_test_exec_outputs", repo_root()) + remove_dir_and_entries(exec_outputs_cache) + defer remove_dir_and_entries(exec_outputs_cache) + expect_fails_with( + t, + `exec { .cmd = "clang", .args = { "--version" }, .outputs = { "nothing.o" } }`, + "declared no output named nothing.o", + "exec_outputs", + ) +} + +@(private = "file") +clang_available :: proc() -> bool { + path_env := os.get_env("PATH", context.temp_allocator) + sep := ";" when ODIN_OS == .Windows else ":" + for dir in strings.split(path_env, sep, context.temp_allocator) { + if dir == "" do continue + if os.exists(strings.concatenate({dir, "/clang"}, context.temp_allocator)) do return true + when ODIN_OS == .Windows { + if os.exists(strings.concatenate({dir, "/clang.exe"}, context.temp_allocator)) do return true + } + } + return false +} diff --git a/src/examples_test.odin b/src/examples_test.odin index f804a15..c18618e 100644 --- a/src/examples_test.odin +++ b/src/examples_test.odin @@ -36,6 +36,10 @@ EXAMPLE_CASES := []Example_Case{ {"cached.hb", "{answer: 42, asking_again_agrees: true, per_argument: {small: 2, large: 11}, file_survives_the_round_trip: true, a_cycle_survives_too: true}"}, {"check-and-invariants.hb", "100"}, {"comparison-and-logic.hb", "{ordered: true, both: true, either: true, mixed: false}"}, + {"folding-a-table.hb", `{sum: 60, in_index_order: "abc", by_key_not_by_entry: "AMZ", equal_tables_agree: true, length: 3}`}, + {"text-slicing.hb", `{length: 7, bytes_would_say: 5, extension: ".c", stem: "cJSON", is_c: true, is_not_h: false, too_short_is_false: false}`}, + {"listing-a-directory.hb", `{all: {"alpha.txt", "beta.md", "gamma.txt"}, only_txt: {"alpha.txt", "gamma.txt"}}`}, + {"workdir-containment.hb", `{root_grants: {io: nothing, exec: nothing, anypath: nothing}, contained_grants: {io: nothing, exec: nothing, workdir: nothing}, reads_inside: "This is the payload for option A.\n"}`}, {"context-permissions.hb", "{ambient: {io: nothing, exec: nothing, anypath: nothing}, io_denied: {exec: nothing, anypath: nothing}, replaced: {}, still_ambient: {io: nothing, exec: nothing, anypath: nothing}}"}, {"cyclic-data.hb", `{round_trip: "Alice", mutual: true, second_hop: "Carol", reordered: 3, same_shape: true}`}, {"files-symlink.hb", `"optiona.txt"`}, @@ -70,7 +74,7 @@ EXAMPLE_CASES := []Example_Case{ // can't be a fixed string, so they get their own tests below. Listed here so // the coverage check counts them as covered rather than missing. @(private = "file") -EXAMPLES_WITH_THEIR_OWN_TEST := []string{"option-picker.hb", "files-sandboxed.hb"} +EXAMPLES_WITH_THEIR_OWN_TEST := []string{"option-picker.hb", "files-sandboxed.hb", "running-a-program.hb"} // examples/link-to-optiona is committed as a symlink, and files-symlink.hb // reads its target. Git only materialises it as a real symlink where it can: @@ -204,7 +208,9 @@ read_dir_names :: proc(dir: string) -> []string { // The is_dir check is not an optimisation: os.open succeeds on a regular file // too, and reading a directory listing out of that handle is not something // every target survives. -@(private = "file") +// Package-visible rather than file-private because builtins_build_test.odin +// needs the same sweep: its `exec` tests build a scratch directory under the +// cache, and leaving one behind makes the next run trip over it. remove_dir_and_entries :: proc(dir: string) { if is_dir, err := fs_stat_is_dir_at(fs_cwd_dir(), dir, true); err == .None && is_dir { for name in read_dir_names(dir) do remove_dir_and_entries(fmt.tprintf("%s/%s", dir, name)) @@ -218,3 +224,46 @@ clear_branch_markers :: proc() { os.remove(fmt.tprintf("%s/examples/branch-%s.marker", repo_root(), branch)) } } + +// running-a-program.hb is the only example that needs something outside this +// repository - it drives `clang`, the same compiler examples/hashmake builds +// cJSON with. It gets its own test rather than an EXAMPLE_CASES row so that a +// checkout without clang skips it with a reason, exactly as files-symlink.hb +// does where symlinks aren't available. +@(test) +test_example_running_a_program_drives_clang :: proc(t: ^testing.T) { + if !command_exists("clang") { + log.info("skipping running-a-program.hb: clang is not on PATH in this environment") + return + } + cache := fmt.tprintf("%s/.examples_test_exec_cache", repo_root()) + defer remove_dir_and_entries(cache) + + path := fmt.tprintf("%s/examples/running-a-program.hb", repo_root()) + formatted, err_msg, ok := eval_source_file(path, false, cache) + testing.expect(t, ok, err_msg) + if !ok do return + defer delete(formatted) + + testing.expect_value( + t, + formatted, + `{compiled: 0, said_its_version: true, ran_what_it_built: "hello from C\n"}`, + ) +} + +@(private = "file") +command_exists :: proc(name: string) -> bool { + path_env := os.get_env("PATH", context.temp_allocator) + for dir in strings.split(path_env, PATH_LIST_SEPARATOR, context.temp_allocator) { + if dir == "" do continue + if os.exists(strings.concatenate({dir, "/", name}, context.temp_allocator)) do return true + when ODIN_OS == .Windows { + if os.exists(strings.concatenate({dir, "/", name, ".exe"}, context.temp_allocator)) do return true + } + } + return false +} + +@(private = "file") +PATH_LIST_SEPARATOR :: ";" when ODIN_OS == .Windows else ":" From 21ba944df31f51a2655af87f9bcc65c1accade87 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:25:44 +0000 Subject: [PATCH 5/8] Teach CI, the WASI smoke test and the playground about the build tooling The submodule is checked out on every job - a partial checkout is a confusing failure mode, and only one job actually needs it - and the Linux job now runs hashmake end to end: it discovers cJSON's sources by listing the submodule, compiles each with clang, links them and runs the result. hashmake lives outside src/, so `odin test src` never builds it; without this it would be the one part of this work nothing checked. Two properties get their own steps rather than being taken on trust. That a second build reuses every entry the first wrote, asserted by counting cache entries rather than by wall-clock, which is not a thing a CI runner can promise. And that a cycle is refused, since a build tool that quietly looped would be worse than one that never ran. running-a-program.hb joins the WASI skip list: WASI cannot start a process at all, so there is no answer for the two targets to agree on. Verified rather than assumed - the wasm build still links with `exec` in it, `exec` under wasmtime reports "running a program is not available on this target", listdir works there, and the smoke test has 31 examples agreeing across the two. The playground manifest skips the vendored checkout: a submodule is one gitlink entry in `git ls-files`, not its contents, so reading it as a file failed outright - and a browser terminal has no use for a C library it cannot compile. hashmake.hb itself is still included. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- .github/workflows/ci.yml | 53 +++++++++++++++++++++++++++++++ .github/workflows/pages.yml | 2 ++ scripts/build_playground_files.py | 7 +++- scripts/wasi_smoke.sh | 5 ++- 4 files changed, 65 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9353fb5..db6f7b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,8 @@ jobs: ODIN_VERSION: dev-2026-08 steps: - uses: actions/checkout@v7 + with: + submodules: recursive # The release tarball's top-level directory is named after the nightly # it was cut from (odin-linux-amd64-nightly+2026-08-06), not after the @@ -67,6 +69,49 @@ jobs: odin build src -out:hb python3 scripts/editor_keys_test.py ./hb + # hashmake lives outside src/, so `odin test src` never builds it. This + # is the end-to-end check: it discovers cJSON's sources by listing the + # submodule, compiles each with clang, links them and runs the result. + - name: hashmake builds and runs the cJSON demo + run: | + odin build tools/hashmake -out:hashmake + cd examples/hashmake + ../../hashmake --graph + ../../hashmake -n + ../../hashmake | tee /tmp/demo.out + grep -q "Version:" /tmp/demo.out + + # The second build must reuse every cache entry the first one wrote. + # Checked by entry count rather than by wall-clock, which is not a thing + # a CI runner can promise. + - name: hashmake rebuilds nothing when nothing changed + run: | + cd examples/hashmake + rm -rf /tmp/hmcache + ../../hashmake --cache-dir /tmp/hmcache >/dev/null + before=$(ls /tmp/hmcache | grep -c "^sha256-") + ../../hashmake --cache-dir /tmp/hmcache >/dev/null + after=$(ls /tmp/hmcache | grep -c "^sha256-") + echo "cache entries: $before then $after" + test "$before" = "$after" + + - name: hashmake refuses a cycle + run: | + mkdir -p /tmp/cyc && cd /tmp/cyc + cat > hashmake.hb <<'EOF' + { + .default = "a", + .targets = { + .a = { .needs = { .x = "b" }, .build = (let p; nothing) }, + .b = { .needs = { .x = "a" }, .build = (let p; nothing) }, + }, + } + EOF + if "$GITHUB_WORKSPACE/hashmake" 2>/tmp/cyc.err; then + echo "expected a cycle to be refused"; exit 1 + fi + grep -q "dependency cycle" /tmp/cyc.err + # The third backend. `odin test` builds natively, so this job is what # actually executes fs_windows.odin, task_native.odin's Windows half and # term_windows.odin - the same suite as the Linux job, on the other native @@ -87,6 +132,8 @@ jobs: run: git config --global core.symlinks true - uses: actions/checkout@v7 + with: + submodules: recursive - name: Cache Odin id: cache-odin @@ -157,6 +204,8 @@ jobs: ODIN_VERSION: dev-2026-08 steps: - uses: actions/checkout@v7 + with: + submodules: recursive - name: Cache Odin id: cache-odin @@ -230,6 +279,8 @@ jobs: WAMR_COMMIT: 16ea74cc6f3671d81db2c4a8dac08fba6eadc73b steps: - uses: actions/checkout@v7 + with: + submodules: recursive - name: Cache Odin id: cache-odin @@ -301,6 +352,8 @@ jobs: ODIN_VERSION: dev-2026-08 steps: - uses: actions/checkout@v7 + with: + submodules: recursive - name: Cache Odin id: cache-odin diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index e50baaa..c1e334e 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -32,6 +32,8 @@ jobs: ODIN_VERSION: dev-2026-08 steps: - uses: actions/checkout@v7 + with: + submodules: recursive - name: Cache Odin id: cache-odin diff --git a/scripts/build_playground_files.py b/scripts/build_playground_files.py index b1d9a9f..176d1fb 100755 --- a/scripts/build_playground_files.py +++ b/scripts/build_playground_files.py @@ -24,7 +24,12 @@ REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) OUT = os.path.join(REPO, "docs", "repo-files.json") SKIP = [re.compile(p) for p in ( - r"^docs/media/", r"^docs/vendor/", r"^docs/hb\.wasm$", r"^docs/repo-files\.json$")] + r"^docs/media/", r"^docs/vendor/", r"^docs/hb\.wasm$", r"^docs/repo-files\.json$", + # A submodule is one gitlink entry in `git ls-files`, not its contents, so + # reading it as a file would fail - and the playground has no use for a + # vendored C library anyway. examples/hashmake's build needs the checkout; + # the browser terminal does not, since it cannot run a compiler. + r"^examples/hashmake/vendor/")] def tracked_paths(): diff --git a/scripts/wasi_smoke.sh b/scripts/wasi_smoke.sh index 2e553f9..d22593c 100755 --- a/scripts/wasi_smoke.sh +++ b/scripts/wasi_smoke.sh @@ -38,7 +38,10 @@ cd "$REPO" # files-sandboxed, option-picker # display real paths, which differ by construction: native shows the # checkout, WASI shows the path inside its preopen. -SKIP="files-sandboxed.hb option-picker.hb" +# running-a-program.hb drives clang: WASI cannot start a process at all (core:os's +# backend answers .Unsupported, which `exec` reports as such), so there is no +# answer for the two targets to agree on. +SKIP="files-sandboxed.hb option-picker.hb running-a-program.hb" ASYNC="async-basics.hb async-branching.hb async-table.hb" failures=0 From ad182a756b6b21fa683cc07d66dde8e42dcc22ae Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:29:42 +0000 Subject: [PATCH 6/8] Document the build builtins, ctx.dir, and hashmake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md makes docs part of a feature rather than follow-up work, and this is that half. SPEC.md gains the design: §9 grows ctx.dir and the three permissions (exec, and the workdir/anypath pair that decide what a handle-less path may reach), including why ctx.dir has to be its own type rather than a directory File - §15 puts the whole ctx into every cache key and §3 hashes a directory File over its contents, so a File there would put the whole project tree into every entry. §16 gains listdir - which resolves half of that section's own TODO, and says why names-only - plus exec, fold and the text builtins, and a new TODO for what exec still cannot do. LANGUAGE.md gains an entry each, with snippets that were run before they were committed, and its "what isn't built yet" list is corrected: directory listing and Table traversal have landed, there are still no loops, and two limits that would otherwise be discovered the hard way are stated - exec collects regular files only, and createfile is still exclusive. GETTING_STARTED.md documents the hashmake CLI, which is where CLAUDE.md routes a tool that isn't demonstrated by a runnable file. tools/hashmake/README.md specifies what a hashmake.hb must evaluate to and what the tool enforces; examples/hashmake/README.md is the worked project, with four things to try - including undoing an edit and watching it come back as a cache hit, which is the property that distinguishes this from a timestamp-based tool. README.md gets the headline: the first thing built *on* the language now runs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- GETTING_STARTED.md | 25 +++++++ LANGUAGE.md | 132 ++++++++++++++++++++++++++++++++++-- README.md | 9 +++ SPEC.md | 44 +++++++++++- examples/README.md | 18 ++++- examples/hashmake/README.md | 72 ++++++++++++++++++++ tools/hashmake/README.md | 98 ++++++++++++++++++++++++++ 7 files changed, 390 insertions(+), 8 deletions(-) create mode 100644 examples/hashmake/README.md create mode 100644 tools/hashmake/README.md diff --git a/GETTING_STARTED.md b/GETTING_STARTED.md index 6942f39..298b539 100644 --- a/GETTING_STARTED.md +++ b/GETTING_STARTED.md @@ -55,6 +55,31 @@ If a freshly built binary refuses to start with *"An Application Control policy - **`./hb --cache-dir ...`** - override where `ctx.cache` writes to, and where `cached` keeps its entries (defaults to your XDG cache dir; `%LOCALAPPDATA%\hashedbuild` on Windows). Handy for a throwaway cache: point it somewhere temporary and `cached` starts from nothing. - **`./hb --help`**, **`./hb --version`** - usage and version. +### `hashmake` - the build tool + +A second binary, built from `tools/hashmake`, that treats a HashedBuild program +as a dependency graph: + +```sh +odin build tools/hashmake -out:hashmake +cd examples/hashmake && ../../hashmake +``` + +That discovers the C sources in a vendored cJSON checkout, compiles each with +clang, links them, and runs the result. Run it a second time and it rebuilds +nothing - not because hashmake remembers, but because each node wraps its work +in `cached`, whose key contains the *content* of the files it read. + +- **`hashmake`** - build the graph's `.default` target. +- **`hashmake ...`** - build these instead. +- **`hashmake --graph`** - print the dependency graph and stop. +- **`hashmake -n`** - print the order targets would be built in, and stop. +- **`hashmake -C `**, **`-f `** - run elsewhere, or read a differently named build file. +- **`hashmake --allow-any-path`** - let the build file read outside its own directory. By default it cannot: hashmake evaluates it with `ctx.dir` set and only the `workdir` permission (LANGUAGE.md's "Where a path is allowed to reach"). +- **`hashmake --cache-dir `** - as `hb`'s. + +`tools/hashmake/README.md` covers what a `hashmake.hb` has to evaluate to. + ## Try each part of the video ### 1. The parser - see the AST diff --git a/LANGUAGE.md b/LANGUAGE.md index 7d8b960..d02f670 100644 --- a/LANGUAGE.md +++ b/LANGUAGE.md @@ -99,6 +99,29 @@ rejected, which `SPEC.md` §3 says it should not be. → `examples/numeric-literals.hb` (§3) +### Measuring and cutting text + +`textlen` and `textslice` count **codepoints, not bytes** — the type is `Utf8`, +and a byte index could cut a character in half. `.start` is 1-based, matching +`[i]`, and reaching past the end is a failure rather than a short answer. + +```hashedbuild +textlen "héllo" // => 5, not 6 +textslice { .text = "cJSON.c", .start = 6, .count = 2 } // => ".c" +``` + +These are the primitives rather than a set of ready-made predicates, because +the predicate you want is a few lines of HashedBuild: + +```hashedbuild +let endswith (let a; + let t a.text; let s a.suffix; + (textlen t) >= (textlen s) + and (textslice { .text = t, .start = (textlen t) - (textlen s) + 1, .count = textlen s }) == s); +``` + +→ `examples/text-slicing.hb` (§16) + ## Operators ```hashedbuild @@ -141,6 +164,32 @@ it exists, use a pattern (below) rather than an access. → `examples/tables-map.hb`, `examples/tables-sequence.hb`, `examples/table-and-concat.hb` (§5) +### Traversing one + +There are no loops, and recursion cannot reach a Table's entries by itself, so +`fold` is how a Table is walked. `.step` is called with `{ .acc, .key, .value }` +and returns the next accumulator: + +```hashedbuild +fold { .table = {10, 20, 30}, .init = 0, .step = (let s; s.acc + s.value) } // => 60 +``` + +**Entries are visited in ascending key order, not the order they were written.** +That buys two things. A sequence (keys 1..N) folds in index order, which is what +makes folding a list of filenames mean anything; and two Tables that compare +equal — §6 ignores entry order — fold to the same answer. + +It is deliberately the only traversal primitive: `map`, `filter` and appending +to a sequence are each a line or two of HashedBuild on top of it, rather than a +builtin apiece. + +```hashedbuild +let seq_len (let t; fold { .table = t, .init = 0, .step = (let s; s.acc + 1) }); +let append (let a; a.seq concat { [(seq_len a.seq) + 1] = a.item }); +``` + +→ `examples/folding-a-table.hb` (§16) + ## Functions Three spellings, all producing ordinary values you can store in a Table, pass @@ -370,8 +419,73 @@ Paths in the single-argument form resolve relative to the source file being run, not to your shell's working directory — so a script behaves the same wherever you invoke it from. +`listdir` gives a directory's entries as a value — names only, sorted byte-wise +so a build does not depend on readdir order: + +```hashedbuild +listdir (loadfile "examples/listing") // => {"alpha.txt", "beta.md", "gamma.txt"} +``` + → `examples/files-sandboxed.hb`, `examples/files-symlink.hb`, -`examples/option-picker.hb` (§3, §16) +`examples/option-picker.hb`, `examples/listing-a-directory.hb` (§3, §16) + +### Where a path is allowed to reach + +A path written **without** a directory handle is governed by a permission, and +which one you hold decides one of three behaviours: + +| permission | `loadfile "x"` resolves | +|---|---| +| `anypath` | anywhere — relative to the source file's directory, or absolute. Granted at the root, so this is the behaviour programs already had. | +| `workdir` | contained to `ctx.dir`, the directory the run is rooted at. `..`, an absolute path, and a symlink pointing outward are refused; `.` resolves to `ctx.dir` rather than through it. | +| neither | refused — only the `{ .dir, .path }` handle forms work. | + +`ctx.dir` is that directory as a handle, usable anywhere a `.dir` is. It is not +a `File`: a `File` of a directory hashes over its contents (§3), and since §15 +puts the whole `ctx` into every cache key, that would put the entire project +tree into every entry. It hashes as a bare tag instead, exactly as `ctx.cache` +does — what you read *through* it are ordinary `File`s that still hash by +content. + +```hashedbuild +loadfile { .dir = ctx.dir, .path = "notes.txt" } +(loadfile "..") chctx chperm { .name = "anypath", .enabled = 1 == 0 } // now a failure +``` + +→ `examples/workdir-containment.hb` (§9, §16) + +### Running a program + +`exec` starts a program in a fresh scratch directory holding nothing but the +inputs it was given, and hands back the outputs it declared **as values**: + +```hashedbuild +exec { + .cmd = "clang", + .args = { "-c", "greet.c", "-o", "greet.o" }, + .inputs = { ["greet.c"] = }, + .outputs = { "greet.o" }, +} +// => { status: 0, stdout: "…", stderr: "…", outputs: { greet.o: } } +``` + +That shape is the point rather than a convenience. A cache key excludes +anything an expression reads at run time (see Caching below), so a thinner exec +that wrote into a directory and let you `loadfile` the result afterwards would +answer with the first run's bytes forever. Here an input is a `File` and a +`File` is its content, so inputs are *in* the key — which is what makes +`cached exec { … }` correct. + +**A non-zero exit is not a failure.** It comes back as `.status`, so you can +`check` it and show `.stderr`. A command that cannot be started, an input that +cannot be written, or a declared output that is not there are all fatal, as is +calling it without `ctx.permissions.exec`. + +Two honest limits: it contains what is *handed to* a program, not what that +program then does — a compiler started this way can read whatever you can — and +WASI cannot start a process at all, where it says so rather than pretending. + +→ `examples/running-a-program.hb`, `examples/hashmake/hashmake.hb` (§16) ## Hashing @@ -616,10 +730,18 @@ that used to be listed here — a directory `File`, a `Function`, and a cyclic value — plus `ctx.cache`. See the Hashing section above for what each of them encodes. A given value has the same digest on every target. -Also absent: `true`/`false` literals, loops of any kind (recursion is the only -repetition there is — see above), a `Bytes`-returning counterpart to -`filetext`, directory listing as a value, and the `#context` implicit name. `SPEC.md` describes several of these as settled -design; none of them run today. +**Directory listing and Table traversal have landed**: `listdir` answers what +is in a directory, and `fold` walks a Table, which is what `map`/`filter`/append +are written on top of. There are still no *loops* — `fold` and recursion are the +whole of repetition, and a fold cannot stop early. + +Also absent: `true`/`false` literals, a `Bytes`-returning counterpart to +`filetext`, and the `#context` implicit name. `SPEC.md` describes several of +these as settled design; none of them run today. + +Two limits worth knowing rather than discovering: `exec`'s `.outputs` collects +regular files only, not directories; and `createfile` is still exclusive, with +no overwrite mode. Removed rather than pending: `serialize` and `serialize_file` were specified in §15 and are gone as of 2026-08-28 — the canonical byte encoding they would have diff --git a/README.md b/README.md index 014d5cf..9fefb3e 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,15 @@ let choice loadfile "choice.txt" |> filetext; Run it with `./hb examples/option-picker.hb` (from anywhere - its paths resolve relative to the script itself, not your shell's current directory), or explore it live with `./hb -i` - a two-to-four-pane terminal editor with a built-in examples picker, a live AST view, and a step-by-step evaluation trace. This one example touches a few of the language's actual design points: files are ordinary values (`loadfile`/`createfile`), branching is built from general composable operators rather than bespoke syntax (`then`/`else` chains into an if/else-if/else), and `error` is a genuinely unrecoverable failure - unlike a failed `then`, no enclosing `else` catches it. +And the first thing built *on* the language now runs too. **[hashmake](tools/hashmake/)** is a build tool whose build files are HashedBuild programs: a `hashmake.hb` evaluates to a dependency graph, and each node is a function from its prerequisites to the artifact it builds. + +```sh +odin build tools/hashmake -out:hashmake +cd examples/hashmake && ../../hashmake +``` + +That vendors [cJSON](https://github.com/DaveGamble/cJSON), finds its C sources by *listing the directory* rather than naming them, compiles each with clang, links them, and runs the result. Run it again and it rebuilds nothing - not because hashmake remembers what it did, but because each node wraps its work in `cached`, whose key holds the *content* of the files it read. Edit one source and exactly that object and the link rebuild; undo the edit and it is a cache hit again, which a timestamp-based tool cannot do. See **[examples/hashmake/](examples/hashmake/)**. + **[LANGUAGE.md](LANGUAGE.md)** is the tour of everything that works today, feature by feature, with a runnable snippet for each and an explicit list of what isn't built yet. **[examples/](examples/)** has a runnable file per feature - all of them executed by the test suite, so they can't drift from the implementation. `SPEC.md` is the full design, including the parts that don't run yet. # Examples diff --git a/SPEC.md b/SPEC.md index d835afa..afabf52 100644 --- a/SPEC.md +++ b/SPEC.md @@ -266,8 +266,18 @@ So the numbered-jump pattern isn't one unified stack — it's per implicit-value Added 2026-08-26, alongside `ctx`/`withctx` above and the filesystem builtins (§16) that are `ctx`'s first real consumer. - **`ctx.permissions`** is a `Table` used as a *set*: it conventionally holds only `nothing` as every value, and a permission is granted iff its key is **present** (not iff its value is "truthy," since `Nothing` has no such notion) — e.g. `ctx.permissions.io` being present at all, regardless of its (always-`nothing`) value, means I/O is allowed. This works because, unlike Lua, a HashedBuild `Table` (§5/§6) genuinely distinguishes "key absent" from "key present with value `nothing`" — the exact distinction a set-of-flags idiom needs, and Lua's conflation of nil-valued and absent keys can't express. -- **The root context** — active at the very start of a program, before any `withctx` — starts with `{ .permissions = { .io = nothing }, .cache = }`: I/O is allowed by default. `withctx` is how you *narrow* permissions around a sub-computation (e.g. before calling `import`ed code, §13), not how you grant them from nothing. Since `withctx` replaces the context wholesale (§7), a program that narrows permissions via a hand-built `Table` rather than `ctx concat {...}` loses `.cache` too unless it explicitly carries it over. +- **The root context** — active at the very start of a program, before any `withctx` — starts with `{ .permissions = { .io = nothing, .exec = nothing, .anypath = nothing }, .cache = , .dir = }`: I/O, starting a program, and unconstrained path resolution are all allowed by default. `withctx` is how you *narrow* permissions around a sub-computation (e.g. before calling `import`ed code, §13), not how you grant them from nothing. Since `withctx` replaces the context wholesale (§7), a program that narrows permissions via a hand-built `Table` rather than `ctx concat {...}` loses `.cache` too unless it explicitly carries it over. - **Builtins read `ctx` live, not captured — the one deliberate exception to the closure-capture rule above.** The capture rule protects a function *from* a caller trying to grant it more authority than it was made with; a builtin like `loadfile` (§16) needs the opposite property — it must see whatever `ctx` is *actually* active at its call site, so that wrapping a call in `... withctx (ctx concat { .permissions = empty })` genuinely denies it from the outside. If builtins captured `ctx` at (interpreter-startup) creation time instead, `withctx` could never restrict them at all. +- **`ctx.permissions.exec`** (added 2026-09-01) gates §16's `exec`. It is deliberately *not* folded into `io`: reading a file and starting an arbitrary program are different amounts of authority, and a program that should be allowed the first and not the second is an ordinary thing to want. Granted at the root alongside `io`. +- **`ctx.permissions.workdir` / `ctx.permissions.anypath`** (added 2026-09-01) decide what a path written **without** a directory handle may reach — `loadfile "x"`, or `createfile` with no `.dir`. Three states, chosen by which key is present: + - **`anypath`** — anywhere: relative to the running source file's own directory, or absolute, with no containment beyond `io`. This is what §16's unsandboxed form always did, and it is granted at the root, so no existing program changes behaviour. + - **`workdir`** — contained to `ctx.dir` (below), by the same component-by-component walk the `{ .dir, .path }` form uses: `..` and an absolute path are refused, a symlink anywhere along the way is refused, and `.` resolves to `ctx.dir` itself rather than through it. Both spellings of a handle-less path share one resolver, so the guarantee cannot hold for one and not the other. + - **neither** — refused outright; only the handle forms work. + + `anypath` subsumes `workdir`, so holding both is not an error, it just means `anypath`. Narrowing is what a host does to contain a program it is about to run: `hashmake` (below) drops `anypath` and grants `workdir` before evaluating a build file, which is why a `hashmake.hb` cannot read outside its own project. +- **`ctx.dir`** (added 2026-09-01) is a handle to the directory the run is rooted at — the source file's own directory for `hb file.hb`, the process's cwd for `-e` and the REPL, the build file's directory for `hashmake`. Accepted wherever a directory handle is (`loadfile`/`createfile`/`symlink`/`readlink`'s `.dir`, and `listdir`), exactly as `ctx.cache` is accepted by `createfile`. + + **It is its own type, not a directory `File`, and this is forced rather than chosen.** §15 puts the whole `ctx` into every `cached` key, and §3 hashes a directory `File` over its contents; a `File` here would therefore put every byte of the project tree into every cache entry, so editing any one file would invalidate all of them — the exact opposite of what a content-addressed build wants. **It hashes as a domain-separated constant** (§15), like `ctx.cache` and for an overlapping reason: hashing it as its *path* would bake the checkout location into every key. Nothing is given up by this, since what a program reads *through* the handle are ordinary `File`s that still hash by content. - **`ctx.cache`** (added 2026-08-27) is its own type, distinct from `File` — see §16 for the full write-up. It's "accepted as a directory" (usable as `createfile`'s `.dir`) without actually being one: it can't be read from, traversed, or passed to `loadfile`/`symlink`/`readlink`, only written to via `createfile`. **It does hash** (resolved 2026-08-31), as a domain-separated constant over an empty payload: it is write-only, unnamed and unlistable, so there is nothing about one cache that distinguishes it from another. This became necessary rather than optional once §15's cache key was defined to include the whole `ctx`, which contains it. Hashing it as its *directory path* was rejected for a concrete reason: the path would then be baked into every key, so moving or copying a cache directory would invalidate everything in it. ## 10. Name scopes / bindings @@ -397,7 +407,37 @@ The four filesystem operations below are gated by [`ctx.permissions.io`](#9-impl - **`readlink { .dir = , .path = }`** — reads the target string of the symlink at `` (contained to ``) without following it, returning it as `Utf8`. - **`filetext `** (added 2026-08-27, partially resolving the TODO below) — the minimal fix for "there's no way to get content back out of a `File`": takes a `File` (a *regular* file — fails for a directory `File`) and returns its content as `Utf8`, failing if the bytes aren't valid UTF-8. Not gated by `io` — the actual read already happened when `loadfile` produced the `File`; this just views already-in-memory bytes as text. `Bytes` extraction (for content that isn't valid UTF-8) is still open, per the TODO below. -> TODO: `filetext` only covers the `Utf8` half of "get content back out of a `File`" - a `Bytes`-returning counterpart is still unspecified, same underlying gap as §3's open `Bytes`↔`Utf8` conversion question. Also unspecified: what a directory `File`'s "listing" looks like as a value (so you can enumerate its entries, not just address a name you already know). +- **`listdir `** (added 2026-09-01, resolving the second half of the TODO below) — takes a directory `File` (fails for a regular one) and returns its entries as a sequence `Table` of names (`Utf8`), sorted byte-wise. Gated by `io`. + + **Names only, not kinds.** What an entry *is* can already be asked by opening it (`loadfile { .dir, .path }`), and a bare sequence is the shape `fold` traverses and `[i]` indexes; a Table of name → tag would need unwrapping at every use for something most callers never consult. Adding kinds later is a widening, which is the direction that stays compatible. **Sorted** for the same reason §3's directory hash sorts: readdir order is a filesystem's private business, and a build whose argument order changed between machines would hash — and so cache — differently on each. + +> TODO: `filetext` only covers the `Utf8` half of "get content back out of a `File`" - a `Bytes`-returning counterpart is still unspecified, same underlying gap as §3's open `Bytes`↔`Utf8` conversion question. + +### Running a program + +**`exec { .cmd, .args, .inputs, .outputs, .stdin }`** (added 2026-09-01) — starts a program and returns `{ .status, .stdout, .stderr, .outputs }`, where `.outputs` is a `Table` of name → `File`. Gated by `ctx.permissions.exec` (§9), and by `io`. + +- **`.cmd`** (`Utf8`, required) is looked up on `PATH` when it is a bare name. A `.cmd` containing a path separator is resolved **against the scratch directory** below — which is the only directory the program can see anyway, and is how one step's output becomes the next step's command. +- **`.args`** — a sequence of `Utf8`; none by default. **`.stdin`** — optional `Utf8` fed to the program. +- **`.inputs`** — either a directory `File`, whose contents become the scratch root, or a `Table` of relative-name → `File` placing each at its key. Empty by default. +- **`.outputs`** — a sequence of relative paths collected after the run, each returned as a `File`. **Regular files only** for now; naming a directory fails saying so. + +**The shape is the point, and it is designed around §15 rather than beside it.** The command runs in a **fresh scratch directory holding nothing but its inputs**, and what comes back are values, never paths. §15's key deliberately excludes anything an expression reads at run time, so a thinner `exec` — one that ran a command in a directory and let the caller `loadfile` the result afterwards — would be *silently wrong* under `cached`, answering with the first run's bytes forever. Here an input is a `File`, a `File` is its content (§3), so the inputs are *in* the key: `cached exec { … }` rebuilds exactly the steps whose inputs changed. This is the first builtin whose signature exists to make caching correct. + +**A non-zero exit is not a failure.** It is returned as `.status`, so a program can `check(r.status == 0, …)` (§11) and surface `.stderr` — the useful thing to do with a compiler that rejected its input. Making it fatal would make `.status` pointless, since §8's failures are uncatchable. Everything else *is* fatal, per this section's rule: a command that cannot be started, an input that cannot be written, a declared output that is not there, a denied permission. + +**Two limits, stated rather than left to be discovered.** Containment covers what is *handed to* a program, not what that program then does: a compiler started here is an ordinary process with the invoking user's authority, and `exec` bounds its inputs, not its reach. And a target with no way to start a process — WASI — fails saying exactly that, rather than reporting a missing program. + +> TODO: collecting a directory as an output, and a way to mark an input executable other than by having produced it as an output, are both unspecified. So is any notion of a program's environment: `exec` passes the parent's, which is not something a content-addressed build should rely on. + +### Traversal and text + +Two more ordinary `Function` values (added 2026-09-01), neither gated, since neither touches the filesystem: + +- **`fold { .table, .init, .step }`** — the one general traversal. `.step` is called with `{ .acc, .key, .value }` and returns the next accumulator. §8 has no loops and recursion cannot reach a `Table`'s entries on its own, so before this the language could not process a collection at all; `map`, `filter` and appending to a sequence are now each a line or two of HashedBuild rather than a builtin apiece. + + **Entries are visited in ascending key order** — numbers first by value, then `Utf8` byte-wise, with any other key kind after those in the order it was written. Not the order the entries were written, and deliberately so: a sequence (§5 — keys 1..N) then folds in *index* order, and two Tables that compare equal (§6, which ignores entry order) fold to the same answer. A fold cannot stop early; that is a real limit, not an oversight to read past. +- **`textlen `** and **`textslice { .text, .start, .count }`** — length and substring, both counting **codepoints, not bytes**, because the type is `Utf8` (§3) and a byte index could split a character into something that is not text. `.start` is 1-based, matching `[i]` (§5); reaching past the end is fatal rather than clamped. They are the primitives deliberately: a suffix test — the thing a build actually wants, to pick source files out of a listing — is four lines written on top of them, and does not need a builtin of its own. **`chperm { .name = , .enabled = }`** (added 2026-08-26) — not gated by `io` itself (it doesn't touch the filesystem, just builds a value); returns a `ctx`-changing function (§7/§9's `chctx`) that, given a context, produces a copy of it with `.permissions.` present (if `` is true) or absent (if false), every other field unchanged. Meant to be used right into `chctx`: ` chctx chperm { .name = "io", .enabled = false }` denies `io` for just ``. Exists specifically so a single-permission edit reads as a small, reusable, named function rather than a `ctx concat {...}` expression rebuilt inline every time. diff --git a/examples/README.md b/examples/README.md index ea744f0..c51d2c0 100644 --- a/examples/README.md +++ b/examples/README.md @@ -17,12 +17,19 @@ Run one with: [LANGUAGE.md](../LANGUAGE.md) walks the same ground feature by feature, with prose around each of these. +Two entries here are not single-value examples and so are not in that table: +`listing/` is a small fixture directory `listing-a-directory.hb` reads, and +`hashmake/` is a whole project - a build description plus a vendored cJSON +checkout - covered by its own test and its own README rather than by an +asserted value. + ## Language basics | Example | Shows | |---|---| | `arithmetic.hb` | Integer vs Float, truncating `/`, `%`, precedence, unary minus | | `numeric-literals.hb` | Hex/octal/binary Integers, `_` grouping, Float exponents | +| `text-slicing.hb` | `textlen`/`textslice`, counting codepoints — and `endswith` built on them | | `strings.hb` | `Utf8` literals and escapes, `concat`, comparison | | `comparison-and-logic.hb` | `==`/`<`/`>`, `and`/`or` — and the absence of boolean literals | | `nothing-and-empty.hb` | `nothing` (unit) vs `empty` (the zero-entry Table) | @@ -33,6 +40,7 @@ prose around each of these. |---|---| | `tables-map.hb` | `.field` and `[computed]` keys, field access, `concat` as update | | `tables-sequence.hb` | `{a, b, c}` shorthand, `[i]` indexing, why `concat` isn't append | +| `folding-a-table.hb` | `fold` — the one traversal, and why it walks in key order | | `table-and-concat.hb` | Overriding a single field of an existing Table | | `table-destructuring.hb` | `is { .a as x }` patterns, and testing for a key without failing | | `sequence-pattern.hb` | `{N}` exact-length patterns with positional binds | @@ -55,6 +63,9 @@ prose around each of these. |---|---| | `files-sandboxed.hb` | Directory handles, contained sub-paths, `filetext`, path display | | `files-symlink.hb` | `readlink` — and why symlinks aren't values of their own | +| `listing-a-directory.hb` | `listdir`, and filtering a listing down to what you want | +| `workdir-containment.hb` | `ctx.dir`, and the three things a handle-less path may reach | +| `running-a-program.hb` | `exec` — a program's inputs and outputs as values | | `hashing.hb` | `sha256`, and the content identity that makes two Files one value | | `hashing-directories.hb` | A directory's hash: its entries, and the one digest that reads | | `hashing-functions.hb` | A closure's hash: its body's shape and the values it captures | @@ -69,7 +80,12 @@ prose around each of these. ## Supporting files `choice.txt`, `optiona.txt`, `optionb.txt` are inputs for `option-picker.hb` and -the async examples; `link-to-optiona` is a symlink `files-symlink.hb` reads. +the async examples; `link-to-optiona` is a symlink `files-symlink.hb` reads; +`listing/` holds three files with fixed names so `listing-a-directory.hb` can +assert a stable answer. + +`hashmake/` is a whole worked project rather than an example file — a build +description and a vendored cJSON checkout. See `hashmake/README.md`. `async-branching.hb` writes `branch-*.marker` files next to itself as evidence of which branches ran. `createfile` is exclusive, so delete them before re-running diff --git a/examples/hashmake/README.md b/examples/hashmake/README.md new file mode 100644 index 0000000..f7caf8d --- /dev/null +++ b/examples/hashmake/README.md @@ -0,0 +1,72 @@ +# Building a real C project with hashmake + +This directory builds [cJSON](https://github.com/DaveGamble/cJSON) — vendored as +a git submodule, pinned to a commit — from source, links it, and runs the +result. It is the worked example for `tools/hashmake`. + +```sh +odin build src -out:hb # if you haven't already +odin build tools/hashmake -out:hashmake +git submodule update --init # if you cloned without --recursive + +cd examples/hashmake +../../hashmake --graph # what depends on what +../../hashmake -n # the order it would build in +../../hashmake # build it, and run what it built +``` + +The last command prints cJSON's own test output — a version banner and a few +formatted JSON documents. + +## The graph + +``` +cJSON.o cJSON_Utils.o test.o + \ | / + \------------------+-----------------/ + | + link (clang -o cjson-demo ... -lm) + | + run (produces no artifact) +``` + +**No C file is named anywhere in `hashmake.hb`.** The sources are found by +listing the checkout and filtering on a suffix: + +```hashedbuild +let sources fold { + .table = listdir cjson, + .init = empty, + .step = (let s; (endswith { .text = s.value, .suffix = ".c" }) then ... else s.acc), +}; +``` + +`endswith` is not a builtin either — it is four lines written on top of +`textlen` and `textslice`. Add a `.c` file to the checkout and it becomes a node +in this graph with nothing edited here. + +The `run` target produces no artifact: it answers with the program's output as +text so hashmake can show it, and hashmake refuses to let anything depend on it. + +## What to try + +**Watch it not rebuild.** Run `../../hashmake` twice. The second run compiles +nothing — every node's `cached` key is unchanged. + +**Watch it rebuild exactly what changed.** Add a comment to one of cJSON's `.c` +files and build again: that one object and the link are rebuilt, the other two +objects are not. + +**Watch it come back.** Undo the edit and build again. It is a *hit*, not a +rebuild — the cache is keyed on content, so restoring a file restores the answer. +A timestamp-based tool would rebuild here. + +**Watch it refuse to escape.** Add `filetext (loadfile "/etc/hostname")` to a +build function. It fails: the build file is contained to this directory unless +you pass `--allow-any-path`. + +## Requirements + +`clang` on `PATH`, and the submodule checked out. This is the one example in the +repository that needs a compiler; the test that runs it skips itself, with a +logged reason, where clang is absent. diff --git a/tools/hashmake/README.md b/tools/hashmake/README.md new file mode 100644 index 0000000..99b5c85 --- /dev/null +++ b/tools/hashmake/README.md @@ -0,0 +1,98 @@ +# hashmake + +A build tool whose build files are HashedBuild programs. + +`hashmake` looks for a `hashmake.hb` in the current directory, evaluates it to a +dependency graph, orders the graph, refuses cycles, and calls each node with +what it asked for. That is the whole of it — in particular it has **no cache of +its own**. Incremental rebuilds come from the language: a node wraps its own +work in `cached` (SPEC.md §15), whose key is the code plus the *values* it +reads, and a `File` is its content (§3). There are no timestamps in this +program. + +## Building it + +```sh +odin build tools/hashmake -out:hashmake +``` + +It imports `src` (the interpreter) directly, so there is nothing to install and +nothing to keep in sync — the tool and the language are built from the same +tree. + +## Usage + +``` +hashmake [options] [target...] + + -C, --directory Run as if started in + -f, --file The build file (default: hashmake.hb) + -n, --dry-run Print the order targets would be built in, and stop + --graph Print the dependency graph, and stop + --allow-any-path Let the build file resolve paths outside its own directory + --cache-dir Where cached entries are kept + -h, --help / --version +``` + +With no target, the graph's `.default` is built. + +## What a `hashmake.hb` must evaluate to + +A HashedBuild program is one expression, and this one evaluates to: + +```hashedbuild +{ + .default = "run", + .targets = { + . = { + .needs = { . = "", ... }, // may be `empty` + .build = , // prerequisites -> artifact + }, + ... + }, +} +``` + +- **`.needs` maps a local alias to the name of the target that produces it.** + hashmake builds each dependency first and hands `.build` exactly + `{ alias -> artifact }` — never a path, and never a filename to look up. +- **`.build` returns the artifact**, which is a `File`. +- **A target that returns something other than a `File` has produced no + artifact.** That is allowed and useful — a "run" target exists for its effect + — but nothing may depend on one, and hashmake says so rather than passing a + non-artifact along. If it returns `Utf8`, hashmake prints it, which is how a + run target's output reaches your terminal. +- A bare `Table` of targets is accepted too, if you don't need `.default`. + +## What it enforces + +- **Cycles are refused before anything is built**, and named in full: + `error: dependency cycle: a -> b -> c -> a`. `--graph` and `-n` check this too, + so you can find a loop without running a compiler. +- **A `.needs` naming a target that doesn't exist** is a clean error, not a + crash. +- **The build file is contained to its own directory.** hashmake evaluates it + with `ctx.dir` set to the build file's directory and only the `workdir` + permission (SPEC.md §9), so `loadfile "../../etc/passwd"` fails. Pass + `--allow-any-path` when a build genuinely needs to reach outside. + + This bounds what the *build description* can read. It does not bound what a + compiler it starts can read — see `exec`'s note in SPEC.md §16. + +## Caching, and what invalidates what + +Nothing is cached unless a node asks. A node that wants to be looks like: + +```hashedbuild +.build = (let prereqs; cached ( + let r exec { .cmd = "clang", .args = { ... }, .inputs = { ... }, .outputs = { "x.o" } }; + check(r.status == 0, "clang failed") r.outputs["x.o"])) +``` + +Because `exec` takes its inputs as `File` values and returns its outputs as +`File` values, the cache key contains the *content* of every input. So editing +one source invalidates that object and everything downstream of it, and nothing +else. Restoring the file restores the hit — which is a thing a timestamp-based +tool cannot do. + +`examples/hashmake/` is a worked example that builds a real C library this way. From 0a21ecdc31f48e505b1026ad44febd837e9e3bfc Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:31:35 +0000 Subject: [PATCH 7/8] Keep running-a-program.hb's test to Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The example declares its linked output as "greet", and Windows produces greet.exe - a declared output that is not there is fatal (§16), by design. GitHub's windows-latest image ships clang, so the existing "skip if clang is missing" guard would not have caught this; the suite would have gone red on a runner nobody was looking at. Teaching the example to pick an extension per platform would put the platform, rather than exec, at the centre of what it demonstrates, so it stays a Linux check and both the test and the example's header say why. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- examples/running-a-program.hb | 4 +++- src/examples_test.odin | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/examples/running-a-program.hb b/examples/running-a-program.hb index 42930c5..1ea523f 100644 --- a/examples/running-a-program.hb +++ b/examples/running-a-program.hb @@ -23,7 +23,9 @@ // // This one shells out to `clang`, so unlike every other example it needs // something outside the repository; the test that runs it skips itself, with a -// logged reason, where clang isn't installed. Evaluates to +// logged reason, where clang isn't installed - and on Windows, where a linked +// program is `greet.exe` and the `.outputs` below would name something that +// isn't there. Evaluates to // { compiled: 0, said_its_version: true, ran_what_it_built: "hello from C\n" }. let source createfile { diff --git a/src/examples_test.odin b/src/examples_test.odin index c18618e..2365c23 100644 --- a/src/examples_test.odin +++ b/src/examples_test.odin @@ -232,6 +232,15 @@ clear_branch_markers :: proc() { // does where symlinks aren't available. @(test) test_example_running_a_program_drives_clang :: proc(t: ^testing.T) { + // Windows names a linked program greet.exe, and the example declares its + // output as "greet" - a declared output that is not there is fatal (§16), by + // design. Teaching the example to pick an extension per platform would put + // the platform, rather than exec, at the centre of what it demonstrates, so + // it stays a Linux check and says so. + when ODIN_OS == .Windows { + log.info("skipping running-a-program.hb: it names a linked output without a platform extension") + return + } if !command_exists("clang") { log.info("skipping running-a-program.hb: clang is not on PATH in this environment") return From e9d549e0e271a9cf1152bf35aae21af147e2dd1a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 09:49:25 +0000 Subject: [PATCH 8/8] Fix the two CI failures: a tree-walking test, and a second skip list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both were mine, and neither was a flake. **Windows.** test_ctx_dir_does_not_hash_as_the_directorys_contents asserted the right property the wrong way: it compared against `sha256 (loadfile ".")`, which hashes the entire checkout - .git and the submodule's internals included - and the runner cannot open everything in there, so it failed with "could not read a directory's entries (Access)". Walking a large tree to prove that nothing about a directory reaches ctx.dir's digest was backwards anyway. It is now asserted directly, on two constructed handles rooted at different paths, which must hash alike: that is the property §15 needs, since the whole ctx goes into every cache key. The other half - that a directory File *does* still hash by content, so reading through ctx.dir is unaffected - is a second test against the three-file fixture rather than the checkout root. **Playground.** running-a-program.hb was added to scripts/wasi_smoke.sh's skip list but not to the one inside scripts/playground_browser_test.py, which I had not noticed was separate. A browser cannot start a process, so `exec` reported exactly that and there was no answer for the two sides to agree on - the same reason, and now the same skip, with a comment on each pointing at the other. Verified rather than assumed: the playground failure was reproduced in a real Chromium and now reports "all playground checks passed". The Windows one cannot be reproduced here, so the fix removes the failing expression outright rather than trying to make it survive - nothing in either replacement reads more than three files. Suite is 253 green, WASI still agrees on 31 examples, and the cJSON build still runs. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01R8FUURkLC3b5axnRKvKUfM --- scripts/playground_browser_test.py | 8 ++++-- src/builtins_build_test.odin | 42 +++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/scripts/playground_browser_test.py b/scripts/playground_browser_test.py index b0c2823..b552297 100644 --- a/scripts/playground_browser_test.py +++ b/scripts/playground_browser_test.py @@ -282,9 +282,13 @@ def run(source, **opts): if os.path.exists(native_hb): # Displayed paths differ by construction (checkout vs preopen), and # async needs a thread-spawn this harness does not give it - both - # covered above. + # covered above. running-a-program.hb drives clang: a browser cannot + # start a process at all, so `exec` there reports exactly that and + # there is no answer for the two to agree on (same reason it is + # skipped in scripts/wasi_smoke.sh). skip = {"files-sandboxed.hb", "option-picker.hb", - "async-basics.hb", "async-branching.hb", "async-table.hb"} + "async-basics.hb", "async-branching.hb", "async-table.hb", + "running-a-program.hb"} compared = 0 for name in sorted(n for n in os.listdir(EXAMPLES) if n.endswith(".hb")): if name in skip: diff --git a/src/builtins_build_test.odin b/src/builtins_build_test.odin index e171733..f318874 100644 --- a/src/builtins_build_test.odin +++ b/src/builtins_build_test.odin @@ -184,18 +184,40 @@ test_ctx_dir_works_as_a_directory_handle :: proc(t: ^testing.T) { // the tree. Hashing as a bare tag is what keeps a cache key stable while the // project changes around it. @(test) -test_ctx_dir_does_not_hash_as_the_directorys_contents :: proc(t: ^testing.T) { - // The same directory, reached two ways: as ctx.dir, and as a directory File. - // The File hashes over every byte of the tree (§3); ctx.dir must not, or the - // whole project would be inside every cache key. - val, ok, err := eval_build(`(sha256 ctx.dir) == (sha256 (loadfile "."))`) +test_ctx_dir_hashes_as_a_constant :: proc(t: ^testing.T) { + // Two handles rooted at different directories must hash alike. That is the + // property the whole incremental story rests on: §15 puts the entire ctx + // into every cache key, so if ctx.dir encoded either its contents or its + // path, every entry would depend on the whole project tree or on where the + // checkout happens to live. + // + // Asserted on constructed values rather than by hashing a real directory: + // the point is that nothing about the directory reaches the digest, and a + // test that walked a tree to show it would be testing the walk. (An earlier + // version compared against `sha256 (loadfile ".")`, which hashed the whole + // checkout - .git included - and failed on Windows for an unrelated reason.) + here := new(Workdir_Value) + here.dir_path = "/some/checkout" + elsewhere := new(Workdir_Value) + elsewhere.dir_path = "/a/quite/different/place" + + a, a_fail := value_digest(Value(here)) + b, b_fail := value_digest(Value(elsewhere)) + testing.expect(t, a_fail.kind == .None, "hashing ctx.dir should not fail") + testing.expect(t, b_fail.kind == .None, "hashing ctx.dir should not fail") + testing.expect(t, a == b, "two ctx.dir handles must hash alike - a path in the key would invalidate a moved checkout") +} + +// The other half of the same promise: a directory File still hashes over its +// contents, so what a program reads *through* ctx.dir is unaffected. Uses the +// three-file fixture rather than the checkout root, which is large and, on +// Windows, contains entries the runner cannot open. +@(test) +test_a_directory_file_still_hashes_by_content :: proc(t: ^testing.T) { + val, ok, err := eval_build(`(sha256 (loadfile "examples/listing")) == (sha256 ctx.dir)`) testing.expect(t, ok, err) if !ok do return - testing.expect( - t, - !val.(bool), - "ctx.dir hashed as its directory's contents - every cached entry would then depend on the whole tree", - ) + testing.expect(t, !val.(bool), "a directory File must not hash as the ctx.dir constant") } // ---- exec ---------------------------------------------------------------------