diff --git a/.github/workflows/windows-perl-inventory-experiment.yml b/.github/workflows/windows-perl-inventory-experiment.yml new file mode 100644 index 00000000..06a3d5de --- /dev/null +++ b/.github/workflows/windows-perl-inventory-experiment.yml @@ -0,0 +1,75 @@ +name: Windows Perl inventory and archive experiment + +on: + push: + branches: [tapish-codex/windows-perl-inventory-20260909] + +permissions: + contents: read + +concurrency: + group: windows-perl-inventory-${{ github.ref }} + cancel-in-progress: false + +jobs: + inventory: + name: Inventory unchanged pinned Windows Perl + runs-on: windows-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Restore normal Windows prerequisites without saving a cache + id: prerequisites + uses: ./.github/actions/setup-windows-native + with: + save-cache: "false" + - name: Inventory installed distribution + shell: pwsh + env: + PERL_CACHE_HIT: ${{ steps.prerequisites.outputs.cache-hit }} + run: >- + python scripts/inventory-windows-perl.py + --root 'C:\Strawberry' + --output "$env:RUNNER_TEMP/perl-inventory/installed.json.gz" + - name: Retain inventory metadata + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: windows-perl-installed-inventory + path: ${{ runner.temp }}/perl-inventory/installed.json.gz + if-no-files-found: error + compression-level: 0 + retention-days: 7 + + # Cold native correctness is retained at 67fa357 in run 34332893760. + # This next stage measures archives only; it does not rerun native tests. + archive-roundtrip: + name: Measure full and reduced Perl archive round trips + runs-on: windows-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Restore unchanged native prerequisites without saving + id: prerequisites + uses: ./.github/actions/setup-windows-native + with: + save-cache: "false" + - name: Check archive and restoration controls + run: python scripts/measure-perl-archive.test.py + - name: Check reused directory-move controls + run: python scripts/measure-perl-cold.test.py + - name: Measure verified local archive round trips + env: + PERL_CACHE_HIT: ${{ steps.prerequisites.outputs.cache-hit }} + run: python scripts/measure-perl-archive.py + - name: Retain archive measurements and restoration evidence + if: ${{ always() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: windows-perl-archive + path: perl-archive-measurements/** + if-no-files-found: error + retention-days: 7 diff --git a/docs/provenance.md b/docs/provenance.md index 506af5af..09bd4682 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -100,3 +100,29 @@ actual image linkage and refusal of a different executable. Apple's defines frame image indexes and binary image UUID/architecture fields. Native CI still disables debug information; retaining a binary does not restore absent source line tables or identify which individual nextest test executed it. + +## Windows Perl inventory experiment + +The installed-distribution inventory script, tests and dedicated experiment +workflow are project-authored under this repository's Apache-2.0 license. +Artifacts contain relative file names, byte counts, SHA-256 digests and scoped +Perl runtime identity. The distribution is supplied by the existing pinned +Windows prerequisite action, retaining its upstream attribution resources. + +The cold-correctness helper reuses the project-authored bounded log collector +and the existing disposable SQLCipher regression test. Its filesystem failure +controls use this test file's own bytes and explicit replacement bytes; they +are not evidence that a reduced Perl runtime builds native dependencies. +The dedicated Windows comparison temporarily moves three inventoried MinGW +build directories on disposable runners, restores their bytes, and retains +fresh native-build timings plus source and installation hashes. It does not +save a distribution or compiler cache, change production prerequisites, or +establish a cache-extraction speed improvement. + +The archive measurement helper and its controls are project-authored. It reuses +the installed inventory, directory-move control and bounded log capture helpers. +Its disposable Windows workflow uses the existing GNU tar/zstd commands and +performs full byte/hash round trips before reporting timings. The prior cold +native comparison is retained in run 34332893760 at commit 67fa357; the archive +stage does not rerun native tests or save production caches. Archive payloads +stay on the disposable runner; uploaded evidence contains metadata and logs. diff --git a/scripts/inventory-windows-perl.py b/scripts/inventory-windows-perl.py new file mode 100644 index 00000000..f7aad778 --- /dev/null +++ b/scripts/inventory-windows-perl.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Read the installed pinned Perl tree; never alter its files or cache.""" +import argparse +from collections import defaultdict +import gzip +import hashlib +import json +import os +from pathlib import Path +import re +import stat +import subprocess +import time + +MAX_FILES = 50000 +MAX_BYTES = 4 * 1024**3 + + +def reject_link(path): + info = path.lstat() + if stat.S_ISLNK(info.st_mode) or getattr(info, "st_file_attributes", 0) & 0x400: + raise ValueError("Perl inventory refuses symlinks and Windows reparse points") + return info + + +def inventory(root): + reject_link(root) + if not root.is_dir(): + raise ValueError("Perl inventory root is not a directory") + files = [] + groups = defaultdict(lambda: {"files": 0, "bytes": 0}) + total = 0 + + def walk_error(error): + raise error + + for directory, directories, names in os.walk(root, onerror=walk_error, followlinks=False): + for name in directories: + reject_link(Path(directory) / name) + for name in names: + path = Path(directory) / name + before = reject_link(path) + if not stat.S_ISREG(before.st_mode): + raise ValueError("Perl inventory encountered a non-regular file") + if len(files) >= MAX_FILES or total + before.st_size > MAX_BYTES: + raise ValueError("Perl inventory exceeded its file/byte bound") + digest = hashlib.sha256() + read_bytes = 0 + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + read_bytes += len(chunk) + after = reject_link(path) + if (before.st_size, before.st_mtime_ns, before.st_ino) != ( + after.st_size, after.st_mtime_ns, after.st_ino + ) or read_bytes != before.st_size: + raise ValueError("Perl inventory file changed during hashing") + relative = path.relative_to(root).as_posix() + parts = relative.split("/") + group = "/".join(parts[:2]) if len(parts) > 2 else parts[0] + files.append({"path": relative, "bytes": read_bytes, "sha256": digest.hexdigest()}) + groups[group]["files"] += 1 + groups[group]["bytes"] += read_bytes + total += read_bytes + if not files: + raise ValueError("Perl inventory root is empty") + return {"total_files": len(files), "total_bytes": total, + "groups": dict(sorted(groups.items())), "files": sorted(files, key=lambda row: row["path"])} + + +def perl_identity(root): + perl = root / "perl/bin/perl.exe" + for path in (root, root / "perl", root / "perl/bin", perl): + reject_link(path) + code = ('print JSON::PP::encode_json({version=>"$^V", arch=>$Config{archname}, ' + 'module=>$INC{"Locale/Maketext/Simple.pm"}})') + result = subprocess.run([str(perl), "-MConfig", "-MJSON::PP", "-MLocale::Maketext::Simple", "-e", code], + capture_output=True, text=True, timeout=30, check=True) + if len(result.stdout) > 4096 or result.stderr: + raise ValueError("Unexpected Perl identity output") + identity = json.loads(result.stdout) + if identity["version"] != "v5.42.2": + raise ValueError("Pinned Perl runtime version mismatch") + module = Path(identity.pop("module")).resolve() + identity["module_path"] = module.relative_to(root.resolve()).as_posix() + return identity + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if os.name != "nt": + raise ValueError("Installed-runtime inventory requires Windows") + if args.output.resolve().is_relative_to(args.root.resolve()): + raise ValueError("Inventory output must be outside the distribution") + commit = os.environ["GITHUB_SHA"] + run_id = os.environ["GITHUB_RUN_ID"] + if not re.fullmatch(r"[0-9a-f]{40}", commit) or not run_id.isdigit(): + raise ValueError("Missing source/run identity") + started = time.monotonic() + identity = perl_identity(args.root) + result = inventory(args.root) + module = identity["module_path"] + if module not in {row["path"] for row in result["files"]}: + raise ValueError("Validated Perl module is absent from the inventory") + result.update({ + "schema_version": 1, "source_sha": commit, "run_id": int(run_id), + "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), + "runner_image": os.environ["ImageVersion"], "runner_os": os.environ["ImageOS"], + "configured_distribution_version": "5.42.2.1", "perl_identity": identity, + "cache_key": "bridge-windows-strawberryperl-5.42.2.1-v1", + "cache_hit": os.environ["PERL_CACHE_HIT"], + "inventory_seconds": round(time.monotonic() - started, 6), + "scope": "Installed distribution inventory and prerequisite identity only; no reduced cache or native-build qualification.", + }) + if result["cache_hit"] not in ("true", "false", ""): + raise ValueError("Unexpected cache restore output") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(gzip.compress((json.dumps(result, indent=2) + "\n").encode(), mtime=0)) + print(json.dumps({key: result[key] for key in + ("source_sha", "run_id", "total_files", "total_bytes", "cache_hit", "perl_identity", "inventory_seconds")})) + + +if __name__ == "__main__": + main() diff --git a/scripts/inventory-windows-perl.test.py b/scripts/inventory-windows-perl.test.py new file mode 100644 index 00000000..af7d574b --- /dev/null +++ b/scripts/inventory-windows-perl.test.py @@ -0,0 +1,42 @@ +import hashlib +import importlib.util +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("inventory_windows_perl", Path(__file__).with_name("inventory-windows-perl.py")) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class InventoryTests(unittest.TestCase): + def test_reads_actual_source_bytes_and_relative_paths(self): + source = Path(__file__).read_bytes() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "perl/bin").mkdir(parents=True) + (root / "perl/bin/source.txt").write_bytes(source) + result = module.inventory(root) + self.assertEqual(result["total_files"], 1) + self.assertEqual(result["total_bytes"], len(source)) + self.assertEqual(result["files"], [{"path": "perl/bin/source.txt", "bytes": len(source), + "sha256": hashlib.sha256(source).hexdigest()}]) + + def test_missing_or_empty_root_cannot_report_zero_files(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + with self.assertRaises(FileNotFoundError): + module.inventory(root / "missing") + with self.assertRaises(ValueError): + module.inventory(root) + + def test_symlink_outside_distribution_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "linked").symlink_to(Path(__file__).resolve()) + with self.assertRaises(ValueError): + module.inventory(root) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/measure-perl-archive.py b/scripts/measure-perl-archive.py new file mode 100644 index 00000000..3248a271 --- /dev/null +++ b/scripts/measure-perl-archive.py @@ -0,0 +1,243 @@ +"""Disposable Windows archive round trips; never save a production cache.""" +from contextlib import contextmanager +import gzip +import hashlib +import importlib.util +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import subprocess +import tarfile +import tempfile + +ROOT = Path(__file__).resolve().parents[1] +spec = importlib.util.spec_from_file_location("perl_cold", ROOT / "scripts/measure-perl-cold.py") +cold = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cold) +SEQUENCE = ("full", "reduced", "full", "reduced", "reduced", "full") + + +def write_archive_manifest(path, root): + # Match @actions/cache's byte write: Windows text translation would add CR + # to the filename GNU tar reads from this one-entry manifest. + path.write_bytes(root.as_posix().encode("utf-8")) + + +def sha(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +@contextmanager +def parked_installation(root): + """Preserve original bytes and refuse to overwrite an unexpected replacement.""" + cold.inventory.reject_link(root) + backup = Path(tempfile.mkdtemp(prefix="bridge-perl-archive-", dir=root.parent)) + original = backup / "original" + root.rename(original) + try: + yield + finally: + if os.path.lexists(root): + raise ValueError("Refusing to overwrite replacement installation; original remains in backup") + original.rename(root) + backup.rmdir() + + +@contextmanager +def extraction_destination(root): + root.mkdir(exist_ok=False) + identity = cold.inventory.reject_link(root).st_ino + try: + yield + finally: + current = cold.inventory.reject_link(root) + if not root.is_dir() or current.st_ino != identity: + raise ValueError("Extraction destination was replaced; refusing cleanup") + # Only this freshly created disposable destination is removed. Refuse + # reparse points or unexpected filesystem objects before recursive cleanup. + count = 0 + def fail(error): + raise error + for directory, directories, files in os.walk(root, onerror=fail, followlinks=False): + for name in directories + files: + info = cold.inventory.reject_link(Path(directory) / name) + count += 1 + if count > 100000 or not (stat.S_ISDIR(info.st_mode) or stat.S_ISREG(info.st_mode)): + raise ValueError("Unexpected extraction cleanup entry") + shutil.rmtree(root) + + +class BlockReader: + """Track the physical stream position without hiding tar-reader lookahead.""" + def __init__(self, stream): + self.stream, self.count, self.tail = stream, 0, b"" + + def read(self, size): + data = self.stream.read(size) + self.count += len(data) + self.tail = (self.tail + data)[-512:] + return data + + +def verify_tar(stream, expected, prefix): + expected = {row["path"]: row for row in expected} + seen = set() + members, last_end = 0, 0 + reader = BlockReader(stream) + with tarfile.open(fileobj=reader, mode="r|", bufsize=512) as archive: + for member in archive: + members += 1 + last_end = member.offset_data + ((member.size + 511) // 512) * 512 + if members > 100000: + raise ValueError("Archive member bound exceeded") + name = member.name.replace("\\", "/").rstrip("/") + if name == prefix and member.isdir(): + continue + if not name.startswith(prefix + "/"): + raise ValueError("Archive member escaped the installation prefix") + relative = name[len(prefix) + 1:] + if ".." in relative.split("/") or PurePosixPath(relative).is_absolute(): + raise ValueError("Unsafe archive member") + if member.isdir(): + continue + if not member.isfile() or relative not in expected or relative in seen: + raise ValueError("Unexpected archive file or link") + row = expected[relative] + if member.size != row["bytes"]: + raise ValueError("Archive file size mismatch") + digest = hashlib.sha256() + content = archive.extractfile(member) + for block in iter(lambda: content.read(1024 * 1024), b""): + digest.update(block) + if digest.hexdigest() != row["sha256"]: + raise ValueError("Archive file hash mismatch") + seen.add(relative) + if seen != set(expected): + raise ValueError("Archive is missing expected files") + # Block-sized reads leave no tarfile lookahead after the first zero block. + # Require both end blocks, then inspect every remaining padding byte. + if reader.count != last_end + 512 or reader.tail != bytes(512): + raise ValueError("Missing or ambiguous first archive end block") + padding = 0 + for block in iter(lambda: stream.read(1024 * 1024), b""): + padding += len(block) + if padding > 1024 * 1024 or any(block): + raise ValueError("Unexpected archive padding") + if padding < 512 or padding % 512: + raise ValueError("Missing or incomplete second archive end block") + return {"verified_files": len(seen), "members": members} + + +def verify_archive(archive, expected, zstd, prefix): + process = subprocess.Popen([str(zstd), "-q", "-d", "-c", str(archive)], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + try: + with process.stdout: + result = verify_tar(process.stdout, expected, prefix) + status = process.wait(timeout=30) + if status: + raise subprocess.CalledProcessError(status, process.args) + return result + finally: + if process.poll() is None: + process.kill() + process.wait() + + +def main(): + if os.name != "nt" or os.environ.get("PERL_CACHE_HIT") != "true": + raise ValueError("Windows and the exact installed-Perl cache are required") + root = Path("C:/Strawberry") + tar = Path("C:/Program Files/Git/usr/bin/tar.exe") + located = shutil.which("zstd") + if located is None: + raise ValueError("Pinned runner zstd is missing") + zstd = Path(located) + tools = {} + for name, path in (("tar", tar), ("zstd", zstd)): + if path.resolve().is_relative_to(root.resolve()): + raise ValueError("Archive tools must remain available while the installation is parked") + cold.inventory.reject_link(path) + result = subprocess.run([str(path), "--version"], capture_output=True, check=True, timeout=30) + if len(result.stdout) + len(result.stderr) > 8192: + raise ValueError("Unexpected archive tool version output") + tools[name] = {"path": str(path), "sha256": sha(path), "version": result.stdout.decode(errors="replace")} + if "GNU tar" not in tools["tar"]["version"]: + raise ValueError("Comparison requires the existing GNU tar path") + output = ROOT / "perl-archive-measurements" + output.mkdir(exist_ok=False) + original = cold.inventory.inventory(root) + identity = cold.inventory.perl_identity(root) + sources = {str(p.relative_to(ROOT)).replace("\\", "/"): sha(p) for p in + (Path(__file__), ROOT / "scripts/measure-perl-cold.py", ROOT / "scripts/inventory-windows-perl.py", + ROOT / "scripts/capture-package-log.py")} + cold.write(output / "source.json", { + "head": os.environ["GITHUB_SHA"], "run_id": int(os.environ["GITHUB_RUN_ID"]), + "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), "image": os.environ["ImageVersion"], + "image_os": os.environ["ImageOS"], "tools": tools, "source_hashes": sources, + "original_tree_sha256": cold.tree_digest(original["files"]), "perl_identity": identity, + "sequence": list(SEQUENCE), "scope": "Local archive extraction with warmed filesystem caches; no network, full CI speedup or production adoption claim.", + }) + (output / "original-inventory.json.gz").write_bytes(gzip.compress(json.dumps(original).encode(), mtime=0)) + archives = {} + try: + # Archives remain local to the disposable runner; only metadata/logs ship. + with tempfile.TemporaryDirectory(prefix="bridge-perl-archives-", dir=root.parent) as temporary: + workspace = Path(temporary) + for case in ("full", "reduced"): + expected = [r for r in original["files"] if case == "full" or + not r["path"].startswith(tuple(p + "/" for p in cold.PREFIXES))] + archive = workspace / f"{case}.tzst" + manifest = workspace / f"{case}-manifest.txt" + write_archive_manifest(manifest, root) + with cold.distribution_variant(root, case == "reduced"): + assert cold.inventory.inventory(root)["files"] == expected + seconds = cold.execute([str(tar), "--posix", "-cf", archive.as_posix(), "--exclude", archive.as_posix(), + "-P", "-C", ROOT.as_posix(), "--files-from", manifest.as_posix(), + "--force-local", "--use-compress-program", "zstd -T0"], output, f"pack-{case}") + assert cold.inventory.inventory(root)["files"] == original["files"] + if archive.stat().st_size > 2 * 1024**3: + raise ValueError("Archive size bound exceeded") + verified = verify_archive(archive, expected, zstd, root.as_posix()) + archives[case] = {"path": archive, "expected": expected, "sha256": sha(archive)} + cold.write(output / f"archive-{case}.json", { + "case": case, "archive_bytes": archive.stat().st_size, "archive_sha256": archives[case]["sha256"], + "packing_seconds": seconds, "input_tree_sha256": cold.tree_digest(expected), **verified, + }) + samples = [] + with parked_installation(root): + for index, case in enumerate(SEQUENCE, 1): + selected = archives[case] + if sha(selected["path"]) != selected["sha256"]: + raise ValueError("Archive changed between trials") + with extraction_destination(root): + seconds = cold.execute([str(tar), "-xf", selected["path"].as_posix(), "-P", "-C", ROOT.as_posix(), + "--force-local", "--use-compress-program", "zstd -d"], output, f"extract-{index}-{case}") + observed = cold.inventory.inventory(root) + assert observed["files"] == selected["expected"], "Extracted bytes differ from the archive input" + assert cold.inventory.perl_identity(root) == identity + samples.append({"index": index, "case": case, "extraction_seconds": seconds, + "archive_sha256": selected["sha256"], "tree_sha256": cold.tree_digest(observed["files"]), + "files": observed["total_files"], "bytes": observed["total_bytes"], "identity_verified": True}) + cold.write(output / "samples.json", samples) + print(json.dumps(samples[-1]), flush=True) + assert cold.inventory.inventory(root)["files"] == original["files"] + cold.write(output / "result.json", {"passed": True, "round_trips": len(samples), "production_cache_saved": False}) + finally: + restored = cold.inventory.inventory(root) + unchanged = all(sha(ROOT / name) == value for name, value in sources.items()) + tools_same = all(sha(Path(row["path"])) == row["sha256"] for row in tools.values()) + cold.write(output / "restoration.json", {"full_tree_restored": restored["files"] == original["files"], + "source_inputs_unchanged": unchanged, "tools_unchanged": tools_same, + "tree_sha256": cold.tree_digest(restored["files"])}) + assert restored["files"] == original["files"] and unchanged and tools_same + + +if __name__ == "__main__": + main() diff --git a/scripts/measure-perl-archive.test.py b/scripts/measure-perl-archive.test.py new file mode 100644 index 00000000..ca026d83 --- /dev/null +++ b/scripts/measure-perl-archive.test.py @@ -0,0 +1,109 @@ +"""Archive/parser and disposable-directory controls; no Windows timing claims.""" +import hashlib +import importlib.util +import io +from pathlib import Path, PureWindowsPath +import tarfile +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("perl_archive", Path(__file__).with_name("measure-perl-archive.py")) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +SOURCE = Path(__file__).read_bytes() +EXPECTED = [{"path": "control.py", "bytes": len(SOURCE), "sha256": hashlib.sha256(SOURCE).hexdigest()}] + + +def archive(name="C:/Strawberry/control.py", content=SOURCE, link=False): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w") as tar: + member = tarfile.TarInfo(name) + member.size = len(content) + if link: + member.type, member.linkname, member.size = tarfile.SYMTYPE, "outside", 0 + tar.addfile(member, None if link else io.BytesIO(content)) + stream.seek(0) + return stream + + +class ArchiveTests(unittest.TestCase): + def test_windows_manifest_has_no_translated_line_ending(self): + with tempfile.TemporaryDirectory() as temp: + path = Path(temp) / "manifest.txt" + module.write_archive_manifest(path, PureWindowsPath("C:/Strawberry")) + self.assertEqual(path.read_bytes(), b"C:/Strawberry") + + def test_parses_actual_source_bytes(self): + self.assertEqual(module.verify_tar(archive(), EXPECTED, "C:/Strawberry")["verified_files"], 1) + + def test_rejects_wrong_hash_and_omitted_source(self): + for rows in ([{**EXPECTED[0], "sha256": "0" * 64}], EXPECTED + [{**EXPECTED[0], "path": "missing"}]): + with self.assertRaises(ValueError): + module.verify_tar(archive(), rows, "C:/Strawberry") + + def test_rejects_escape_and_link(self): + for stream in (archive("C:/outside/control.py"), archive("C:/Strawberry/../control.py"), archive(link=True)): + with self.assertRaises(ValueError): + module.verify_tar(stream, EXPECTED, "C:/Strawberry") + + def test_rejects_nonzero_padding_inside_and_beyond_reader_buffer(self): + raw = archive().getvalue() + first_end = 512 + ((len(SOURCE) + 511) // 512) * 512 + for offset in (first_end + 512, len(raw) + 512): + altered = bytearray(raw) + if len(altered) < offset + 17: + altered.extend(bytes(offset + 17 - len(altered))) + altered[offset:offset + 17] = b"nonzero-padding!!" + with self.assertRaises(ValueError): + module.verify_tar(io.BytesIO(altered), EXPECTED, "C:/Strawberry") + + def test_rejects_missing_end_blocks(self): + raw = archive().getvalue() + first_end = 512 + ((len(SOURCE) + 511) // 512) * 512 + for length in (first_end, first_end + 512): + with self.assertRaises(ValueError): + module.verify_tar(io.BytesIO(raw[:length]), EXPECTED, "C:/Strawberry") + + def test_body_failure_cleans_partial_directories_and_restores_original(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "distribution" + root.mkdir() + (root / "control").write_bytes(SOURCE) + with self.assertRaisesRegex(RuntimeError, "control failure"): + with module.parked_installation(root): + with module.extraction_destination(root): + (root / "empty/child").mkdir(parents=True) + raise RuntimeError("control failure") + self.assertEqual((root / "control").read_bytes(), SOURCE) + self.assertEqual(list(Path(temp).glob("bridge-perl-archive-*")), []) + + def test_replaced_extraction_directory_is_not_deleted(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "extracted" + moved = Path(temp) / "moved" + with self.assertRaisesRegex(ValueError, "destination was replaced"): + with module.extraction_destination(root): + (root / "control").write_bytes(SOURCE) + root.rename(moved) + root.mkdir() + (root / "replacement").write_bytes(b"unexpected") + self.assertEqual((root / "replacement").read_bytes(), b"unexpected") + self.assertEqual((moved / "control").read_bytes(), SOURCE) + + def test_collision_never_overwrites_replacement_or_original(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) / "distribution" + root.mkdir() + (root / "control").write_bytes(SOURCE) + with self.assertRaisesRegex(ValueError, "Refusing to overwrite"): + with module.parked_installation(root): + root.mkdir() + (root / "replacement").write_bytes(b"unexpected") + self.assertEqual((root / "replacement").read_bytes(), b"unexpected") + backups = list(Path(temp).glob("bridge-perl-archive-*/original/control")) + self.assertEqual(len(backups), 1) + self.assertEqual(backups[0].read_bytes(), SOURCE) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/measure-perl-cold.py b/scripts/measure-perl-cold.py new file mode 100644 index 00000000..72105cf8 --- /dev/null +++ b/scripts/measure-perl-cold.py @@ -0,0 +1,177 @@ +"""Cold native correctness control for the installed Perl directory reduction.""" +import argparse +from contextlib import contextmanager +import gzip +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import time +import tomllib + +ROOT = Path(__file__).resolve().parents[1] +PREFIXES = ("c/include", "c/libexec", "c/x86_64-w64-mingw32") +TEST = "db::encrypted::tests::sqlcipher_encrypts_contents_and_rejects_the_wrong_key" +CARGO = ["rustup", "run", "1.96.0", "cargo"] + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +inventory = load("perl_inventory", "inventory-windows-perl.py") +capture = load("bounded_capture", "capture-package-log.py") + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def tree_digest(rows): + return digest(json.dumps(rows, sort_keys=True, separators=(",", ":")).encode()) + + +@contextmanager +def distribution_variant(root, reduced): + """Move only the three inventoried directories, restoring even on failure.""" + for prefix in PREFIXES: + path = root / prefix + inventory.reject_link(path) + if not path.is_dir(): + raise ValueError("Expected full-distribution control directory missing") + moved = [] + backup = None + try: + if reduced: + # A sibling stays on the same volume: renames preserve bytes/metadata. + backup = Path(tempfile.mkdtemp(prefix="bridge-perl-cold-", dir=root.parent)) + for prefix in PREFIXES: + source, destination = root / prefix, backup / prefix + destination.parent.mkdir(parents=True, exist_ok=True) + source.rename(destination) + moved.append((source, destination)) + yield + finally: + errors = [] + for source, destination in reversed(moved): + try: + if os.path.lexists(source): + raise ValueError("Unexpected path appeared in a moved directory; refusing overwrite") + destination.rename(source) + except (OSError, ValueError) as error: + errors.append(error) + if errors: + raise ExceptionGroup("Perl restoration incomplete; original bytes remain in the sibling backup", errors) + if backup is not None: + (backup / "c").rmdir() + backup.rmdir() + + +def execute(command, output, name): + started = time.monotonic() + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process: + capture.capture(process.stdout, output / name) + status = process.wait() + if status: + raise subprocess.CalledProcessError(status, command) + return round(time.monotonic() - started, 6) + + +def write(path, value): + path.write_text(json.dumps(value, indent=2) + "\n") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--case", choices=("full", "reduced"), required=True) + args = parser.parse_args() + if os.name != "nt": + raise ValueError("Windows MSVC experiment only") + target = ROOT / "src-tauri/target" + if Path(os.environ["CARGO_TARGET_DIR"]).resolve() != target.resolve(): + raise ValueError("Experiment requires the explicit source-local Cargo target") + if os.environ.get("CARGO_BUILD_TARGET"): + raise ValueError("Experiment requires the default Windows MSVC target") + if os.environ.get("PERL_CACHE_HIT") != "true": + raise ValueError("Comparison requires the exact installed-distribution cache") + for variable in ("RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER", "OPENSSL_NO_VENDOR"): + assert not os.environ.get(variable), f"Unexpected experiment input: {variable}" + root = Path("C:/Strawberry") + assert Path(os.environ["OPENSSL_SRC_PERL"]).resolve() == (root / "perl/bin/perl.exe").resolve() + output = ROOT / "perl-cold-measurements" / args.case + output.mkdir(parents=True, exist_ok=False) + manifest = ROOT / "src-tauri/Cargo.toml" + lock = ROOT / "src-tauri/Cargo.lock" + inputs = {str(path.relative_to(ROOT)): digest(path.read_bytes()) for path in + (manifest, lock, ROOT / "src-tauri/src/db/encrypted.rs")} + packages = {p["name"]: p["version"] for p in tomllib.loads(lock.read_text())["package"] + if p["name"] in ("openssl-src", "openssl-sys", "libsqlite3-sys")} + assert packages["openssl-src"] == "300.6.1+3.6.3" + write(output / "source.json", { + "head": os.environ["GITHUB_SHA"], "run_id": int(os.environ["GITHUB_RUN_ID"]), + "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), "case": args.case, + "image": os.environ["ImageVersion"], "image_os": os.environ["ImageOS"], + "perl_cache_hit": os.environ["PERL_CACHE_HIT"], + "input_hashes": inputs, "native_packages": packages, + "excluded_prefixes": list(PREFIXES) if args.case == "reduced" else [], + "scope": "Cold native correctness only; elapsed times are diagnostics, not a speed comparison.", + }) + original = inventory.inventory(root) + original_identity = inventory.perl_identity(root) + (output / "original-inventory.json.gz").write_bytes(gzip.compress(json.dumps(original).encode(), mtime=0)) + try: + with distribution_variant(root, args.case == "reduced"): + active = inventory.inventory(root) + expected = [row for row in original["files"] if args.case == "full" or + not row["path"].startswith(tuple(p + "/" for p in PREFIXES))] + assert active["files"] == expected, "Unexpected change outside selected exclusions" + assert inventory.perl_identity(root) == original_identity + write(output / "active-tree.json", { + "original_tree_sha256": tree_digest(original["files"]), + "active_tree_sha256": tree_digest(active["files"]), + "original_files": original["total_files"], "active_files": active["total_files"], + "original_bytes": original["total_bytes"], "active_bytes": active["total_bytes"], + "perl_identity": original_identity, "only_selected_exclusions": True, + }) + print(json.dumps({"case": args.case, "phase": "tree_verified", "files": active["total_files"]}), flush=True) + execute(CARGO + ["fetch", "--locked", "--manifest-path", str(manifest)], output, "fetch.log") + execute(CARGO + ["clean", "--manifest-path", str(manifest), "--release", "-p", "openssl-src", + "-p", "openssl-sys", "-p", "libsqlite3-sys"], output, "clean.log") + timings = target / "cargo-timings" + previous = set(timings.glob("cargo-timing-*.html")) + elapsed = execute(CARGO + ["test", "--locked", "--manifest-path", str(manifest), "--release", "--lib", + TEST, "--timings", "--", "--exact", "--nocapture"], output, "test.log") + reports = set(timings.glob("cargo-timing-*.html")) - previous + assert len(reports) == 1, "Missing or ambiguous native timing report" + report = reports.pop().read_text() + (output / "cargo-timing.html").write_text(report) + match = re.search(r"const UNIT_DATA = (\[.*?\]);", report, re.S) + assert match, "Compiler timing units missing" + units = json.loads(match.group(1)) + native = [u for u in units if u["name"] in ("openssl-sys", "libsqlite3-sys") and u["mode"] == "run-custom-build"] + assert {u["name"] for u in native} == {"openssl-sys", "libsqlite3-sys"} + assert all(u["duration"] > 0 for u in native), "Native builds were not fresh" + log = (output / "test.log/build-tail.log").read_text(errors="replace") + assert f"test {TEST} ... ok" in log + assert "test result: ok. 1 passed; 0 failed; 0 ignored" in log + write(output / "result.json", {"test": TEST, "passed": True, "command_seconds": elapsed, + "fresh_native_units": native}) + print(json.dumps({"case": args.case, "phase": "cold_cipher_test_passed"}), flush=True) + finally: + restored = inventory.inventory(root) + unchanged = all(digest((ROOT / name).read_bytes()) == expected for name, expected in inputs.items()) + write(output / "restoration.json", {"full_tree_restored": restored["files"] == original["files"], + "source_inputs_unchanged": unchanged, + "tree_sha256": tree_digest(restored["files"])}) + assert restored["files"] == original["files"] and unchanged, "Experiment input restoration failed" + + +if __name__ == "__main__": + main() diff --git a/scripts/measure-perl-cold.test.py b/scripts/measure-perl-cold.test.py new file mode 100644 index 00000000..3b2dc04a --- /dev/null +++ b/scripts/measure-perl-cold.test.py @@ -0,0 +1,68 @@ +"""Filesystem failure controls; these do not qualify a reduced Perl runtime.""" +import importlib.util +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("measure_perl_cold", Path(__file__).with_name("measure-perl-cold.py")) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class DistributionTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) / "distribution" + for prefix in module.PREFIXES: + directory = self.root / prefix + directory.mkdir(parents=True) + (directory / "control").write_bytes(Path(__file__).read_bytes()) + self.original = module.inventory.inventory(self.root)["files"] + + def assert_restored(self): + self.assertEqual(module.inventory.inventory(self.root)["files"], self.original) + self.assertEqual(list(self.root.parent.glob("bridge-perl-cold-*")), []) + + def test_full_control_is_unchanged(self): + with module.distribution_variant(self.root, False): + self.assert_restored() + self.assert_restored() + + def test_reduction_restores_after_body_failure(self): + with self.assertRaisesRegex(RuntimeError, "control failure"): + with module.distribution_variant(self.root, True): + self.assertTrue(all(not (self.root / p).exists() for p in module.PREFIXES)) + raise RuntimeError("control failure") + self.assert_restored() + + def test_partial_move_failure_restores_prior_moves(self): + rename = Path.rename + def fail_second(path, target): + if path == self.root / module.PREFIXES[1]: + raise OSError("control move failure") + return rename(path, target) + with patch.object(Path, "rename", fail_second): + with self.assertRaisesRegex(OSError, "control move failure"): + with module.distribution_variant(self.root, True): + self.fail("Partial reduction must not reach the build") + self.assert_restored() + + def test_collision_preserves_bytes_and_restores_other_directories(self): + collision = self.root / module.PREFIXES[-1] + with self.assertRaises(ExceptionGroup) as raised: + with module.distribution_variant(self.root, True): + collision.write_bytes(b"unexpected replacement") + self.assertEqual(len(raised.exception.exceptions), 1) + self.assertIsInstance(raised.exception.exceptions[0], ValueError) + self.assertEqual(collision.read_bytes(), b"unexpected replacement") + for prefix in module.PREFIXES[:-1]: + self.assertEqual((self.root / prefix / "control").read_bytes(), Path(__file__).read_bytes()) + backups = list(self.root.parent.glob("bridge-perl-cold-*")) + self.assertEqual(len(backups), 1) + self.assertEqual((backups[0] / module.PREFIXES[-1] / "control").read_bytes(), Path(__file__).read_bytes()) + + +if __name__ == "__main__": + unittest.main()