diff --git a/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md b/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md index 94f7401..a2f3b7e 100644 --- a/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md +++ b/.claude/skills/porting-to-canyonos-core/references/runtime-contract.md @@ -127,7 +127,13 @@ workflow_launcher.py Also avoid a yaml basename that shadows a different source module imported by an adapter. The validator checks deterministic flat-name collisions. -File sweep and editable-install behavior are runtime capabilities. For nested +File sweep and editable-install behavior are runtime capabilities. Where the +full project-file sweep is available, every project file ships at its relative +path, not only `.py` -- data files, prompts, and framework config included. It +leaves behind hidden files and directories, the generated `stubs/`, +`grpc_stubs/`, and `docker_container/`, host caches and virtualenvs, private key +material, and the root filenames the build context owns. A file the source opens +at runtime must therefore not be hidden or named like key material. For nested imports, follow [packaging.md](packaging.md). ## Dependencies and protobuf diff --git a/tests/test_stub_generator.py b/tests/test_stub_generator.py index fb01f2e..f6b989b 100644 --- a/tests/test_stub_generator.py +++ b/tests/test_stub_generator.py @@ -1,16 +1,20 @@ +import io import os import sys import tempfile import unittest +from contextlib import redirect_stdout from pathlib import Path import yaml sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +from ventis import stub_generator from ventis.stub_generator import ( BASE_AGENT_REQUIREMENTS, BASE_WORKFLOW_REQUIREMENTS, + _sweep_project_files, generate_docker, generate_workflow_docker, ) @@ -86,5 +90,273 @@ def test_per_workflow_requirements_are_appended_to_base(self): self.assertEqual(requirements, BASE_WORKFLOW_REQUIREMENTS + ["yfinance"]) +def _write(path, content="x"): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + return path + + +class ProjectSweepTests(unittest.TestCase): + """The sweep carries the whole project, not only its .py files.""" + + def _swept(self, project_dir): + with redirect_stdout(io.StringIO()): + return {rel for _, rel in _sweep_project_files(str(project_dir))} + + def _swept_with_output(self, project_dir): + buffer = io.StringIO() + with redirect_stdout(buffer): + swept = {rel for _, rel in _sweep_project_files(str(project_dir))} + return swept, buffer.getvalue() + + def test_non_python_files_are_swept_with_their_layout(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "notes.txt") + _write(project / "pyproject.toml") + _write(project / "langgraph.json") + _write(project / "agent.py") + _write(project / "docs" / "manual.pdf") + _write(project / "src" / "pkg" / "prompts" / "system.md") + + swept = self._swept(project) + + self.assertEqual( + swept, + { + "notes.txt", + "pyproject.toml", + "langgraph.json", + "agent.py", + os.path.join("docs", "manual.pdf"), + os.path.join("src", "pkg", "prompts", "system.md"), + }, + ) + + def test_generated_hidden_and_host_local_paths_are_left_behind(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "keep.txt") + _write(project / ".env", "OPENAI_API_KEY=real") + _write(project / ".config" / "settings.json") + _write(project / "stubs" / "Old.py") + _write(project / "grpc_stubs" / "old_pb2.py") + _write(project / "docker_container" / "Agent" / "Dockerfile") + _write(project / "__pycache__" / "agent.cpython-311.pyc") + _write(project / "venv" / "lib" / "site.py") + _write(project / "node_modules" / "left-pad" / "index.js") + _write(project / "proj.egg-info" / "PKG-INFO") + _write(project / "compiled.pyc") + _write(project / "client.pem", "-----BEGIN PRIVATE KEY-----") + + swept = self._swept(project) + + self.assertEqual(swept, {"keep.txt"}) + + def test_generated_directory_names_are_only_reserved_at_the_root(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "stubs" / "Generated.py") + _write(project / "src" / "stubs" / "handwritten.py") + + swept = self._swept(project) + + self.assertEqual(swept, {os.path.join("src", "stubs", "handwritten.py")}) + + def test_symlinks_are_not_followed_and_are_reported(self): + with tempfile.TemporaryDirectory() as tmpdir: + outside = Path(tmpdir) / "outside" + _write(outside / "secret.txt", "not ours") + project = Path(tmpdir) / "project" + _write(project / "real.txt") + (project / "link.txt").symlink_to(project / "real.txt") + (project / "escape").symlink_to(outside) + + swept, output = self._swept_with_output(project) + + self.assertEqual(swept, {"real.txt"}) + self.assertIn("link.txt", output) + self.assertIn("escape", output) + + def test_project_requirements_does_not_replace_the_generated_one(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "requirements.txt", "yfinance==0.1\n") + yaml_path = project / "ExampleAgent.yaml" + yaml_path.write_text(yaml.safe_dump({"agent": {"name": "ExampleAgent"}})) + agent_file = _write(project / "agent.py", "print('ok')\n") + output_dir = os.path.join(tmpdir, "out") + + generate_docker( + str(yaml_path), + str(agent_file), + output_dir=output_dir, + project_dir=str(project), + ) + + requirements = _read_requirements(output_dir) + + self.assertEqual(requirements, BASE_AGENT_REQUIREMENTS) + + def test_agent_context_receives_the_swept_project(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + yaml_path = project / "ExampleAgent.yaml" + yaml_path.write_text(yaml.safe_dump({"agent": {"name": "ExampleAgent"}})) + agent_file = _write(project / "agent.py", "print('ok')\n") + _write(project / "data" / "handbook.pdf", "%PDF-1.4") + output_dir = os.path.join(tmpdir, "out") + + generate_docker( + str(yaml_path), + str(agent_file), + output_dir=output_dir, + project_dir=str(project), + ) + + copied = Path(output_dir) / "data" / "handbook.pdf" + + self.assertEqual(copied.read_text(), "%PDF-1.4") + + def test_workflow_context_receives_the_swept_project(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + workflow_file = _write(project / "workflow.py", "print('ok')\n") + _write(project / "config" / "langgraph.json", "{}") + output_dir = os.path.join(tmpdir, "out") + + generate_workflow_docker( + str(workflow_file), + [], + output_dir=output_dir, + project_dir=str(project), + ) + + copied = Path(output_dir) / "config" / "langgraph.json" + + self.assertEqual(copied.read_text(), "{}") + + def test_private_keys_are_recognized_by_armor_not_by_name(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "id_rsa", "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n") + _write(project / "server.pem", "-----BEGIN RSA PRIVATE KEY-----\nabc\n") + _write(project / "keystore.p12", "binary-ish") + _write(project / "ca.pem", "-----BEGIN CERTIFICATE-----\nabc\n") + _write(project / "notes.key", "this is a text file about keys") + + swept, output = self._swept_with_output(project) + + self.assertEqual(swept, {"ca.pem", "notes.key"}) + self.assertIn("id_rsa", output) + self.assertIn("keystore.p12", output) + + def test_skipped_hidden_paths_are_reported(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "agent.py") + _write(project / ".env", "OPENAI_API_KEY=real") + _write(project / ".streamlit" / "config.toml") + + swept, output = self._swept_with_output(project) + + self.assertEqual(swept, {"agent.py"}) + self.assertIn(".env", output) + self.assertIn(".streamlit", output) + + def test_an_oversized_context_is_reported(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "dataset.bin", "x" * 4096) + _write(project / "agent.py") + + original = stub_generator._LARGE_CONTEXT_BYTES + stub_generator._LARGE_CONTEXT_BYTES = 1024 + try: + swept, output = self._swept_with_output(project) + finally: + stub_generator._LARGE_CONTEXT_BYTES = original + + self.assertEqual(swept, {"dataset.bin", "agent.py"}) + self.assertIn("dataset.bin", output) + + def test_a_normal_project_reports_nothing(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "agent.py") + _write(project / "notes.txt") + _write(project / "__pycache__" / "agent.cpython-311.pyc") + + _, output = self._swept_with_output(project) + + self.assertEqual(output, "") + + def test_the_build_context_is_not_swept_into_itself(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + yaml_path = project / "ExampleAgent.yaml" + yaml_path.write_text(yaml.safe_dump({"agent": {"name": "ExampleAgent"}})) + agent_file = _write(project / "agent.py", "print('ok')\n") + # Not docker_container/, so nothing but exclude_dir keeps this out. + output_dir = project / "build_context" + + generate_docker( + str(yaml_path), + str(agent_file), + output_dir=str(output_dir), + project_dir=str(project), + ) + + nested = list(output_dir.rglob("build_context")) + + self.assertEqual(nested, []) + + def test_an_empty_file_does_not_break_the_sweep(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + # An empty __init__.py is in nearly every Python project, and it + # used to make the largest-file bookkeeping compare a path to None. + _write(project / "src" / "__init__.py", "") + _write(project / "src" / "agent.py", "print('ok')\n") + + swept = self._swept(project) + + self.assertEqual( + swept, + {os.path.join("src", "__init__.py"), os.path.join("src", "agent.py")}, + ) + + def test_ordinary_repo_furniture_is_not_reported(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "agent.py") + _write(project / ".gitignore", "*.pyc") + _write(project / ".git" / "config") + _write(project / ".venv" / "pyvenv.cfg") + _write(project / ".mypy_cache" / "cache.json") + + swept, output = self._swept_with_output(project) + + self.assertEqual(swept, {"agent.py"}) + self.assertEqual(output, "") + + def test_host_local_directories_are_reported_because_they_used_to_ship(self): + with tempfile.TemporaryDirectory() as tmpdir: + project = Path(tmpdir) + _write(project / "agent.py") + # These held .py files that the old .py-only sweep shipped, so + # dropping them is a behavior change and has to be visible. + _write(project / "venv" / "lib" / "site.py") + _write(project / "proj.egg-info" / "PKG-INFO") + _write(project / "__pycache__" / "agent.cpython-311.pyc") + + swept, output = self._swept_with_output(project) + + self.assertEqual(swept, {"agent.py"}) + self.assertIn("venv", output) + self.assertIn("proj.egg-info", output) + self.assertNotIn("__pycache__", output) + + if __name__ == "__main__": unittest.main() diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 54a46b9..cc8f55b 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -271,25 +271,198 @@ def _format_source(source): return "\n".join(formatted) + "\n" +# What the sweep leaves out. These lists are hardcoded with no project override +# because the build context is assembled by hand, so Docker's own ignore +# mechanism never gets to run. + # Directories ventis build itself generates inside a project -- never swept. _GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} - -def _sweep_py_files(project_dir): - """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" +# A macOS virtualenv or cache is dead weight in a linux image, at best. +_SKIPPED_DIRS = {"__pycache__", "node_modules", "venv", "site-packages"} + +# The generator writes these into the context itself, requirements.txt before +# the copy runs -- a project file of the same name at the root would win. +_RESERVED_CONTEXT_NAMES = { + "Dockerfile", + "requirements.txt", + "workflow_launcher.py", +} + +_SKIPPED_SUFFIXES = (".pyc", ".pyo", ".pyd") + +# Binary keystores carry no armor to detect them by. +_KEYSTORE_SUFFIXES = (".p12", ".pfx", ".jks") + +# Enough to see PEM armor past any header the tool that wrote it left. +_KEY_ARMOR_SCAN_BYTES = 4096 + +# Past this, say so: usually a dataset, a checkpoint, or a virtualenv under a +# name _SKIPPED_DIRS does not know. +_LARGE_CONTEXT_BYTES = 100 * 1024 * 1024 + +# Hidden paths every repo has. Held back like all hidden paths, but not worth +# saying so on every build: a note that always fires is wallpaper, and it takes +# the one that matters down with it. Guessing wrong here costs a line of output, +# not a missing file -- which is why the guess lives here and not above. +_UNREMARKABLE_HIDDEN = { + ".git", + ".gitignore", + ".gitattributes", + ".gitmodules", + ".dockerignore", + ".editorconfig", + ".python-version", + ".venv", + ".idea", + ".vscode", + ".DS_Store", +} + + +def _is_unremarkable_hidden(name): + """Whether this hidden path is the tooling furniture every repo has, rather than project content.""" + return name in _UNREMARKABLE_HIDDEN or name.endswith("_cache") + + +def _looks_like_private_key(path, fname): + """Whether this file is private key material that must not be baked into an image. + + Matching on names is theater -- an OpenSSH key is called `id_rsa`, with no + extension at all -- so PEM material is found by its armor instead. A + certificate is public and ships; only a PRIVATE KEY block is held back. + + Not a secret scanner: `credentials.json` still ships, because nothing can + recognize it. Credentials belong in the container environment either way. + """ + if fname.endswith(_KEYSTORE_SUFFIXES): + return True + try: + with open(path, "rb") as handle: + head = handle.read(_KEY_ARMOR_SCAN_BYTES) + except OSError: + return False + return b"-----BEGIN" in head and b"PRIVATE KEY-----" in head + + +def _format_path_sample(paths, limit=5): + """One line naming up to `limit` of these paths, with a count standing in for the rest.""" + shown = ", ".join(sorted(paths)[:limit]) + if len(paths) > limit: + shown += f", (+{len(paths) - limit} more)" + return shown + + +def _sweep_project_files(project_dir, exclude_dir=None): + """Recursively collect (abs_src, rel_dst) for every project file under project_dir, preserving its directory structure. + + Not only .py: source opens PDFs, prompts, and framework config at runtime, + and a file missing from the image fails only once the agent is serving. + + Symlinks are never followed. A link to /etc or to a home directory would + copy files from outside the project into the image, so both the file and the + directory case are pruned here rather than left to `os.walk` defaulting to + `followlinks=False` -- a security property should not rest on a default + someone can flip. + + Every exclusion is reported except the ones that could never have held + shippable content -- __pycache__, bytecode, and the build's own output. A + file that silently fails to arrive is the bug this sweep exists to fix, so + dropping one quietly just moves it somewhere harder to find. + + `exclude_dir` is the build context being assembled, which sits under the + project root. Matching it by resolved path rather than name is what keeps + this working for a caller that picks an output directory outside + `_GENERATED_DIRS`. + """ swept = [] + hidden = [] + symlinks = [] + host_local = [] + private_keys = [] + reserved = [] + total_bytes = 0 + largest = (0, None) + context_dir = os.path.realpath(exclude_dir) if exclude_dir else None + for root, dirs, files in os.walk(project_dir): - dirs[:] = [ - d - for d in dirs - if not d.startswith(".") - and not (root == project_dir and d in _GENERATED_DIRS) - ] + at_root = root == project_dir + + kept_dirs = [] + for name in dirs: + abs_dir = os.path.join(root, name) + if context_dir and os.path.realpath(abs_dir) == context_dir: + continue + rel_dir = os.path.relpath(abs_dir, project_dir) + os.sep + if os.path.islink(abs_dir): + symlinks.append(rel_dir) + elif name.startswith("."): + if not _is_unremarkable_hidden(name): + hidden.append(rel_dir) + elif name in _SKIPPED_DIRS or name.endswith(".egg-info"): + if name != "__pycache__": + host_local.append(rel_dir) + elif not (at_root and name in _GENERATED_DIRS): + kept_dirs.append(name) + dirs[:] = kept_dirs + for fname in files: abs_src = os.path.join(root, fname) - if fname.endswith(".py") and not os.path.islink(abs_src): - rel_dst = os.path.relpath(abs_src, project_dir) - swept.append((abs_src, rel_dst)) + rel_dst = os.path.relpath(abs_src, project_dir) + if fname.startswith("."): + if not _is_unremarkable_hidden(fname): + hidden.append(rel_dst) + continue + if os.path.islink(abs_src): + symlinks.append(rel_dst) + continue + if fname.endswith(_SKIPPED_SUFFIXES): + continue + if _looks_like_private_key(abs_src, fname): + private_keys.append(rel_dst) + continue + if at_root and fname in _RESERVED_CONTEXT_NAMES: + reserved.append(rel_dst) + continue + swept.append((abs_src, rel_dst)) + try: + size = os.path.getsize(abs_src) + except OSError: + size = 0 + total_bytes += size + if size > largest[0]: + largest = (size, rel_dst) + + if hidden: + print( + f" Note: {len(hidden)} hidden path(s) not copied into the image: " + f"{_format_path_sample(hidden)}. Move anything the agent opens at runtime out of " + f"a dotted path." + ) + if symlinks: + print( + f" Note: {len(symlinks)} symlink(s) not followed into the image: " + f"{_format_path_sample(symlinks)}. Copy the target in if the agent needs it." + ) + if host_local: + print( + f" Note: {len(host_local)} host-local path(s) not copied into the image: " + f"{_format_path_sample(host_local)}. The image installs its own dependencies." + ) + for rel_dst in sorted(private_keys): + print(f" Warning: not copying private key material into the image: {rel_dst}") + for rel_dst in sorted(reserved): + print( + f" Warning: the build context owns '{os.path.basename(rel_dst)}', so the " + f"project's own copy is not included in the image" + ) + if total_bytes > _LARGE_CONTEXT_BYTES: + print( + f" Warning: sweeping {total_bytes // (1024 * 1024)} MB into this image, " + f"largest is {largest[1]} at {largest[0] // (1024 * 1024)} MB. Everything " + f"under the project root ships unless it is hidden or generated." + ) + return swept @@ -345,7 +518,7 @@ def generate_docker( output_dir: Optional output directory (default: docker_container//). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). stub_files: Optional list of agent stub files to copy into the context. - project_dir: Optional project root to sweep for extra .py helper files. + project_dir: Optional project root whose files are swept into the context. stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. requirements: Optional list of extra pip packages this agent needs. """ @@ -370,10 +543,10 @@ def generate_docker( with open(os.path.join(output_dir, "requirements.txt"), "w") as f: f.write(requirements_txt) - # Sweep the project for extra .py helper files not on the explicit list below. + # Sweep the whole project first; the explicit list below is copied on top. files_to_copy = [] if project_dir: - files_to_copy += _sweep_py_files(project_dir) + files_to_copy += _sweep_project_files(project_dir, exclude_dir=output_dir) # Copy general agent files files_to_copy += [ @@ -472,7 +645,7 @@ def generate_workflow_docker( stub_files: List of stub file paths to include. output_dir: Optional output directory (default: docker_container/Workflow/). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - project_dir: Optional project root to sweep for extra .py helper files. + project_dir: Optional project root whose files are swept into the context. stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. requirements: Optional list of extra pip packages this workflow needs. """ @@ -496,8 +669,10 @@ def generate_workflow_docker( # ---- Copy source files into the build context ------------------------ workflow_basename = os.path.basename(workflow_file) - # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = _sweep_py_files(project_dir) if project_dir else [] + # Sweep the whole project first; the explicit list below is copied on top. + files_to_copy = ( + _sweep_project_files(project_dir, exclude_dir=output_dir) if project_dir else [] + ) files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename),