Sweep the whole project into images, not only .py

.claude/skills/porting-to-canyonos-core/references/runtime-contract.md

@@ -127,7 +127,13 @@ workflow_launcher.py
 Also avoid a yaml basename that shadows a different source module imported by an
 adapter. The validator checks deterministic flat-name collisions.
 
-File sweep and editable-install behavior are runtime capabilities. For nested
+File sweep and editable-install behavior are runtime capabilities. Where the
+full project-file sweep is available, every project file ships at its relative
+path, not only `.py` -- data files, prompts, and framework config included. It
+leaves behind hidden files and directories, the generated `stubs/`,
+`grpc_stubs/`, and `docker_container/`, host caches and virtualenvs, private key
+material, and the root filenames the build context owns. A file the source opens
+at runtime must therefore not be hidden or named like key material. For nested
 imports, follow [packaging.md](packaging.md).
 
 ## Dependencies and protobuf

tests/test_stub_generator.py

@@ -1,16 +1,20 @@
+import io
 import os
 import sys
 import tempfile
 import unittest
+from contextlib import redirect_stdout
 from pathlib import Path
 
 import yaml
 
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
+from ventis import stub_generator
 from ventis.stub_generator import (
     BASE_AGENT_REQUIREMENTS,
     BASE_WORKFLOW_REQUIREMENTS,
+    _sweep_project_files,
     generate_docker,
     generate_workflow_docker,
 )
@@ -86,5 +90,273 @@ def test_per_workflow_requirements_are_appended_to_base(self):
         self.assertEqual(requirements, BASE_WORKFLOW_REQUIREMENTS + ["yfinance"])
 
 
+def _write(path, content="x"):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(content)
+    return path
+
+
+class ProjectSweepTests(unittest.TestCase):
+    """The sweep carries the whole project, not only its .py files."""
+
+    def _swept(self, project_dir):
+        with redirect_stdout(io.StringIO()):
+            return {rel for _, rel in _sweep_project_files(str(project_dir))}
+
+    def _swept_with_output(self, project_dir):
+        buffer = io.StringIO()
+        with redirect_stdout(buffer):
+            swept = {rel for _, rel in _sweep_project_files(str(project_dir))}
+        return swept, buffer.getvalue()
+
+    def test_non_python_files_are_swept_with_their_layout(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "notes.txt")
+            _write(project / "pyproject.toml")
+            _write(project / "langgraph.json")
+            _write(project / "agent.py")
+            _write(project / "docs" / "manual.pdf")
+            _write(project / "src" / "pkg" / "prompts" / "system.md")
+
+            swept = self._swept(project)
+
+        self.assertEqual(
+            swept,
+            {
+                "notes.txt",
+                "pyproject.toml",
+                "langgraph.json",
+                "agent.py",
+                os.path.join("docs", "manual.pdf"),
+                os.path.join("src", "pkg", "prompts", "system.md"),
+            },
+        )
+
+    def test_generated_hidden_and_host_local_paths_are_left_behind(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "keep.txt")
+            _write(project / ".env", "OPENAI_API_KEY=real")
+            _write(project / ".config" / "settings.json")
+            _write(project / "stubs" / "Old.py")
+            _write(project / "grpc_stubs" / "old_pb2.py")
+            _write(project / "docker_container" / "Agent" / "Dockerfile")
+            _write(project / "__pycache__" / "agent.cpython-311.pyc")
+            _write(project / "venv" / "lib" / "site.py")
+            _write(project / "node_modules" / "left-pad" / "index.js")
+            _write(project / "proj.egg-info" / "PKG-INFO")
+            _write(project / "compiled.pyc")
+            _write(project / "client.pem", "-----BEGIN PRIVATE KEY-----")
+
+            swept = self._swept(project)
+
+        self.assertEqual(swept, {"keep.txt"})
+
+    def test_generated_directory_names_are_only_reserved_at_the_root(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "stubs" / "Generated.py")
+            _write(project / "src" / "stubs" / "handwritten.py")
+
+            swept = self._swept(project)
+
+        self.assertEqual(swept, {os.path.join("src", "stubs", "handwritten.py")})
+
+    def test_symlinks_are_not_followed_and_are_reported(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            outside = Path(tmpdir) / "outside"
+            _write(outside / "secret.txt", "not ours")
+            project = Path(tmpdir) / "project"
+            _write(project / "real.txt")
+            (project / "link.txt").symlink_to(project / "real.txt")
+            (project / "escape").symlink_to(outside)
+
+            swept, output = self._swept_with_output(project)
+
+        self.assertEqual(swept, {"real.txt"})
+        self.assertIn("link.txt", output)
+        self.assertIn("escape", output)
+
+    def test_project_requirements_does_not_replace_the_generated_one(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "requirements.txt", "yfinance==0.1\n")
+            yaml_path = project / "ExampleAgent.yaml"
+            yaml_path.write_text(yaml.safe_dump({"agent": {"name": "ExampleAgent"}}))
+            agent_file = _write(project / "agent.py", "print('ok')\n")
+            output_dir = os.path.join(tmpdir, "out")
+
+            generate_docker(
+                str(yaml_path),
+                str(agent_file),
+                output_dir=output_dir,
+                project_dir=str(project),
+            )
+
+            requirements = _read_requirements(output_dir)
+
+        self.assertEqual(requirements, BASE_AGENT_REQUIREMENTS)
+
+    def test_agent_context_receives_the_swept_project(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            yaml_path = project / "ExampleAgent.yaml"
+            yaml_path.write_text(yaml.safe_dump({"agent": {"name": "ExampleAgent"}}))
+            agent_file = _write(project / "agent.py", "print('ok')\n")
+            _write(project / "data" / "handbook.pdf", "%PDF-1.4")
+            output_dir = os.path.join(tmpdir, "out")
+
+            generate_docker(
+                str(yaml_path),
+                str(agent_file),
+                output_dir=output_dir,
+                project_dir=str(project),
+            )
+
+            copied = Path(output_dir) / "data" / "handbook.pdf"
+
+            self.assertEqual(copied.read_text(), "%PDF-1.4")
+
+    def test_workflow_context_receives_the_swept_project(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            workflow_file = _write(project / "workflow.py", "print('ok')\n")
+            _write(project / "config" / "langgraph.json", "{}")
+            output_dir = os.path.join(tmpdir, "out")
+
+            generate_workflow_docker(
+                str(workflow_file),
+                [],
+                output_dir=output_dir,
+                project_dir=str(project),
+            )
+
+            copied = Path(output_dir) / "config" / "langgraph.json"
+
+            self.assertEqual(copied.read_text(), "{}")
+
+    def test_private_keys_are_recognized_by_armor_not_by_name(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "id_rsa", "-----BEGIN OPENSSH PRIVATE KEY-----\nabc\n")
+            _write(project / "server.pem", "-----BEGIN RSA PRIVATE KEY-----\nabc\n")
+            _write(project / "keystore.p12", "binary-ish")
+            _write(project / "ca.pem", "-----BEGIN CERTIFICATE-----\nabc\n")
+            _write(project / "notes.key", "this is a text file about keys")
+
+            swept, output = self._swept_with_output(project)
+
+        self.assertEqual(swept, {"ca.pem", "notes.key"})
+        self.assertIn("id_rsa", output)
+        self.assertIn("keystore.p12", output)
+
+    def test_skipped_hidden_paths_are_reported(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "agent.py")
+            _write(project / ".env", "OPENAI_API_KEY=real")
+            _write(project / ".streamlit" / "config.toml")
+
+            swept, output = self._swept_with_output(project)
+
+        self.assertEqual(swept, {"agent.py"})
+        self.assertIn(".env", output)
+        self.assertIn(".streamlit", output)
+
+    def test_an_oversized_context_is_reported(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "dataset.bin", "x" * 4096)
+            _write(project / "agent.py")
+
+            original = stub_generator._LARGE_CONTEXT_BYTES
+            stub_generator._LARGE_CONTEXT_BYTES = 1024
+            try:
+                swept, output = self._swept_with_output(project)
+            finally:
+                stub_generator._LARGE_CONTEXT_BYTES = original
+
+        self.assertEqual(swept, {"dataset.bin", "agent.py"})
+        self.assertIn("dataset.bin", output)
+
+    def test_a_normal_project_reports_nothing(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "agent.py")
+            _write(project / "notes.txt")
+            _write(project / "__pycache__" / "agent.cpython-311.pyc")
+
+            _, output = self._swept_with_output(project)
+
+        self.assertEqual(output, "")
+
+    def test_the_build_context_is_not_swept_into_itself(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            yaml_path = project / "ExampleAgent.yaml"
+            yaml_path.write_text(yaml.safe_dump({"agent": {"name": "ExampleAgent"}}))
+            agent_file = _write(project / "agent.py", "print('ok')\n")
+            # Not docker_container/, so nothing but exclude_dir keeps this out.
+            output_dir = project / "build_context"
+
+            generate_docker(
+                str(yaml_path),
+                str(agent_file),
+                output_dir=str(output_dir),
+                project_dir=str(project),
+            )
+
+            nested = list(output_dir.rglob("build_context"))
+
+        self.assertEqual(nested, [])
+
+    def test_an_empty_file_does_not_break_the_sweep(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            # An empty __init__.py is in nearly every Python project, and it
+            # used to make the largest-file bookkeeping compare a path to None.
+            _write(project / "src" / "__init__.py", "")
+            _write(project / "src" / "agent.py", "print('ok')\n")
+
+            swept = self._swept(project)
+
+        self.assertEqual(
+            swept,
+            {os.path.join("src", "__init__.py"), os.path.join("src", "agent.py")},
+        )
+
+    def test_ordinary_repo_furniture_is_not_reported(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "agent.py")
+            _write(project / ".gitignore", "*.pyc")
+            _write(project / ".git" / "config")
+            _write(project / ".venv" / "pyvenv.cfg")
+            _write(project / ".mypy_cache" / "cache.json")
+
+            swept, output = self._swept_with_output(project)
+
+        self.assertEqual(swept, {"agent.py"})
+        self.assertEqual(output, "")
+
+    def test_host_local_directories_are_reported_because_they_used_to_ship(self):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            project = Path(tmpdir)
+            _write(project / "agent.py")
+            # These held .py files that the old .py-only sweep shipped, so
+            # dropping them is a behavior change and has to be visible.
+            _write(project / "venv" / "lib" / "site.py")
+            _write(project / "proj.egg-info" / "PKG-INFO")
+            _write(project / "__pycache__" / "agent.cpython-311.pyc")
+
+            swept, output = self._swept_with_output(project)
+
+        self.assertEqual(swept, {"agent.py"})
+        self.assertIn("venv", output)
+        self.assertIn("proj.egg-info", output)
+        self.assertNotIn("__pycache__", output)
+
+
 if __name__ == "__main__":
     unittest.main()