From 872d72b722386bb0182894e2cc59e1914f49c5c6 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Thu, 3 Sep 2026 13:33:17 -0700 Subject: [PATCH] Support .car artifact layout with legacy fallback --- .gitignore | 4 + README.md | 30 +++---- tests/run_tests.sh | 4 +- tests/test_cli.py | 103 +++++++++++++++++++++-- ventis/cli.py | 108 +++++++++++++++++-------- ventis/controller/global_controller.py | 12 +-- 6 files changed, 197 insertions(+), 64 deletions(-) diff --git a/.gitignore b/.gitignore index 085b4a3..1d7998b 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,10 @@ env/ Thumbs.db ._* +# Canyon artifacts. `.car` is generated from the application source by the +# porting skill and `ventis build`; it is never committed. +.car/ + # Generated stubs stubs/ grpc_stubs/ diff --git a/README.md b/README.md index 81d944f..cc07005 100644 --- a/README.md +++ b/README.md @@ -39,26 +39,26 @@ cd my-app ``` This command creates a new directory `my-app` with the following structure: ``` -├── agents/ # Agent implementations and YAML definitions -│ ├── example_agent.py -│ └── example_agent.yaml -├── workflows/ # Workflow scripts (deployed as REST APIs) -│ └── example_workflow.py -├── config/ -│ ├── global_controller.yaml # Deployment configuration -│ └── policy.yaml # Access control rules -├── stubs/ # Generated agent stubs (auto-generated) -├── grpc_stubs/ # Generated gRPC stubs (auto-generated) -└── README.md # Readme for the project +├── .car/ +│ ├── app/ # Source copy used for builds +│ ├── config/ +│ │ ├── global_controller.yaml +│ │ ├── example_agent.yaml # Agent declaration +│ │ └── policy.yaml +│ ├── stubs/ +│ ├── grpc_stubs/ +│ └── docker_container/ +└── README.md ``` The Readme in the newly created project directory provides a quick overview of the project and how to use it. Including how to add new files etc. We provide some overview in next few steps. #### Step 2: Define Your Agents -Place your agent logic (`.py`) and definitions (`.yaml`) in the `agents/` directory. +Agent declarations live under `.car/config/`. The source used for builds is +copied to `.car/app/` by `canyonos integrate`. -- **`agents/my_agent.yaml`**: Defines methods and schemas. -- **`agents/my_agent.py`**: Contains the actual Python implementation. +- **`.car/config/my_agent.yaml`**: Defines methods and schemas. +- **`.car/app/path/to/my_agent.py`**: Contains the agent implementation. We have provided an example of a finance agent and a market research agent in the `examples/` directory. To run the example, copy files into your newly created project directory from within the your my-app directory with the command - @@ -69,7 +69,7 @@ cp -r ../examples/* ./ ## Deployment Guide #### Step 1: Configure the Global Controller -Edit `config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. +Edit `.car/config/global_controller.yaml` in your project directory to list the agents you want to deploy, their `provider`, `replicas`, and resource limits. Add a per-agent `requirements: [pkg, ...]` list for any extra pip packages that agent's code imports — only a small base list (grpc, redis, pyyaml, psutil, etc.) is installed by default. #### Step 1.1: Passing secrets to agents (optional) diff --git a/tests/run_tests.sh b/tests/run_tests.sh index e9f8386..cf7537a 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -30,8 +30,8 @@ cd "$TEST_DIR" echo ">> 1. Generating new project..." ventis new-project $PROJECT_NAME cd $PROJECT_NAME -grep -v 'gpu:' config/global_controller.yaml > config/global_controller.yaml.tmp -mv config/global_controller.yaml.tmp config/global_controller.yaml +grep -v 'gpu:' .car/config/global_controller.yaml > .car/config/global_controller.yaml.tmp +mv .car/config/global_controller.yaml.tmp .car/config/global_controller.yaml echo ">> 2. Building agents (ventis build)..." ventis build diff --git a/tests/test_cli.py b/tests/test_cli.py index 406b95d..93e361f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -81,6 +81,35 @@ def test_deploy_runs_ec2_preflight_for_ec2_config( preflight.assert_called_once_with(config, os.getcwd()) controller.run.assert_called_once_with() + @patch("atexit.register") + @patch("signal.signal") + @patch("ventis.cli._ensure_grpc_stubs_importable") + @patch("ventis.cli._preflight_ec2_deploy") + def test_deploy_uses_car_when_present( + self, preflight, ensure_grpc, _signal_patch, _atexit_patch + ): + controller = MagicMock() + controller_module = self._fake_controller_module(controller) + args = SimpleNamespace(config=".car/config/global_controller.yaml") + + with tempfile.TemporaryDirectory() as tmpdir, patch( + "ventis.cli.os.path.isfile", return_value=True + ), patch( + "ventis.cli._load_config", return_value={"agents": []} + ), patch.dict( + sys.modules, {"ventis.controller.global_controller": controller_module} + ): + Path(tmpdir, ".car").mkdir() + cwd = os.getcwd() + os.chdir(tmpdir) + try: + cli.cmd_deploy(args) + finally: + os.chdir(cwd) + + ensure_grpc.assert_called_once_with(os.path.join(os.path.realpath(tmpdir), ".car")) + preflight.assert_not_called() + @patch("ventis.cli._ensure_grpc_stubs_importable") @patch("ventis.cli._require_docker_for_ec2") def test_preflight_does_not_require_ssh_fields(self, require_docker, ensure_grpc): @@ -107,7 +136,10 @@ def _run_build( Returns (docker_calls, generate_docker_mock, generate_workflow_docker_mock). """ - config_path = project_dir / "config" / "global_controller.yaml" + artifact_root = ( + project_dir / ".car" if (project_dir / ".car").is_dir() else project_dir + ) + config_path = artifact_root / "config" / "global_controller.yaml" args = SimpleNamespace(config=str(config_path)) docker_calls = [] @@ -115,16 +147,29 @@ def fake_run(cmd, check): docker_calls.append(cmd) return SimpleNamespace(returncode=0) + def fake_glob(pattern): + if pattern.endswith("*.proto"): + return ["proto/a.proto"] + if agent_yaml_paths: + self.assertEqual( + os.path.realpath(Path(pattern).parent), + os.path.realpath(Path(agent_yaml_paths[0]).parent), + ) + return agent_yaml_paths + + def fake_generate_stub(yaml_path, _output_path): + with open(yaml_path) as f: + self.assertIn("agent", yaml.safe_load(f)) + with ( patch( "ventis.cli._get_package_dir", return_value=str(project_dir / "package"), ), + patch("ventis.cli.glob.glob", side_effect=fake_glob), patch( - "ventis.cli.glob.glob", - side_effect=[agent_yaml_paths, ["proto/a.proto"]], + "ventis.stub_generator.generate_stub", side_effect=fake_generate_stub ), - patch("ventis.stub_generator.generate_stub"), patch("ventis.stub_generator.generate_docker") as generate_docker, patch( "ventis.stub_generator.generate_workflow_docker" @@ -240,6 +285,32 @@ def test_build_uses_buildx_bake_when_available(self): ) self.assertEqual(targets["workflow"]["tags"], ["ventis-workflow"]) + def test_build_uses_car_when_present(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + artifact_root = project_dir / ".car" + source_root = artifact_root / "app" + source_root.mkdir(parents=True) + source_yaml = self._write_agent_and_workflow_config(source_root) + source_root.joinpath("config").rename(artifact_root / "config") + agent_yaml = artifact_root / "config" / source_yaml.name + source_yaml.rename(agent_yaml) + + manifest = artifact_root / "config" / "global_controller.yaml" + _, generate_docker, generate_workflow_docker = self._run_build( + project_dir, [str(manifest), str(agent_yaml)], buildx_available=True + ) + + for call in (generate_docker, generate_workflow_docker): + self.assertEqual( + os.path.realpath(call.call_args.kwargs["project_dir"]), + os.path.realpath(source_root), + ) + self.assertEqual( + os.path.realpath(generate_docker.call_args.kwargs["output_dir"]), + os.path.realpath(artifact_root / "docker_container" / "ExampleAgent"), + ) + def test_build_with_no_agents_builds_nothing(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) @@ -252,7 +323,7 @@ def test_build_with_no_agents_builds_nothing(self): self.assertFalse(any(call[0] == "docker" for call in docker_calls)) - def test_build_skips_agent_without_entrypoint(self): + def test_build_fails_when_stub_cannot_be_generated(self): with tempfile.TemporaryDirectory() as tmpdir: project_dir = Path(tmpdir) (project_dir / "config").mkdir() @@ -268,9 +339,8 @@ def test_build_skips_agent_without_entrypoint(self): ) ) - docker_calls, _, _ = self._run_build(project_dir, [], buildx_available=True) - - self.assertFalse(any(call[0] == "docker" for call in docker_calls)) + with self.assertRaises(SystemExit): + self._run_build(project_dir, [], buildx_available=True) def _write_requirements_config(self, project_dir): """Scaffold one plain agent, one agent with `requirements`, one workflow with `requirements`.""" @@ -373,5 +443,22 @@ def test_build_ignores_non_list_requirements(self): self.assertEqual(generate_docker.call_args.kwargs["requirements"], []) +class CliCleanTests(unittest.TestCase): + def test_clean_uses_car_when_present(self): + with tempfile.TemporaryDirectory() as tmpdir: + project_dir = Path(tmpdir) + (project_dir / ".car" / "stubs").mkdir(parents=True) + (project_dir / "stubs").mkdir() + cwd = os.getcwd() + os.chdir(project_dir) + try: + cli.cmd_clean(SimpleNamespace()) + finally: + os.chdir(cwd) + + self.assertFalse((project_dir / ".car" / "stubs").exists()) + self.assertTrue((project_dir / "stubs").exists()) + + if __name__ == "__main__": unittest.main() diff --git a/ventis/cli.py b/ventis/cli.py index 4b5c95e..1f0f3b6 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -19,7 +19,8 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" -DEFAULT_CONFIG_PATH = "config/global_controller.yaml" +ARTIFACT_DIR_NAME = ".car" +SOURCE_DIR_NAME = "app" EC2_REQUIRED_CONFIG_KEYS = ( "ami_id", "subnet_id", @@ -51,6 +52,10 @@ def _load_config(config_path): return yaml.safe_load(f) +def _artifact_prefix(root): + return ARTIFACT_DIR_NAME if os.path.isdir(os.path.join(root, ARTIFACT_DIR_NAME)) else "" + + def _normalize_requirements(agent_cfg): """Return an agent's `requirements` list, or [] if absent/null/malformed.""" requirements = agent_cfg.get("requirements") or [] @@ -173,12 +178,27 @@ def cmd_new_project(args): logger.error("Templates directory not found at %s", templates_dir) sys.exit(1) - # Copy the entire templates tree into the new project - shutil.copytree(templates_dir, project_dir) + artifact_root = os.path.join(project_dir, ARTIFACT_DIR_NAME) + source_root = os.path.join(artifact_root, SOURCE_DIR_NAME) + shutil.copytree(templates_dir, source_root) + + source_config = os.path.join(source_root, "config") + artifact_config = os.path.join(artifact_root, "config") + if os.path.isdir(source_config): + shutil.move(source_config, artifact_root) + else: + os.makedirs(artifact_config) + + source_agents = os.path.join(source_root, "agents") + for declaration in glob.glob(os.path.join(source_agents, "*.yaml")): + shutil.move(declaration, artifact_config) + + readme = os.path.join(source_root, "README.md") + if os.path.isfile(readme): + shutil.move(readme, project_dir) - # Create empty output directories - os.makedirs(os.path.join(project_dir, "stubs"), exist_ok=True) - os.makedirs(os.path.join(project_dir, "grpc_stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "stubs"), exist_ok=True) + os.makedirs(os.path.join(artifact_root, "grpc_stubs"), exist_ok=True) logger.info("Created new Ventis project: %s", project_dir) logger.info("") @@ -197,7 +217,7 @@ def cmd_build(args): Generate stubs, compile gRPC protos, generate Docker contexts, and build Docker images. - Must be run from the project root (where config/ lives). + Must be run from the project root. """ config_path = args.config if not os.path.isfile(config_path): @@ -206,14 +226,19 @@ def cmd_build(args): config = _load_config(config_path) agents = config.get("agents", []) - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir + source_root = os.path.join(artifact_root, SOURCE_DIR_NAME) if prefix else project_dir package_dir = _get_package_dir() # -------------------------------------------------------------- # # Step 1: Discover agent YAML files and generate Python stubs # # -------------------------------------------------------------- # - agents_dir = os.path.join(project_dir, "agents") - stubs_dir = os.path.join(project_dir, "stubs") + declarations_dir = os.path.join( + artifact_root, "config" if prefix else "agents" + ) + stubs_dir = os.path.join(artifact_root, "stubs") os.makedirs(stubs_dir, exist_ok=True) from ventis.stub_generator import ( @@ -222,12 +247,12 @@ def cmd_build(args): generate_workflow_docker, ) - yaml_files = glob.glob(os.path.join(agents_dir, "*.yaml")) + yaml_files = glob.glob(os.path.join(declarations_dir, "*.yaml")) if not yaml_files: - logger.warning("No agent YAML files found in %s", agents_dir) + logger.warning("No agent YAML files found in %s", declarations_dir) import yaml - + # Looks up a config entry's YAML and to map stubs to entrypoints. yaml_by_name = {} for yaml_path in yaml_files: @@ -237,6 +262,16 @@ def cmd_build(args): yaml_by_name[name] = yaml_path entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + missing_stubs = [ + a["name"] + for a in agents + if a.get("type", "agent") != "workflow" + and (a["name"] not in yaml_by_name or not a.get("entrypoint")) + ] + if missing_stubs: + logger.error("Cannot generate stubs for agents: %s", ", ".join(missing_stubs)) + sys.exit(1) + stub_entrypoints = { f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] for n, p in yaml_by_name.items() @@ -244,7 +279,7 @@ def cmd_build(args): } stub_paths = [] - for yaml_path in yaml_files: + for yaml_path in yaml_by_name.values(): base_name = os.path.splitext(os.path.basename(yaml_path))[0] output_path = os.path.join(stubs_dir, f"{base_name}.py") logger.info("Generating stub: %s -> %s", yaml_path, output_path) @@ -254,7 +289,7 @@ def cmd_build(args): # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # - grpc_stubs_dir = os.path.join(project_dir, "grpc_stubs") + grpc_stubs_dir = os.path.join(artifact_root, "grpc_stubs") os.makedirs(grpc_stubs_dir, exist_ok=True) proto_dir = os.path.join(package_dir, "controller", "proto") @@ -292,12 +327,12 @@ def cmd_build(args): ) continue - workflow_path = os.path.join(project_dir, workflow_file) + workflow_path = os.path.join(source_root, workflow_file) if not os.path.isfile(workflow_path): logger.error("Workflow file not found: %s", workflow_path) continue - docker_context = os.path.join(project_dir, "docker_container", "Workflow") + docker_context = os.path.join(artifact_root, "docker_container", "Workflow") logger.info("Generating workflow Docker context for '%s'", agent_name) generate_workflow_docker( workflow_path, @@ -305,7 +340,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), - project_dir=project_dir, + project_dir=source_root, stub_entrypoints=stub_entrypoints, requirements=_normalize_requirements(agent_cfg), ) @@ -319,7 +354,7 @@ def cmd_build(args): ) continue - agent_file = os.path.join(project_dir, entrypoint) + agent_file = os.path.join(source_root, entrypoint) if not os.path.isfile(agent_file): logger.error("Agent file not found: %s", agent_file) continue @@ -333,7 +368,7 @@ def cmd_build(args): ) continue - docker_context = os.path.join(project_dir, "docker_container", agent_name) + docker_context = os.path.join(artifact_root, "docker_container", agent_name) logger.info("Generating Docker context for '%s'", agent_name) generate_docker( matching_yaml, @@ -341,7 +376,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, - project_dir=project_dir, + project_dir=source_root, stub_entrypoints=stub_entrypoints, requirements=_normalize_requirements(agent_cfg), ) @@ -360,7 +395,7 @@ def cmd_build(args): if not bake_targets: logger.info("No Docker images to build.") elif _docker_available() and _docker_available(("docker", "buildx", "version")): - docker_container_dir = os.path.join(project_dir, "docker_container") + docker_container_dir = os.path.join(artifact_root, "docker_container") os.makedirs(docker_container_dir, exist_ok=True) bake_file_path = os.path.join(docker_container_dir, "docker-bake.json") _write_bake_file(bake_targets, bake_file_path, _docker_platform()) @@ -407,15 +442,17 @@ def cmd_deploy(args): sys.exit(1) config = _load_config(config_path) - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir - _ensure_grpc_stubs_importable(project_dir) + _ensure_grpc_stubs_importable(artifact_root) if any( agent.get("provider", "local").upper() == "EC2" for agent in config.get("agents", []) ): - _preflight_ec2_deploy(config, project_dir) + _preflight_ec2_deploy(config, artifact_root) from ventis.controller.global_controller import GlobalController @@ -456,12 +493,14 @@ def cmd_clean(args): """ Remove generated stubs, gRPC files, and Docker build contexts. """ - project_dir = os.getcwd() + project_dir = os.path.abspath(os.getcwd()) + prefix = _artifact_prefix(project_dir) + artifact_root = os.path.join(project_dir, prefix) if prefix else project_dir paths_to_clean = [ - os.path.join(project_dir, "stubs"), - os.path.join(project_dir, "grpc_stubs"), - os.path.join(project_dir, "docker_container"), + os.path.join(artifact_root, "stubs"), + os.path.join(artifact_root, "grpc_stubs"), + os.path.join(artifact_root, "docker_container"), ] for path in paths_to_clean: @@ -483,6 +522,9 @@ def cmd_clean(args): def main(): + default_config_path = os.path.join( + _artifact_prefix(os.getcwd()), "config", "global_controller.yaml" + ) parser = argparse.ArgumentParser( prog="ventis", description="Ventis — Distributed Agent Orchestration Framework", @@ -505,8 +547,8 @@ def main(): build.add_argument( "-c", "--config", - default=DEFAULT_CONFIG_PATH, - help=f"Path to global controller config (default: {DEFAULT_CONFIG_PATH})", + default=default_config_path, + help=f"Path to global controller config (default: {default_config_path})", ) build.set_defaults(func=cmd_build) @@ -518,8 +560,8 @@ def main(): deploy.add_argument( "-c", "--config", - default=DEFAULT_CONFIG_PATH, - help=f"Path to global controller config (default: {DEFAULT_CONFIG_PATH})", + default=default_config_path, + help=f"Path to global controller config (default: {default_config_path})", ) deploy.set_defaults(func=cmd_deploy) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 1e24f10..e4369ac 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -26,8 +26,8 @@ from ventis.utils.redis_client import RedisClient from ventis.utils.grpc_options import GRPC_CHANNEL_OPTIONS -# Add generated grpc_stubs from the local project to the path -sys.path.insert(0, os.path.abspath("grpc_stubs")) +_artifact_prefix = ".car" if os.path.isdir(".car") else "" +sys.path.insert(0, os.path.abspath(os.path.join(_artifact_prefix, "grpc_stubs"))) import local_controler_pb2 import local_controler_pb2_grpc import grpc @@ -744,9 +744,9 @@ def stop(self): if __name__ == "__main__": - script_dir = os.path.dirname(os.path.abspath(__file__)) - project_root = os.path.join(script_dir, "..", "..") - default_config = os.path.join(project_root, "config", "global_controller.yaml") + default_config = os.path.join( + _artifact_prefix, "config", "global_controller.yaml" + ) import argparse @@ -755,7 +755,7 @@ def stop(self): "-c", "--config", default=default_config, - help="Path to the YAML config file (default: config/global_controller.yaml)", + help=f"Path to the YAML config file (default: {default_config})", ) args = parser.parse_args()