From 4d28f95d8bb56f70fb2507642278975948ad448f Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 17:26:35 -0700 Subject: [PATCH 01/13] includes all the files when ventis build --- ventis/cli.py | 2 ++ ventis/stub_generator.py | 73 ++++++++++++++++++++++++++++++++++------ 2 files changed, 64 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 3aceb18..9ffc149 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -274,6 +274,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), + project_dir=project_dir, ) else: @@ -316,6 +317,7 @@ def cmd_build(args): output_dir=docker_context, grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, + project_dir=project_dir, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..a571ccc 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -263,8 +263,36 @@ def _format_source(source): return "\n".join(formatted) + "\n" +# Directories ventis build itself generates inside a project -- never swept. +_GENERATED_DIRS = {"docker_container", "stubs", "grpc_stubs"} + + +def _sweep_py_files(project_dir): + """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" + swept = [] + for root, dirs, files in os.walk(project_dir): + dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + for fname in files: + if fname.endswith(".py"): + abs_src = os.path.join(root, fname) + rel_dst = os.path.relpath(abs_src, project_dir) + swept.append((abs_src, rel_dst)) + return swept + + +def _stub_destination(stub_file, project_dir): + """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" + basename = os.path.basename(stub_file) + return os.path.join("agents", basename) if project_dir else basename + + def generate_docker( - yaml_path, agent_file, output_dir=None, grpc_stubs_dir=None, stub_files=None + yaml_path, + agent_file, + output_dir=None, + grpc_stubs_dir=None, + stub_files=None, + project_dir=None, ): """ Generate a minimal Docker build context for an agent. @@ -278,6 +306,7 @@ def generate_docker( output_dir: Optional output directory (default: docker_container//). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -301,8 +330,13 @@ def generate_docker( with open(os.path.join(output_dir, "requirements.txt"), "w") as f: f.write(requirements) + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + # Copy general agent files - files_to_copy = [ + files_to_copy += [ # (source_path, destination_filename) (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -322,13 +356,13 @@ def generate_docker( (os.path.join(script_dir, "llm", "bedrock.py"), "bedrock.py"), ] - # Copy provided agent stubs + # Copy provided agent stubs, overwriting the swept real file at the same path if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), os.path.basename(stub_file)) + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) ) - + files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) # Copy gRPC generated stubs if they exist @@ -339,7 +373,9 @@ def generate_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") @@ -377,7 +413,12 @@ def generate_docker( def generate_workflow_docker( - workflow_file, stub_files, output_dir=None, grpc_stubs_dir=None, api_port=8080 + workflow_file, + stub_files, + output_dir=None, + grpc_stubs_dir=None, + api_port=8080, + project_dir=None, ): """ Generate a Docker build context for a workflow. @@ -391,6 +432,7 @@ def generate_workflow_docker( stub_files: List of stub file paths to include. output_dir: Optional output directory (default: docker_container/Workflow/). grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -417,7 +459,12 @@ def generate_workflow_docker( # ---- Copy source files into the build context ------------------------ workflow_basename = os.path.basename(workflow_file) - files_to_copy = [ + # Sweep the project for extra .py helper files not on the explicit list below. + files_to_copy = [] + if project_dir: + files_to_copy += _sweep_py_files(project_dir) + + files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), @@ -437,9 +484,11 @@ def generate_workflow_docker( ], ] - # Copy stub files + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: - files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) + files_to_copy.append( + (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -449,7 +498,9 @@ def generate_workflow_docker( for src, dst in files_to_copy: if os.path.isfile(src): - shutil.copy2(src, os.path.join(output_dir, dst)) + dest_path = os.path.join(output_dir, dst) + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) else: print(f" Warning: source file not found, skipping: {src}") From 23e3928d01bc5996fd1cd4e086d45212c283de0e Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 18:23:51 -0700 Subject: [PATCH 02/13] fixed some bugs --- ventis/cli.py | 19 ++++++++++++++ ventis/stub_generator.py | 54 +++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 9ffc149..cb9ee0a 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -220,6 +220,23 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) + # Map each stub's basename to its agent's declared entrypoint, so a stub + # overwrites the exact real file it replaces instead of guessing its path. + stub_entrypoints = {} + for agent_cfg in agents: + entrypoint = agent_cfg.get("entrypoint") + if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: + continue + for yaml_path in yaml_files: + import yaml + + with open(yaml_path) as f: + ydata = yaml.safe_load(f) + if ydata.get("agent", {}).get("name") == agent_cfg["name"]: + base_name = os.path.splitext(os.path.basename(yaml_path))[0] + stub_entrypoints[f"{base_name}.py"] = entrypoint + break + # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -275,6 +292,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, api_port=agent_cfg.get("api_port", 8080), project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) else: @@ -318,6 +336,7 @@ def cmd_build(args): grpc_stubs_dir=grpc_stubs_dir, stub_files=stub_paths, project_dir=project_dir, + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a571ccc..a56080d 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -271,7 +271,12 @@ def _sweep_py_files(project_dir): """Recursively collect (abs_src, rel_dst) for every .py file under project_dir, preserving its directory structure.""" swept = [] for root, dirs, files in os.walk(project_dir): - dirs[:] = [d for d in dirs if d not in _GENERATED_DIRS and not d.startswith(".")] + dirs[:] = [ + d + for d in dirs + if not d.startswith(".") + and not (root == project_dir and d in _GENERATED_DIRS) + ] for fname in files: if fname.endswith(".py"): abs_src = os.path.join(root, fname) @@ -280,10 +285,15 @@ def _sweep_py_files(project_dir): return swept -def _stub_destination(stub_file, project_dir): - """Mirror a stub to agents/, overwriting the real agent file cli.py always puts there; flat if no project sweep.""" +def _stub_destination(stub_file, stub_entrypoints): + """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" basename = os.path.basename(stub_file) - return os.path.join("agents", basename) if project_dir else basename + entrypoint = stub_entrypoints.get(basename) + if entrypoint: + normalized = entrypoint.replace("\\", "/") + if not normalized.startswith("/") and ".." not in normalized.split("/"): + return entrypoint + return basename def generate_docker( @@ -293,6 +303,7 @@ def generate_docker( grpc_stubs_dir=None, stub_files=None, project_dir=None, + stub_entrypoints=None, ): """ Generate a minimal Docker build context for an agent. @@ -301,12 +312,13 @@ def generate_docker( source files needed to run the agent with its own local controller. Args: - yaml_path: Path to the YAML agent definition. - agent_file: Path to the original Python agent implementation. - output_dir: Optional output directory (default: docker_container//). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - stub_files: Optional list of agent stub files to copy into the context. - project_dir: Optional project root to sweep for extra .py helper files. + yaml_path: Path to the YAML agent definition. + agent_file: Path to the original Python agent implementation. + output_dir: Optional output directory (default: docker_container//). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + stub_files: Optional list of agent stub files to copy into the context. + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ with open(yaml_path, "r") as f: config = yaml.safe_load(f) @@ -360,7 +372,10 @@ def generate_docker( if stub_files: for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) @@ -419,6 +434,7 @@ def generate_workflow_docker( grpc_stubs_dir=None, api_port=8080, project_dir=None, + stub_entrypoints=None, ): """ Generate a Docker build context for a workflow. @@ -428,11 +444,12 @@ def generate_workflow_docker( with its own local controller. Args: - workflow_file: Path to the workflow Python file. - stub_files: List of stub file paths to include. - output_dir: Optional output directory (default: docker_container/Workflow/). - grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). - project_dir: Optional project root to sweep for extra .py helper files. + workflow_file: Path to the workflow Python file. + stub_files: List of stub file paths to include. + output_dir: Optional output directory (default: docker_container/Workflow/). + grpc_stubs_dir: Optional path to compiled gRPC stubs (default: /grpc_stubs). + project_dir: Optional project root to sweep for extra .py helper files. + stub_entrypoints: Optional {stub_basename: entrypoint} map for exact stub placement. """ script_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.join(script_dir, "..") @@ -487,7 +504,10 @@ def generate_workflow_docker( # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( - (os.path.abspath(stub_file), _stub_destination(stub_file, project_dir)) + ( + os.path.abspath(stub_file), + _stub_destination(stub_file, stub_entrypoints or {}), + ) ) # Copy gRPC generated stubs if they exist From 95240ca941167a11c6da6cc791ca4a2fb13c7ebe Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Tue, 25 Aug 2026 19:17:55 -0700 Subject: [PATCH 03/13] ventis build: sweep project .py files into Docker build contexts generate_docker()/generate_workflow_docker() now recursively sweep every .py file under the project directory into the build context, preserving directory structure, so helper files that aren't declared as an agent entrypoint still make it into the image. Generated dirs (docker_container/, stubs/, grpc_stubs/) are excluded at the project root only, not at every depth. Stub files are placed at their agent's declared entrypoint path (mapped from global_controller.yaml) instead of a hardcoded guess, so a stub overwrites the exact real file it replaces. Guards against absolute and '..'-containing entrypoints, symlinked sources, and symlinked-destination escapes, with warnings on unsafe or unmapped stubs. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 45 +++++++++++++++--------------------- ventis/stub_generator.py | 49 ++++++++++++++++++++++------------------ 2 files changed, 45 insertions(+), 49 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index cb9ee0a..4c9badf 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -212,6 +212,23 @@ def cmd_build(args): if not yaml_files: logger.warning("No agent YAML files found in %s", agents_dir) + import yaml + + # Looks up a config entry's YAML and to map stubs to entrypoints. + yaml_by_name = {} + for yaml_path in yaml_files: + with open(yaml_path) as f: + name = yaml.safe_load(f).get("agent", {}).get("name") + if name: + yaml_by_name[name] = yaml_path + + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -220,23 +237,6 @@ def cmd_build(args): generate_stub(yaml_path, output_path) stub_paths.append(output_path) - # Map each stub's basename to its agent's declared entrypoint, so a stub - # overwrites the exact real file it replaces instead of guessing its path. - stub_entrypoints = {} - for agent_cfg in agents: - entrypoint = agent_cfg.get("entrypoint") - if agent_cfg.get("type", "agent") == "workflow" or not entrypoint: - continue - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_cfg["name"]: - base_name = os.path.splitext(os.path.basename(yaml_path))[0] - stub_entrypoints[f"{base_name}.py"] = entrypoint - break - # -------------------------------------------------------------- # # Step 2: Compile gRPC protobuf stubs # # -------------------------------------------------------------- # @@ -310,16 +310,7 @@ def cmd_build(args): continue # Find matching YAML by agent name - matching_yaml = None - for yaml_path in yaml_files: - import yaml - - with open(yaml_path) as f: - ydata = yaml.safe_load(f) - if ydata.get("agent", {}).get("name") == agent_name: - matching_yaml = yaml_path - break - + matching_yaml = yaml_by_name.get(agent_name) if not matching_yaml: logger.warning( "No YAML definition found for agent '%s', skipping Docker", diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index a56080d..9ea6c99 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -278,24 +278,43 @@ def _sweep_py_files(project_dir): and not (root == project_dir and d in _GENERATED_DIRS) ] for fname in files: - if fname.endswith(".py"): - abs_src = os.path.join(root, fname) + abs_src = os.path.join(root, fname) + if fname.endswith(".py") and not os.path.islink(abs_src): rel_dst = os.path.relpath(abs_src, project_dir) swept.append((abs_src, rel_dst)) return swept def _stub_destination(stub_file, stub_entrypoints): - """Mirror a stub to its agent's declared entrypoint path, overwriting the real file; flat if unknown, absolute, or containing '..'.""" + """Where to copy a stub so it overwrites the real file it replaces, falling back to flat if that's unsafe.""" basename = os.path.basename(stub_file) entrypoint = stub_entrypoints.get(basename) if entrypoint: normalized = entrypoint.replace("\\", "/") if not normalized.startswith("/") and ".." not in normalized.split("/"): - return entrypoint + return normalized + print(f" Warning: unsafe entrypoint '{entrypoint}' for stub {basename}, placing flat instead") + elif stub_entrypoints: + print(f" Warning: no entrypoint mapping for stub {basename}, placing flat instead") return basename +def _copy_files(output_dir, files_to_copy): + """Copy each (src, dst) pair into output_dir, refusing to write outside it (e.g. via a symlinked destination parent).""" + real_output_dir = os.path.realpath(output_dir) + for src, dst in files_to_copy: + if not os.path.isfile(src): + print(f" Warning: source file not found, skipping: {src}") + continue + dest_path = os.path.join(output_dir, dst) + real_dest = os.path.realpath(dest_path) + if os.path.commonpath([real_output_dir, real_dest]) != real_output_dir: + print(f" Warning: destination escapes build context, skipping: {dst}") + continue + os.makedirs(os.path.dirname(dest_path), exist_ok=True) + shutil.copy2(src, dest_path) + + def generate_docker( yaml_path, agent_file, @@ -386,13 +405,7 @@ def generate_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # Copy the YAML definition too shutil.copy2( @@ -477,9 +490,7 @@ def generate_workflow_docker( workflow_basename = os.path.basename(workflow_file) # Sweep the project for extra .py helper files not on the explicit list below. - files_to_copy = [] - if project_dir: - files_to_copy += _sweep_py_files(project_dir) + files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ (os.path.abspath(workflow_file), workflow_basename), @@ -500,7 +511,7 @@ def generate_workflow_docker( for name in ("gpu_metrics.py", "session_logging.py") ], ] - + # Copy stub files, overwriting the swept real file at the same path for stub_file in stub_files: files_to_copy.append( @@ -516,13 +527,7 @@ def generate_workflow_docker( if fname.endswith(".py"): files_to_copy.append((os.path.join(grpc_stubs_dir, fname), fname)) - for src, dst in files_to_copy: - if os.path.isfile(src): - dest_path = os.path.join(output_dir, dst) - os.makedirs(os.path.dirname(dest_path), exist_ok=True) - shutil.copy2(src, dest_path) - else: - print(f" Warning: source file not found, skipping: {src}") + _copy_files(output_dir, files_to_copy) # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading From 69d5405c24485ff51db960ad7843e496d91354ce Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:06:31 -0700 Subject: [PATCH 04/13] Fix missing os import in metrics_agent.py Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json From 653bee84d8999a233bf71ce1e75e0ae2d1d44057 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 13:23:55 -0700 Subject: [PATCH 05/13] WIP: OTel exporter testing + portfolio merge-conflict fix (pre-pull checkpoint) Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 174 ++++++++++ OTel_Exporter/__init__.py | 0 OTel_Exporter/convert.py | 96 ++++++ OTel_Exporter/db.py | 185 ++++++++++ OTel_Exporter/otel_exporter.py | 98 ++++++ .../helloworld/workflow/example_workflow.py | 4 +- examples/portfolio/agents/advisor_agent.py | 16 +- examples/portfolio/agents/intent_agent.py | 35 -- examples/portfolio/agents/llm_agent.py | 50 --- examples/portfolio/agents/llm_agent.yaml | 14 - examples/portfolio/agents/metrics_agent.py | 1 + .../portfolio/config/global_controller.yaml | 32 +- examples/portfolio/config/policy.yaml | 1 - pyproject.toml | 6 +- requirements.txt | 4 + uv.lock | 320 ++++++++++++++++++ ventis/controller/global_controller.py | 57 +++- ventis/controller/utils/process_supervisor.py | 59 ++++ ventis/deploy.py | 2 +- ventis/stub_generator.py | 9 +- 20 files changed, 1020 insertions(+), 143 deletions(-) create mode 100644 OTel_Exporter/DESIGN.md create mode 100644 OTel_Exporter/__init__.py create mode 100644 OTel_Exporter/convert.py create mode 100644 OTel_Exporter/db.py create mode 100644 OTel_Exporter/otel_exporter.py delete mode 100644 examples/portfolio/agents/llm_agent.py delete mode 100644 examples/portfolio/agents/llm_agent.yaml create mode 100644 ventis/controller/utils/process_supervisor.py diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md new file mode 100644 index 0000000..73c4da2 --- /dev/null +++ b/OTel_Exporter/DESIGN.md @@ -0,0 +1,174 @@ +# OTLP Exporter for Ventis GlobalController — Design + +Status: **implemented (single-table design)**. `GlobalController` writes futures into a +`waiting` table (SQLite); a GC-supervised, GC-restarted OTel Exporter process reads +finished/unsent rows, converts each to an OTel span, and hands it to a real +`BatchSpanProcessor`/`OTLPSpanExporter`. Batching, serialization, and sending are all +OTel SDK code — the only custom pieces are the row→span conversion and durable +sent-tracking. This doc is a design/rationale reference; the actual files +(`otel_exporter.py`, `db.py`, `convert.py`, `ventis/controller/utils/process_supervisor.py`) +are the source of truth for current behavior. + +## Context +Ventis futures need to reach an external OTLP-compatible tracing backend. Design: a +separate OTLP Exporter process, spawned and supervised by GlobalController, that reads +unsent finished future rows from a local SQLite DB, converts them into OTel spans, and +hands them to the OTel SDK's own batching/export machinery, which ships them to an +external OTLP Receiver (out of scope here — assumed to be a separate, already-addressable +service). + +Decisions (final status): +- **Process model**: a true separate OS process, spawned and supervised by + GlobalController (not an in-process thread) — via `ProcessSupervisor` + (`ventis/controller/utils/process_supervisor.py`, built): `register`/`start_all` to + spawn, `check_and_respawn` (called from GC's existing poll tick, guarded on + `self.running` to avoid a shutdown race) to restart it if it ever dies unexpectedly, + `terminate_all` (called from GC's `stop()`) to shut it down cleanly. Rationale: fault + isolation from GC's core polling/health loop and independent restart, at low added + complexity since SQLite is already the entire hand-off boundary between the two. +- **Config**: implemented via a new `otel:` section in `global_controller.yaml` + (`protocol`/`endpoint`/`headers`), *not* by making `otel_exporter.py` itself + config-aware. `GlobalController` translates that section into the OTel SDK's own + standard env vars (`OTEL_EXPORTER_OTLP_PROTOCOL`/`_ENDPOINT`/`_HEADERS`) and passes + them to the exporter subprocess via `ProcessSupervisor.register(..., env=...)`. The + exporter still just constructs `OTLPSpanExporter()` with no explicit args (endpoint + and headers are resolved by the SDK itself from those env vars, same as always) and + reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly, to pick the gRPC vs HTTP exporter + class — the one piece of protocol selection the plain SDK classes don't do on their + own. Deliberately vendor-neutral: no backend name (Postgres, Langfuse, or otherwise) + appears anywhere in `otel_exporter.py`; the destination is 100% deploy-time config, + set once in `global_controller.yaml` and never touched by app code again. The + originally-planned `database.url` repurposing (below, kept for history) was decided + against — env-var configuration is the SDK's own idiomatic mechanism, so no + exporter-side config plumbing was added, only a GC-side YAML→env-var translation. + Does not (yet) support simultaneous multi-destination export — see "Known gaps". +- **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own + SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + `_poll_controllers` *alongside* (not instead of) the existing + `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled + from the dashboard/cost table. +- **Two tables collapsed into one**: an earlier version of this design had a second + `queue` table (`waiting` → promote → `queue` → drain → send). Collapsed once it became + clear `BatchSpanProcessor` already provides its own in-memory queue — the only thing a + second table added was durability across the exporter's own process restarts, which a + `sent` column on `waiting` alone provides just as well, with less code. See `db.py`'s + module docstring. +- **Span construction**: settled — spans are built as `ReadableSpan` objects directly + (bypassing `Tracer`/`TracerProvider` entirely, no `IdGenerator` workaround needed for + either `trace_id` or `span_id`). Confirmed working via `ConsoleSpanExporter` during + development and via real (though unreachable) OTLP export attempts. + +## Implementation summary + +### 1. Config +`global_controller.yaml` gains an optional `otel:` section: +```yaml +otel: + protocol: grpc # or http + endpoint: otlp-pg-receiver.railway.internal:4317 + headers: {} # e.g. Authorization: "Basic " for a backend needing auth +``` +`GlobalController._otel_exporter_env()` translates this into +`OTEL_EXPORTER_OTLP_PROTOCOL`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` +and hands them to `ProcessSupervisor.register("otel_exporter", ..., env=...)`, which now +supports an `env` param (merged on top of the parent process's own environment, not a +replacement). Omitting `otel:` entirely falls back to whatever ambient env the exporter +subprocess would otherwise inherit, same as before this change. + +`otel_exporter.py` itself reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly (to pick +which `OTLPSpanExporter` class to import — gRPC or HTTP; the plain SDK classes don't +self-select this the way `opentelemetry-instrument`'s auto-config does). Endpoint and +headers are never read directly — `OTLPSpanExporter()` is still constructed with no +explicit args, letting the SDK resolve those from the same env vars itself, exactly as +before this change. `BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000)` +— the flush delay is explicitly overridden from the SDK default (5000ms) to 1000ms; +`max_export_batch_size` is left at the SDK default (512), which already approximates the +original "500 spans" batching ask without any override needed. + +### 2. `OTel_Exporter/otel_exporter.py` +A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM +stays responsive), calling `_send_pending()` each tick: +- `SELECT * FROM waiting WHERE finished_at IS NOT NULL AND (sent IS NULL OR sent = 0)`. +- Per row, each isolated in its own try/except (one malformed row is logged and skipped, + never blocks the rest of the batch): `convert.waiting_row_to_span(row)` → + `_processor.on_end(span)` → `db.mark_sent(future_id)` immediately — atomic per row, not + batched at the end, so a crash mid-poll can't leave an already-sent row unmarked (which + would cause a duplicate send on the next run). +- `_processor` is constructed once at startup; no `TracerProvider` is used at all, since + spans are hand-built and handed straight to the processor via `on_end()`. +- `_processor.shutdown()` on exit, flushing any pending batch. + +### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +`future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is +the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel +`trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs +truncation). No hashing — just hex-decode and truncate (deterministic, pure): +```python +trace_id = int(row["session_id"], 16) +span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") +parent_span_id = int.from_bytes(bytes.fromhex(row["parent_id"])[:8], "big") if row["parent_id"] else None +``` +Spans are assembled as plain `ReadableSpan(name=..., context=SpanContext(...), parent=SpanContext(...) or None, attributes=..., events=..., status=..., start_time=..., end_time=...)` +— no `Tracer`, no `IdGenerator`. Failed rows get a hand-built `exception` `Event` (using +the SDK's own `EXCEPTION_TYPE`/`EXCEPTION_MESSAGE` constants from `opentelemetry.sdk.trace`, +not hardcoded strings — `record_exception()` can't be used retrospectively since there's +no live exception object, only strings) plus `Status(StatusCode.ERROR, description=...)`. + +**Attribute naming**: `model`/`input_token_count`/`output_token_count` are set under the +real, current OTel GenAI semantic-convention keys — `gen_ai.request.model`/ +`gen_ai.usage.input_tokens`/`gen_ai.usage.output_tokens` — verified against the actual +spec (`open-telemetry/semantic-conventions`), not assumed. `cpu`/`gpu`/ +`execution_time_ms`/`queue_time_ms`/`token_count` keep plain names deliberately: none of +them have an OTel GenAI equivalent (cpu/gpu/queue-time are Ventis infra concepts, and +`token_count`, an input+output sum, isn't part of the spec at all — inventing a +`gen_ai.*`-shaped name for any of these would fabricate a standard rather than follow +one. `cached_tokens`/`cache_hit_ratio` exist on the `waiting` row but aren't exported to +attributes at all yet — a separate, pre-existing gap, not touched here. + +### 4. Process supervisor — `ventis/controller/utils/process_supervisor.py` (built) +`ProcessSupervisor`: `register(name, argv, env=None)` declares a process spec (`env`, +when given, is merged on top of — not a replacement for — the parent's own environment); +`start_all()` spawns everything registered; `check_and_respawn()` restarts anything that +exited, replaying the same argv/env (called from GC's `_poll_controllers`, guarded by +`if self.running:` so a SIGTERM mid-tick can't cause it to resurrect a process +`terminate_all()` just intentionally killed); `terminate_all()` terminates every managed +process (all `.terminate()` calls first, then `.wait()` on each, falling back to +`.kill()`), called from GC's `stop()`. Adding a future second daemon is one more +`register()` call — no new spawn/monitor/terminate code needed. + +### 5. Dependencies (all added) +`opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, +`opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` +config work, since `protocol: http` now needs that package importable). + +## Known gaps (not yet built) +- Spans carry no explicit `resource`/`instrumentation_scope` — would show as + `service.name=unknown_service` at a real backend. +- No simultaneous multi-destination export — `otel:` configures exactly one + destination; sending to two backends at once would mean registering a second, + separately-configured `otel_exporter` subprocess (same script, different env), not + something the exporter or its config format do today. +- `waiting` grows unboundedly: sent rows are never pruned, and futures that never finish + (`finished_at` never arrives) also stay forever, invisible and un-expiring. +- `error_name` is always `NULL` — Ventis's own Redis writer never records a distinct + exception-type field, only a message string. +- No committed test suite — all verification during development was ad hoc scripts, not + `pytest` files under `tests/`. +- Never verified against a live OTLP receiver — only against a refused connection + (confirmed the SDK's real retry/error-handling path is exercised correctly). +- No retry-limit/quarantine for a permanently malformed row — it logs an error every poll + forever rather than being given up on. + +## Verification approach used during development +- Row→span conversion: ad hoc scripts asserting deterministic id derivation, correct + parent/child linkage, correct `ERROR` status + `exception` event on failed rows, and + passing hand-built spans through `ConsoleSpanExporter().export([span])` to confirm the + SDK accepts them without error. +- Pipeline correctness: seeded `waiting` with mixes of finished/still-running/malformed/ + failed rows, ran the real `otel_exporter.py` subprocess, and inspected the resulting + `sent` flags and log output directly — including confirming a second run does not + re-send already-sent rows, and that a malformed row is skipped without blocking others. +- Process supervision: unit-tested `ProcessSupervisor` against a dummy process (spawn, + kill, confirm respawn with a new PID, confirm clean `terminate_all`) and + integration-tested it managing the real `otel_exporter.py` process. +- Scoped to the local provider throughout — no EC2 needed. diff --git a/OTel_Exporter/__init__.py b/OTel_Exporter/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/OTel_Exporter/convert.py b/OTel_Exporter/convert.py new file mode 100644 index 0000000..8e062a4 --- /dev/null +++ b/OTel_Exporter/convert.py @@ -0,0 +1,96 @@ +"""Convert a `waiting` table row (see db.py) into an OTel ReadableSpan. + +Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects +directly instead of going through Tracer.start_span() -- there's no live tracer here, +futures already finished (sometimes in another process), so this is a historical-row +conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +the SDK's usual advice against constructing ReadableSpan by hand. +""" + +from opentelemetry.sdk.trace import EXCEPTION_MESSAGE, EXCEPTION_TYPE, Event, ReadableSpan +from opentelemetry.trace import SpanContext, SpanKind, TraceFlags +from opentelemetry.trace.status import Status, StatusCode + +_SAMPLED = TraceFlags(TraceFlags.SAMPLED) + + +def to_epoch_nanos(unix_seconds): + """Convert a unix-epoch-seconds float (as stored in waiting) to OTel's ns int.""" + if unix_seconds is None: + return None + return round(float(unix_seconds) * 1e9) + + +def waiting_row_to_span(row): + """Convert one waiting row (dict-like, column names as keys) into a ReadableSpan. + + Rows without finished_at are accepted but produce a span with end_time=None -- + filtering to finished rows is the caller's responsibility, not this function's. + """ + # sqlite3.Row supports row["col"] but not row.get("col") -- normalize once so the + # rest of this function can use .get() freely for optional fields. + row = dict(row) + + trace_id = int(row["session_id"], 16) + span_id = int.from_bytes(bytes.fromhex(row["future_id"])[:8], "big") + parent_id = row.get("parent_id") + parent_span_id = ( + int.from_bytes(bytes.fromhex(parent_id)[:8], "big") if parent_id else None + ) + + context = SpanContext( + trace_id=trace_id, span_id=span_id, is_remote=False, trace_flags=_SAMPLED + ) + parent = ( + SpanContext( + trace_id=trace_id, span_id=parent_span_id, is_remote=False, trace_flags=_SAMPLED + ) + if parent_span_id + else None + ) + + events = [] + status = Status(StatusCode.UNSET) + if row["failed"]: + events.append( + Event( + name="exception", + attributes={ + EXCEPTION_TYPE: row.get("error_name") or "RuntimeError", + EXCEPTION_MESSAGE: row.get("error_message") or "", + }, + timestamp=to_epoch_nanos(row.get("finished_at")), + ) + ) + status = Status(StatusCode.ERROR, description=row.get("error_message")) + + # model/input/output use real OTel GenAI semconv names; cpu/gpu/execution_time_ms/ + # queue_time_ms/token_count have no semconv equivalent (Ventis infra concepts, or -- + # for token_count -- a derived sum the spec doesn't define), so they keep plain names + # rather than being forced into a fake gen_ai.* one. See DESIGN.md. + attributes = { + k: v + for k, v in { + "gen_ai.request.model": row.get("model"), + "cpu": row.get("cpu"), + "gpu": row.get("gpu"), + "execution_time_ms": row.get("execution_time_ms"), + "queue_time_ms": row.get("queue_time_ms"), + "gen_ai.usage.input_tokens": row.get("input_token_count"), + "gen_ai.usage.output_tokens": row.get("output_token_count"), + "token_count": row.get("token_count"), + }.items() + if v is not None + } + + return ReadableSpan( + name=row.get("agent_id") or "unknown_agent", + context=context, + parent=parent, + attributes=attributes, + events=events, + status=status, + kind=SpanKind.INTERNAL, + start_time=to_epoch_nanos(row.get("started_at")), + end_time=to_epoch_nanos(row.get("finished_at")), + ) diff --git a/OTel_Exporter/db.py b/OTel_Exporter/db.py new file mode 100644 index 0000000..4bb301d --- /dev/null +++ b/OTel_Exporter/db.py @@ -0,0 +1,185 @@ +"""SQLite schema and writes for the OTel export pipeline's waiting table. + +`waiting` holds future rows as GlobalController observes them (including still-running +ones). There's no separate queue table -- OTel's own BatchSpanProcessor already queues +and batches spans in memory, so the only thing we need to track durably is which rows +have already been sent, which the `sent` column on this same table provides. (An earlier +version of this pipeline had a second `queue` table for that; collapsed away since it +wasn't doing anything BatchSpanProcessor doesn't already do -- see DESIGN.md.) +""" + +import os +import sqlite3 + +from ventis.controller.utils import pricing + +DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "otel_queue.db") + +# Demo-only multipliers for scaling displayed costs; not real recorded costs. Kept +# deliberately standalone/duplicated from telemetry_logging.py's identical constants +# (rather than importing them) so this module has no dependency on it -- keep these in +# sync by hand if the multipliers there ever change. +_TOKEN_COST_MULTIPLIER = 10000 +_SERVER_COST_MULTIPLIER = 100000 + +# Timestamps are stored as unix epoch seconds (matching the Redis future hash fields +# they're read from), not as SQLite datetime strings. Column set mirrors +# runtime_information 1:1 (see telemetry_logging.py) plus this pipeline's own additions +# (error_name/error_message/sent). +_TABLE_COLUMNS = """ + future_id TEXT PRIMARY KEY, + parent_id TEXT, + session_id TEXT NOT NULL, + project_id TEXT, + agent_id TEXT, + model TEXT, + cpu REAL, + gpu REAL, + started_at TIMESTAMP, + finished_at TIMESTAMP, + execution_time_ms INTEGER, + queue_time_ms INTEGER, + input_token_count INTEGER, + output_token_count INTEGER, + token_count INTEGER, + errors INTEGER, + failed BOOLEAN, + server_cost REAL, + token_cost REAL, + total_cost REAL, + cached_tokens INTEGER, + cache_hit_ratio REAL, + error_name TEXT, + error_message TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + sent BOOLEAN DEFAULT 0 +""" + + +def init_db(db_path=DB_PATH): + """Create the waiting table if it doesn't already exist.""" + conn = sqlite3.connect(db_path) + try: + conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") + conn.commit() + finally: + conn.close() + + +# `sent` is deliberately excluded here so re-upserting a waiting row (e.g. GC +# re-writing it from Redis) never resets it back to unsent. +_COLUMNS = [ + "future_id", "parent_id", "session_id", "project_id", "agent_id", "model", + "cpu", "gpu", "started_at", "finished_at", "execution_time_ms", "queue_time_ms", + "input_token_count", "output_token_count", "token_count", "errors", + "failed", "server_cost", "token_cost", "total_cost", + "cached_tokens", "cache_hit_ratio", "error_name", "error_message", +] + +_WAITING_UPSERT = """ + INSERT INTO waiting ({cols}) VALUES ({placeholders}) + ON CONFLICT(future_id) DO UPDATE SET {updates} +""".format( + cols=", ".join(_COLUMNS), + placeholders=", ".join(f":{c}" for c in _COLUMNS), + updates=", ".join(f"{c}=excluded.{c}" for c in _COLUMNS if c != "future_id"), +) + + +def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH): + """Upsert future rows (as returned by telemetry_logging.pull_runtime_information) + into the waiting table. Unlike runtime_information, rows without finished_at are + kept (not skipped) -- that's what "waiting" means here. `redis_client` is only used + to look up the executing agent's instance type for server-cost pricing, mirroring + send_runtime_information; pass None to skip cost lookups (server_cost stays 0).""" + if not rows: + return + conn = sqlite3.connect(db_path) + try: + for raw in rows: + fid = raw.get("future_id") + session_id = raw.get("request_id") + if not fid or not session_id: + continue + agent_id = raw.get("agent") + started_at = float(raw.get("created_at") or 0) or None + finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None + execution_time_ms = ( + round((finished_at - started_at) * 1000) + if finished_at and started_at + else None + ) + input_token_count = int(float(raw.get("input_token_count") or 0)) + output_token_count = int(float(raw.get("output_token_count") or 0)) + token_count = int(float(raw.get("token_count") or 0)) + cached_tokens = int(float(raw.get("input_cache_tokens") or 0)) + + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER + ) + # Server cost needs an elapsed duration -- only available once finished. + if finished_at and started_at: + server_cost = ( + pricing.compute_server_cost( + redis_client.get(f"agent:{agent_id}:instance_type") + if redis_client is not None and agent_id + else None, + finished_at - started_at, + ) + * _SERVER_COST_MULTIPLIER + ) + else: + server_cost = 0.0 + + conn.execute( + _WAITING_UPSERT, + { + "future_id": fid, + "parent_id": raw.get("parent") or None, + "session_id": session_id, + "project_id": project_id, + "agent_id": agent_id, + "model": raw.get("model"), + "cpu": float(raw.get("cpu_resource") or 0), + "gpu": float(raw.get("gpu_resource") or 0), + "started_at": started_at, + "finished_at": finished_at, + "execution_time_ms": execution_time_ms, + "queue_time_ms": ( + round(float(raw["queue_time"]) * 1000) + if raw.get("queue_time") + else None + ), + "input_token_count": input_token_count, + "output_token_count": output_token_count, + "token_count": token_count, + "errors": int(raw.get("errors") or 0), + "failed": bool(int(raw.get("failed") or 0)), + "server_cost": server_cost, + "token_cost": token_cost, + "total_cost": server_cost + token_cost, + "cached_tokens": cached_tokens, + "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, + "error_name": raw.get("error_name"), + "error_message": raw.get("error_message"), + }, + ) + conn.commit() + finally: + conn.close() + + +def mark_sent(future_id, db_path=DB_PATH): + """Mark one waiting row sent. Call this immediately after successfully handing its + span to the batch processor -- one row, one commit -- so a crash between two rows' + sends can't leave an already-sent row unmarked (which would cause a duplicate send + on the next run).""" + conn = sqlite3.connect(db_path) + try: + conn.execute("UPDATE waiting SET sent = 1 WHERE future_id = ?", (future_id,)) + conn.commit() + finally: + conn.close() diff --git a/OTel_Exporter/otel_exporter.py b/OTel_Exporter/otel_exporter.py new file mode 100644 index 0000000..b1c7f1a --- /dev/null +++ b/OTel_Exporter/otel_exporter.py @@ -0,0 +1,98 @@ +"""Entrypoint for the OTLP Exporter process. + +Each poll tick: read finished, not-yet-sent rows from `waiting`, convert each to a span, +hand it to a BatchSpanProcessor/OTLPSpanExporter, and mark it sent -- batching, OTLP +serialization, and sending are all the SDK's own code, not ours (see DESIGN.md). Each +row's send-and-mark-sent is atomic and happens immediately after its own successful +send, not batched at the end, so a crash mid-poll can't leave an already-sent row +unmarked (which would cause a duplicate send next run). `OTLPSpanExporter()` takes no +explicit endpoint/headers here -- it falls back to the SDK's own standard +`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars, or localhost:4317, +per the SDK's own default behavior. GlobalController sets those env vars (plus +`OTEL_EXPORTER_OTLP_PROTOCOL`, which this module reads itself below to pick the gRPC vs +HTTP class) from `global_controller.yaml`'s `otel:` section when it spawns this process; +this file has no YAML/app-config awareness of its own, only standard OTel env vars -- +see DESIGN.md. +""" + +import logging +import os +import signal +import sqlite3 +import time + +# Protocol is the one thing the SDK's own exporter classes don't self-select from +# OTEL_EXPORTER_OTLP_PROTOCOL -- endpoint/headers/auth stay fully env-var-driven via +# each class's own defaults; see OTel_Exporter/DESIGN.md. +if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").startswith("http"): + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +else: + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +import convert +import db + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +_running = True +_processor = None +POLL_INTERVAL_SECONDS = 5 + + +def _handle_shutdown(signum, frame): + global _running + _running = False + + +def _send_pending(): + """Convert and send each finished, not-yet-sent waiting row.""" + conn = sqlite3.connect(db.DB_PATH) + conn.row_factory = sqlite3.Row + try: + rows = conn.execute( + "SELECT * FROM waiting WHERE finished_at IS NOT NULL " + "AND (sent IS NULL OR sent = 0)" + ).fetchall() + finally: + conn.close() + if not rows: + return + sent_count = 0 + for row in rows: + try: + span = convert.waiting_row_to_span(row) + _processor.on_end(span) + except Exception as e: + logger.error( + "Skipping waiting row %s -- failed to send: %s", row["future_id"], e + ) + continue + db.mark_sent(row["future_id"]) + sent_count += 1 + logger.info("Sent %d span(s) to the batch processor.", sent_count) + + +def main(): + global _processor + signal.signal(signal.SIGTERM, _handle_shutdown) + signal.signal(signal.SIGINT, _handle_shutdown) + db.init_db() + _processor = BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000) + logger.info("OTel exporter process started.") + last_poll = 0 + while _running: + if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + try: + _send_pending() + except Exception as e: + logger.warning("Poll cycle failed (non-fatal): %s", e) + last_poll = time.time() + time.sleep(1) + _processor.shutdown() + logger.info("OTel exporter process exiting.") + + +if __name__ == "__main__": + main() diff --git a/examples/helloworld/workflow/example_workflow.py b/examples/helloworld/workflow/example_workflow.py index 8bd4600..842fe80 100644 --- a/examples/helloworld/workflow/example_workflow.py +++ b/examples/helloworld/workflow/example_workflow.py @@ -15,11 +15,11 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "grpc_stubs")) from deploy import deploy -from example_agent_stub import ExampleAgentStub +from example_agent import ExampleAgent def main(name: str = "World"): - agent = ExampleAgentStub() + agent = ExampleAgent() greeting = agent.hello(name=name) return {"greeting": greeting.value()} diff --git a/examples/portfolio/agents/advisor_agent.py b/examples/portfolio/agents/advisor_agent.py index 2db763f..5cc31ae 100644 --- a/examples/portfolio/agents/advisor_agent.py +++ b/examples/portfolio/agents/advisor_agent.py @@ -11,13 +11,10 @@ # If the LLM is unavailable (returns an empty string), it falls back to a # deterministic templated summary so the pipeline still returns. # -# Resource profile: cheap CPU; the LLM cost sits in LLMAgent, not here. +# Resource profile: cheap CPU, single call per request, on the critical path. -import sys import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from llm_agent import LLMAgent try: from ventis.llm.bedrock import call_bedrock except ImportError: @@ -27,16 +24,14 @@ class AdvisorAgent(object): def __init__(self): self.tools = [self.summarize] - self.llm = LLMAgent() + self.model_id = os.environ.get( + "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" + ) + self.region = os.environ.get("AWS_REGION", "us-east-1") def summarize(self, holdings: dict, metrics: dict, risk: dict) -> str: """Write a short plain-English briefing on the portfolio.""" prompt = self._build_prompt(holdings, metrics, risk) - text = self.llm.complete( - prompt=prompt, max_tokens=400, temperature=0.2 - ).value() - if not text: - print("AdvisorAgent: LLM returned no output; using templated summary.") try: response = call_bedrock( model_id=self.model_id, @@ -48,7 +43,6 @@ def summarize(self, holdings: dict, metrics: dict, risk: dict) -> str: except Exception as e: print(f"AdvisorAgent: Bedrock call failed ({e}); using templated summary.") return self._fallback_summary(metrics, risk) - return text def _build_prompt(self, holdings: dict, metrics: dict, risk: dict) -> str: lines = ["You are a portfolio analyst. Given the figures below, write a " diff --git a/examples/portfolio/agents/intent_agent.py b/examples/portfolio/agents/intent_agent.py index 972c8af..1eb15d7 100644 --- a/examples/portfolio/agents/intent_agent.py +++ b/examples/portfolio/agents/intent_agent.py @@ -7,17 +7,6 @@ # -> {"holdings": {"AAPL": 0.4, "MSFT": 0.35, "NVDA": 0.25}, # "lookback_days": 180} # -<<<<<<< HEAD -# The actual model call lives in the shared LLMAgent (remote, resolved via -# .value()) — this agent only builds the prompt and parses the result, so no -# Bedrock boilerplate lives here. If the LLM is unavailable or returns -# unparseable output, parse() raises: there is no fallback, the request fails -# loudly rather than guessing at the holdings. Weights are renormalized to 1.0. -# -# Resource profile: cheap CPU; the LLM cost sits in LLMAgent, not here. - -import sys -======= # Calls AWS Bedrock (Converse API) via ventis.llm.bedrock -- same pattern as # AdvisorAgent -- so token/cost telemetry gets recorded onto this execution's # future::metrics hash. Configure with env vars: @@ -31,20 +20,14 @@ # Resource profile: cheap CPU, single call per request, on the critical path # before the fan-out. ->>>>>>> remotes/origin/telemetry-signals import os import re import json -<<<<<<< HEAD -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "stubs")) -from llm_agent import LLMAgent -======= try: from ventis.llm.bedrock import call_bedrock except ImportError: from bedrock import call_bedrock ->>>>>>> remotes/origin/telemetry-signals DEFAULT_LOOKBACK_DAYS = 365 @@ -52,15 +35,6 @@ class IntentAgent(object): def __init__(self): self.tools = [self.parse] -<<<<<<< HEAD - self.llm = LLMAgent() - - def parse(self, query: str) -> dict: - """Parse a natural-language portfolio request into holdings + lookback.""" - text = self.llm.complete( - prompt=self._build_prompt(query), max_tokens=300, temperature=0.0 - ).value() -======= self.model_id = os.environ.get( "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" ) @@ -75,7 +49,6 @@ def parse(self, query: str) -> dict: region=self.region, ) text = response["output"]["message"]["content"][0]["text"] ->>>>>>> remotes/origin/telemetry-signals if not text: raise ValueError("IntentAgent: LLM returned no output for the request.") @@ -148,15 +121,7 @@ def _sanitize(self, parsed: dict) -> dict: if __name__ == "__main__": -<<<<<<< HEAD - # Assumes the LLMAgent stub (Future-returning) is on the path, as it is - # inside the deployed pipeline. - agent = IntentAgent() - print(agent.parse( - "Analyze 40% Apple, 35% Microsoft and 25% Nvidia over the last 6 months" -======= agent = IntentAgent() print(agent.parse( query="Analyze 40% Apple, 35% Microsoft and 25% Nvidia over the last 6 months" ->>>>>>> remotes/origin/telemetry-signals )) diff --git a/examples/portfolio/agents/llm_agent.py b/examples/portfolio/agents/llm_agent.py deleted file mode 100644 index d42e4fb..0000000 --- a/examples/portfolio/agents/llm_agent.py +++ /dev/null @@ -1,50 +0,0 @@ -# LLM Agent -# -# Shared inference node. Owns all the AWS Bedrock (Converse API) plumbing so no -# other agent has to carry boto3 boilerplate — they just call complete(prompt) -# and get text back. Configure with env vars: -# BEDROCK_MODEL_ID (default: meta.llama3-8b-instruct-v1:0) -# AWS_REGION (default: us-east-1) -# -# On any failure (no boto3, no creds, model not enabled) it returns an empty -# string; callers decide how to degrade (templated summary, regex parse, etc.). -# -# Resource profile: LLM-bound. This is the only node that talks to Bedrock, so -# it's the natural place to scale inference capacity independently. - -import os - - -class LLMAgent(object): - def __init__(self): - self.tools = [self.complete] - self.model_id = os.environ.get( - "BEDROCK_MODEL_ID", "meta.llama3-8b-instruct-v1:0" - ) - self.region = os.environ.get("AWS_REGION", "us-east-1") - - def complete( - self, prompt: str, max_tokens: int = 400, temperature: float = 0.2 - ) -> str: - """Run a single-turn completion on Bedrock; '' on any failure.""" - try: - import boto3 - - client = boto3.client("bedrock-runtime", region_name=self.region) - response = client.converse( - modelId=self.model_id, - messages=[{"role": "user", "content": [{"text": prompt}]}], - inferenceConfig={ - "maxTokens": max_tokens, - "temperature": temperature, - }, - ) - return response["output"]["message"]["content"][0]["text"] - except Exception as e: - print(f"LLMAgent: Bedrock call failed ({e}).") - return "" - - -if __name__ == "__main__": - agent = LLMAgent() - print(agent.complete("Say hello in one short sentence.", max_tokens=50)) \ No newline at end of file diff --git a/examples/portfolio/agents/llm_agent.yaml b/examples/portfolio/agents/llm_agent.yaml deleted file mode 100644 index ccc0095..0000000 --- a/examples/portfolio/agents/llm_agent.yaml +++ /dev/null @@ -1,14 +0,0 @@ -agent: - name: LLMAgent - functions: - - name: complete - description: Run a single-turn completion on Bedrock; '' on any failure. - arguments: - - name: prompt - type: str - - name: max_tokens - type: int - - name: temperature - type: float - returns: - type: str diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index 374a869..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -8,6 +8,7 @@ # downstream RiskAgent can build the portfolio covariance. # # Resource profile: cheap CPU, high fan-out — one compute() call per holding. +import os import sys import json diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index 29e8de9..9a0a9a5 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -6,26 +6,6 @@ # reflect each stage's real cost so the scheduler has placement decisions to make. agents: -<<<<<<< HEAD - # Shared inference node. Owns all Bedrock plumbing; IntentAgent and - # AdvisorAgent delegate their model calls here. LLM-bound — scale replicas - # to match inference demand. - - name: LLMAgent - host: localhost - port: 8075 - redis_port: 6379 - replicas: 1 - resources: - cpu: 1 - memory: 512 - entrypoint: agents/llm_agent.py - - # Stage 0: parse the free-text request into structured holdings + lookback - # window (calls LLMAgent). Cheap CPU, one call per request, on the critical - # path before the fan-out. - - name: IntentAgent - host: localhost - port: 8076 # Stage 0: parse the free-text request into structured holdings + lookback # window (calls Bedrock directly via ventis.llm.bedrock). Cheap CPU, one # call per request, on the critical path before the fan-out. @@ -36,8 +16,6 @@ agents: cpu: 1 memory: 256 entrypoint: agents/intent_agent.py - - # Stage 0: price history fetch. Network/IO-bound, cheap CPU. Called by provider: EC2 instance_type: t3.micro @@ -107,3 +85,13 @@ redis: host: localhost port: 6379 db: 0 + +# EC2 defaults for `provider: EC2` replicas. +ec2: + region: us-east-1 + ami_id: ami-031ff6df47f26b546 + subnet_id: subnet-0638ac6d79d488124 + security_group_ids: + - sg-025daf3a98e06cef3 + ssh_user: ubuntu + ssh_private_key_path: ~/.ssh/ventis_ec2 diff --git a/examples/portfolio/config/policy.yaml b/examples/portfolio/config/policy.yaml index 573c91b..834c88e 100644 --- a/examples/portfolio/config/policy.yaml +++ b/examples/portfolio/config/policy.yaml @@ -14,7 +14,6 @@ rules: - match: {} access: - Workflow - - LLMAgent - IntentAgent - PriceAgent - MetricsAgent diff --git a/pyproject.toml b/pyproject.toml index 2efc1ab..8410b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,10 @@ dependencies = [ "pyyaml", "flask", "psutil", + "opentelemetry-api>=1.44.0", + "opentelemetry-sdk>=1.44.0", + "opentelemetry-exporter-otlp-proto-grpc>=1.44.0", + "opentelemetry-exporter-otlp-proto-http>=1.44.0", ] [project.scripts] @@ -23,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*"] +include = ["ventis*", "OTel_Exporter*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index b0dd97e..dd7a254 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,7 @@ ipython sqlalchemy psycopg[binary] psutil +opentelemetry-api +opentelemetry-sdk +opentelemetry-exporter-otlp-proto-grpc +opentelemetry-exporter-otlp-proto-http diff --git a/uv.lock b/uv.lock index 9b7b188..ba10707 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,11 @@ version = 1 revision = 3 requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version < '3.13'", +] [[package]] name = "async-timeout" @@ -48,6 +53,178 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3d/82/6f8fbbea47b773734ba0199643d2d851e5c2f75bc3699fe99db8af344d96/botocore-1.43.58-py3-none-any.whl", hash = "sha256:f516159f0732da8249206163ccea3bd1f82ad2a9d184fe6ed447e1abdba4330e", size = 15426503, upload-time = "2026-07-28T19:34:53.508Z" }, ] +[[package]] +name = "certifi" +version = "2026.7.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/aa/554e2614f38fc34c58ff1d0911ae8535ad2516440d5482d76fe59f1088b0/charset_normalizer-3.5.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa", size = 369072, upload-time = "2026-08-15T08:16:22.964Z" }, + { url = "https://files.pythonhosted.org/packages/03/6d/439231dfc3ccfa6f8c06477b7da2219cbd41a2de3d49084df8ec7b5100f2/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda", size = 251142, upload-time = "2026-08-15T08:16:24.81Z" }, + { url = "https://files.pythonhosted.org/packages/55/53/7d819bd23a00ef45039146fa2cce1daa2f0771e758c5653ee1f6edac91ed/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45", size = 240714, upload-time = "2026-08-15T08:16:26.392Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2c/45847198c16f4b38090cc7423b2b6a9008e438704d8ab413211832498d31/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab", size = 279637, upload-time = "2026-08-15T08:16:27.961Z" }, + { url = "https://files.pythonhosted.org/packages/69/2b/d8be3523ddf9f0b0f3e56d1359034aa10653a4d11564c697f802b4775766/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a", size = 276543, upload-time = "2026-08-15T08:16:29.399Z" }, + { url = "https://files.pythonhosted.org/packages/32/cd/4f564b8f132de25db594efc706897069f016790cea63a5669c9df2675f64/charset_normalizer-3.5.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3", size = 261644, upload-time = "2026-08-15T08:16:30.722Z" }, + { url = "https://files.pythonhosted.org/packages/f5/e3/38b975422534a608f98c360e79c2f07c763d66dd4272300d45fb1fee54b0/charset_normalizer-3.5.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb", size = 259609, upload-time = "2026-08-15T08:16:32.248Z" }, + { url = "https://files.pythonhosted.org/packages/87/bd/fbc24d825c66f1c74f6ccdea3742c3d8354a4888e86d1315a197fee69061/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f", size = 252457, upload-time = "2026-08-15T08:16:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/b9/2d/918d0e98a0e679469ed05bb2d90c2088b4d315bb612969d8499f76fb5210/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea", size = 242240, upload-time = "2026-08-15T08:16:35.396Z" }, + { url = "https://files.pythonhosted.org/packages/20/c8/c36f6e0b2dfec351bd38cbc05362697e58bcd073d7dbd95154290c9714ce/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f", size = 280308, upload-time = "2026-08-15T08:16:36.825Z" }, + { url = "https://files.pythonhosted.org/packages/ca/7b/311b3e02e8c4092400c449c850a760d8c45d900983c83a70cc07208c551d/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182", size = 258679, upload-time = "2026-08-15T08:16:38.22Z" }, + { url = "https://files.pythonhosted.org/packages/b9/90/082cc45599c392f28c036a497f49e0634041a785fc3849c80ccf396d096f/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa", size = 277221, upload-time = "2026-08-15T08:16:39.62Z" }, + { url = "https://files.pythonhosted.org/packages/58/ad/b9aecf38d805cbcf84fa94f14c5d972a16561e20296a11dc799a5dcf3763/charset_normalizer-3.5.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818", size = 263799, upload-time = "2026-08-15T08:16:40.885Z" }, + { url = "https://files.pythonhosted.org/packages/b7/23/b38a20598d5a825f85d9d7636860e56ff0db1479f86497a6e485aa9326f7/charset_normalizer-3.5.1-cp310-cp310-win32.whl", hash = "sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20", size = 182037, upload-time = "2026-08-15T08:16:42.198Z" }, + { url = "https://files.pythonhosted.org/packages/d2/21/83fffb77864408b8bf0fe1ca603926401d6f8775a8e150b39aacc9958f8a/charset_normalizer-3.5.1-cp310-cp310-win_amd64.whl", hash = "sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d", size = 206030, upload-time = "2026-08-15T08:16:43.787Z" }, + { url = "https://files.pythonhosted.org/packages/86/2e/b93135b5034b1157fb29554b0d06d4844ce62282f0e0a14036f93d7ee2e7/charset_normalizer-3.5.1-cp310-cp310-win_arm64.whl", hash = "sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5", size = 185092, upload-time = "2026-08-15T08:16:45.177Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b6/034f6802e9c3f6418966cfabb7db8c9252cc2429c5098f41cc43af804149/charset_normalizer-3.5.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30", size = 363585, upload-time = "2026-08-15T08:16:46.646Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fa/6a7e2a7c4b5451912b8c417732df79574354443592a88d616de03da66ae5/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488", size = 251189, upload-time = "2026-08-15T08:16:48.287Z" }, + { url = "https://files.pythonhosted.org/packages/a4/c8/ab42b07cfd82e919f427fcfaa7c41abae8242833ad1aad66d42bae40b669/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22", size = 239724, upload-time = "2026-08-15T08:16:49.67Z" }, + { url = "https://files.pythonhosted.org/packages/e7/80/b9348b5d3041209f98b4cdad7655766369233f1d533f4f4f7558e9717bec/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731", size = 280078, upload-time = "2026-08-15T08:16:51.228Z" }, + { url = "https://files.pythonhosted.org/packages/82/38/083a24028304bc85bb9e376fed801178423dcbb67495f73b6ea0624e1894/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c", size = 276650, upload-time = "2026-08-15T08:16:52.625Z" }, + { url = "https://files.pythonhosted.org/packages/0d/35/731ac04aa0a097fc1c97f0994c375bdb230c6c96619db794208fe664e9ce/charset_normalizer-3.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8", size = 262325, upload-time = "2026-08-15T08:16:54.085Z" }, + { url = "https://files.pythonhosted.org/packages/f5/28/c2028e7021fb89c6e56868ed0e387b8e9aa811abdd2ab3208d6578d2c930/charset_normalizer-3.5.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486", size = 261140, upload-time = "2026-08-15T08:16:55.604Z" }, + { url = "https://files.pythonhosted.org/packages/28/f0/0c0ceec6d98b7daa62e361e418135d59685811d79ba11529aad5cdf15e84/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f", size = 252791, upload-time = "2026-08-15T08:16:57.103Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3e/48f4cd187b1c33189d86039e9cbe4f92c05454175504b44ff81806d4d1bf/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c", size = 240730, upload-time = "2026-08-15T08:16:58.418Z" }, + { url = "https://files.pythonhosted.org/packages/42/85/f9e22af69af67c54cce42be9455d9c81294f918b4ccc454db01f66efcac2/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18", size = 280791, upload-time = "2026-08-15T08:16:59.918Z" }, + { url = "https://files.pythonhosted.org/packages/fd/4c/9044135f42127630b6fa742feb51256353f6ab87a78f2fdd1de3de955a7f/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5", size = 259598, upload-time = "2026-08-15T08:17:01.421Z" }, + { url = "https://files.pythonhosted.org/packages/ba/ed/1dd7cfebb4e75812934c49ca3b79757d11948053f7937ab7070c151f3c55/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b", size = 278217, upload-time = "2026-08-15T08:17:02.782Z" }, + { url = "https://files.pythonhosted.org/packages/bf/eb/239c84503cc9e3ba6eb34686a24bc66e84f3924efdd7e38e751a19f6bc10/charset_normalizer-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6", size = 263417, upload-time = "2026-08-15T08:17:04.216Z" }, + { url = "https://files.pythonhosted.org/packages/37/ab/4e4510e1e288478e2c8333131d1c1382382ba8cd2165053c79e39d1da961/charset_normalizer-3.5.1-cp311-cp311-win32.whl", hash = "sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b", size = 181774, upload-time = "2026-08-15T08:17:05.58Z" }, + { url = "https://files.pythonhosted.org/packages/e3/57/32f0ccea59e8612057c61d6fd22ef2cb63cca93c9fe594094919696ac170/charset_normalizer-3.5.1-cp311-cp311-win_amd64.whl", hash = "sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9", size = 206653, upload-time = "2026-08-15T08:17:07.075Z" }, + { url = "https://files.pythonhosted.org/packages/17/d4/b65c433fc521e58b5f54293982a5e51c05cb5f2dd3f1c7a6acb65b75324e/charset_normalizer-3.5.1-cp311-cp311-win_arm64.whl", hash = "sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10", size = 185630, upload-time = "2026-08-15T08:17:08.502Z" }, + { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" }, + { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" }, + { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" }, + { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" }, + { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" }, + { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" }, + { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" }, + { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" }, + { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" }, + { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" }, + { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" }, + { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" }, + { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" }, + { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" }, + { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" }, + { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" }, + { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" }, + { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" }, + { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" }, + { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" }, + { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" }, + { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" }, + { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" }, + { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" }, + { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" }, + { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" }, + { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" }, + { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" }, + { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" }, + { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" }, + { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" }, + { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" }, + { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" }, + { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" }, + { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" }, + { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" }, + { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" }, + { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" }, + { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" }, + { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" }, + { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" }, + { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" }, + { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" }, + { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" }, + { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" }, + { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -86,6 +263,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/90/fb8f1c84537fbf210c1f53a53ae473a805f6599c5a40b93c1bbadd211f7a/googleapis_common_protos-1.75.2.tar.gz", hash = "sha256:8829a3d1e4508c5b7b9a6b9525f7fccff611f8531644579a76466c29295d4bb2", size = 154083, upload-time = "2026-08-25T19:19:13.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5b/1c9e55363c3b1890a98cae813de5b4ea327845756cd8fb7ee690140c7eac/googleapis_common_protos-1.75.2-py3-none-any.whl", hash = "sha256:6b83302f554ea93a0f48409c7fc2050f954bcbcddb7e3a9c76d4a823cb22920e", size = 307002, upload-time = "2026-08-25T19:18:08.927Z" }, +] + [[package]] name = "greenlet" version = "3.5.4" @@ -280,6 +469,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/30/b1/d1f150b2ab3b4ae9932c05104fe1edbcb7fbf505587ea8db99e49341a05f/grpcio_tools-1.83.0-cp314-cp314-win_amd64.whl", hash = "sha256:b8c9686b0c19f70b63d8d6cfeff5ad3480bdedecd60f14711fe43950f5397253", size = 1224199, upload-time = "2026-07-23T15:22:16.064Z" }, ] +[[package]] +name = "idna" +version = "3.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/f7/abb373e5757eaec4b922b92f97ec8d6d7e057cf06778247604fbc4e7c3f3/idna-3.19.tar.gz", hash = "sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15", size = 215237, upload-time = "2026-08-18T05:14:24.27Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, +] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -395,6 +593,105 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/47/80d9e9d468dc5de3af5096f5ccdb065fa4dd1470f74495cc53e59e397f47/opentelemetry_exporter_otlp_proto_grpc-1.44.0.tar.gz", hash = "sha256:40d1ae9e03fcc36de3cbac610cc99f35894938bff9cfd90fc4ec68bd85448463", size = 27225, upload-time = "2026-07-16T15:25:38.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/29/6ae42ba32b153ae0a44ae125f0caff2188bbe62d99c82d1768da30864e72/opentelemetry_exporter_otlp_proto_grpc-1.44.0-py3-none-any.whl", hash = "sha256:6a1a645ea182a2f59440c51fa8301d309f3324a8f9d65f8395584b064b67ee4e", size = 19624, upload-time = "2026-07-16T15:25:19.096Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -606,6 +903,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/0a/c2345ebf1ebe70840ce3f6c6ee612f8fa749cfbd1b03069c53bf0c62aaad/redis-8.0.1-py3-none-any.whl", hash = "sha256:47daa35a058c23468d6437f17a8c76882cb316b838ef763036af99b96cedd743", size = 502406, upload-time = "2026-06-23T14:52:36.137Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "s3transfer" version = "0.19.2" @@ -727,6 +1039,10 @@ dependencies = [ { name = "flask" }, { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "psutil" }, { name = "psycopg", extra = ["binary"] }, { name = "pyyaml" }, @@ -740,6 +1056,10 @@ requires-dist = [ { name = "flask" }, { name = "grpcio" }, { name = "grpcio-tools" }, + { name = "opentelemetry-api", specifier = ">=1.44.0" }, + { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.44.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.44.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.44.0" }, { name = "psutil" }, { name = "psycopg", extras = ["binary"] }, { name = "pyyaml" }, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 4e416b4..496ae1a 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,6 +3,7 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit +import importlib.util import logging import signal import subprocess @@ -15,6 +16,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.process_supervisor import ProcessSupervisor from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -100,6 +102,31 @@ def __init__(self, config_path): self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() + # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # supervised so it gets restarted if it ever exits unexpectedly. + otel_exporter_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "OTel_Exporter", + ) + otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") + self.process_supervisor = ProcessSupervisor() + # `otel:` in global_controller.yaml maps straight to the OTel SDK's own + # standard env vars, not app-specific args -- the exporter subprocess itself + # stays a plain vendor-neutral OTel process; see OTel_Exporter/DESIGN.md. + otel_env = self._otel_exporter_env(self.config.get("otel", {})) + self.process_supervisor.register( + "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + ) + self.process_supervisor.start_all() + + # waiting table GC writes future data into (see OTel_Exporter/db.py); the + # exporter process itself calls init_db() to create the table. + otel_db_spec = importlib.util.spec_from_file_location( + "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") + ) + self._otel_db = importlib.util.module_from_spec(otel_db_spec) + otel_db_spec.loader.exec_module(self._otel_db) + # ------------------------------------------------------------------ # # Stale container cleanup # # ------------------------------------------------------------------ # @@ -143,6 +170,22 @@ def _load_config(config_path): with open(config_path, "r") as f: return yaml.safe_load(f) + @staticmethod + def _otel_exporter_env(otel_cfg): + """Translate global_controller.yaml's `otel:` section into standard OTLP env + vars for the exporter subprocess; returns None if `otel:` is absent/empty so + the subprocess falls back to the SDK's own defaults untouched.""" + env = {} + if otel_cfg.get("protocol"): + env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] + if otel_cfg.get("endpoint"): + env["OTEL_EXPORTER_OTLP_ENDPOINT"] = otel_cfg["endpoint"] + if otel_cfg.get("headers"): + env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( + f"{k}={v}" for k, v in otel_cfg["headers"].items() + ) + return env or None + @staticmethod def _get_replica_placements(ctrl): """Normalize replicas into a list of (host, port) placements.""" @@ -407,14 +450,25 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ + # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which + # terminates every managed process) via the signal handler before this line is + # reached -- without the guard, this could respawn a process just intentionally + # killed. See OTel_Exporter/DESIGN.md. + if self.running: + self.process_supervisor.check_and_respawn() + for instance in self.instance_manager.list_instances(): name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) send_runtime_information( - pull_runtime_information(node_redis), + future_rows, node_redis, self.config.get("database", {}).get("url"), ) @@ -654,6 +708,7 @@ def stop(self): self.running = False self._stop_docker_agents() self._stop_redis_containers() + self.process_supervisor.terminate_all() logger.info("Global controller shut down.") diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py new file mode 100644 index 0000000..8f5724c --- /dev/null +++ b/ventis/controller/utils/process_supervisor.py @@ -0,0 +1,59 @@ +"""Registry for OS processes GlobalController spawns and supervises. + +register() + start_all() spawn processes; check_and_respawn() (call from GC's existing +poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown +path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not +calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +shutdown-race note). +""" + +import logging +import os +import subprocess + +logger = logging.getLogger(__name__) + + +class ProcessSupervisor: + def __init__(self): + self._specs = {} # name -> (argv, env) tuple + self._procs = {} # name -> subprocess.Popen + + def register(self, name, argv, env=None): + """Declare a process to manage. Does not start it -- call start_all() once + everything is registered. `env`, if given, is merged on top of (not a + replacement for) this process's own environment, so the child still inherits + PATH etc.""" + self._specs[name] = (argv, env) + + def start_all(self): + for name, (argv, env) in self._specs.items(): + self._start(name, argv, env) + + def _start(self, name, argv, env=None): + merged_env = {**os.environ, **env} if env else None + self._procs[name] = subprocess.Popen(argv, env=merged_env) + + def check_and_respawn(self): + """Restart any registered process that has exited.""" + for name, proc in list(self._procs.items()): + if proc.poll() is not None: + logger.warning( + "Managed process %r exited (code %s), respawning", + name, + proc.returncode, + ) + argv, env = self._specs[name] + self._start(name, argv, env) + + def terminate_all(self, timeout=10): + """Terminate every managed process, falling back to kill() on timeout.""" + for proc in self._procs.values(): + proc.terminate() + for name, proc in self._procs.items(): + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + self._procs.clear() diff --git a/ventis/deploy.py b/ventis/deploy.py index b6ac721..148d3a4 100644 --- a/ventis/deploy.py +++ b/ventis/deploy.py @@ -9,7 +9,7 @@ import ventis def my_workflow(query: str): - finance = FinanceAgentStub() + finance = FinanceAgent() price = finance.get_stock_price(ticker=query) return {"price": price.value()} diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 8c956ef..d9480be 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -164,13 +164,12 @@ def _build_stub_class(agent_config): Build an AST node for the entire stub class. Generates a class like: - class FinanceAgentStub(object): + class FinanceAgent(object): def __init__(self): pass ...stub methods... """ - # class_name = agent_config["name"] + "Stub" - class_name = agent_config["name"] + class_name = agent_config["name"] functions = agent_config.get("functions", []) # __init__ method: simple pass, no gRPC setup needed. @@ -241,7 +240,7 @@ def generate_stub(yaml_path, output_path): with open(output_path, "w") as f: f.write(source) - class_name = agent_config["name"] + "Stub" + class_name = agent_config["name"] print(f"Generated stub class '{class_name}' -> {output_path}") return source @@ -521,7 +520,7 @@ def start_lc(): "-o", "--output", default=None, - help="Output path for the generated stub file (default: stubs/_stub.py)", + help="Output path for the generated stub file (default: stubs/.py)", ) parser.add_argument( "--agent-file", From 4af43ae3abb8da914050f9f83c61cbf2badebf17 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Tue, 25 Aug 2026 20:40:54 -0700 Subject: [PATCH 06/13] [Feature] Pass env / secrets into agent containers Users had no way to get API keys (OpenAI, Anthropic, embedding models) into an agent container. Add a top-level `env_file` key to global_controller.yaml pointing at a local .env file, which reaches every container as `docker run --env-file`. - resolve_env_file validates the path before anything launches, so a missing .env fails at deploy time instead of deep inside a container. Relative paths resolve against the project root, matching entrypoint. - env_file_args is a context manager owning the local-vs-remote decision and the cleanup, so both runtimes share one code path. Local containers read the original file; remote containers get a copy that is deleted as soon as `docker run` returns, whether or not it succeeded. - GlobalController._push_file streams the file over ssh under `umask 077` rather than scp, so the copy is never briefly world-readable and the secret never lands in a command line. _run_cmd's ssh options moved to a shared _ssh_args. --env-file is appended after the explicit -e VENTIS_* flags; Docker gives those precedence regardless of order, so a stray VENTIS_* line in someone's .env cannot break agent wiring. Closes #50 --- ventis/cli.py | 10 ++ .../cloud_provider_logic/EC2/_runtime.py | 12 ++- .../cloud_provider_logic/Local/_runtime.py | 12 ++- ventis/controller/global_controller.py | 94 +++++++++++++------ ventis/controller/utils/env_file.py | 92 ++++++++++++++++++ 5 files changed, 185 insertions(+), 35 deletions(-) create mode 100644 ventis/controller/utils/env_file.py diff --git a/ventis/cli.py b/ventis/cli.py index b43a6b3..6d85e1f 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -16,6 +16,8 @@ import subprocess import sys +from ventis.controller.utils.env_file import resolve_env_file + logging.basicConfig(level=logging.INFO) logger = logging.getLogger("ventis") DEFAULT_DOCKER_PLATFORM = "linux/amd64" @@ -397,6 +399,14 @@ def cmd_deploy(args): config = _load_config(config_path) project_dir = os.getcwd() + # Fail here rather than after a fleet of containers is already up without + # the API keys they need. + try: + resolve_env_file(config, base_dir=project_dir) + except ValueError as e: + logger.error("%s", e) + sys.exit(1) + _ensure_grpc_stubs_importable(project_dir) if any( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 4d5f766..9955fa2 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -23,6 +23,7 @@ import boto3 +from ventis.controller.utils.env_file import env_file_args from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.utils.redis_client import RedisClient @@ -285,8 +286,15 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, cmd.extend(["-e", f"VENTIS_DATABASE_URL={db_url}"]) if project_id: cmd.extend(["-e", f"VENTIS_PROJECT_ID={project_id}"]) - cmd.append(image) - result = _controller._run_cmd(cmd, host, user=ssh_user) + + # User secrets from `env_file`. Explicit -e flags above still win over + # anything in the file. + with env_file_args( + _controller, host, ssh_user, container_name, is_local=False + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _controller._run_cmd(cmd, host, user=ssh_user) if result.returncode != 0: raise RuntimeError( f"SSH bootstrap failed on {host}: {(result.stderr or result.stdout or '').strip()}" diff --git a/ventis/controller/cloud_provider_logic/Local/_runtime.py b/ventis/controller/cloud_provider_logic/Local/_runtime.py index 963eef3..a387f7b 100644 --- a/ventis/controller/cloud_provider_logic/Local/_runtime.py +++ b/ventis/controller/cloud_provider_logic/Local/_runtime.py @@ -8,6 +8,8 @@ import logging +from ventis.controller.utils.env_file import env_file_args + logger = logging.getLogger(__name__) DEFAULT_HOST = "localhost" @@ -110,9 +112,15 @@ def bootstrap_instance(provisioned, spec, replica_index, agent_id): cmd.extend(["--memory", f"{resources['memory']}m"]) if resources.get("gpu"): cmd.extend(["--gpus", str(resources["gpu"])]) - cmd.append(image) - result = _require_controller()._run_cmd(cmd, host, user) + # User secrets from `env_file`. Explicit -e flags above still win, so a + # stray VENTIS_* line in someone's .env cannot break agent wiring. + with env_file_args( + _require_controller(), host, user, runtime_id, _is_local_host(host) + ) as env_args: + cmd.extend(env_args) + cmd.append(image) + result = _require_controller()._run_cmd(cmd, host, user) if result.returncode != 0: raise RuntimeError(f"Failed to launch {runtime_id}") diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 1e24f10..241daff 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -4,6 +4,7 @@ import atexit import logging +import shlex import signal import subprocess import threading @@ -16,6 +17,7 @@ import yaml from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs +from ventis.controller.utils.env_file import resolve_env_file from ventis.controller.utils.redis_utils import _wait_for_redis from ventis.controller.utils.telemetry_logging import ( assign_project_id, @@ -64,6 +66,9 @@ class GlobalController(object): def __init__(self, config_path): self.config_path = config_path self.config = self._load_config(config_path) + # Validate before launching anything: an agent that boots without its + # API keys fails deep inside a container, where it is expensive to debug. + self.env_file_path = resolve_env_file(self.config) redis_cfg = self.config.get("redis", {}) self.redis = RedisClient( @@ -174,6 +179,7 @@ def reload_config(self): """Reload the config file and rebuild the routing table.""" logger.info("Reloading config from %s", self.config_path) self.config = self._load_config(self.config_path) + self.env_file_path = resolve_env_file(self.config) self.controllers = self.config.get("agents", []) self.poll_interval = self.config.get("poll_interval", 5) assign_project_id(self.config.get("project_id", 0)) @@ -642,6 +648,28 @@ def _send(instance): # Runtime launching # # ------------------------------------------------------------------ # + def _ssh_args(self, host, user=None): + """Return the `ssh ... target` prefix used to reach a remote host.""" + ssh_key_path = os.path.expanduser( + self.config.get("ec2", {}).get("ssh_private_key_path", "~/.ssh/ventis_ec2") + ) + return [ + "ssh", + "-o", + "StrictHostKeyChecking=no", + "-o", + "IdentitiesOnly=yes", + "-o", + "ConnectTimeout=10", + "-o", + "ServerAliveInterval=10", + "-o", + "ServerAliveCountMax=3", + "-i", + ssh_key_path, + f"{user}@{host}" if user else host, + ] + def _run_cmd(self, cmd, host, user=None): """ Run a command locally or on a remote host via SSH. @@ -656,41 +684,45 @@ def _run_cmd(self, cmd, host, user=None): """ is_local = _is_local_host(host) if is_local: - return subprocess.run( - cmd, capture_output=True, text=True, timeout=180 - ) - else: - ssh_key_path = os.path.expanduser( - self.config.get("ec2", {}).get( - "ssh_private_key_path", "~/.ssh/ventis_ec2" - ) - ) - ssh_target = f"{user}@{host}" if user else host - remote_cmd = " ".join(cmd) - if cmd and cmd[0] == "docker": - remote_cmd = f"sudo {remote_cmd}" - return subprocess.run( - [ - "ssh", - "-o", - "StrictHostKeyChecking=no", - "-o", - "IdentitiesOnly=yes", - "-o", - "ConnectTimeout=10", - "-o", - "ServerAliveInterval=10", - "-o", - "ServerAliveCountMax=3", - "-i", - ssh_key_path, - ssh_target, - remote_cmd, - ], + return subprocess.run(cmd, capture_output=True, text=True, timeout=180) + + remote_cmd = " ".join(cmd) + if cmd and cmd[0] == "docker": + remote_cmd = f"sudo {remote_cmd}" + return subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + capture_output=True, + text=True, + timeout=180, + ) + + def _push_file(self, local_path, remote_path, host, user=None): + """ + Copy a local file to a remote host over SSH. + + Streams the bytes through `cat` under `umask 077` rather than using + `scp`, so a secrets file is never briefly world-readable on the far + side. + + Returns: + subprocess.CompletedProcess + """ + remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + with open(local_path, "rb") as f: + result = subprocess.run( + self._ssh_args(host, user) + [remote_cmd], + stdin=f, capture_output=True, text=True, timeout=180, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to copy {local_path} to {host}:{remote_path}: " + f"{(result.stderr or result.stdout or '').strip()}" ) + return result def launch_docker_agents(self): """Launch all configured runtimes through InstanceManager.""" diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py new file mode 100644 index 0000000..c83f770 --- /dev/null +++ b/ventis/controller/utils/env_file.py @@ -0,0 +1,92 @@ +""" +Pass user secrets (API keys and friends) into agent containers. + +The user points `env_file` in `config/global_controller.yaml` at a local +`.env` file. Containers on this machine read that file directly; containers +on a remote host get a short-lived 0600 copy. Either way the file reaches +Docker as `--env-file`. +""" + +import logging +import os +from contextlib import contextmanager + +logger = logging.getLogger(__name__) + +REMOTE_ENV_DIR = "/tmp" + + +def resolve_env_file(config, base_dir=None): + """ + Return the absolute path of the configured env file, or None when unset. + + Relative paths resolve against `base_dir` (default: the current working + directory), matching how `entrypoint` and `workflow_file` are resolved. + + Raises: + ValueError: the file is configured but unusable. Deploy should fail + here rather than start a fleet of agents with no API keys. + """ + raw = config.get("env_file") + if not raw: + return None + + path = os.path.expanduser(str(raw)) + if not os.path.isabs(path): + path = os.path.join(base_dir or os.getcwd(), path) + path = os.path.abspath(path) + + if not os.path.exists(path): + raise ValueError(f"env_file does not exist: {path} (from env_file: {raw})") + if not os.path.isfile(path): + raise ValueError(f"env_file is not a file: {path} (from env_file: {raw})") + if not os.access(path, os.R_OK): + raise ValueError(f"env_file is not readable: {path}") + return path + + +def remote_env_path(container_name): + """Where a remote host holds this container's copy of the env file.""" + return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + + +@contextmanager +def env_file_args(controller, host, user, container_name, is_local): + """ + Yield the `docker run` flags that hand the user's env file to a container. + + A container on this machine reads the original file. A container on a + remote host gets a 0600 copy, deleted as soon as the `with` body ends -- + success or failure, since by then the container holds the variables + itself. Keep that body tight around `docker run` so the copy is never + on the host longer than it has to be. + + Yields an empty list when no `env_file` is configured. + """ + env_file_path = getattr(controller, "env_file_path", None) + if not env_file_path: + yield [] + return + + if is_local: + yield ["--env-file", env_file_path] + return + + remote_path = remote_env_path(container_name) + controller._push_file(env_file_path, remote_path, host, user=user) + try: + yield ["--env-file", remote_path] + finally: + _remove_remote_copy(controller, remote_path, host, user) + + +def _remove_remote_copy(controller, remote_path, host, user): + """Delete a remote copy. Best effort -- never masks the caller's error.""" + try: + result = controller._run_cmd(["rm", "-f", remote_path], host, user=user) + if getattr(result, "returncode", 0) != 0: + logger.warning("Failed to delete env file copy %s on %s", remote_path, host) + except Exception as e: + logger.warning( + "Failed to delete env file copy %s on %s: %s", remote_path, host, e + ) From 087ee15b66e967937297580fc551c121c7301a20 Mon Sep 17 00:00:00 2001 From: Nick Huo Date: Wed, 26 Aug 2026 13:28:52 -0700 Subject: [PATCH 07/13] Harden the remote env file copy against a hostile /tmp Two holes in the remote staging path, both found reviewing the feature commit. `umask 077` only governs files the shell creates, and `>` follows symlinks -- so it did not actually guarantee a 0600 copy. The destination path is fully predictable (`/tmp/ventis-env-ventis-ec2--`), so a local user on the remote host could pre-create it world-readable, or point it at a file of their own, and collect the API keys. Remove whatever sits at the path before writing; `rm -f` unlinks a symlink rather than following it, so `cat >` then creates a fresh file under the umask. `_run_cmd` joins its argv with spaces and hands the result to a remote shell unquoted. `_push_file` quoted its path but the cleanup `rm` did not, so a container name containing a space split the `rm` into two arguments that matched nothing -- it exited 0 while the secrets file stayed on the host, and the returncode check logged nothing. Scrub the name down to [A-Za-z0-9_.-] in remote_env_path, which also closes the same gap in the `--env-file` argument and in any future use of that path. Still open, tracked separately: a push that dies mid-transfer can leave a copy behind, since the cleanup only covers the `docker run` that follows. On EC2 the instance is terminated on that path, which disposes of it. --- ventis/controller/global_controller.py | 9 ++++++++- ventis/controller/utils/env_file.py | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 241daff..0b30307 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -704,10 +704,17 @@ def _push_file(self, local_path, remote_path, host, user=None): `scp`, so a secrets file is never briefly world-readable on the far side. + Anything already sitting at the destination is removed first: `umask` + only governs files the shell creates, and `>` follows symlinks. Without + the `rm`, a local user on the remote host could pre-create the path + world-readable, or point it at a file of their own, and collect + whatever we write there. + Returns: subprocess.CompletedProcess """ - remote_cmd = f"umask 077; cat > {shlex.quote(remote_path)}" + quoted = shlex.quote(remote_path) + remote_cmd = f"umask 077; rm -f {quoted}; cat > {quoted}" with open(local_path, "rb") as f: result = subprocess.run( self._ssh_args(host, user) + [remote_cmd], diff --git a/ventis/controller/utils/env_file.py b/ventis/controller/utils/env_file.py index c83f770..6cd77b6 100644 --- a/ventis/controller/utils/env_file.py +++ b/ventis/controller/utils/env_file.py @@ -9,11 +9,13 @@ import logging import os +import re from contextlib import contextmanager logger = logging.getLogger(__name__) REMOTE_ENV_DIR = "/tmp" +_UNSAFE_PATH_CHARS = re.compile(r"[^A-Za-z0-9_.-]") def resolve_env_file(config, base_dir=None): @@ -46,8 +48,17 @@ def resolve_env_file(config, base_dir=None): def remote_env_path(container_name): - """Where a remote host holds this container's copy of the env file.""" - return f"{REMOTE_ENV_DIR}/ventis-env-{container_name}" + """ + Where a remote host holds this container's copy of the env file. + + The name is scrubbed down to a shell-safe alphabet. This path is + interpolated into remote commands that `_run_cmd` joins with spaces and + hands to a shell unquoted, so a container name carrying a space would + split the cleanup `rm` into two harmless arguments -- it would exit 0 + while the secrets stayed on the host, with nothing in the log to say so. + """ + safe_name = _UNSAFE_PATH_CHARS.sub("-", container_name) + return f"{REMOTE_ENV_DIR}/ventis-env-{safe_name}" @contextmanager From db0ba260c9cc1f88943d98836ce556fb7b458817 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:05:41 -0700 Subject: [PATCH 08/13] WIP: OTel multi-destination fan-out (Railway+Langfuse+Grafana) + cleanup-race fix (pre-pull checkpoint) Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 102 ++++-- OTel_Exporter/convert.py | 11 +- OTel_Exporter/db.py | 44 ++- OTel_Exporter/otel_exporter.py | 266 +++++++++++++-- .../portfolio/config/global_controller.yaml | 18 + pyproject.toml | 5 + tests/test_otel_exporter_fanout.py | 318 ++++++++++++++++++ tests/test_otel_exporter_fields.py | 99 ++++++ uv.lock | 128 +++++++ ventis/controller/global_controller.py | 173 +++++++++- 10 files changed, 1074 insertions(+), 90 deletions(-) create mode 100644 tests/test_otel_exporter_fanout.py create mode 100644 tests/test_otel_exporter_fields.py diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md index 73c4da2..86a2a95 100644 --- a/OTel_Exporter/DESIGN.md +++ b/OTel_Exporter/DESIGN.md @@ -1,6 +1,6 @@ # OTLP Exporter for Ventis GlobalController — Design -Status: **implemented (single-table design)**. `GlobalController` writes futures into a +Status: **implemented (single-table design; multi-destination fan-out in progress)**. `GlobalController` writes futures into a `waiting` table (SQLite); a GC-supervised, GC-restarted OTel Exporter process reads finished/unsent rows, converts each to an OTel span, and hands it to a real `BatchSpanProcessor`/`OTLPSpanExporter`. Batching, serialization, and sending are all @@ -41,7 +41,12 @@ Decisions (final status): originally-planned `database.url` repurposing (below, kept for history) was decided against — env-var configuration is the SDK's own idiomatic mechanism, so no exporter-side config plumbing was added, only a GC-side YAML→env-var translation. - Does not (yet) support simultaneous multi-destination export — see "Known gaps". + The initial multi-destination extension uses one `otel.destinations` list and one + independent exporter/`BatchSpanProcessor` pair per destination. gRPC and HTTP + destinations may be mixed in the same list. The legacy single-destination fields + remain supported through the original standard-environment-variable path. + Configuration is read at exporter startup; changing it requires a + GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing @@ -64,39 +69,47 @@ Decisions (final status): `global_controller.yaml` gains an optional `otel:` section: ```yaml otel: - protocol: grpc # or http - endpoint: otlp-pg-receiver.railway.internal:4317 - headers: {} # e.g. Authorization: "Basic " for a backend needing auth + destinations: + - name: railway + protocol: grpc # or http + endpoint: otlp-pg-receiver.railway.internal:4317 + headers: {} + - name: langfuse + protocol: http + endpoint: https://cloud.langfuse.com/api/public/otel/v1/traces + headers: {} # e.g. Authorization: "Basic " ``` -`GlobalController._otel_exporter_env()` translates this into -`OTEL_EXPORTER_OTLP_PROTOCOL`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` -and hands them to `ProcessSupervisor.register("otel_exporter", ..., env=...)`, which now -supports an `env` param (merged on top of the parent process's own environment, not a -replacement). Omitting `otel:` entirely falls back to whatever ambient env the exporter -subprocess would otherwise inherit, same as before this change. +`GlobalController._otel_exporter_env()` translates each destination into the exporter +process's destination configuration and hands it to `ProcessSupervisor.register( +"otel_exporter", ..., env=...)`, which supports an `env` param (merged on top of the +parent process's own environment, not a replacement). The legacy single-destination +`protocol`/`endpoint`/`headers` form remains valid and continues through the SDK's +standard OTLP environment variables. Omitting `otel:` entirely falls back to whatever +ambient env the exporter subprocess would otherwise inherit, same as before this +change. -`otel_exporter.py` itself reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly (to pick -which `OTLPSpanExporter` class to import — gRPC or HTTP; the plain SDK classes don't -self-select this the way `opentelemetry-instrument`'s auto-config does). Endpoint and -headers are never read directly — `OTLPSpanExporter()` is still constructed with no -explicit args, letting the SDK resolve those from the same env vars itself, exactly as -before this change. `BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000)` +`otel_exporter.py` parses the destination configuration at startup and constructs the +appropriate OTLP exporter for each entry (gRPC or HTTP), passing that destination's +endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis=1000)` — the flush delay is explicitly overridden from the SDK default (5000ms) to 1000ms; `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. ### 2. `OTel_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM -stays responsive), calling `_send_pending()` each tick: +stays responsive), calling `_send_pending()` each tick. At startup it constructs one +independent OTLP exporter and `BatchSpanProcessor` for each configured destination; +each pair may use a different protocol, endpoint, and headers: - `SELECT * FROM waiting WHERE finished_at IS NOT NULL AND (sent IS NULL OR sent = 0)`. - Per row, each isolated in its own try/except (one malformed row is logged and skipped, never blocks the rest of the batch): `convert.waiting_row_to_span(row)` → - `_processor.on_end(span)` → `db.mark_sent(future_id)` immediately — atomic per row, not - batched at the end, so a crash mid-poll can't leave an already-sent row unmarked (which - would cause a duplicate send on the next run). -- `_processor` is constructed once at startup; no `TracerProvider` is used at all, since - spans are hand-built and handed straight to the processor via `on_end()`. -- `_processor.shutdown()` on exit, flushing any pending batch. + `on_end(span)` on every configured processor → `db.mark_sent(future_id)` immediately. + The row is marked after it has been queued to all processors. `sent` therefore means + **queued to every configured destination**, not remotely acknowledged; this is the + initial best-effort delivery contract and retains the existing single boolean schema. +- Each processor is constructed once at startup; no `TracerProvider` is used at all, + since spans are hand-built and handed straight to the processors via `on_end()`. +- Every processor is shut down on exit, flushing its pending batch independently. ### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is @@ -117,7 +130,11 @@ no live exception object, only strings) plus `Status(StatusCode.ERROR, descripti **Attribute naming**: `model`/`input_token_count`/`output_token_count` are set under the real, current OTel GenAI semantic-convention keys — `gen_ai.request.model`/ `gen_ai.usage.input_tokens`/`gen_ai.usage.output_tokens` — verified against the actual -spec (`open-telemetry/semantic-conventions`), not assumed. `cpu`/`gpu`/ +spec (`open-telemetry/semantic-conventions`), not assumed. Submitted `args` and the +completed `result` are stored in `waiting.input`/`waiting.output` as valid JSON text and +exported under Langfuse's documented `langfuse.observation.input`/ +`langfuse.observation.output` attributes. The span name is the stable logical +`service.method`, not the executing instance's UUID. `cpu`/`gpu`/ `execution_time_ms`/`queue_time_ms`/`token_count` keep plain names deliberately: none of them have an OTel GenAI equivalent (cpu/gpu/queue-time are Ventis infra concepts, and `token_count`, an input+output sum, isn't part of the spec at all — inventing a @@ -136,24 +153,45 @@ process (all `.terminate()` calls first, then `.wait()` on each, falling back to `.kill()`), called from GC's `stop()`. Adding a future second daemon is one more `register()` call — no new spawn/monitor/terminate code needed. -### 5. Dependencies (all added) +### 5. Poll/cleanup race fix (`ventis/controller/global_controller.py`) +GC's cleanup thread used to run on its own `cleanup_interval` timer (default 10s), +fully independent of the poll loop's `poll_interval` (default 5s) that writes futures +into `waiting`. On a fast-completing request, cleanup could delete a session's Redis +future keys before the next poll tick ever read them, so those futures never reached +`waiting` at all — silently dropped from every OTel destination, not just one. +Reproduced live: a fast request left only 1 of 6 agent calls in `waiting`. Fixed by +having the poll loop signal a `threading.Event` (`_cleanup_ready`) right after each +tick; the cleanup thread waits on that event instead of sleeping on its own timer, so +cleanup only ever runs immediately after a poll has already captured that tick's state. +Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is a +fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall +the poll loop's health checks and OTel writes. + +### 6. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). ## Known gaps (not yet built) +- `WriteResult()` passes an undefined `error_message` variable to its fan-out callback, + which can interrupt remote consumer propagation after the callback hash is persisted. +- Redis records failure text under `error`, but the waiting-table writer reads + `error_name`/`error_message`, so exported exception details are usually empty. +- Rows are marked `sent` immediately after `BatchSpanProcessor.on_end()` accepts them, + before the asynchronous OTLP export is confirmed; a later delivery failure can lose a + span while leaving `sent = 1`. - Spans carry no explicit `resource`/`instrumentation_scope` — would show as `service.name=unknown_service` at a real backend. -- No simultaneous multi-destination export — `otel:` configures exactly one - destination; sending to two backends at once would mean registering a second, - separately-configured `otel_exporter` subprocess (same script, different env), not - something the exporter or its config format do today. +- Destination-specific delivery acknowledgement/retry state is not tracked yet: + `sent` only records that the span was queued to all configured processors, so an + asynchronous export failure can still lose a span until a later delivery-state design + is added. - `waiting` grows unboundedly: sent rows are never pruned, and futures that never finish (`finished_at` never arrives) also stay forever, invisible and un-expiring. - `error_name` is always `NULL` — Ventis's own Redis writer never records a distinct exception-type field, only a message string. -- No committed test suite — all verification during development was ad hoc scripts, not - `pytest` files under `tests/`. +- Test coverage is still limited; the waiting-field migration/normalization/conversion + path is covered, but the exporter process and live OTLP delivery are not. - Never verified against a live OTLP receiver — only against a refused connection (confirmed the SDK's real retry/error-handling path is exercised correctly). - No retry-limit/quarantine for a permanently malformed row — it logs an error every poll diff --git a/OTel_Exporter/convert.py b/OTel_Exporter/convert.py index 8e062a4..b7fb493 100644 --- a/OTel_Exporter/convert.py +++ b/OTel_Exporter/convert.py @@ -64,10 +64,9 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # model/input/output use real OTel GenAI semconv names; cpu/gpu/execution_time_ms/ - # queue_time_ms/token_count have no semconv equivalent (Ventis infra concepts, or -- - # for token_count -- a derived sum the spec doesn't define), so they keep plain names - # rather than being forced into a fake gen_ai.* one. See DESIGN.md. + # Model and token usage use OTel GenAI semantic-convention names. Observation + # input/output use Langfuse's documented JSON-string attributes. The remaining + # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. attributes = { k: v for k, v in { @@ -79,12 +78,14 @@ def waiting_row_to_span(row): "gen_ai.usage.input_tokens": row.get("input_token_count"), "gen_ai.usage.output_tokens": row.get("output_token_count"), "token_count": row.get("token_count"), + "langfuse.observation.input": row.get("input"), + "langfuse.observation.output": row.get("output"), }.items() if v is not None } return ReadableSpan( - name=row.get("agent_id") or "unknown_agent", + name=row.get("name") or row.get("agent_id") or "unknown_agent", context=context, parent=parent, attributes=attributes, diff --git a/OTel_Exporter/db.py b/OTel_Exporter/db.py index 4bb301d..a8a675f 100644 --- a/OTel_Exporter/db.py +++ b/OTel_Exporter/db.py @@ -8,6 +8,7 @@ wasn't doing anything BatchSpanProcessor doesn't already do -- see DESIGN.md.) """ +import json import os import sqlite3 @@ -51,16 +52,31 @@ cache_hit_ratio REAL, error_name TEXT, error_message TEXT, + name TEXT, + input TEXT, + output TEXT, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, sent BOOLEAN DEFAULT 0 """ +_MIGRATION_COLUMNS = { + "name": "TEXT", + "input": "TEXT", + "output": "TEXT", +} + def init_db(db_path=DB_PATH): - """Create the waiting table if it doesn't already exist.""" + """Create the waiting table and add columns missing from older databases.""" conn = sqlite3.connect(db_path) try: conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") + existing_columns = { + row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() + } + for column, column_type in _MIGRATION_COLUMNS.items(): + if column not in existing_columns: + conn.execute(f"ALTER TABLE waiting ADD COLUMN {column} {column_type}") conn.commit() finally: conn.close() @@ -74,6 +90,7 @@ def init_db(db_path=DB_PATH): "input_token_count", "output_token_count", "token_count", "errors", "failed", "server_cost", "token_cost", "total_cost", "cached_tokens", "cache_hit_ratio", "error_name", "error_message", + "name", "input", "output", ] _WAITING_UPSERT = """ @@ -86,6 +103,17 @@ def init_db(db_path=DB_PATH): ) +def _normalize_json_text(value): + """Return JSON text, encoding legacy scalar strings that are not valid JSON.""" + if value is None or value == "": + return None + try: + json.loads(value) + except (json.JSONDecodeError, TypeError): + return json.dumps(value) + return value + + def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH): """Upsert future rows (as returned by telemetry_logging.pull_runtime_information) into the waiting table. Unlike runtime_information, rows without finished_at are @@ -113,6 +141,17 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH output_token_count = int(float(raw.get("output_token_count") or 0)) token_count = int(float(raw.get("token_count") or 0)) cached_tokens = int(float(raw.get("input_cache_tokens") or 0)) + service = raw.get("service") + method = raw.get("method") + name = raw.get("name") or ".".join( + part for part in (service, method) if part + ) + result = raw.get("result") + # Compatibility with pre-consolidation deployments, where completion + # metrics live in future:{id}:metrics but result lives in future:{id}. + # Unified hashes already include result and avoid this extra read. + if not result and finished_at and redis_client is not None: + result = redis_client.hget(f"future:{fid}", "result") token_cost = ( pricing.compute_token_cost( @@ -165,6 +204,9 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, "error_name": raw.get("error_name"), "error_message": raw.get("error_message"), + "name": name or agent_id or "unknown_agent", + "input": _normalize_json_text(raw.get("args")), + "output": _normalize_json_text(result), }, ) conn.commit() diff --git a/OTel_Exporter/otel_exporter.py b/OTel_Exporter/otel_exporter.py index b1c7f1a..cad2af8 100644 --- a/OTel_Exporter/otel_exporter.py +++ b/OTel_Exporter/otel_exporter.py @@ -1,33 +1,31 @@ -"""Entrypoint for the OTLP Exporter process. - -Each poll tick: read finished, not-yet-sent rows from `waiting`, convert each to a span, -hand it to a BatchSpanProcessor/OTLPSpanExporter, and mark it sent -- batching, OTLP -serialization, and sending are all the SDK's own code, not ours (see DESIGN.md). Each -row's send-and-mark-sent is atomic and happens immediately after its own successful -send, not batched at the end, so a crash mid-poll can't leave an already-sent row -unmarked (which would cause a duplicate send next run). `OTLPSpanExporter()` takes no -explicit endpoint/headers here -- it falls back to the SDK's own standard -`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_EXPORTER_OTLP_HEADERS` env vars, or localhost:4317, -per the SDK's own default behavior. GlobalController sets those env vars (plus -`OTEL_EXPORTER_OTLP_PROTOCOL`, which this module reads itself below to pick the gRPC vs -HTTP class) from `global_controller.yaml`'s `otel:` section when it spawns this process; -this file has no YAML/app-config awareness of its own, only standard OTel env vars -- -see DESIGN.md. +"""Entrypoint for the OTLP exporter process. + +Each poll tick reads finished, not-yet-sent rows from ``waiting``, converts each to a +span, hands it to every configured BatchSpanProcessor, and marks it sent only after +all processors accept it. Batching, OTLP serialization, and sending remain the SDK's +responsibility (see DESIGN.md). + +GlobalController may provide a JSON list in ``VENTIS_OTEL_DESTINATIONS``. That is a +Ventis-specific configuration because the standard OTEL exporter environment +variables describe only one destination. If it is absent, the original single +destination behavior is retained: the exporter class and its settings are selected +from the standard OTEL environment variables and SDK defaults. """ +import json import logging +import math import os import signal import sqlite3 import time -# Protocol is the one thing the SDK's own exporter classes don't self-select from -# OTEL_EXPORTER_OTLP_PROTOCOL -- endpoint/headers/auth stay fully env-var-driven via -# each class's own defaults; see OTel_Exporter/DESIGN.md. -if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").startswith("http"): - from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -else: - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as GrpcOTLPSpanExporter, +) +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as HttpOTLPSpanExporter, +) from opentelemetry.sdk.trace.export import BatchSpanProcessor import convert @@ -38,7 +36,165 @@ _running = True _processor = None +_processors = [] POLL_INTERVAL_SECONDS = 5 +DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" + + +def _normalize_protocol(protocol): + """Return the exporter family for a configured protocol name.""" + if not isinstance(protocol, str) or not protocol.strip(): + raise ValueError("destination protocol must be a non-empty string") + normalized = protocol.strip().lower().replace("_", "-") + if normalized in {"grpc", "otlp/grpc", "grpc/protobuf", "grpc-protobuf"}: + return "grpc" + if normalized in { + "http", + "http/protobuf", + "http-protobuf", + "http/proto", + "http+protobuf", + "protobuf", + }: + return "http" + raise ValueError( + f"unsupported destination protocol {protocol!r}; expected grpc or http/protobuf" + ) + + +def _validate_destination(destination, index): + if not isinstance(destination, dict): + raise ValueError(f"destination {index} must be an object") + + name = destination.get("name") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"destination {index} name must be a non-empty string") + + protocol = _normalize_protocol(destination.get("protocol")) + endpoint = destination.get("endpoint") + if not isinstance(endpoint, str) or not endpoint.strip(): + raise ValueError(f"destination {name!r} endpoint must be a non-empty string") + + headers = destination.get("headers") + if headers is not None: + if not isinstance(headers, dict): + raise ValueError(f"destination {name!r} headers must be an object") + if any( + not isinstance(key, str) + or not key.strip() + or not isinstance(value, str) + for key, value in headers.items() + ): + raise ValueError( + f"destination {name!r} headers must map non-empty strings to strings" + ) + headers = dict(headers) + + insecure = destination.get("insecure") + if insecure is not None and not isinstance(insecure, bool): + raise ValueError(f"destination {name!r} insecure must be a boolean") + + timeout = destination.get("timeout") + if timeout is not None: + if isinstance(timeout, bool) or not isinstance(timeout, (int, float)): + raise ValueError(f"destination {name!r} timeout must be a positive number") + if not math.isfinite(timeout) or timeout <= 0: + raise ValueError(f"destination {name!r} timeout must be a positive number") + + return { + "name": name.strip(), + "protocol": protocol, + "endpoint": endpoint.strip(), + "headers": headers, + "insecure": insecure, + "timeout": timeout, + } + + +def _configured_destinations(): + """Parse and validate the Ventis multi-destination environment variable. + + ``None`` means no Ventis-specific configuration was supplied, so callers can + preserve legacy OTEL environment-variable behavior. An empty or malformed value + is an explicit configuration error and fails startup rather than silently + exporting to the wrong destination. + """ + raw = os.environ.get(DESTINATIONS_ENV) + if raw is None: + return None + try: + destinations = json.loads(raw) + except (TypeError, json.JSONDecodeError) as exc: + raise ValueError(f"{DESTINATIONS_ENV} must contain a JSON list") from exc + if not isinstance(destinations, list) or not destinations: + raise ValueError(f"{DESTINATIONS_ENV} must contain a non-empty JSON list") + + validated = [] + names = set() + for index, destination in enumerate(destinations): + validated_destination = _validate_destination(destination, index) + name = validated_destination["name"] + if name in names: + raise ValueError(f"destination names must be unique; duplicate {name!r}") + names.add(name) + validated.append(validated_destination) + return validated + + +def _build_exporter(destination): + """Construct one explicitly configured exporter without logging credentials.""" + kwargs = { + "endpoint": destination["endpoint"], + } + if destination["headers"] is not None: + kwargs["headers"] = destination["headers"] + if destination["timeout"] is not None: + kwargs["timeout"] = destination["timeout"] + + if destination["protocol"] == "grpc": + if destination["insecure"] is not None: + kwargs["insecure"] = destination["insecure"] + return GrpcOTLPSpanExporter(**kwargs) + + if destination["insecure"] is not None: + logger.warning( + "Destination %s specifies insecure=%s, which is ignored for HTTP exporters.", + destination["name"], + destination["insecure"], + ) + return HttpOTLPSpanExporter(**kwargs) + + +def _build_processors(): + """Build destination processors, or one legacy processor when unconfigured.""" + destinations = _configured_destinations() + if destinations is None: + protocol = os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() + exporter_class = ( + HttpOTLPSpanExporter if protocol.startswith("http") else GrpcOTLPSpanExporter + ) + return [("legacy", BatchSpanProcessor(exporter_class(), schedule_delay_millis=1000))] + + processors = [] + try: + for destination in destinations: + exporter = _build_exporter(destination) + processors.append( + ( + destination["name"], + BatchSpanProcessor(exporter, schedule_delay_millis=1000), + ) + ) + logger.info( + "Configured OTel destination %s (%s).", + destination["name"], + destination["protocol"], + ) + except Exception: + for _, processor in processors: + processor.shutdown() + raise + return processors def _handle_shutdown(signum, frame): @@ -48,6 +204,14 @@ def _handle_shutdown(signum, frame): def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" + processors = _processors + if not processors and _processor is not None: + # Compatibility for callers that configured the pre-fan-out singular + # ``_processor`` directly (the normal startup path always populates both). + processors = [("legacy", _processor)] + if not processors: + raise RuntimeError("OTel exporter has no configured processors") + conn = sqlite3.connect(db.DB_PATH) conn.row_factory = sqlite3.Row try: @@ -63,35 +227,63 @@ def _send_pending(): for row in rows: try: span = convert.waiting_row_to_span(row) - _processor.on_end(span) except Exception as e: logger.error( - "Skipping waiting row %s -- failed to send: %s", row["future_id"], e + "Skipping waiting row %s -- failed to convert: %s", row["future_id"], e ) continue + + failed_destinations = [] + for destination_name, processor in processors: + try: + processor.on_end(span) + except Exception as e: + # Still offer the span to the remaining processors. The row is only + # acknowledged when every destination accepted it, so a failed + # destination will be retried by the next poll. + failed_destinations.append(destination_name) + logger.error( + "Destination %s rejected waiting row %s: %s", + destination_name, + row["future_id"], + e, + ) + if failed_destinations: + continue db.mark_sent(row["future_id"]) sent_count += 1 - logger.info("Sent %d span(s) to the batch processor.", sent_count) + logger.info("Queued %d span(s) for all configured OTel destinations.", sent_count) def main(): - global _processor + global _processor, _processors signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() - _processor = BatchSpanProcessor(OTLPSpanExporter(), schedule_delay_millis=1000) - logger.info("OTel exporter process started.") - last_poll = 0 - while _running: - if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + _processors = _build_processors() + # Keep the old singular module variable available to integrations that imported + # it, while all sending uses the destination-aware collection above. + _processor = _processors[0][1] + logger.info("OTel exporter process started with %d destination(s).", len(_processors)) + try: + last_poll = 0 + while _running: + if time.time() - last_poll >= POLL_INTERVAL_SECONDS: + try: + _send_pending() + except Exception as e: + logger.warning("Poll cycle failed (non-fatal): %s", e) + last_poll = time.time() + time.sleep(1) + finally: + for destination_name, processor in _processors: try: - _send_pending() + processor.shutdown() except Exception as e: - logger.warning("Poll cycle failed (non-fatal): %s", e) - last_poll = time.time() - time.sleep(1) - _processor.shutdown() - logger.info("OTel exporter process exiting.") + logger.error( + "Failed to shut down OTel destination %s: %s", destination_name, e + ) + logger.info("OTel exporter process exiting.") if __name__ == "__main__": diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index d61ad42..ab5af4d 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -78,6 +78,24 @@ agents: provider: EC2 instance_type: t3.micro + +otel: + destinations: + - name: railway + protocol: grpc + endpoint: yamanote.proxy.rlwy.net:19803 + insecure: true + headers: {} + - name: langfuse + protocol: http + endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces + headers: {} + - name: grafana + protocol: http + endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces + headers: + Authorization: Basic ${GRAFANA_OTLP_HEADERS} + # Polling interval in seconds poll_interval: 5 diff --git a/pyproject.toml b/pyproject.toml index 8410b24..9ae1234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,3 +58,8 @@ allowed-unresolved-imports = [ "*_stub", "*_agent_stub", ] + +[dependency-groups] +dev = [ + "pytest>=9.1.1", +] diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py new file mode 100644 index 0000000..ed1b423 --- /dev/null +++ b/tests/test_otel_exporter_fanout.py @@ -0,0 +1,318 @@ +"""Focused tests for the Ventis OTel exporter fan-out configuration.""" + +import json +import os +import sqlite3 +import sys +import tempfile +import types +import unittest +from unittest.mock import MagicMock, patch + + +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +# ``otel_exporter.py`` is also executed as a script from its own directory and +# therefore imports ``convert`` and ``db`` as top-level modules. +sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) + +import db # noqa: E402 +import otel_exporter # noqa: E402 + + +# The generated local-controller protobuf modules are build artifacts and are +# not present in a source checkout. The static config helper does not use them, +# so provide the tiny import-time surface needed to test it in isolation. +if "local_controler_pb2" not in sys.modules: + local_pb2 = types.ModuleType("local_controler_pb2") + local_pb2.JsonResponse = object + sys.modules["local_controler_pb2"] = local_pb2 +if "local_controler_pb2_grpc" not in sys.modules: + local_pb2_grpc = types.ModuleType("local_controler_pb2_grpc") + local_pb2_grpc.LocalControllerStub = object + sys.modules["local_controler_pb2_grpc"] = local_pb2_grpc + + +class OTelExporterFanoutTests(unittest.TestCase): + def setUp(self): + self.db_file = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + self.db_path = self.db_file.name + self.db_file.close() + db.init_db(self.db_path) + + def tearDown(self): + os.unlink(self.db_path) + + @staticmethod + def _destination_config(): + return [ + { + "name": "railway", + "protocol": "grpc", + "endpoint": "receiver.example:4317", + "headers": {"x-api-key": "railway-key"}, + "insecure": True, + "timeout": 3.5, + }, + { + "name": "langfuse", + "protocol": "http/protobuf", + "endpoint": "https://langfuse.example/api/public/otel", + "headers": {"authorization": "Basic secret"}, + "timeout": 7, + }, + ] + + def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): + grpc_exporter = object() + http_exporter = object() + grpc_processor = MagicMock(name="grpc_processor") + http_processor = MagicMock(name="http_processor") + destinations = self._destination_config() + + with patch.dict( + os.environ, + {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, + clear=True, + ), patch.object( + otel_exporter, + "GrpcOTLPSpanExporter", + return_value=grpc_exporter, + ) as grpc_constructor, patch.object( + otel_exporter, + "HttpOTLPSpanExporter", + return_value=http_exporter, + ) as http_constructor, patch.object( + otel_exporter, + "BatchSpanProcessor", + side_effect=[grpc_processor, http_processor], + ) as processor_constructor: + processors = otel_exporter._build_processors() + + self.assertEqual( + processors, [("railway", grpc_processor), ("langfuse", http_processor)] + ) + grpc_constructor.assert_called_once_with( + endpoint="receiver.example:4317", + headers={"x-api-key": "railway-key"}, + timeout=3.5, + insecure=True, + ) + http_constructor.assert_called_once_with( + endpoint="https://langfuse.example/api/public/otel", + headers={"authorization": "Basic secret"}, + timeout=7, + ) + self.assertEqual( + processor_constructor.call_args_list, + [ + unittest.mock.call(grpc_exporter, schedule_delay_millis=1000), + unittest.mock.call(http_exporter, schedule_delay_millis=1000), + ], + ) + + def test_build_processors_preserves_legacy_single_destination_fallback(self): + http_exporter = object() + processor = MagicMock(name="legacy_processor") + with patch.dict( + os.environ, + {"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, + clear=True, + ), patch.object( + otel_exporter, "HttpOTLPSpanExporter", return_value=http_exporter + ) as constructor, patch.object( + otel_exporter, "BatchSpanProcessor", return_value=processor + ): + result = otel_exporter._build_processors() + + self.assertEqual(result, [("legacy", processor)]) + constructor.assert_called_once_with() + + def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): + invalid_values = [ + "not-json", + json.dumps([]), + json.dumps( + [ + { + "name": "same", + "protocol": "grpc", + "endpoint": "one:4317", + }, + { + "name": "same", + "protocol": "http/protobuf", + "endpoint": "https://two", + }, + ] + ), + ] + for raw in invalid_values: + with self.subTest(raw=raw), patch.dict( + os.environ, {otel_exporter.DESTINATIONS_ENV: raw}, clear=True + ): + with self.assertRaises(ValueError): + otel_exporter._configured_destinations() + + def test_controller_expands_env_and_builds_langfuse_basic_auth(self): + from ventis.controller.global_controller import GlobalController + + with patch.dict( + os.environ, + { + "LANGFUSE_BASE_URL": "https://us.cloud.langfuse.com", + "LANGFUSE_PUBLIC_KEY": "public", + "LANGFUSE_SECRET_KEY": "secret", + }, + clear=True, + ): + env = GlobalController._otel_exporter_env( + { + "destinations": [ + { + "name": "langfuse", + "protocol": "http/protobuf", + "endpoint": "${LANGFUSE_BASE_URL}/api/public/otel/v1/traces", + } + ] + } + ) + + destination = json.loads(env[otel_exporter.DESTINATIONS_ENV])[0] + self.assertEqual( + destination["endpoint"], + "https://us.cloud.langfuse.com/api/public/otel/v1/traces", + ) + self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") + + def test_controller_env_serializes_destinations_and_keeps_legacy_mapping(self): + # Importing the controller is intentionally local: this test remains + # runnable in the exporter-only environment used by the focused suite. + from ventis.controller.global_controller import GlobalController + + destinations = self._destination_config() + env = GlobalController._otel_exporter_env( + { + "protocol": "grpc", + "endpoint": "legacy.example:4317", + "headers": {"x-tenant": "demo"}, + "destinations": destinations, + } + ) + self.assertEqual(env["OTEL_EXPORTER_OTLP_PROTOCOL"], "grpc") + self.assertEqual(env["OTEL_EXPORTER_OTLP_ENDPOINT"], "legacy.example:4317") + self.assertEqual(env["OTEL_EXPORTER_OTLP_HEADERS"], "x-tenant=demo") + self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) + + def test_controller_rejects_invalid_destinations_before_starting_child(self): + from ventis.controller.global_controller import GlobalController + + invalid_destinations = [ + [], + [{"name": "railway", "protocol": "grpc"}], + [ + {"name": "same", "protocol": "grpc", "endpoint": "one:4317"}, + { + "name": "same", + "protocol": "http/protobuf", + "endpoint": "https://two", + }, + ], + [{"name": "bad", "protocol": "smtp", "endpoint": "example"}], + ] + for destinations in invalid_destinations: + with self.subTest(destinations=destinations), self.assertRaises(ValueError): + GlobalController._otel_exporter_env( + {"destinations": destinations} + ) + + def _insert_pending_row(self): + conn = sqlite3.connect(self.db_path) + try: + conn.execute( + """ + INSERT INTO waiting ( + future_id, session_id, started_at, finished_at, failed, + name, input, output, sent + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0) + """, + ( + "00112233445566778899aabbccddeeff", + "ffeeddccbbaa99887766554433221100", + 1.0, + 2.0, + 0, + "PriceAgent.get_history", + '{"ticker":"NVDA"}', + '{"price":100}', + ), + ) + conn.commit() + finally: + conn.close() + + def test_send_pending_delivers_the_same_span_to_every_processor(self): + self._insert_pending_row() + first = MagicMock(name="first") + second = MagicMock(name="second") + with patch.object(otel_exporter.db, "DB_PATH", self.db_path), patch.object( + otel_exporter.db, "mark_sent" + ) as mark_sent: + otel_exporter._processors = [("railway", first), ("langfuse", second)] + otel_exporter._processor = None + otel_exporter._send_pending() + + first.on_end.assert_called_once() + second.on_end.assert_called_once() + self.assertIs(first.on_end.call_args.args[0], second.on_end.call_args.args[0]) + mark_sent.assert_called_once_with("00112233445566778899aabbccddeeff") + + def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_failure(self): + self._insert_pending_row() + failed = MagicMock(name="failed") + failed.on_end.side_effect = RuntimeError("destination unavailable") + remaining = MagicMock(name="remaining") + with patch.object(otel_exporter.db, "DB_PATH", self.db_path), patch.object( + otel_exporter.db, "mark_sent" + ) as mark_sent: + otel_exporter._processors = [("railway", failed), ("langfuse", remaining)] + otel_exporter._processor = None + otel_exporter._send_pending() + + failed.on_end.assert_called_once() + remaining.on_end.assert_called_once() + mark_sent.assert_not_called() + + conn = sqlite3.connect(self.db_path) + try: + self.assertEqual(conn.execute("SELECT sent FROM waiting").fetchone()[0], 0) + finally: + conn.close() + + def test_processor_construction_failure_shuts_down_already_built_processors(self): + first_processor = MagicMock(name="first_processor") + destinations = self._destination_config() + with patch.dict( + os.environ, + {otel_exporter.DESTINATIONS_ENV: json.dumps(destinations)}, + clear=True, + ), patch.object( + otel_exporter, + "GrpcOTLPSpanExporter", + return_value=object(), + ), patch.object( + otel_exporter, + "HttpOTLPSpanExporter", + side_effect=RuntimeError("bad HTTP exporter"), + ), patch.object( + otel_exporter, + "BatchSpanProcessor", + return_value=first_processor, + ): + with self.assertRaisesRegex(RuntimeError, "bad HTTP exporter"): + otel_exporter._build_processors() + + first_processor.shutdown.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py new file mode 100644 index 0000000..4a62c58 --- /dev/null +++ b/tests/test_otel_exporter_fields.py @@ -0,0 +1,99 @@ +import json +import os +import sqlite3 +import tempfile +import unittest +from unittest.mock import patch + +from OTel_Exporter import convert, db + + +class OTelExporterFieldTests(unittest.TestCase): + def setUp(self): + handle = tempfile.NamedTemporaryFile(suffix=".db", delete=False) + self.db_path = handle.name + handle.close() + + def tearDown(self): + os.unlink(self.db_path) + + def test_init_db_migrates_existing_waiting_table(self): + with sqlite3.connect(self.db_path) as conn: + conn.execute( + "CREATE TABLE waiting (future_id TEXT PRIMARY KEY, session_id TEXT NOT NULL)" + ) + + db.init_db(self.db_path) + + with sqlite3.connect(self.db_path) as conn: + columns = { + row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() + } + self.assertTrue({"name", "input", "output"}.issubset(columns)) + + def test_fields_are_normalized_and_added_to_span(self): + db.init_db(self.db_path) + raw = { + "future_id": "00112233445566778899aabbccddeeff", + "request_id": "ffeeddccbbaa99887766554433221100", + "service": "PriceAgent", + "method": "get_history", + "args": '{"ticker": "NVDA"}', + "result": "plain text result", + "created_at": "1.0", + "finished_at": "2.0", + "failed": "0", + } + + with patch.object(db.pricing, "compute_token_cost", return_value=0.0): + db.write_waiting_rows([raw], db_path=self.db_path) + + with sqlite3.connect(self.db_path) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM waiting").fetchone() + + self.assertEqual(row["name"], "PriceAgent.get_history") + self.assertEqual(row["input"], raw["args"]) + self.assertEqual(json.loads(row["output"]), raw["result"]) + + span = convert.waiting_row_to_span(row) + self.assertEqual(span.name, "PriceAgent.get_history") + self.assertEqual(span.attributes["langfuse.observation.input"], raw["args"]) + self.assertEqual( + span.attributes["langfuse.observation.output"], row["output"] + ) + + def test_split_hash_result_is_loaded_for_finished_rows(self): + class SplitHashRedis: + def hget(self, key, field): + self.request = (key, field) + return '{"recommendation": "hold"}' + + def get(self, key): + return None + + db.init_db(self.db_path) + redis = SplitHashRedis() + raw = { + "future_id": "11112222333344445555666677778888", + "request_id": "88887777666655554444333322221111", + "service": "AdvisorAgent", + "method": "summarize", + "args": '{"risk": "moderate"}', + "result": "", + "created_at": "1.0", + "finished_at": "2.0", + "failed": "0", + } + + with patch.object(db.pricing, "compute_token_cost", return_value=0.0): + db.write_waiting_rows([raw], redis_client=redis, db_path=self.db_path) + + with sqlite3.connect(self.db_path) as conn: + output = conn.execute("SELECT output FROM waiting").fetchone()[0] + self.assertEqual(redis.request, (f"future:{raw['future_id']}", "result")) + self.assertEqual(json.loads(output), {"recommendation": "hold"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index ba10707..9b86805 100644 --- a/uv.lock +++ b/uv.lock @@ -246,6 +246,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + [[package]] name = "flask" version = "3.1.3" @@ -478,6 +490,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/57/b0/0e52c878c53f245edd3a11020f20979b3f490f245af532c7cae3027754b5/idna-3.19-py3-none-any.whl", hash = "sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4", size = 68550, upload-time = "2026-08-18T05:14:22.343Z" }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + [[package]] name = "itsdangerous" version = "2.2.0" @@ -692,6 +713,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, ] +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + [[package]] name = "protobuf" version = "7.35.1" @@ -815,6 +854,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1003,6 +1069,60 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, ] +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" @@ -1050,6 +1170,11 @@ dependencies = [ { name = "sqlalchemy" }, ] +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + [package.metadata] requires-dist = [ { name = "boto3" }, @@ -1067,6 +1192,9 @@ requires-dist = [ { name = "sqlalchemy" }, ] +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=9.1.1" }] + [[package]] name = "werkzeug" version = "3.1.8" diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index b84614b..9947680 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,15 +3,18 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit +import base64 import importlib.util +import json import logging +import math +import os +import re import signal import subprocess +import sys import threading import time -import json -import sys -import os from concurrent.futures import ThreadPoolExecutor import yaml @@ -102,7 +105,8 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() @@ -114,22 +118,23 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # `otel:` in global_controller.yaml maps straight to the OTel SDK's own - # standard env vars, not app-specific args -- the exporter subprocess itself - # stays a plain vendor-neutral OTel process; see OTel_Exporter/DESIGN.md. + # Legacy `otel:` fields map straight to the OTel SDK's own standard env vars. + # A `destinations` list is additionally passed as one Ventis-specific JSON + # variable; the exporter subprocess remains a plain OTel process otherwise. otel_env = self._otel_exporter_env(self.config.get("otel", {})) self.process_supervisor.register( "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env ) - self.process_supervisor.start_all() - # waiting table GC writes future data into (see OTel_Exporter/db.py); the - # exporter process itself calls init_db() to create the table. + # Initialize/migrate the waiting table synchronously before either the GC or + # exporter process can access it. otel_db_spec = importlib.util.spec_from_file_location( "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") ) self._otel_db = importlib.util.module_from_spec(otel_db_spec) otel_db_spec.loader.exec_module(self._otel_db) + self._otel_db.init_db() + self.process_supervisor.start_all() # ------------------------------------------------------------------ # # Stale container cleanup # @@ -177,15 +182,50 @@ def _cleanup_stale_containers(self): @staticmethod def _load_config(config_path): - """Load the YAML config file.""" + """Load the YAML config file after importing root .env values.""" + project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) + GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: return yaml.safe_load(f) + @staticmethod + def _load_dotenv(path): + """Load simple KEY=VALUE entries without overriding existing environment values.""" + if not os.path.isfile(path): + return + with open(path, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + if key and key not in os.environ: + os.environ[key] = value + + @staticmethod + def _expand_otel_value(value): + if isinstance(value, str): + return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) + if isinstance(value, dict): + return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + if isinstance(value, list): + return [GlobalController._expand_otel_value(item) for item in value] + return value + @staticmethod def _otel_exporter_env(otel_cfg): """Translate global_controller.yaml's `otel:` section into standard OTLP env - vars for the exporter subprocess; returns None if `otel:` is absent/empty so - the subprocess falls back to the SDK's own defaults untouched.""" + vars for the exporter subprocess. When present, ``destinations`` is passed as + JSON for the exporter to construct a fan-out. Returns None if `otel:` is + absent/empty so the subprocess falls back to the SDK's own defaults untouched. + + The legacy protocol/endpoint/headers mappings intentionally remain unchanged + for existing configurations. + """ env = {} if otel_cfg.get("protocol"): env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] @@ -195,8 +235,109 @@ def _otel_exporter_env(otel_cfg): env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( f"{k}={v}" for k, v in otel_cfg["headers"].items() ) + + if "destinations" in otel_cfg: + destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + for destination in destinations: + if destination.get("name") == "langfuse": + public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") + secret_key = os.environ.get("LANGFUSE_SECRET_KEY") + if public_key and secret_key: + auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode() + destination.setdefault("headers", {})["Authorization"] = f"Basic {auth}" + GlobalController._validate_otel_destinations(destinations) + try: + env["VENTIS_OTEL_DESTINATIONS"] = json.dumps(destinations) + except (TypeError, ValueError) as exc: + # Do not include the offending value: destination configs commonly + # contain credentials in headers. + raise ValueError( + "otel.destinations must contain JSON-serializable values" + ) from exc return env or None + @staticmethod + def _validate_otel_destinations(destinations): + """Validate the shape of the optional exporter fan-out configuration. + + Keep this validation deliberately structural: destination-specific options + are interpreted by the exporter. Error messages identify only the location + and type, never destination contents or header values. + """ + if not isinstance(destinations, list): + raise ValueError("otel.destinations must be a list") + if not destinations: + raise ValueError("otel.destinations must not be empty") + + names = set() + for index, destination in enumerate(destinations): + if not isinstance(destination, dict): + raise ValueError( + f"otel.destinations[{index}] must be a mapping" + ) + + for field in ("name", "protocol", "endpoint"): + value = destination.get(field) + if not isinstance(value, str) or not value.strip(): + raise ValueError( + f"otel.destinations[{index}].{field} must be a non-empty string" + ) + + name = destination["name"].strip() + if name in names: + raise ValueError(f"otel.destinations contains duplicate name {name!r}") + names.add(name) + + protocol = destination["protocol"].strip().lower().replace("_", "-") + if protocol not in { + "grpc", + "otlp/grpc", + "grpc/protobuf", + "grpc-protobuf", + "http", + "http/protobuf", + "http-protobuf", + "http/proto", + "http+protobuf", + "protobuf", + }: + raise ValueError( + f"otel.destinations[{index}].protocol must be grpc or http/protobuf" + ) + + if "headers" in destination: + headers = destination["headers"] + if not isinstance(headers, dict): + raise ValueError( + f"otel.destinations[{index}].headers must be a mapping" + ) + if any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in headers.items() + ): + raise ValueError( + f"otel.destinations[{index}].headers keys and values must be strings" + ) + + if "insecure" in destination and not isinstance( + destination["insecure"], bool + ): + raise ValueError( + f"otel.destinations[{index}].insecure must be a boolean" + ) + + if "timeout" in destination: + timeout = destination["timeout"] + if ( + isinstance(timeout, bool) + or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) + or timeout <= 0 + ): + raise ValueError( + f"otel.destinations[{index}].timeout must be a positive number" + ) + @staticmethod def _get_replica_placements(ctrl): """Normalize replicas into a list of (host, port) placements.""" @@ -479,6 +620,7 @@ def run(self): self._poll_controllers() except Exception as e: logger.warning("Polling loop encountered an error: %s", e) + self._cleanup_ready.set() time.sleep(self.poll_interval) except KeyboardInterrupt: self.stop() @@ -638,9 +780,10 @@ def _get_lc_stub(self, endpoint): return self._lc_stubs[endpoint] def _cleanup_loop(self): - """Background thread: periodically trigger cleanup of completed requests.""" + """Background thread: trigger cleanup right after each poll tick, or every cleanup_interval as a fallback.""" while True: - time.sleep(self.cleanup_interval) + self._cleanup_ready.wait(timeout=self.cleanup_interval) + self._cleanup_ready.clear() try: self._trigger_cleanup() except Exception as e: From 098548c60ee297032ba7cb7c3074726955917dd3 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:24:08 -0700 Subject: [PATCH 09/13] Dedupe 'import os' from PR #51 merge (both sides added it independently) Co-Authored-By: Claude Sonnet 5 --- examples/portfolio/agents/metrics_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/portfolio/agents/metrics_agent.py b/examples/portfolio/agents/metrics_agent.py index d5f722e..28069ac 100644 --- a/examples/portfolio/agents/metrics_agent.py +++ b/examples/portfolio/agents/metrics_agent.py @@ -10,7 +10,6 @@ # Resource profile: cheap CPU, high fan-out — one compute() call per holding. import os import sys -import os import json import math From 0b9546cbe324a99b34589e5e5b497cdad09951e4 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 18:49:27 -0700 Subject: [PATCH 10/13] Fix PR #51 regression: disable entrypoint-based stub relocation This project's agents/workflow import each other's stubs by flat module name, not by the exporting agent's own entrypoint path. Applying _stub_destination's entrypoint-mirroring broke both the Workflow (ModuleNotFoundError: intent_agent) and agent-to-agent calls (MetricsAgent -> price_agent) on live redeploy. Keeps PR #51's actual fix (project_dir sweep for unstubbed helper files) intact. Co-Authored-By: Claude Sonnet 5 --- ventis/cli.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ventis/cli.py b/ventis/cli.py index 28c4fd7..920df15 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -229,8 +229,8 @@ def cmd_build(args): logger.warning("No agent YAML files found in %s", agents_dir) import yaml - - # Looks up a config entry's YAML and to map stubs to entrypoints. + + # Looks up a config entry's YAML by agent name. yaml_by_name = {} for yaml_path in yaml_files: with open(yaml_path) as f: @@ -238,13 +238,6 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path - entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} - stub_entrypoints = { - f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] - for n, p in yaml_by_name.items() - if entrypoints_by_name.get(n) - } - stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -309,7 +302,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + # A workflow script imports stubs by flat module name (e.g. `from + # intent_agent import ...`), not by the agent's own entrypoint path. + stub_entrypoints=None, ) else: @@ -345,7 +340,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - stub_entrypoints=stub_entrypoints, + # Same reasoning as the workflow call above: this project's agents + # import each other's stubs by flat module name, not entrypoint path. + stub_entrypoints=None, ) bake_targets.append( From 7b3b167b27e9e8889edf099de517dc82f5662a13 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Wed, 26 Aug 2026 21:40:17 -0700 Subject: [PATCH 11/13] Parallelize per-instance polling in _poll_controllers Metrics/telemetry latency scaled with instance count x per-instance round-trip time since every instance was polled sequentially, one blocking the next, with the following tick only starting after the whole pass finished. Extracted the per-instance body into _poll_one_instance (whole body wrapped in one top-level try/except, since ThreadPoolExecutor.map() re-raises on first exception when results are consumed) and run all instances concurrently via the same ThreadPoolExecutor pattern _trigger_cleanup already used. Co-Authored-By: Claude Sonnet 5 --- OTel_Exporter/DESIGN.md | 16 +- ventis/controller/global_controller.py | 217 +++++++++++++------------ 2 files changed, 130 insertions(+), 103 deletions(-) diff --git a/OTel_Exporter/DESIGN.md b/OTel_Exporter/DESIGN.md index 86a2a95..418ca2a 100644 --- a/OTel_Exporter/DESIGN.md +++ b/OTel_Exporter/DESIGN.md @@ -167,7 +167,21 @@ Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall the poll loop's health checks and OTel writes. -### 6. Dependencies (all added) +### 6. Parallelized per-instance polling (`ventis/controller/global_controller.py`) +`_poll_controllers` used to loop over every instance sequentially -- Redis reads, an +OTel sqlite write, and up to two Postgres writes per instance, one instance fully +blocking the next, with the following poll tick only starting after the whole pass +finished. Total metrics/telemetry latency scaled with instance count x round-trip +time, not the configured `poll_interval`. Fixed by extracting the per-instance body +into `_poll_one_instance` (its whole body wrapped in one top-level try/except, since +`ThreadPoolExecutor.map()` re-raises on first exception when results are consumed) +and running all instances concurrently via the same `ThreadPoolExecutor` pattern +`_trigger_cleanup` already used. Known, pre-existing, previously acknowledged in +commit `a6694d9`'s own message but never actually fixed (a same-named follow-up +branch was found to contain no real threading changes) -- see company-memory for +the investigation. + +### 7. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index bdec683..da29783 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -643,118 +643,131 @@ def _poll_controllers(self): if self.running: self.process_supervisor.check_and_respawn() - for instance in self.instance_manager.list_instances(): + # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer + # gates every other instance's poll -- see OTel_Exporter/DESIGN.md. + instances = self.instance_manager.list_instances() + if instances: + with ThreadPoolExecutor(max_workers=len(instances)) as executor: + list(executor.map(self._poll_one_instance, instances)) + + def _poll_one_instance(self, instance): + """Poll and persist one instance's runtime/metrics/health data; never raises.""" + try: name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, + except Exception as e: + logger.warning("Failed to poll instance %s: %s", instance, e) + return + + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" + + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now - - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, - ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) - else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status + else: + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses # From b5d6e4dcbc6aee7265e1dc01080f70da39eb90f2 Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Fri, 28 Aug 2026 13:58:05 -0700 Subject: [PATCH 12/13] rough draft --- .../portfolio/config/global_controller.yaml | 25 +++++---- .../portfolio/workflow/portfolio_workflow.py | 1 - pyproject.toml | 2 +- requirements.txt | 2 - tests/test_otel_exporter_fanout.py | 2 +- tests/test_otel_exporter_fields.py | 2 +- .../OTLP_Exporter}/DESIGN.md | 6 +- ventis/OTLP_Exporter/SCHEMA.md | 56 +++++++++++++++++++ .../OTLP_Exporter}/__init__.py | 0 .../OTLP_Exporter}/convert.py | 24 ++++++-- {OTel_Exporter => ventis/OTLP_Exporter}/db.py | 4 +- .../OTLP_Exporter}/otel_exporter.py | 0 ventis/OTLP_Exporter/otel_queue.db | 0 ventis/cli.py | 21 +++++-- .../cloud_provider_logic/EC2/_runtime.py | 5 +- ventis/controller/global_controller.py | 27 +++++---- ventis/controller/utils/process_supervisor.py | 2 +- ventis/stub_generator.py | 40 +++++++++++-- 18 files changed, 167 insertions(+), 52 deletions(-) rename {OTel_Exporter => ventis/OTLP_Exporter}/DESIGN.md (98%) create mode 100644 ventis/OTLP_Exporter/SCHEMA.md rename {OTel_Exporter => ventis/OTLP_Exporter}/__init__.py (100%) rename {OTel_Exporter => ventis/OTLP_Exporter}/convert.py (72%) rename {OTel_Exporter => ventis/OTLP_Exporter}/db.py (98%) rename {OTel_Exporter => ventis/OTLP_Exporter}/otel_exporter.py (100%) create mode 100644 ventis/OTLP_Exporter/otel_queue.db diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index ab5af4d..edee85f 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -83,18 +83,18 @@ otel: destinations: - name: railway protocol: grpc - endpoint: yamanote.proxy.rlwy.net:19803 + endpoint: ${RAILWAY_OTLP_ENDPOINT} insecure: true headers: {} - - name: langfuse - protocol: http - endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces - headers: {} - name: grafana protocol: http endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} +# - name: langfuse +# protocol: http +# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces +# headers: {} # Polling interval in seconds poll_interval: 5 @@ -107,10 +107,13 @@ redis: # EC2 defaults for `provider: EC2` replicas. ec2: - region: us-east-1 - ami_id: ami-031ff6df47f26b546 - subnet_id: subnet-0638ac6d79d488124 + region: ${EC2_REGION} + ami_id: ${EC2_AMI_ID} + subnet_id: ${EC2_SUBNET_ID} security_group_ids: - - sg-025daf3a98e06cef3 - ssh_user: ubuntu - ssh_private_key_path: ~/.ssh/ventis_ec2 + - ${EC2_SECURITY_GROUP_ID} + ssh_user: ${EC2_SSH_USER} + ssh_private_key_path: ${EC2_SSH_PRIVATE_KEY_PATH} + +database: + url: ${DATABASE_URL} diff --git a/examples/portfolio/workflow/portfolio_workflow.py b/examples/portfolio/workflow/portfolio_workflow.py index 299a3f5..b8b684a 100644 --- a/examples/portfolio/workflow/portfolio_workflow.py +++ b/examples/portfolio/workflow/portfolio_workflow.py @@ -46,7 +46,6 @@ def main( advisor = AdvisorAgent() # Stage 0: parse the free-text request into structured holdings + window. - intent = intent_agent.parse(query=query).value() # parse() returns a dict, but a Future's .value() only ever gives back the # raw string ventis stored in Redis -- same deserialization requirement as # every other dict-returning agent call below. diff --git a/pyproject.toml b/pyproject.toml index 9ae1234..40cb43a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ requires = ["setuptools>=64"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] -include = ["ventis*", "OTel_Exporter*"] +include = ["ventis*"] [tool.setuptools.package-data] ventis = [ diff --git a/requirements.txt b/requirements.txt index dd7a254..f06e0b0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ grpcio-tools redis pyyaml flask -ipdb -ipython sqlalchemy psycopg[binary] psutil diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index ed1b423..0691d61 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -13,7 +13,7 @@ ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) # ``otel_exporter.py`` is also executed as a script from its own directory and # therefore imports ``convert`` and ``db`` as top-level modules. -sys.path.insert(0, os.path.join(ROOT, "OTel_Exporter")) +sys.path.insert(0, os.path.join(ROOT, "ventis", "OTLP_Exporter")) import db # noqa: E402 import otel_exporter # noqa: E402 diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index 4a62c58..ff8cad6 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -5,7 +5,7 @@ import unittest from unittest.mock import patch -from OTel_Exporter import convert, db +from ventis.OTLP_Exporter import convert, db class OTelExporterFieldTests(unittest.TestCase): diff --git a/OTel_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md similarity index 98% rename from OTel_Exporter/DESIGN.md rename to ventis/OTLP_Exporter/DESIGN.md index 418ca2a..d433fe0 100644 --- a/OTel_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -48,7 +48,7 @@ Decisions (final status): Configuration is read at exporter startup; changing it requires a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own - SQLite file (`OTel_Exporter/otel_queue.db`, see `db.py`), written by GC's existing + SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing `send_runtime_information` write. Keeps this pipeline's schema/state fully decoupled from the dashboard/cost table. @@ -95,7 +95,7 @@ endpoint and headers to the SDK. `BatchSpanProcessor(..., schedule_delay_millis= `max_export_batch_size` is left at the SDK default (512), which already approximates the original "500 spans" batching ask without any override needed. -### 2. `OTel_Exporter/otel_exporter.py` +### 2. `ventis/OTLP_Exporter/otel_exporter.py` A plain loop, polling every `POLL_INTERVAL_SECONDS` (5s, checked every 1s so SIGTERM stays responsive), calling `_send_pending()` each tick. At startup it constructs one independent OTLP exporter and `BatchSpanProcessor` for each configured destination; @@ -111,7 +111,7 @@ each pair may use a different protocol, endpoint, and headers: since spans are hand-built and handed straight to the processors via `on_end()`. - Every processor is shut down on exit, flushing its pending batch independently. -### 3. Future row → OTel span conversion (`OTel_Exporter/convert.py`) +### 3. Future row → OTel span conversion (`ventis/OTLP_Exporter/convert.py`) `future_id` maps to OTel `span_id`, not `trace_id` — `session_id` (== `request_id`) is the one that maps to `trace_id`. Both are `uuid4().hex` (32 hex chars / 16 bytes); OTel `trace_id` is 128-bit (16 bytes, fits directly) and `span_id` is 64-bit (8 bytes, needs diff --git a/ventis/OTLP_Exporter/SCHEMA.md b/ventis/OTLP_Exporter/SCHEMA.md new file mode 100644 index 0000000..38f8664 --- /dev/null +++ b/ventis/OTLP_Exporter/SCHEMA.md @@ -0,0 +1,56 @@ +# OTel span storage schema + +Each emitted OTel span is stored as one `otel_spans` record. This chart also defines a +one-to-one `otel_span_attributes` projection, linked by the same `span_id`, for the +known exporter attributes. The current receiver retains the raw `attributes` JSONB map; +the child table is the relational schema described in `otel_spans_schema.txt`. + +```text +┌───────────────────────────┐ ┌────────────────────────────────┐ +│ otel_spans │ │ otel_span_attributes │ +├───────────────────────────┤ ├────────────────────────────────┤ +│ PK span_id │────1:1───│ PK/FK span_id │ +│ trace_id │ │ model and agent ID │ +│ parent_span_id │ │ CPU and GPU │ +│ name │ │ timing and token usage │ +│ kind │ │ input and output │ +│ start/end time (ns) │ │ project and error count │ +│ status code/message │ │ server/token/total cost │ +│ attributes (JSONB) │ │ cache tokens/hit ratio │ +│ events (JSONB) │ └────────────────────────────────┘ +└───────────────────────────┘ +``` + +## `otel_spans` + +| Column | Meaning | +| --- | --- | +| `span_id` | Unique identifier for this span. | +| `trace_id` | Identifier shared by all spans in the same trace. | +| `parent_span_id` | Parent span; empty for a root span. | +| `name` | Operation name, such as an agent method. | +| `kind` | OTel role of the work; current exporter spans are `SPAN_KIND_INTERNAL`. | +| `start_time_unix_nano` / `end_time_unix_nano` | Raw Unix timestamps in nanoseconds. | +| `status_code` / `status_message` | OTel outcome: normally `STATUS_CODE_UNSET`, or `STATUS_CODE_ERROR` with an error message. | +| `attributes` | Complete raw OTel attribute map (JSONB). | +| `events` | OTel events, including any `exception` event (JSONB). | + +## `otel_span_attributes` + +This one-to-one projection mirrors every attribute currently emitted by +`convert.waiting_row_to_span()`. Fields are nullable because OTel omits an attribute +whose source value is `None`. + +| Group | Columns | +| --- | --- | +| Model and agent | `gen_ai.request.model`, `gen_ai.agent.id` | +| Resources and timing | `cpu`, `gpu`, `execution_time_ms`, `queue_time_ms` | +| Token usage | `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `token_count`, `gen_ai.usage.cache_read.input_tokens` | +| Input and output | `langfuse.observation.input`, `langfuse.observation.output` | +| Project and errors | `project_id`, `error_count` | +| Costs | `server_cost`, `token_cost`, `gen_ai.usage.cost` | +| Cache | `cache_hit_ratio` | + +The DBML source is [`../otel_spans_schema.txt`](../otel_spans_schema.txt). + +Successful spans use `STATUS_CODE_UNSET` rather than `STATUS_CODE_OK` because OpenTelemetry reserves `OK` for application- or operator-validated success, while instrumentation normally sets a status only when it records an error. diff --git a/OTel_Exporter/__init__.py b/ventis/OTLP_Exporter/__init__.py similarity index 100% rename from OTel_Exporter/__init__.py rename to ventis/OTLP_Exporter/__init__.py diff --git a/OTel_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py similarity index 72% rename from OTel_Exporter/convert.py rename to ventis/OTLP_Exporter/convert.py index b7fb493..66da972 100644 --- a/OTel_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -3,7 +3,7 @@ Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects directly instead of going through Tracer.start_span() -- there's no live tracer here, futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See OTel_Exporter/DESIGN.md for why this deviates from +conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from the SDK's usual advice against constructing ReadableSpan by hand. """ @@ -64,9 +64,17 @@ def waiting_row_to_span(row): ) status = Status(StatusCode.ERROR, description=row.get("error_message")) - # Model and token usage use OTel GenAI semantic-convention names. Observation - # input/output use Langfuse's documented JSON-string attributes. The remaining - # Ventis infrastructure values have no GenAI equivalent, so they keep plain names. + # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention + # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). + # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute + # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details + # attribute is currently broken -- see langfuse/langfuse#11030). Observation + # input/output use Langfuse's documented JSON-string attributes. `errors` is named + # error_count, not "errors"/"error", to avoid colliding with OTel's reserved + # error.* namespace (error.type etc.), which describes a single error, not a + # count. The remaining Ventis-specific values (project_id, server/token cost + # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep + # plain names. attributes = { k: v for k, v in { @@ -80,6 +88,14 @@ def waiting_row_to_span(row): "token_count": row.get("token_count"), "langfuse.observation.input": row.get("input"), "langfuse.observation.output": row.get("output"), + "project_id": row.get("project_id"), + "gen_ai.agent.id": row.get("agent_id"), + "error_count": row.get("errors"), + "server_cost": row.get("server_cost"), + "token_cost": row.get("token_cost"), + "gen_ai.usage.cost": row.get("total_cost"), + "gen_ai.usage.cache_read.input_tokens": row.get("cached_tokens"), + "cache_hit_ratio": row.get("cache_hit_ratio"), }.items() if v is not None } diff --git a/OTel_Exporter/db.py b/ventis/OTLP_Exporter/db.py similarity index 98% rename from OTel_Exporter/db.py rename to ventis/OTLP_Exporter/db.py index a8a675f..e6a1834 100644 --- a/OTel_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -130,7 +130,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH if not fid or not session_id: continue agent_id = raw.get("agent") - started_at = float(raw.get("created_at") or 0) or None + started_at = float(raw.get("created_at") or 0) finished_at = float(raw["finished_at"]) if raw.get("finished_at") else None execution_time_ms = ( round((finished_at - started_at) * 1000) @@ -160,7 +160,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _TOKEN_COST_MULTIPLIER ) # Server cost needs an elapsed duration -- only available once finished. - if finished_at and started_at: + if finished_at is not None: server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") diff --git a/OTel_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py similarity index 100% rename from OTel_Exporter/otel_exporter.py rename to ventis/OTLP_Exporter/otel_exporter.py diff --git a/ventis/OTLP_Exporter/otel_queue.db b/ventis/OTLP_Exporter/otel_queue.db new file mode 100644 index 0000000..e69de29 diff --git a/ventis/cli.py b/ventis/cli.py index 920df15..31a32e0 100644 --- a/ventis/cli.py +++ b/ventis/cli.py @@ -238,6 +238,15 @@ def cmd_build(args): if name: yaml_by_name[name] = yaml_path + # Maps each generated stub's basename to its agent's entrypoint path, so a + # stub can also be placed at its nested, entrypoint-mirrored location. + entrypoints_by_name = {a["name"]: a.get("entrypoint") for a in agents} + stub_entrypoints = { + f"{os.path.splitext(os.path.basename(p))[0]}.py": entrypoints_by_name[n] + for n, p in yaml_by_name.items() + if entrypoints_by_name.get(n) + } + stub_paths = [] for yaml_path in yaml_files: base_name = os.path.splitext(os.path.basename(yaml_path))[0] @@ -302,9 +311,9 @@ def cmd_build(args): api_port=agent_cfg.get("api_port", 8080), requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # A workflow script imports stubs by flat module name (e.g. `from - # intent_agent import ...`), not by the agent's own entrypoint path. - stub_entrypoints=None, + # Stubs are placed both flat and at their entrypoint-mirrored path, + # so both flat and nested import styles resolve to the stub. + stub_entrypoints=stub_entrypoints, ) else: @@ -340,9 +349,9 @@ def cmd_build(args): stub_files=stub_paths, requirements=_normalize_requirements(agent_cfg), project_dir=project_dir, - # Same reasoning as the workflow call above: this project's agents - # import each other's stubs by flat module name, not entrypoint path. - stub_entrypoints=None, + # Same reasoning as the workflow call above: stubs are placed both + # flat and at their entrypoint-mirrored path. + stub_entrypoints=stub_entrypoints, ) bake_targets.append( diff --git a/ventis/controller/cloud_provider_logic/EC2/_runtime.py b/ventis/controller/cloud_provider_logic/EC2/_runtime.py index 9955fa2..7e4e9ab 100644 --- a/ventis/controller/cloud_provider_logic/EC2/_runtime.py +++ b/ventis/controller/cloud_provider_logic/EC2/_runtime.py @@ -239,11 +239,12 @@ def _bootstrap_instance(host, spec, replica_index, cfg, redis_host, redis_port, logger.info("Transferring image %s to %s", image, host) result = subprocess.run( "set -o pipefail; " - f"docker save {shlex.quote(image)} | ssh -o StrictHostKeyChecking=no " + f"docker save {shlex.quote(image)} | zstd -T0 | ssh -o StrictHostKeyChecking=no " f"-o IdentitiesOnly=yes -o ConnectTimeout=10 " f"-o ServerAliveInterval=10 -o ServerAliveCountMax=3 " f"-i {shlex.quote(key)} " - f"{shlex.quote(f'{ssh_user}@{host}')} 'sudo docker load'", + f"{shlex.quote(f'{ssh_user}@{host}')} " + "'set -o pipefail; zstd -d | sudo docker load'", shell=True, capture_output=True, text=True, diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index da29783..6584ef8 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -110,16 +110,16 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see OTel_Exporter/DESIGN.md. + # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() - # Spawn the OTLP exporter as a separate process (see OTel_Exporter/DESIGN.md), + # Spawn the OTLP exporter as a separate process (see ventis/OTLP_Exporter/DESIGN.md), # supervised so it gets restarted if it ever exits unexpectedly. otel_exporter_dir = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), - "OTel_Exporter", + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "OTLP_Exporter", ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() @@ -191,7 +191,12 @@ def _load_config(config_path): project_root = os.path.abspath(os.path.join(os.path.dirname(config_path), "..")) GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: - return yaml.safe_load(f) + config = yaml.safe_load(f) + if "ec2" in config: + config["ec2"] = GlobalController._expand_env_value(config["ec2"]) + if "database" in config: + config["database"] = GlobalController._expand_env_value(config["database"]) + return config @staticmethod def _load_dotenv(path): @@ -212,13 +217,13 @@ def _load_dotenv(path): os.environ[key] = value @staticmethod - def _expand_otel_value(value): + def _expand_env_value(value): if isinstance(value, str): return re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", lambda m: os.environ.get(m.group(1), m.group(0)), value) if isinstance(value, dict): - return {key: GlobalController._expand_otel_value(item) for key, item in value.items()} + return {key: GlobalController._expand_env_value(item) for key, item in value.items()} if isinstance(value, list): - return [GlobalController._expand_otel_value(item) for item in value] + return [GlobalController._expand_env_value(item) for item in value] return value @staticmethod @@ -242,7 +247,7 @@ def _otel_exporter_env(otel_cfg): ) if "destinations" in otel_cfg: - destinations = GlobalController._expand_otel_value(otel_cfg["destinations"]) + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) for destination in destinations: if destination.get("name") == "langfuse": public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") @@ -639,12 +644,12 @@ def _poll_controllers(self): # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which # terminates every managed process) via the signal handler before this line is # reached -- without the guard, this could respawn a process just intentionally - # killed. See OTel_Exporter/DESIGN.md. + # killed. See ventis/OTLP_Exporter/DESIGN.md. if self.running: self.process_supervisor.check_and_respawn() # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer - # gates every other instance's poll -- see OTel_Exporter/DESIGN.md. + # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. instances = self.instance_manager.list_instances() if instances: with ThreadPoolExecutor(max_workers=len(instances)) as executor: diff --git a/ventis/controller/utils/process_supervisor.py b/ventis/controller/utils/process_supervisor.py index 8f5724c..8c061bc 100644 --- a/ventis/controller/utils/process_supervisor.py +++ b/ventis/controller/utils/process_supervisor.py @@ -3,7 +3,7 @@ register() + start_all() spawn processes; check_and_respawn() (call from GC's existing poll tick) restarts any that exit unexpectedly; terminate_all() (call from GC's shutdown path) stops them all cleanly. Deliberately GC-agnostic -- callers are responsible for not -calling check_and_respawn() during their own shutdown (see OTel_Exporter/DESIGN.md's +calling check_and_respawn() during their own shutdown (see ventis/OTLP_Exporter/DESIGN.md's shutdown-race note). """ diff --git a/ventis/stub_generator.py b/ventis/stub_generator.py index 803fc2d..4647619 100644 --- a/ventis/stub_generator.py +++ b/ventis/stub_generator.py @@ -17,9 +17,7 @@ import yaml # Packages every agent container needs regardless of its specific business logic. -# grpcio-tools/pyyaml/ipdb/ipython aren't needed/used, but keeping to keep the scope constrained right now -# - Leave a comment if you want me to remove these, I kept them in since you originally had them but they aren't used -BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "ipdb", "ipython", "boto3"] +BASE_AGENT_REQUIREMENTS = ["grpcio", "grpcio-tools", "redis", "pyyaml", "psutil", "boto3"] # Workflow will always require these BASE_WORKFLOW_REQUIREMENTS = BASE_AGENT_REQUIREMENTS + ["flask", "sqlalchemy", "psycopg[binary]"] @@ -322,6 +320,21 @@ def _copy_files(output_dir, files_to_copy): shutil.copy2(src, dest_path) +def _write_entrypoint_file(src, dest_path, project_dir): + """Copy an entrypoint file to dest_path, injecting a sys.path entry for its + original sibling directory so a co-located, non-stub helper import still resolves.""" + original_dir = os.path.dirname(os.path.relpath(src, project_dir)) if project_dir else "" + if not original_dir: + shutil.copy2(src, dest_path) + return + injection = ( + f"import sys, os\n" + f"sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), {original_dir!r}))\n" + ) + with open(src) as f, open(dest_path, "w") as out: + out.write(injection + f.read()) + + def generate_docker( yaml_path, agent_file, @@ -405,8 +418,7 @@ def generate_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) - - files_to_copy.append((os.path.abspath(agent_file), os.path.basename(agent_file))) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -416,6 +428,14 @@ def generate_docker( _copy_files(output_dir, files_to_copy) + # Copy the agent's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(agent_file), + os.path.join(output_dir, os.path.basename(agent_file)), + project_dir, + ) + # Copy the YAML definition too shutil.copy2( os.path.abspath(yaml_path), @@ -499,7 +519,6 @@ def generate_workflow_docker( files_to_copy = _sweep_py_files(project_dir) if project_dir else [] files_to_copy += [ - (os.path.abspath(workflow_file), workflow_basename), (os.path.join(script_dir, "future.py"), "future.py"), (os.path.join(script_dir, "ventis_context.py"), "ventis_context.py"), (os.path.join(script_dir, "deploy.py"), "deploy.py"), @@ -527,6 +546,7 @@ def generate_workflow_docker( _stub_destination(stub_file, stub_entrypoints or {}), ) ) + files_to_copy.append((os.path.abspath(stub_file), os.path.basename(stub_file))) # Copy gRPC generated stubs if they exist if os.path.isdir(grpc_stubs_dir): @@ -536,6 +556,14 @@ def generate_workflow_docker( _copy_files(output_dir, files_to_copy) + # Copy the workflow's own real file last, so it wins over any same-named stub + # copy above; inject a sys.path entry so its own sibling helpers still resolve. + _write_entrypoint_file( + os.path.abspath(workflow_file), + os.path.join(output_dir, workflow_basename), + project_dir, + ) + # ---- workflow_launcher.py -------------------------------------------- launcher = f"""import threading import time From cd3a5214099e3bdb4493f26820cafa684b8af2fd Mon Sep 17 00:00:00 2001 From: Saaketh Sodanapalli Date: Mon, 31 Aug 2026 13:58:37 -0700 Subject: [PATCH 13/13] cleaned up OTel Exporter --- .../portfolio/config/global_controller.yaml | 4 - tests/test_otel_exporter_fanout.py | 58 +-- tests/test_otel_exporter_fields.py | 36 +- ventis/OTLP_Exporter/DESIGN.md | 73 ++-- ventis/OTLP_Exporter/convert.py | 17 +- ventis/OTLP_Exporter/db.py | 61 +-- ventis/OTLP_Exporter/otel_exporter.py | 67 +-- ventis/controller/global_controller.py | 385 ++++++------------ 8 files changed, 216 insertions(+), 485 deletions(-) diff --git a/examples/portfolio/config/global_controller.yaml b/examples/portfolio/config/global_controller.yaml index edee85f..96f371c 100644 --- a/examples/portfolio/config/global_controller.yaml +++ b/examples/portfolio/config/global_controller.yaml @@ -91,10 +91,6 @@ otel: endpoint: ${GRAFANA_OTLP_ENDPOINT}/v1/traces headers: Authorization: Basic ${GRAFANA_OTLP_HEADERS} -# - name: langfuse -# protocol: http -# endpoint: ${LANGFUSE_BASE_URL}/api/public/otel/v1/traces -# headers: {} # Polling interval in seconds poll_interval: 5 diff --git a/tests/test_otel_exporter_fanout.py b/tests/test_otel_exporter_fanout.py index 0691d61..96ca1b5 100644 --- a/tests/test_otel_exporter_fanout.py +++ b/tests/test_otel_exporter_fanout.py @@ -110,22 +110,10 @@ def test_build_processors_constructs_mixed_exporters_with_explicit_args(self): ], ) - def test_build_processors_preserves_legacy_single_destination_fallback(self): - http_exporter = object() - processor = MagicMock(name="legacy_processor") - with patch.dict( - os.environ, - {"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, - clear=True, - ), patch.object( - otel_exporter, "HttpOTLPSpanExporter", return_value=http_exporter - ) as constructor, patch.object( - otel_exporter, "BatchSpanProcessor", return_value=processor - ): - result = otel_exporter._build_processors() - - self.assertEqual(result, [("legacy", processor)]) - constructor.assert_called_once_with() + def test_build_processors_raises_when_destinations_env_unset(self): + with patch.dict(os.environ, {}, clear=True): + with self.assertRaisesRegex(RuntimeError, "otel.destinations is required"): + otel_exporter._build_processors() def test_configured_destinations_rejects_malformed_empty_and_duplicate_values(self): invalid_values = [ @@ -184,46 +172,20 @@ def test_controller_expands_env_and_builds_langfuse_basic_auth(self): ) self.assertEqual(destination["headers"]["Authorization"], "Basic cHVibGljOnNlY3JldA==") - def test_controller_env_serializes_destinations_and_keeps_legacy_mapping(self): + def test_controller_env_serializes_destinations_only(self): # Importing the controller is intentionally local: this test remains # runnable in the exporter-only environment used by the focused suite. from ventis.controller.global_controller import GlobalController destinations = self._destination_config() - env = GlobalController._otel_exporter_env( - { - "protocol": "grpc", - "endpoint": "legacy.example:4317", - "headers": {"x-tenant": "demo"}, - "destinations": destinations, - } - ) - self.assertEqual(env["OTEL_EXPORTER_OTLP_PROTOCOL"], "grpc") - self.assertEqual(env["OTEL_EXPORTER_OTLP_ENDPOINT"], "legacy.example:4317") - self.assertEqual(env["OTEL_EXPORTER_OTLP_HEADERS"], "x-tenant=demo") + env = GlobalController._otel_exporter_env({"destinations": destinations}) + self.assertEqual(set(env), {otel_exporter.DESTINATIONS_ENV}) self.assertEqual(json.loads(env[otel_exporter.DESTINATIONS_ENV]), destinations) - def test_controller_rejects_invalid_destinations_before_starting_child(self): + def test_controller_env_is_none_when_otel_not_configured(self): from ventis.controller.global_controller import GlobalController - invalid_destinations = [ - [], - [{"name": "railway", "protocol": "grpc"}], - [ - {"name": "same", "protocol": "grpc", "endpoint": "one:4317"}, - { - "name": "same", - "protocol": "http/protobuf", - "endpoint": "https://two", - }, - ], - [{"name": "bad", "protocol": "smtp", "endpoint": "example"}], - ] - for destinations in invalid_destinations: - with self.subTest(destinations=destinations), self.assertRaises(ValueError): - GlobalController._otel_exporter_env( - {"destinations": destinations} - ) + self.assertIsNone(GlobalController._otel_exporter_env({})) def _insert_pending_row(self): conn = sqlite3.connect(self.db_path) @@ -258,7 +220,6 @@ def test_send_pending_delivers_the_same_span_to_every_processor(self): otel_exporter.db, "mark_sent" ) as mark_sent: otel_exporter._processors = [("railway", first), ("langfuse", second)] - otel_exporter._processor = None otel_exporter._send_pending() first.on_end.assert_called_once() @@ -275,7 +236,6 @@ def test_send_pending_attempts_remaining_processors_and_leaves_row_unsent_on_fai otel_exporter.db, "mark_sent" ) as mark_sent: otel_exporter._processors = [("railway", failed), ("langfuse", remaining)] - otel_exporter._processor = None otel_exporter._send_pending() failed.on_end.assert_called_once() diff --git a/tests/test_otel_exporter_fields.py b/tests/test_otel_exporter_fields.py index ff8cad6..b74177d 100644 --- a/tests/test_otel_exporter_fields.py +++ b/tests/test_otel_exporter_fields.py @@ -17,12 +17,7 @@ def setUp(self): def tearDown(self): os.unlink(self.db_path) - def test_init_db_migrates_existing_waiting_table(self): - with sqlite3.connect(self.db_path) as conn: - conn.execute( - "CREATE TABLE waiting (future_id TEXT PRIMARY KEY, session_id TEXT NOT NULL)" - ) - + def test_init_db_creates_waiting_table_with_full_schema(self): db.init_db(self.db_path) with sqlite3.connect(self.db_path) as conn: @@ -63,17 +58,8 @@ def test_fields_are_normalized_and_added_to_span(self): span.attributes["langfuse.observation.output"], row["output"] ) - def test_split_hash_result_is_loaded_for_finished_rows(self): - class SplitHashRedis: - def hget(self, key, field): - self.request = (key, field) - return '{"recommendation": "hold"}' - - def get(self, key): - return None - + def test_error_message_is_wired_from_redis_error_field(self): db.init_db(self.db_path) - redis = SplitHashRedis() raw = { "future_id": "11112222333344445555666677778888", "request_id": "88887777666655554444333322221111", @@ -83,16 +69,24 @@ def get(self, key): "result": "", "created_at": "1.0", "finished_at": "2.0", - "failed": "0", + "failed": "1", + "error": "agent exploded", } with patch.object(db.pricing, "compute_token_cost", return_value=0.0): - db.write_waiting_rows([raw], redis_client=redis, db_path=self.db_path) + db.write_waiting_rows([raw], db_path=self.db_path) with sqlite3.connect(self.db_path) as conn: - output = conn.execute("SELECT output FROM waiting").fetchone()[0] - self.assertEqual(redis.request, (f"future:{raw['future_id']}", "result")) - self.assertEqual(json.loads(output), {"recommendation": "hold"}) + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM waiting").fetchone() + + self.assertEqual(row["error_message"], "agent exploded") + + span = convert.waiting_row_to_span(row) + self.assertEqual(span.status.description, "agent exploded") + self.assertEqual( + span.events[0].attributes["exception.message"], "agent exploded" + ) if __name__ == "__main__": diff --git a/ventis/OTLP_Exporter/DESIGN.md b/ventis/OTLP_Exporter/DESIGN.md index d433fe0..ec2957d 100644 --- a/ventis/OTLP_Exporter/DESIGN.md +++ b/ventis/OTLP_Exporter/DESIGN.md @@ -27,26 +27,22 @@ Decisions (final status): isolation from GC's core polling/health loop and independent restart, at low added complexity since SQLite is already the entire hand-off boundary between the two. - **Config**: implemented via a new `otel:` section in `global_controller.yaml` - (`protocol`/`endpoint`/`headers`), *not* by making `otel_exporter.py` itself - config-aware. `GlobalController` translates that section into the OTel SDK's own - standard env vars (`OTEL_EXPORTER_OTLP_PROTOCOL`/`_ENDPOINT`/`_HEADERS`) and passes - them to the exporter subprocess via `ProcessSupervisor.register(..., env=...)`. The - exporter still just constructs `OTLPSpanExporter()` with no explicit args (endpoint - and headers are resolved by the SDK itself from those env vars, same as always) and - reads only `OTEL_EXPORTER_OTLP_PROTOCOL` directly, to pick the gRPC vs HTTP exporter - class — the one piece of protocol selection the plain SDK classes don't do on their - own. Deliberately vendor-neutral: no backend name (Postgres, Langfuse, or otherwise) - appears anywhere in `otel_exporter.py`; the destination is 100% deploy-time config, - set once in `global_controller.yaml` and never touched by app code again. The - originally-planned `database.url` repurposing (below, kept for history) was decided - against — env-var configuration is the SDK's own idiomatic mechanism, so no - exporter-side config plumbing was added, only a GC-side YAML→env-var translation. - The initial multi-destination extension uses one `otel.destinations` list and one - independent exporter/`BatchSpanProcessor` pair per destination. gRPC and HTTP - destinations may be mixed in the same list. The legacy single-destination fields - remain supported through the original standard-environment-variable path. - Configuration is read at exporter startup; changing it requires a - GlobalController/exporter restart. + holding a `destinations` list, *not* by making `otel_exporter.py` itself + config-aware. `GlobalController` serializes that list to JSON and passes it to the + exporter subprocess as a single `VENTIS_OTEL_DESTINATIONS` env var via + `ProcessSupervisor.register(..., env=...)`. The exporter builds one independent + exporter/`BatchSpanProcessor` pair per destination, picking the gRPC vs HTTP + exporter class from each destination's `protocol` field. gRPC and HTTP destinations + may be mixed in the same list. Deliberately vendor-neutral: no backend name + (Postgres, Langfuse, or otherwise) appears anywhere in `otel_exporter.py`; the + destination is 100% deploy-time config, set once in `global_controller.yaml` and + never touched by app code again. The originally-planned `database.url` repurposing + (below, kept for history) was decided against — env-var configuration is the SDK's + own idiomatic mechanism, so no exporter-side config plumbing was added, only a + GC-side YAML→env-var translation. If `otel.destinations` is absent, GlobalController + logs that no OTel metrics collection will happen and skips starting the exporter + subprocess entirely. Configuration is read at exporter startup; changing it requires + a GlobalController/exporter restart. - **Data source**: NOT `runtime_information` — a dedicated `waiting` table in its own SQLite file (`ventis/OTLP_Exporter/otel_queue.db`, see `db.py`), written by GC's existing `_poll_controllers` *alongside* (not instead of) the existing @@ -77,16 +73,19 @@ otel: - name: langfuse protocol: http endpoint: https://cloud.langfuse.com/api/public/otel/v1/traces - headers: {} # e.g. Authorization: "Basic " + headers: + Authorization: Basic ${LANGFUSE_OTLP_HEADERS} # deployer pre-encodes public:secret ``` -`GlobalController._otel_exporter_env()` translates each destination into the exporter -process's destination configuration and hands it to `ProcessSupervisor.register( +`GlobalController._otel_exporter_env()` translates the `destinations` list into +`VENTIS_OTEL_DESTINATIONS` and hands it to `ProcessSupervisor.register( "otel_exporter", ..., env=...)`, which supports an `env` param (merged on top of the -parent process's own environment, not a replacement). The legacy single-destination -`protocol`/`endpoint`/`headers` form remains valid and continues through the SDK's -standard OTLP environment variables. Omitting `otel:` entirely falls back to whatever -ambient env the exporter subprocess would otherwise inherit, same as before this -change. +parent process's own environment, not a replacement). If `otel.destinations` is +absent, `_otel_exporter_env()` returns `None` and `GlobalController.__init__` skips +registering the exporter subprocess entirely, logging that no OTel metrics +collection will happen. No shape +validation is duplicated on the GlobalController side (deliberately: keep this side +simple, `otel_exporter.py` itself validates destination shape at subprocess startup, +and raises if invoked directly without `VENTIS_OTEL_DESTINATIONS` set). `otel_exporter.py` parses the destination configuration at startup and constructs the appropriate OTLP exporter for each entry (gRPC or HTTP), passing that destination's @@ -167,21 +166,7 @@ Cleanup stays on its own thread (the event's `wait(timeout=cleanup_interval)` is fallback, not the primary trigger) so a slow/hung instance during cleanup can't stall the poll loop's health checks and OTel writes. -### 6. Parallelized per-instance polling (`ventis/controller/global_controller.py`) -`_poll_controllers` used to loop over every instance sequentially -- Redis reads, an -OTel sqlite write, and up to two Postgres writes per instance, one instance fully -blocking the next, with the following poll tick only starting after the whole pass -finished. Total metrics/telemetry latency scaled with instance count x round-trip -time, not the configured `poll_interval`. Fixed by extracting the per-instance body -into `_poll_one_instance` (its whole body wrapped in one top-level try/except, since -`ThreadPoolExecutor.map()` re-raises on first exception when results are consumed) -and running all instances concurrently via the same `ThreadPoolExecutor` pattern -`_trigger_cleanup` already used. Known, pre-existing, previously acknowledged in -commit `a6694d9`'s own message but never actually fixed (a same-named follow-up -branch was found to contain no real threading changes) -- see company-memory for -the investigation. - -### 7. Dependencies (all added) +### 6. Dependencies (all added) `opentelemetry-api`, `opentelemetry-sdk`, `opentelemetry-exporter-otlp-proto-grpc`, `opentelemetry-exporter-otlp-proto-http` (the last one added alongside the `otel:` config work, since `protocol: http` now needs that package importable). @@ -189,8 +174,6 @@ config work, since `protocol: http` now needs that package importable). ## Known gaps (not yet built) - `WriteResult()` passes an undefined `error_message` variable to its fan-out callback, which can interrupt remote consumer propagation after the callback hash is persisted. -- Redis records failure text under `error`, but the waiting-table writer reads - `error_name`/`error_message`, so exported exception details are usually empty. - Rows are marked `sent` immediately after `BatchSpanProcessor.on_end()` accepts them, before the asynchronous OTLP export is confirmed; a later delivery failure can lose a span while leaving `sent = 1`. diff --git a/ventis/OTLP_Exporter/convert.py b/ventis/OTLP_Exporter/convert.py index 66da972..5e6ac74 100644 --- a/ventis/OTLP_Exporter/convert.py +++ b/ventis/OTLP_Exporter/convert.py @@ -1,10 +1,7 @@ -"""Convert a `waiting` table row (see db.py) into an OTel ReadableSpan. +"""Converts a future into an OTel ReadableSpan. -Pure function, no I/O, no batching, no network calls. Builds ReadableSpan objects -directly instead of going through Tracer.start_span() -- there's no live tracer here, -futures already finished (sometimes in another process), so this is a historical-row -conversion, not live tracing. See ventis/OTLP_Exporter/DESIGN.md for why this deviates from -the SDK's usual advice against constructing ReadableSpan by hand. +Pure function, no I/O, no batching, no network calls. Futures already finished, so this is just a +conversion. """ from opentelemetry.sdk.trace import EXCEPTION_MESSAGE, EXCEPTION_TYPE, Event, ReadableSpan @@ -66,13 +63,7 @@ def waiting_row_to_span(row): # Model, token, agent, and cache-read usage use OTel GenAI semantic-convention # names (see https://opentelemetry.io/docs/specs/semconv/registry/attributes/gen-ai/). - # total_cost uses gen_ai.usage.cost, which isn't yet an official semconv attribute - # but is the name Langfuse's OTel ingestion actually reads (its JSON cost_details - # attribute is currently broken -- see langfuse/langfuse#11030). Observation - # input/output use Langfuse's documented JSON-string attributes. `errors` is named - # error_count, not "errors"/"error", to avoid colliding with OTel's reserved - # error.* namespace (error.type etc.), which describes a single error, not a - # count. The remaining Ventis-specific values (project_id, server/token cost + # total_cost uses gen_ai.usage.cost. The remaining Ventis-specific values (project_id, server/token cost # breakdown, cache_hit_ratio) have no GenAI or Langfuse equivalent, so they keep # plain names. attributes = { diff --git a/ventis/OTLP_Exporter/db.py b/ventis/OTLP_Exporter/db.py index e6a1834..005ba71 100644 --- a/ventis/OTLP_Exporter/db.py +++ b/ventis/OTLP_Exporter/db.py @@ -12,21 +12,18 @@ import os import sqlite3 -from ventis.controller.utils import pricing +from ventis.controller.utils import pricing +# Will need to eventually delete dependency on this and move to OTLP +# It is currently stored here for backcompat with the old telemetry collecting + DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "otel_queue.db") -# Demo-only multipliers for scaling displayed costs; not real recorded costs. Kept -# deliberately standalone/duplicated from telemetry_logging.py's identical constants -# (rather than importing them) so this module has no dependency on it -- keep these in -# sync by hand if the multipliers there ever change. +# Demo-only multipliers for scaling displayed costs, DELETE FOR MORE ACCURATE METRICS _TOKEN_COST_MULTIPLIER = 10000 _SERVER_COST_MULTIPLIER = 100000 -# Timestamps are stored as unix epoch seconds (matching the Redis future hash fields -# they're read from), not as SQLite datetime strings. Column set mirrors -# runtime_information 1:1 (see telemetry_logging.py) plus this pipeline's own additions -# (error_name/error_message/sent). +# Table schema _TABLE_COLUMNS = """ future_id TEXT PRIMARY KEY, parent_id TEXT, @@ -55,28 +52,14 @@ name TEXT, input TEXT, output TEXT, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, sent BOOLEAN DEFAULT 0 """ -_MIGRATION_COLUMNS = { - "name": "TEXT", - "input": "TEXT", - "output": "TEXT", -} - - def init_db(db_path=DB_PATH): - """Create the waiting table and add columns missing from older databases.""" + """Create the waiting table if it doesn't already exist.""" conn = sqlite3.connect(db_path) try: conn.execute(f"CREATE TABLE IF NOT EXISTS waiting ({_TABLE_COLUMNS})") - existing_columns = { - row[1] for row in conn.execute("PRAGMA table_info(waiting)").fetchall() - } - for column, column_type in _MIGRATION_COLUMNS.items(): - if column not in existing_columns: - conn.execute(f"ALTER TABLE waiting ADD COLUMN {column} {column_type}") conn.commit() finally: conn.close() @@ -147,20 +130,16 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH part for part in (service, method) if part ) result = raw.get("result") - # Compatibility with pre-consolidation deployments, where completion - # metrics live in future:{id}:metrics but result lives in future:{id}. - # Unified hashes already include result and avoid this extra read. - if not result and finished_at and redis_client is not None: - result = redis_client.hget(f"future:{fid}", "result") - - token_cost = ( - pricing.compute_token_cost( - raw.get("model"), input_token_count, output_token_count - ) - * _TOKEN_COST_MULTIPLIER - ) - # Server cost needs an elapsed duration -- only available once finished. + + # Cost figures are only meaningful once the future has finished, so skip + # computing them until then rather than recomputing on every poll. if finished_at is not None: + token_cost = ( + pricing.compute_token_cost( + raw.get("model"), input_token_count, output_token_count + ) + * _TOKEN_COST_MULTIPLIER + ) server_cost = ( pricing.compute_server_cost( redis_client.get(f"agent:{agent_id}:instance_type") @@ -171,6 +150,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH * _SERVER_COST_MULTIPLIER ) else: + token_cost = 0.0 server_cost = 0.0 conn.execute( @@ -203,7 +183,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH "cached_tokens": cached_tokens, "cache_hit_ratio": cached_tokens / token_count if token_count else 0.0, "error_name": raw.get("error_name"), - "error_message": raw.get("error_message"), + "error_message": raw.get("error") or raw.get("error_message"), "name": name or agent_id or "unknown_agent", "input": _normalize_json_text(raw.get("args")), "output": _normalize_json_text(result), @@ -215,10 +195,7 @@ def write_waiting_rows(rows, redis_client=None, project_id=None, db_path=DB_PATH def mark_sent(future_id, db_path=DB_PATH): - """Mark one waiting row sent. Call this immediately after successfully handing its - span to the batch processor -- one row, one commit -- so a crash between two rows' - sends can't leave an already-sent row unmarked (which would cause a duplicate send - on the next run).""" + """Mark one waiting row sent. Atomic Operation""" conn = sqlite3.connect(db_path) try: conn.execute("UPDATE waiting SET sent = 1 WHERE future_id = ?", (future_id,)) diff --git a/ventis/OTLP_Exporter/otel_exporter.py b/ventis/OTLP_Exporter/otel_exporter.py index cad2af8..eafb786 100644 --- a/ventis/OTLP_Exporter/otel_exporter.py +++ b/ventis/OTLP_Exporter/otel_exporter.py @@ -5,11 +5,8 @@ all processors accept it. Batching, OTLP serialization, and sending remain the SDK's responsibility (see DESIGN.md). -GlobalController may provide a JSON list in ``VENTIS_OTEL_DESTINATIONS``. That is a -Ventis-specific configuration because the standard OTEL exporter environment -variables describe only one destination. If it is absent, the original single -destination behavior is retained: the exporter class and its settings are selected -from the standard OTEL environment variables and SDK defaults. +GlobalController provides a JSON list in ``VENTIS_OTEL_DESTINATIONS``, required because +the standard OTEL exporter environment variables describe only one destination. """ import json @@ -35,33 +32,11 @@ logger = logging.getLogger(__name__) _running = True -_processor = None _processors = [] POLL_INTERVAL_SECONDS = 5 DESTINATIONS_ENV = "VENTIS_OTEL_DESTINATIONS" -def _normalize_protocol(protocol): - """Return the exporter family for a configured protocol name.""" - if not isinstance(protocol, str) or not protocol.strip(): - raise ValueError("destination protocol must be a non-empty string") - normalized = protocol.strip().lower().replace("_", "-") - if normalized in {"grpc", "otlp/grpc", "grpc/protobuf", "grpc-protobuf"}: - return "grpc" - if normalized in { - "http", - "http/protobuf", - "http-protobuf", - "http/proto", - "http+protobuf", - "protobuf", - }: - return "http" - raise ValueError( - f"unsupported destination protocol {protocol!r}; expected grpc or http/protobuf" - ) - - def _validate_destination(destination, index): if not isinstance(destination, dict): raise ValueError(f"destination {index} must be an object") @@ -70,7 +45,7 @@ def _validate_destination(destination, index): if not isinstance(name, str) or not name.strip(): raise ValueError(f"destination {index} name must be a non-empty string") - protocol = _normalize_protocol(destination.get("protocol")) + protocol = destination.get("protocol") # must be exactly "grpc" or "http" endpoint = destination.get("endpoint") if not isinstance(endpoint, str) or not endpoint.strip(): raise ValueError(f"destination {name!r} endpoint must be a non-empty string") @@ -112,13 +87,7 @@ def _validate_destination(destination, index): def _configured_destinations(): - """Parse and validate the Ventis multi-destination environment variable. - - ``None`` means no Ventis-specific configuration was supplied, so callers can - preserve legacy OTEL environment-variable behavior. An empty or malformed value - is an explicit configuration error and fails startup rather than silently - exporting to the wrong destination. - """ + """Parse and validate the Ventis multi-destination environment variable.""" raw = os.environ.get(DESTINATIONS_ENV) if raw is None: return None @@ -142,18 +111,15 @@ def _configured_destinations(): def _build_exporter(destination): - """Construct one explicitly configured exporter without logging credentials.""" + """Construct one OTLP exporter.""" kwargs = { "endpoint": destination["endpoint"], } - if destination["headers"] is not None: - kwargs["headers"] = destination["headers"] - if destination["timeout"] is not None: - kwargs["timeout"] = destination["timeout"] + if destination["headers"] is not None: kwargs["headers"] = destination["headers"] # fmt: skip + if destination["timeout"] is not None: kwargs["timeout"] = destination["timeout"] # fmt: skip if destination["protocol"] == "grpc": - if destination["insecure"] is not None: - kwargs["insecure"] = destination["insecure"] + if destination["insecure"] is not None: kwargs["insecure"] = destination["insecure"] # fmt: skip return GrpcOTLPSpanExporter(**kwargs) if destination["insecure"] is not None: @@ -166,14 +132,10 @@ def _build_exporter(destination): def _build_processors(): - """Build destination processors, or one legacy processor when unconfigured.""" + """Build one exporter/BatchSpanProcessor pair per configured destination.""" destinations = _configured_destinations() if destinations is None: - protocol = os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc").lower() - exporter_class = ( - HttpOTLPSpanExporter if protocol.startswith("http") else GrpcOTLPSpanExporter - ) - return [("legacy", BatchSpanProcessor(exporter_class(), schedule_delay_millis=1000))] + raise RuntimeError(f"{DESTINATIONS_ENV} is not set; otel.destinations is required") processors = [] try: @@ -205,10 +167,6 @@ def _handle_shutdown(signum, frame): def _send_pending(): """Convert and send each finished, not-yet-sent waiting row.""" processors = _processors - if not processors and _processor is not None: - # Compatibility for callers that configured the pre-fan-out singular - # ``_processor`` directly (the normal startup path always populates both). - processors = [("legacy", _processor)] if not processors: raise RuntimeError("OTel exporter has no configured processors") @@ -256,14 +214,11 @@ def _send_pending(): def main(): - global _processor, _processors + global _processors signal.signal(signal.SIGTERM, _handle_shutdown) signal.signal(signal.SIGINT, _handle_shutdown) db.init_db() _processors = _build_processors() - # Keep the old singular module variable available to integrations that imported - # it, while all sending uses the destination-aware collection above. - _processor = _processors[0][1] logger.info("OTel exporter process started with %d destination(s).", len(_processors)) try: last_poll = 0 diff --git a/ventis/controller/global_controller.py b/ventis/controller/global_controller.py index 6584ef8..ba53881 100644 --- a/ventis/controller/global_controller.py +++ b/ventis/controller/global_controller.py @@ -3,11 +3,8 @@ # Periodically polls Redis to check controller health and updates the routing table. import atexit -import base64 -import importlib.util import json import logging -import math import os import re import shlex @@ -19,6 +16,7 @@ from concurrent.futures import ThreadPoolExecutor import yaml +from ventis.OTLP_Exporter import db as otel_db from ventis.controller.instance_manager import InstanceManager from ventis.controller.utils.agent_specs import write_agent_specs from ventis.controller.utils.env_file import resolve_env_file @@ -110,7 +108,7 @@ def __init__(self, config_path): len(self.controllers), ) - # Start background cleanup thread, woken by each poll tick rather than its own timer -- see ventis/OTLP_Exporter/DESIGN.md. + # Start background cleanup thread self._cleanup_ready = threading.Event() self._cleanup_thread = threading.Thread(target=self._cleanup_loop, daemon=True) self._cleanup_thread.start() @@ -123,21 +121,19 @@ def __init__(self, config_path): ) otel_exporter_script = os.path.join(otel_exporter_dir, "otel_exporter.py") self.process_supervisor = ProcessSupervisor() - # Legacy `otel:` fields map straight to the OTel SDK's own standard env vars. - # A `destinations` list is additionally passed as one Ventis-specific JSON - # variable; the exporter subprocess remains a plain OTel process otherwise. + + # Passing OTel info from yaml file to process, so process doesn't have external facing logic otel_env = self._otel_exporter_env(self.config.get("otel", {})) - self.process_supervisor.register( - "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env - ) + if otel_env is not None: + self.process_supervisor.register( + "otel_exporter", [sys.executable, otel_exporter_script], env=otel_env + ) + else: + logger.info("otel.destinations not configured -- no OTel metrics collection will happen.") # Initialize/migrate the waiting table synchronously before either the GC or # exporter process can access it. - otel_db_spec = importlib.util.spec_from_file_location( - "otel_queue_db", os.path.join(otel_exporter_dir, "db.py") - ) - self._otel_db = importlib.util.module_from_spec(otel_db_spec) - otel_db_spec.loader.exec_module(self._otel_db) + self._otel_db = otel_db self._otel_db.init_db() self.process_supervisor.start_all() @@ -192,10 +188,7 @@ def _load_config(config_path): GlobalController._load_dotenv(os.path.join(project_root, ".env")) with open(config_path, "r") as f: config = yaml.safe_load(f) - if "ec2" in config: - config["ec2"] = GlobalController._expand_env_value(config["ec2"]) - if "database" in config: - config["database"] = GlobalController._expand_env_value(config["database"]) + config = GlobalController._expand_env_value(config) return config @staticmethod @@ -228,125 +221,21 @@ def _expand_env_value(value): @staticmethod def _otel_exporter_env(otel_cfg): - """Translate global_controller.yaml's `otel:` section into standard OTLP env - vars for the exporter subprocess. When present, ``destinations`` is passed as - JSON for the exporter to construct a fan-out. Returns None if `otel:` is - absent/empty so the subprocess falls back to the SDK's own defaults untouched. - - The legacy protocol/endpoint/headers mappings intentionally remain unchanged - for existing configurations. - """ - env = {} - if otel_cfg.get("protocol"): - env["OTEL_EXPORTER_OTLP_PROTOCOL"] = otel_cfg["protocol"] - if otel_cfg.get("endpoint"): - env["OTEL_EXPORTER_OTLP_ENDPOINT"] = otel_cfg["endpoint"] - if otel_cfg.get("headers"): - env["OTEL_EXPORTER_OTLP_HEADERS"] = ",".join( - f"{k}={v}" for k, v in otel_cfg["headers"].items() - ) - - if "destinations" in otel_cfg: - destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) - for destination in destinations: - if destination.get("name") == "langfuse": - public_key = os.environ.get("LANGFUSE_PUBLIC_KEY") - secret_key = os.environ.get("LANGFUSE_SECRET_KEY") - if public_key and secret_key: - auth = base64.b64encode(f"{public_key}:{secret_key}".encode()).decode() - destination.setdefault("headers", {})["Authorization"] = f"Basic {auth}" - GlobalController._validate_otel_destinations(destinations) - try: - env["VENTIS_OTEL_DESTINATIONS"] = json.dumps(destinations) - except (TypeError, ValueError) as exc: - # Do not include the offending value: destination configs commonly - # contain credentials in headers. - raise ValueError( - "otel.destinations must contain JSON-serializable values" - ) from exc - return env or None - - @staticmethod - def _validate_otel_destinations(destinations): - """Validate the shape of the optional exporter fan-out configuration. - - Keep this validation deliberately structural: destination-specific options - are interpreted by the exporter. Error messages identify only the location - and type, never destination contents or header values. + """Translate global_controller.yaml's `otel:` section into the exporter + subprocess's env. Returns None if `otel.destinations` is absent, so the + caller skips starting the exporter subprocess entirely. Destination + shape/protocol is validated by the exporter subprocess itself + (otel_exporter.py), not duplicated here. """ - if not isinstance(destinations, list): - raise ValueError("otel.destinations must be a list") - if not destinations: - raise ValueError("otel.destinations must not be empty") - - names = set() - for index, destination in enumerate(destinations): - if not isinstance(destination, dict): - raise ValueError( - f"otel.destinations[{index}] must be a mapping" - ) - - for field in ("name", "protocol", "endpoint"): - value = destination.get(field) - if not isinstance(value, str) or not value.strip(): - raise ValueError( - f"otel.destinations[{index}].{field} must be a non-empty string" - ) - - name = destination["name"].strip() - if name in names: - raise ValueError(f"otel.destinations contains duplicate name {name!r}") - names.add(name) - - protocol = destination["protocol"].strip().lower().replace("_", "-") - if protocol not in { - "grpc", - "otlp/grpc", - "grpc/protobuf", - "grpc-protobuf", - "http", - "http/protobuf", - "http-protobuf", - "http/proto", - "http+protobuf", - "protobuf", - }: - raise ValueError( - f"otel.destinations[{index}].protocol must be grpc or http/protobuf" - ) - - if "headers" in destination: - headers = destination["headers"] - if not isinstance(headers, dict): - raise ValueError( - f"otel.destinations[{index}].headers must be a mapping" - ) - if any( - not isinstance(key, str) or not isinstance(value, str) - for key, value in headers.items() - ): - raise ValueError( - f"otel.destinations[{index}].headers keys and values must be strings" - ) - - if "insecure" in destination and not isinstance( - destination["insecure"], bool - ): - raise ValueError( - f"otel.destinations[{index}].insecure must be a boolean" - ) - - if "timeout" in destination: - timeout = destination["timeout"] - if ( - isinstance(timeout, bool) - or not isinstance(timeout, (int, float)) - or not math.isfinite(timeout) - or timeout <= 0 - ): - raise ValueError( - f"otel.destinations[{index}].timeout must be a positive number" - ) + if "destinations" not in otel_cfg: + return None + destinations = GlobalController._expand_env_value(otel_cfg["destinations"]) + try: + return {"VENTIS_OTEL_DESTINATIONS": json.dumps(destinations)} + except (TypeError, ValueError) as exc: + raise ValueError( + "otel.destinations must contain JSON-serializable values" + ) from exc @staticmethod def _get_replica_placements(ctrl): @@ -641,138 +530,124 @@ def _poll_controllers(self): Check the health of each registered controller replica via its node's Redis. Also retrieves the request calls made in each instance. """ - # Guarded on self.running: a SIGTERM can interrupt mid-tick and run stop() (which - # terminates every managed process) via the signal handler before this line is - # reached -- without the guard, this could respawn a process just intentionally - # killed. See ventis/OTLP_Exporter/DESIGN.md. + # Prevents a process from restarting if a deliberate kill-cmd happens if self.running: self.process_supervisor.check_and_respawn() - # Polled in parallel, one instance's slow Redis/Postgres round-trip no longer - # gates every other instance's poll -- see ventis/OTLP_Exporter/DESIGN.md. - instances = self.instance_manager.list_instances() - if instances: - with ThreadPoolExecutor(max_workers=len(instances)) as executor: - list(executor.map(self._poll_one_instance, instances)) - - def _poll_one_instance(self, instance): - """Poll and persist one instance's runtime/metrics/health data; never raises.""" - try: + for instance in self.instance_manager.list_instances(): name = instance["agent_name"] host = instance["host"] port = instance["host_port"] node_redis = self._get_node_redis_for(host) - except Exception as e: - logger.warning("Failed to poll instance %s: %s", instance, e) - return - - try: - future_rows = pull_runtime_information(node_redis) - self._otel_db.write_waiting_rows( - future_rows, node_redis, self.config.get("project_id", 0) - ) - send_runtime_information( - future_rows, - node_redis, - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write runtime information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - agent_host = self._agent_host_key(host) - status_key = f"controller:{agent_host}:{port}:status" - metrics_key = f"controller:{agent_host}:{port}:metrics" + try: + future_rows = pull_runtime_information(node_redis) + self._otel_db.write_waiting_rows( + future_rows, node_redis, self.config.get("project_id", 0) + ) - # Getting metrics from local controllers - # See LocalController._execute_locally - try: - metrics = node_redis.hgetall(metrics_key) - if metrics: - now = time.time() - requests_served = int(float(metrics.get("requests_served") or 0)) - elapsed = now - self._last_metrics_poll_time.get( - (host, port), now - self.poll_interval + # This is now legacy, keeping it for now, but will remove this later + send_runtime_information( + future_rows, + node_redis, + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write runtime information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, ) - throughput = requests_served / elapsed if elapsed > 0 else 0.0 - self._last_metrics_poll_time[(host, port)] = now + agent_host = self._agent_host_key(host) + status_key = f"controller:{agent_host}:{port}:status" + metrics_key = f"controller:{agent_host}:{port}:metrics" - try: - send_agent_information( - [ - { - **instance, - **metrics, - "requests_served": requests_served, - "throughput": throughput, - } - ], - self.config.get("database", {}).get("url"), - ) - except Exception as e: - logger.warning( - "Failed to write agent information for instance %s (%s:%s) " - "(non-fatal): %s", - name, - host, - port, - e, - ) - else: - # Only clear the accumulated counters once they've actually been persisted - node_redis.hset_multiple( - metrics_key, - {"full_failures": 0, "error_count": 0, "requests_served": 0}, + # Getting metrics from local controllers + # See LocalController._execute_locally + try: + metrics = node_redis.hgetall(metrics_key) + if metrics: + now = time.time() + requests_served = int(float(metrics.get("requests_served") or 0)) + elapsed = now - self._last_metrics_poll_time.get( + (host, port), now - self.poll_interval ) - except Exception as e: - logger.warning( - "Failed to poll metrics for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + throughput = requests_served / elapsed if elapsed > 0 else 0.0 + self._last_metrics_poll_time[(host, port)] = now + + try: + send_agent_information( + [ + { + **instance, + **metrics, + "requests_served": requests_served, + "throughput": throughput, + } + ], + self.config.get("database", {}).get("url"), + ) + except Exception as e: + logger.warning( + "Failed to write agent information for instance %s (%s:%s) " + "(non-fatal): %s", + name, + host, + port, + e, + ) + else: + # Only clear the accumulated counters once they've actually been persisted + node_redis.hset_multiple( + metrics_key, + {"full_failures": 0, "error_count": 0, "requests_served": 0}, + ) + except Exception as e: + logger.warning( + "Failed to poll metrics for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) - try: - status = node_redis.get(status_key) or "unknown" - prev = self._last_status.get((host, port)) + try: + status = node_redis.get(status_key) or "unknown" + prev = self._last_status.get((host, port)) - if status != prev: - if status == "healthy": - logger.info( - "Controller %s (%s:%s) is now healthy.", name, host, port - ) - self._on_controller_healthy(name, host, port) - else: - logger.warning( - "Controller %s (%s:%s) status changed: %s -> %s", - name, - host, - port, - prev or "(none)", - status, - ) - self._on_controller_unhealthy(name, host, port) - self._last_status[(host, port)] = status - else: - # No change — healthy stays quiet, unhealthy stays quiet too - if status == "healthy": - self._on_controller_healthy(name, host, port) + if status != prev: + if status == "healthy": + logger.info( + "Controller %s (%s:%s) is now healthy.", name, host, port + ) + self._on_controller_healthy(name, host, port) + else: + logger.warning( + "Controller %s (%s:%s) status changed: %s -> %s", + name, + host, + port, + prev or "(none)", + status, + ) + self._on_controller_unhealthy(name, host, port) + self._last_status[(host, port)] = status else: - self._on_controller_unhealthy(name, host, port) - except Exception as e: - logger.warning( - "Failed to poll status for instance %s (%s:%s): %s", - name, - host, - port, - e, - ) + # No change — healthy stays quiet, unhealthy stays quiet too + if status == "healthy": + self._on_controller_healthy(name, host, port) + else: + self._on_controller_unhealthy(name, host, port) + except Exception as e: + logger.warning( + "Failed to poll status for instance %s (%s:%s): %s", + name, + host, + port, + e, + ) # ------------------------------------------------------------------ # # Extensibility hooks — override in subclasses #