From 7b5da97459a62c229c525d14c2d16fd9e329c006 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Tue, 1 Sep 2026 15:35:47 -0400 Subject: [PATCH 01/10] rev: add optional Langfuse invocation observability --- EXAMPLES.md | 37 ++++ README.md | 41 +++++ TCT/interfaces/cli.py | 6 +- TCT/interfaces/invocation.py | 46 ++++- TCT/interfaces/mcp.py | 9 +- TCT/interfaces/observability.py | 104 +++++++++++ pyproject.toml | 3 + tests/test_cli.py | 24 ++- tests/test_observability.py | 132 ++++++++++++++ tests/test_server.py | 1 + uv.lock | 294 +++++++++++++++++++++++++++++--- 11 files changed, 661 insertions(+), 36 deletions(-) create mode 100644 TCT/interfaces/observability.py create mode 100644 tests/test_observability.py diff --git a/EXAMPLES.md b/EXAMPLES.md index da6e65f..1f712eb 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -243,3 +243,40 @@ python main.py Existing Python imports from `TCT.server` also remain compatibility aliases for the MCP server and its registered tools. + +## Observe agent-facing calls with Langfuse + +Install observability separately from the core library. Include `mcp` when the +MCP server is needed: + +```bash +uv sync --extra langfuse --extra mcp +``` + +Set the standard Langfuse credentials, then use the same CLI and MCP commands: + +```bash +export LANGFUSE_PUBLIC_KEY=your-public-key +export LANGFUSE_SECRET_KEY=your-secret-key +export LANGFUSE_BASE_URL=https://cloud.langfuse.com +export LANGFUSE_TRACING_ENVIRONMENT=development + +uv run tct normalize-nodes --query CHEBI:15365 +uv run tct-server +``` + +Every call made through either adapter is represented as a Langfuse `tool` +observation. The observation includes the interface, tool name, bound input +arguments (including defaults), and a JSON-compatible successful result. No +per-tool decorator is needed because both adapters call the same invocation +function. + +To run normally without emitting observations while retaining credentials in +the environment: + +```bash +TCT_LANGFUSE_ENABLED=false uv run tct name-lookup --query aspirin +``` + +Direct developer calls such as `TCT.name_lookup(...)` do not cross the +agent-facing invocation boundary and are not traced by this integration. diff --git a/README.md b/README.md index 806ccb4..764383d 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,12 @@ To develop or run the MCP server from a source checkout: uv sync --extra mcp ``` +To observe CLI and MCP tool invocations with Langfuse: + +```bash +uv sync --extra mcp --extra langfuse +``` + ## Python, CLI, and MCP interfaces TCT exposes one curated set of well-documented operations through three @@ -179,6 +185,41 @@ parameters, and defaults shown by the CLI. A typical client configuration is: When running from a source checkout, run `uv sync --extra mcp` first and use `uv run tct-server`. +### Optional Langfuse observability + +The CLI and MCP adapters can create one Langfuse `tool` observation for each +call made through their shared invocation boundary. No TCT function is +decorated: direct Python library calls remain uninstrumented and importing TCT +does not require the Langfuse SDK. + +Install the optional extra and configure the standard Langfuse environment +variables: + +```bash +uv sync --extra mcp --extra langfuse + +export LANGFUSE_PUBLIC_KEY=your-public-key +export LANGFUSE_SECRET_KEY=your-secret-key +export LANGFUSE_BASE_URL=https://cloud.langfuse.com + +uv run tct name-lookup --query aspirin +uv run tct-server +``` + +Tracing turns on automatically when both keys are present. Set +`TCT_LANGFUSE_ENABLED=false` to disable it explicitly, or set it to `true` to +enable it when credentials are supplied by another Langfuse-supported +mechanism. `LANGFUSE_TRACING_ENVIRONMENT` can distinguish deployments such as +`ci`, `staging`, and `production` in Langfuse; it is independent of +`TCT_ENVIRONMENT`, which selects TCT service endpoints. + +Observations are named `tct.tool.`, tagged with the `cli` or `mcp` +interface, and include normalized arguments, defaults, and successful results. +The original exception crosses the observation boundary on failure before the +CLI or MCP adapter converts it to its stable interface error. Because this can +record biomedical queries and service responses, configure Langfuse according +to the data-handling requirements of the deployment. + ### Shared tool capabilities The table uses CLI kebab-case spellings; MCP publishes the corresponding diff --git a/TCT/interfaces/cli.py b/TCT/interfaces/cli.py index 3a8148f..8e60888 100644 --- a/TCT/interfaces/cli.py +++ b/TCT/interfaces/cli.py @@ -17,6 +17,7 @@ dumps_result, invoke, ) +from .observability import flush_observability class _StringOrListAction(argparse.Action): @@ -171,10 +172,13 @@ def main(argv: list[str] | None = None) -> int: tool = values.pop("_tool") command = values.pop("command") try: - result = invoke(tool, **values) + result = invoke(tool, _interface="cli", **values) except ToolInvocationError as error: print(f"{parser.prog}: {command}: {error}", file=sys.stderr) return 1 + finally: + # The CLI is short-lived, so ensure queued observations are delivered. + flush_observability() try: output = dumps_result(result) except ResultSerializationError as error: diff --git a/TCT/interfaces/invocation.py b/TCT/interfaces/invocation.py index 9f9b05d..fc880d3 100644 --- a/TCT/interfaces/invocation.py +++ b/TCT/interfaces/invocation.py @@ -2,6 +2,7 @@ from __future__ import annotations +import inspect import json import math from collections.abc import Callable, Mapping @@ -9,6 +10,8 @@ from enum import Enum from typing import Any +from .observability import observe_tool + class ToolInvocationError(RuntimeError): """Represent a shared tool failure before an interface translates it.""" @@ -27,10 +30,47 @@ def __init__(self, cause: Exception) -> None: super().__init__(str(cause)) -def invoke(tool: Callable[..., Any], /, *args: Any, **kwargs: Any) -> Any: - """Invoke a shared tool and normalize its ordinary failure boundary.""" +def _trace_value(value: Any) -> Any: + """Prepare trace data without letting conversion break a tool call.""" + try: + return to_jsonable(value) + except Exception as error: + return {"serialization_error": str(error), "value": repr(value)} + + +def _trace_input( + tool: Callable[..., Any], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Any: + bound = inspect.signature(tool).bind(*args, **kwargs) + bound.apply_defaults() + return _trace_value(dict(bound.arguments)) + + +def invoke( + tool: Callable[..., Any], + /, + *args: Any, + _interface: str | None = None, + **kwargs: Any, +) -> Any: + """Invoke and optionally observe a tool at an interface boundary.""" try: - return tool(*args, **kwargs) + metadata = { + "tct.interface": _interface or "shared", + "tct.module": tool.__module__, + "tct.tool": tool.__name__, + } + with observe_tool( + name=f"tct.tool.{tool.__name__}", + input_factory=lambda: _trace_input(tool, args, kwargs), + metadata=metadata, + ) as observation: + result = tool(*args, **kwargs) + if observation is not None: + observation.update(output=_trace_value(result)) + return result except ToolInvocationError: raise except Exception as error: diff --git a/TCT/interfaces/mcp.py b/TCT/interfaces/mcp.py index 7377849..d9716f5 100644 --- a/TCT/interfaces/mcp.py +++ b/TCT/interfaces/mcp.py @@ -18,6 +18,7 @@ from . import tools as shared_tools from .invocation import ToolInvocationError, invoke as invoke_tool +from .observability import flush_observability mcp = FastMCP("TCT") @@ -51,7 +52,7 @@ def _register_tool( @wraps(tool) def invoke(*args: Any, **kwargs: Any) -> Any: try: - return invoke_tool(tool, *args, **kwargs) + return invoke_tool(tool, *args, _interface="mcp", **kwargs) except ToolInvocationError as error: raise McpError( ErrorData( @@ -72,7 +73,11 @@ def invoke(*args: Any, **kwargs: Any) -> Any: def main() -> None: """Entry point for the installed ``tct-server`` command.""" - mcp.run() + try: + mcp.run() + finally: + # The SDK batches events while the long-running server is active. + flush_observability() __all__ = ["main", "mcp", *[tool.__name__ for tool in shared_tools.TOOLS]] diff --git a/TCT/interfaces/observability.py b/TCT/interfaces/observability.py new file mode 100644 index 0000000..4d91759 --- /dev/null +++ b/TCT/interfaces/observability.py @@ -0,0 +1,104 @@ +"""Optional observability for agent-facing TCT interfaces. + +This module deliberately imports Langfuse only when tracing is enabled. The +core library and its shared tool functions therefore remain independent of +the observability SDK. +""" + +from __future__ import annotations + +import importlib +import os +from collections.abc import Callable, Generator, Mapping +from contextlib import contextmanager +from typing import Any + + +_ENABLED_VARIABLE = "TCT_LANGFUSE_ENABLED" +_TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) +_FALSE_VALUES = frozenset({"0", "false", "no", "off"}) + + +class ObservabilityConfigurationError(RuntimeError): + """Report an invalid or incomplete optional observability setup.""" + + +def langfuse_enabled(environ: Mapping[str, str] | None = None) -> bool: + """Return whether Langfuse tracing is enabled for interface invocations. + + ``TCT_LANGFUSE_ENABLED`` takes precedence when set. Otherwise, tracing is + enabled automatically when both standard Langfuse API keys are present. + """ + variables = os.environ if environ is None else environ + configured = variables.get(_ENABLED_VARIABLE) + if configured is not None: + normalized = configured.strip().lower() + if normalized in _TRUE_VALUES: + return True + if normalized in _FALSE_VALUES: + return False + raise ObservabilityConfigurationError( + f"{_ENABLED_VARIABLE} must be one of: " + "1, true, yes, on, 0, false, no, off" + ) + return bool( + variables.get("LANGFUSE_PUBLIC_KEY") + and variables.get("LANGFUSE_SECRET_KEY") + ) + + +def _get_langfuse_client() -> Any | None: + if not langfuse_enabled(): + return None + try: + langfuse = importlib.import_module("langfuse") + except ModuleNotFoundError as error: + if error.name != "langfuse": + raise + raise ObservabilityConfigurationError( + "Langfuse tracing is enabled but its SDK is not installed; " + "install TCT with the 'langfuse' extra" + ) from error + return langfuse.get_client() + + +@contextmanager +def observe_tool( + *, + name: str, + input_factory: Callable[[], Any], + metadata: Mapping[str, Any], +) -> Generator[Any | None, None, None]: + """Open a Langfuse tool observation, or yield ``None`` when disabled.""" + client = _get_langfuse_client() + if client is None: + yield None + return + + with client.start_as_current_observation( + as_type="tool", + name=name, + input=input_factory(), + metadata=dict(metadata), + ) as observation: + yield observation + + +def flush_observability() -> None: + """Flush enabled tracing without importing Langfuse in untraced runs.""" + try: + client = _get_langfuse_client() + except ObservabilityConfigurationError: + # Invocation reports setup errors with the relevant tool context. A + # cleanup attempt must not replace that useful interface error. + return + if client is not None: + client.flush() + + +__all__ = [ + "ObservabilityConfigurationError", + "flush_observability", + "langfuse_enabled", + "observe_tool", +] diff --git a/pyproject.toml b/pyproject.toml index a9a4f55..9da61b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,9 @@ vision = [ mcp = [ "fastmcp>=2.12.2", ] +langfuse = [ + "langfuse>=3.63.0", +] [project.scripts] tct = "TCT.interfaces.cli:main" diff --git a/tests/test_cli.py b/tests/test_cli.py index 2f78673..e5e8e20 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,7 +2,6 @@ import argparse import json -from pathlib import Path import pytest @@ -110,6 +109,27 @@ def echo(message: str, repeat: int = 1) -> dict[str, list[str]]: assert json.loads(capsys.readouterr().out) == {"messages": ["hello", "hello"]} +def test_cli_attributes_invocation_and_flushes_observability(monkeypatch, capsys): + """Short-lived CLI calls identify their adapter and deliver queued spans.""" + calls = [] + + def example() -> str: + """Return an example.""" + return "unused" + + def fake_invoke(tool, **kwargs): + calls.append((tool, kwargs)) + return "observed" + + monkeypatch.setattr(tools, "TOOLS", (example,)) + monkeypatch.setattr(cli, "invoke", fake_invoke) + monkeypatch.setattr(cli, "flush_observability", lambda: calls.append("flush")) + + assert cli.main(["example"]) == 0 + assert json.loads(capsys.readouterr().out) == "observed" + assert calls == [(example, {"_interface": "cli"}), "flush"] + + def test_cli_reports_tool_failures_without_a_traceback(monkeypatch, capsys): """Invocation failures become concise CLI errors and a nonzero result.""" @@ -147,4 +167,4 @@ def invalid_result() -> object: assert captured.out == "" assert captured.err == ( "tct: invalid-result: could not serialize result: invalid result\n" - ) \ No newline at end of file + ) diff --git a/tests/test_observability.py b/tests/test_observability.py new file mode 100644 index 0000000..ac147c4 --- /dev/null +++ b/tests/test_observability.py @@ -0,0 +1,132 @@ +"""Tests for optional Langfuse instrumentation at interface boundaries.""" + +from contextlib import contextmanager + +import pytest + +from TCT.interfaces import invocation, observability +from TCT.interfaces.invocation import ToolInvocationError + + +def test_langfuse_activation_uses_standard_keys_and_explicit_override(): + """Credentials enable tracing unless the TCT override disables it.""" + credentials = { + "LANGFUSE_PUBLIC_KEY": "public", + "LANGFUSE_SECRET_KEY": "secret", + } + + assert observability.langfuse_enabled(credentials) is True + assert observability.langfuse_enabled( + {**credentials, "TCT_LANGFUSE_ENABLED": "false"} + ) is False + assert observability.langfuse_enabled({"TCT_LANGFUSE_ENABLED": "yes"}) is True + assert observability.langfuse_enabled({}) is False + + +def test_invalid_langfuse_activation_value_is_actionable(): + """Configuration mistakes fail with the relevant variable name.""" + with pytest.raises( + observability.ObservabilityConfigurationError, + match="TCT_LANGFUSE_ENABLED", + ): + observability.langfuse_enabled({"TCT_LANGFUSE_ENABLED": "perhaps"}) + + +def test_enabled_tracing_requires_only_the_optional_install(monkeypatch): + """A base installation imports normally and explains an enabled missing SDK.""" + monkeypatch.setenv("TCT_LANGFUSE_ENABLED", "true") + + def missing_langfuse(name): + raise ModuleNotFoundError(name="langfuse") + + monkeypatch.setattr(observability.importlib, "import_module", missing_langfuse) + + with pytest.raises( + observability.ObservabilityConfigurationError, + match="install TCT with the 'langfuse' extra", + ): + with observability.observe_tool( + name="tct.tool.example", + input_factory=dict, + metadata={}, + ): + pass + + +def test_invoke_records_tool_input_output_and_interface(monkeypatch): + """One boundary supplies Langfuse data for every registered callable.""" + captured = {} + + class Observation: + def update(self, **values): + captured["update"] = values + + @contextmanager + def fake_observe_tool(*, name, input_factory, metadata): + captured.update( + name=name, + input=input_factory(), + metadata=metadata, + ) + yield Observation() + + monkeypatch.setattr(invocation, "observe_tool", fake_observe_tool) + + def combine(left: str, right: str = "default") -> tuple[str, str]: + return left, right + + result = invocation.invoke(combine, "value", _interface="mcp") + + assert result == ("value", "default") + assert captured == { + "name": "tct.tool.combine", + "input": {"left": "value", "right": "default"}, + "metadata": { + "tct.interface": "mcp", + "tct.module": __name__, + "tct.tool": "combine", + }, + "update": {"output": ["value", "default"]}, + } + + +def test_tool_errors_cross_the_observation_before_normalization(monkeypatch): + """Langfuse sees the original exception while adapters keep stable errors.""" + captured = {} + + @contextmanager + def fake_observe_tool(**kwargs): + try: + yield object() + except Exception as error: + captured["error"] = error + raise + + monkeypatch.setattr(invocation, "observe_tool", fake_observe_tool) + cause = ValueError("failed") + + def fail() -> None: + raise cause + + with pytest.raises(ToolInvocationError) as error: + invocation.invoke(fail, _interface="cli") + + assert captured["error"] is cause + assert error.value.cause is cause + + +def test_disabled_observability_does_not_evaluate_trace_input(monkeypatch): + """Untraced core/interface calls avoid serialization work and SDK imports.""" + monkeypatch.delenv("TCT_LANGFUSE_ENABLED", raising=False) + monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) + monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) + + class Value: + def to_dict(self): + raise AssertionError("trace input should not be serialized") + + def identity(value): + return value + + value = Value() + assert invocation.invoke(identity, value) is value diff --git a/tests/test_server.py b/tests/test_server.py index 5ead3bf..836dafa 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -85,6 +85,7 @@ def fake_invoke(tool, *args, **kwargs): "query": "aspirin", "return_top_response": True, "return_synonyms": False, + "_interface": "mcp", }, ) ] diff --git a/uv.lock b/uv.lock index 317cb15..d1cc34f 100644 --- a/uv.lock +++ b/uv.lock @@ -62,7 +62,7 @@ resolution-markers = [ "python_full_version >= '3.14'", ] dependencies = [ - { name = "cffi", marker = "python_full_version >= '3.14'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/184b8ccce6683b0aa2fbb7ba5683ea4b9c5763f1356347f1312c32e3c66e/argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3", size = 1779911, upload-time = "2021-12-01T08:52:55.68Z" } wheels = [ @@ -88,7 +88,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "cffi", marker = "python_full_version < '3.14'" }, + { name = "cffi" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/2d/db8af0df73c1cf454f71b2bbe5e356b8c1f8041c979f505b3d3186e520a9/argon2_cffi_bindings-25.1.0.tar.gz", hash = "sha256:b957f3e6ea4d55d820e40ff76f450952807013d361a65d7f28acc0acbf29229d", size = 1783441, upload-time = "2025-07-30T10:02:05.147Z" } wheels = [ @@ -183,6 +183,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, ] +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001, upload-time = "2022-10-05T19:19:32.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148, upload-time = "2022-10-05T19:19:30.546Z" }, +] + [[package]] name = "beautifulsoup4" version = "4.15.0" @@ -390,7 +399,7 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/54/eb9bfc647b19f2009dd5c7f5ec51c4e6ca831725f1aea7a993034f483147/contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54", size = 13466130, upload-time = "2025-04-15T17:47:53.79Z" } wheels = [ @@ -462,7 +471,7 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -906,6 +915,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cf/58/8acf1b3e91c58313ce5cb67df61001fc9dcd21be4fadb76c1a2d540e09ed/fqdn-1.5.1-py3-none-any.whl", hash = "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014", size = 9121, upload-time = "2021-03-11T07:16:28.351Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/90/fb8f1c84537fbf210c1f53a53ae473a805f6599c5a40b93c1bbadd211f7a/googleapis_common_protos-1.75.2.tar.gz", hash = "sha256:8829a3d1e4508c5b7b9a6b9525f7fccff611f8531644579a76466c29295d4bb2", size = 154083, upload-time = "2026-08-25T19:19:13.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/5b/1c9e55363c3b1890a98cae813de5b4ea327845756cd8fb7ee690140c7eac/googleapis_common_protos-1.75.2-py3-none-any.whl", hash = "sha256:6b83302f554ea93a0f48409c7fc2050f954bcbcddb7e3a9c76d4a823cb22920e", size = 307002, upload-time = "2026-08-25T19:18:08.927Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1041,17 +1062,17 @@ resolution-markers = [ "python_full_version < '3.11'", ] dependencies = [ - { name = "colorama", marker = "python_full_version < '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.11'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, - { name = "jedi", marker = "python_full_version < '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.11'" }, - { name = "pexpect", marker = "python_full_version < '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.11'" }, - { name = "pygments", marker = "python_full_version < '3.11'" }, - { name = "stack-data", marker = "python_full_version < '3.11'" }, - { name = "traitlets", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/85/31/10ac88f3357fc276dc8a64e8880c82e80e7459326ae1d0a211b40abf6665/ipython-8.37.0.tar.gz", hash = "sha256:ca815841e1a41a1e6b73a0b08f3038af9b2252564d01fc405356d34033012216", size = 5606088, upload-time = "2025-05-31T16:39:09.613Z" } wheels = [ @@ -1068,17 +1089,17 @@ resolution-markers = [ "python_full_version == '3.11.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "pygments", marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments" }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6e/71/a86262bf5a68bf211bcc71fe302af7e05f18a2852fdc610a854d20d085e6/ipython-9.5.0.tar.gz", hash = "sha256:129c44b941fe6d9b82d36fc7a7c18127ddb1d6f02f78f867f402e2e3adde3113", size = 4389137, upload-time = "2025-08-29T12:15:21.519Z" } wheels = [ @@ -1090,7 +1111,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", marker = "python_full_version >= '3.11'" }, + { name = "pygments" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1566,6 +1587,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/e9/0d4add7873a73e462aeb45c036a2dead2562b825aa46ba326727b3f31016/kiwisolver-1.4.9-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:fb940820c63a9590d31d88b815e7a3aa5915cad3ce735ab45f0c730b39547de1", size = 73929, upload-time = "2025-08-10T21:27:48.236Z" }, ] +[[package]] +name = "langfuse" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backoff" }, + { name = "httpx" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "typing-extensions" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/51/ed5569bc2dcc8fe767e3b315207a7511ad23d00d811f02bbf0d6f80bf906/langfuse-4.15.1.tar.gz", hash = "sha256:70cb47529a6ba78383c4f2a197eb3d3ab9d39529c9e6b568d392150c1f40dbb9", size = 432353, upload-time = "2026-08-28T07:55:15.894Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/f9/7160bafcfe9797575359a9e77f810cf26f57f55a7c5ceda602fd9e353ddf/langfuse-4.15.1-py3-none-any.whl", hash = "sha256:795760693a62895157b6f5d6c80f1ad524fd12380995f06f82fc3a627b88c8d4", size = 789929, upload-time = "2026-08-28T07:55:14.034Z" }, +] + [[package]] name = "lark" version = "1.3.1" @@ -2214,6 +2255,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/dd/b3fd642260cb17532f66cc1e8250f3507d1e580483e209dc1e9d13bd980d/openapi_spec_validator-0.7.2-py3-none-any.whl", hash = "sha256:4bbdc0894ec85f1d1bea1d6d9c8b2c3c8d7ccaa13577ef40da9c006c9fd0eb60", size = 39713, upload-time = "2025-06-07T14:48:54.077Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "overrides" version = "7.7.0" @@ -2479,6 +2601,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, ] +[[package]] +name = "protobuf" +version = "7.36.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/86/73/f66c748df06e7fe24e658eddd600d19c4b40bad836c97ce2d0ad9851fb6b/protobuf-7.36.1.tar.gz", hash = "sha256:d0f6470f0ce2b84e3feaea2d4b816378b37ba4d4aa08a274305373de93e2d524", size = 512499, upload-time = "2026-08-31T22:40:04.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/6c/3a54a58f2948b0f485df9ecdd06590f15d0a7abf46a89d50c3de709ff4ff/protobuf-7.36.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:3cf2ee25d006cee57294a1196ea43b37feb78e0dcd1e8af5c1aeddb777655aca", size = 456046, upload-time = "2026-08-31T22:39:56.865Z" }, + { url = "https://files.pythonhosted.org/packages/6e/08/9f9548793c771095245c0eeaf0c84b76b16a5ef26158043d456f551344a0/protobuf-7.36.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:43d3d37b1eb24c113b9b7d02008cac44e423f00b611b7781ae998d7623972969", size = 344226, upload-time = "2026-08-31T22:39:58.263Z" }, + { url = "https://files.pythonhosted.org/packages/fe/51/1bdbd612fa3c51e42ea6f45d05e84dc748ddcd2663e1aa1e89a00a33facd/protobuf-7.36.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:39c518c05586c016d7874ff6079ee115bcec1ea5fbb1d177fbf7867ef4c67e44", size = 357229, upload-time = "2026-08-31T22:39:59.198Z" }, + { url = "https://files.pythonhosted.org/packages/22/df/c799fe7a05ef16ba853a59db01f3a2c5f7d0676469589ccc4874f76a2a88/protobuf-7.36.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:97198b77e369a0abd8e262b8f6c7266c55ddb796a3a12c76d7b8881188ed83aa", size = 343228, upload-time = "2026-08-31T22:40:00.179Z" }, + { url = "https://files.pythonhosted.org/packages/3d/33/d4724ec5d86d496fe4108e220618aa0837ad3423a7af7ccdd9684ccc77c8/protobuf-7.36.1-cp310-abi3-win32.whl", hash = "sha256:0b53ce95272aad50ad25d7ff03373743209822e8ba42ea7fad27d2bee1547d00", size = 443002, upload-time = "2026-08-31T22:40:01.32Z" }, + { url = "https://files.pythonhosted.org/packages/db/37/155788a0d8daded960375af604202805308169f9b859419ea0aa370946e2/protobuf-7.36.1-cp310-abi3-win_amd64.whl", hash = "sha256:51139351435d9b43d88a55eaa49fb6f737fbb478fb0cbf2cf694d1a04a9d3363", size = 456518, upload-time = "2026-08-31T22:40:02.494Z" }, + { url = "https://files.pythonhosted.org/packages/39/ca/c47f91d3cab175b01fd8c4f0d80fdf8613be876cc616e66ad281a59c5ddf/protobuf-7.36.1-py3-none-any.whl", hash = "sha256:7d951e46b3f963d6c264c367c437921de9d5aedd9c3f9612b9077736b4e3ad5c", size = 179813, upload-time = "2026-08-31T22:40:03.54Z" }, +] + [[package]] name = "psutil" version = "7.2.2" @@ -3299,6 +3436,9 @@ dependencies = [ ] [package.optional-dependencies] +langfuse = [ + { name = "langfuse" }, +] mcp = [ { name = "fastmcp" }, ] @@ -3327,6 +3467,7 @@ requires-dist = [ { name = "fastmcp", marker = "extra == 'mcp'", specifier = ">=2.12.2" }, { name = "igraph" }, { name = "ipycytoscape", marker = "extra == 'vision'" }, + { name = "langfuse", marker = "extra == 'langfuse'", specifier = ">=3.63.0" }, { name = "matplotlib", marker = "extra == 'vision'" }, { name = "networkx", marker = "extra == 'vision'" }, { name = "numpy" }, @@ -3337,7 +3478,7 @@ requires-dist = [ { name = "seaborn", marker = "extra == 'vision'" }, { name = "zstandard" }, ] -provides-extras = ["vision", "mcp"] +provides-extras = ["vision", "mcp", "langfuse"] [package.metadata.requires-dev] dev = [ @@ -3569,6 +3710,103 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/51/5447876806d1088a0f8f71e16542bf350918128d0a69437df26047c8e46f/widgetsnbextension-4.0.14-py3-none-any.whl", hash = "sha256:4875a9eaf72fbf5079dc372a51a9f268fc38d46f767cbf85c43a36da5cb9b575", size = 2196503, upload-time = "2025-04-10T13:01:23.086Z" }, ] +[[package]] +name = "wrapt" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/65/ba/8dc25478ed234dacc7d83c671634f347d0bdfb65bf0502f41879cf2f15a9/wrapt-2.4.0.tar.gz", hash = "sha256:7082fc1f94b020ac275870c4af71b09cff22876fe6e9c4c0ad01ea21d217b288", size = 161179, upload-time = "2026-08-30T04:41:51.424Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/51/8698f7646b9e6f4deca78995c59df25760c047eacfa595262205497b0b07/wrapt-2.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:643e45aa88698c8aae938c50e61940985d4ab9e53ea666d3e8e4eb86a4820d0f", size = 95416, upload-time = "2026-08-30T04:39:07.662Z" }, + { url = "https://files.pythonhosted.org/packages/c0/3c/a4620a03518eb133131aa54ae6973ba273a6f7244f2f771002be5f2db938/wrapt-2.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4597d19904b4aa97331d8bb651ac626d9397727e717942cf11bd7699ff97aa45", size = 95897, upload-time = "2026-08-30T04:39:09.455Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e6/ce8d8da15f825d55523ef1af03763de15d62c559c826516726fcafa072b6/wrapt-2.4.0-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:635cc171ddfd72edff10e295a02daa65edaa1c0ba619ad11eeed15cd2258c5df", size = 209578, upload-time = "2026-08-30T04:39:11.028Z" }, + { url = "https://files.pythonhosted.org/packages/a5/83/d5bf17b460fcf280f22ddfdfb8b674f5c213e46cad048d4bb2cfd43ce58d/wrapt-2.4.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:932a8892265df7b71257c30e5752635bc1f06a8c4e264024ff031bdf9bb10918", size = 212288, upload-time = "2026-08-30T04:39:12.601Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ec/bda24f3b18046274dde04feb56fd80ae1bf85f89110ddc86ad45d0dd2f06/wrapt-2.4.0-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5b8fff692f74782de89ba9d7b526a7cc398569b6a988ddc848159cc033c86237", size = 201612, upload-time = "2026-08-30T04:39:14.32Z" }, + { url = "https://files.pythonhosted.org/packages/a3/98/e8d30eaef0f531831c91e6ec6cf9c2c95ada3f8111463a393bf395445c78/wrapt-2.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a27d9653e0f88aa06598954337a545fc3f75bc811df897157a8614846d18d9c", size = 210648, upload-time = "2026-08-30T04:39:15.948Z" }, + { url = "https://files.pythonhosted.org/packages/45/9d/eb138df0a2d85953885a6b14768d3416e55f94728bb45b1fc7fa3dcde246/wrapt-2.4.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:9c884240e7415d3a384e70a15ceea0e884cc9289bcc254afd6412d4e7cf99c47", size = 199700, upload-time = "2026-08-30T04:39:17.697Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fe/38c3d26493869740aec6f4f5eebd51df0781ee703d554310e4e4d12ea2ce/wrapt-2.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37ba372e9ae71ec43e165b5db05f52e71f7c07dafb9d6a254ef7128112dce751", size = 199774, upload-time = "2026-08-30T04:39:19.16Z" }, + { url = "https://files.pythonhosted.org/packages/b0/07/820553cbd7a94abf1fb3d324f19c09e7bcd73d8f36fca6164f6997c9130c/wrapt-2.4.0-cp310-cp310-win32.whl", hash = "sha256:07daab5babb7edaf89413f5c8bd638474540fb2643b5dfb685bdc0680c96803a", size = 91196, upload-time = "2026-08-30T04:39:20.705Z" }, + { url = "https://files.pythonhosted.org/packages/a2/67/fab3ec749a0bb831ab0993d3eff2ca90032b858ab9e0c0b1932f663e7e43/wrapt-2.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:ef9797bf7c6f9ad9d294538c4f9a64ef3dbbadb63590a9a067393fd49ba28b0f", size = 96080, upload-time = "2026-08-30T04:39:22.274Z" }, + { url = "https://files.pythonhosted.org/packages/fb/05/4295a347b2f8772cead2b45f0d4049902242fcb46a5f4488c0e8b0951681/wrapt-2.4.0-cp310-cp310-win_arm64.whl", hash = "sha256:11ccb5f3de2047ef91408464abdc04682e40e7d7bc9614885d2abcaa7e2ef149", size = 92994, upload-time = "2026-08-30T04:39:23.63Z" }, + { url = "https://files.pythonhosted.org/packages/44/f0/f2f25fe8d516e63354ce4b027d4dc8d824bbf1f5f173f0bb83ce1bcbf706/wrapt-2.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a67ec80d15ac199d4a9a04a33f3039a1c219c9bf1c07b1b0422497613f167fb9", size = 95620, upload-time = "2026-08-30T04:39:25.116Z" }, + { url = "https://files.pythonhosted.org/packages/81/29/8e1d699fd15591e58f375e1eb5ce444aa955645edc53d09d86cf41d8aa2e/wrapt-2.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fc1b2cebd6d8db9b4ac0adc817c08b4901922e85604ae2a69aecb5217b2c09d8", size = 95795, upload-time = "2026-08-30T04:39:26.619Z" }, + { url = "https://files.pythonhosted.org/packages/ff/81/63c2fde1f11d008596ef86631afb37a8cb250eec62382003b7d12efd0071/wrapt-2.4.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e52c6a5be3284719e53b629ccfa565c146e604e861de35e861c94f7622806eb5", size = 217752, upload-time = "2026-08-30T04:39:28.301Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e9/373bc7c86eb41f6ab2e5608afac0bda11130b870c4dff4f4d1f25ffafe8a/wrapt-2.4.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9905bceb7b2833559518574ad6259d2ec9ffd111a0aa330ca685db74478e1ae3", size = 219872, upload-time = "2026-08-30T04:39:30.085Z" }, + { url = "https://files.pythonhosted.org/packages/49/95/a599d1095b6a271ef91ebb6852f7b4cfc7462d0aff7f4cc1fc3e6437193d/wrapt-2.4.0-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:abc347e92f9202c8ac1d5c1626a800fd5e56e13433f0651b26dddda5b421ac79", size = 205822, upload-time = "2026-08-30T04:39:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/71/59/14ea2e24c2546da9a08cf9da8fcb2a8ada40ddca0c9a4f26f6a559e49efb/wrapt-2.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:52f01626f1d2bc54585954cd8b4931f81003b0ac8dad61c741f43014bc9a0f0b", size = 217806, upload-time = "2026-08-30T04:39:33.671Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e8/ff294f964325a6451ff413aa918f5d55a197303b813d0ee0a16ecf3c9bd1/wrapt-2.4.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:811a36628d8b76724b980d508d576e5c5ecae1073b6ec4b4eb21646921906fe6", size = 203892, upload-time = "2026-08-30T04:39:35.103Z" }, + { url = "https://files.pythonhosted.org/packages/fe/45/34c1b0172f0c36305c26c2ffddc8f0bae7a43d78d6b32b00b1b043f77fec/wrapt-2.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b33df90f3d1e5b1c8811830b11a3e718b4f3a2823b748fa9be1688cb82b193f1", size = 207087, upload-time = "2026-08-30T04:39:36.55Z" }, + { url = "https://files.pythonhosted.org/packages/bc/a2/db3b55e29d04b685c761b1d6aca9f84a9c4f7e1c93543f23ba8a180a2a3f/wrapt-2.4.0-cp311-cp311-win32.whl", hash = "sha256:be535bdfbedda84cb8ebc6a80955dfd03d46840c13470486bd038f089e38b172", size = 91301, upload-time = "2026-08-30T04:39:38.114Z" }, + { url = "https://files.pythonhosted.org/packages/c6/55/c9fd1bf55e144082da6d62313d38f1449707bca16b76af4abbd5492f91e6/wrapt-2.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:a1117c63a39ba4d1b884e658089e512412d5174217ea1b4fe570977e42a5b129", size = 96308, upload-time = "2026-08-30T04:39:39.432Z" }, + { url = "https://files.pythonhosted.org/packages/6e/89/68d6c10590e74c496046f9fcbbb6ef80a2eca823f924305bf79acb65cccd/wrapt-2.4.0-cp311-cp311-win_arm64.whl", hash = "sha256:637fd6a18bb668a0c27b4767dcbc2fa93119c90da735bd2669fdde2d7b59fab3", size = 92839, upload-time = "2026-08-30T04:39:40.845Z" }, + { url = "https://files.pythonhosted.org/packages/f0/22/581a0b44349d5babe526c958f365b8126e0fbd8fc2810e80446c47358050/wrapt-2.4.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ef4e2d6e399ce6eecc80179a6b9ef6544f121288f95fc132bc36c9d9503903af", size = 96374, upload-time = "2026-08-30T04:39:42.335Z" }, + { url = "https://files.pythonhosted.org/packages/5d/90/095984648cec62a786bb27c0b50f6cfa5856d1e073ba1006fe148d190084/wrapt-2.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b9b32d5e4f0a179cef5075cc79b79d6d3482c44c434c12969e48c6719e06d95", size = 96178, upload-time = "2026-08-30T04:39:43.789Z" }, + { url = "https://files.pythonhosted.org/packages/f8/fd/b20e3cb3cab35131b515edf18e8cd777dff680fc76fc00919481f4e536af/wrapt-2.4.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d7dbbdbfdacb85c2d962fa52db791c77943fd777d600d74c95af2d53b32f5a94", size = 227806, upload-time = "2026-08-30T04:39:45.264Z" }, + { url = "https://files.pythonhosted.org/packages/08/75/c8dfba5e0caf17cd0718a0cbbe76cb85e637a2d65183fb728232419f6fca/wrapt-2.4.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:39cd68df4dff79f5336f9c745c06259d204bcb42d504040c9c91eac9e2abb39c", size = 229004, upload-time = "2026-08-30T04:39:47.068Z" }, + { url = "https://files.pythonhosted.org/packages/42/05/d4853fbd33e5860b10d5aec690f563547a92a82e61fb8bb2d4ece1ce3570/wrapt-2.4.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2a9f1a2f75bb95257cc5744e255e10a5a86e923f328b40ad3dbf9d8d03430013", size = 208934, upload-time = "2026-08-30T04:39:48.73Z" }, + { url = "https://files.pythonhosted.org/packages/a3/66/23d0e8de9b411fd198af5121627587563657370c8d509fbe5ea8adb3df79/wrapt-2.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8763ad01e3725b7751a4575f38bbcc19c0aa0822fec91c5c5bd21ce3ce7e1d2b", size = 225709, upload-time = "2026-08-30T04:39:50.287Z" }, + { url = "https://files.pythonhosted.org/packages/01/37/3b357bc90530d510ae59ae7ac48265c482ae899e47637ca4436645688b40/wrapt-2.4.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9125c6dbe8b88c00dd8ef4fc1e55757e8eb4720b6b2b2cc610a45bd32bd28c57", size = 207090, upload-time = "2026-08-30T04:39:51.78Z" }, + { url = "https://files.pythonhosted.org/packages/6d/0c/d8a5c6dbcc2d221308223bcea4130c6332454a855cb4dbd5dcb2360b13b2/wrapt-2.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:28f5de1526831b8f173889a436e289fe181ede8c66c9feb669d1aca8fd602eaf", size = 216269, upload-time = "2026-08-30T04:39:53.641Z" }, + { url = "https://files.pythonhosted.org/packages/92/93/cc9fc8fef1d3d25edaa1c2dc2337b556dc1d0613ddc1c4a6fe9ee08ad705/wrapt-2.4.0-cp312-cp312-win32.whl", hash = "sha256:a9ca1cdb3f7facb4990c7739ea5afbaceeb6728d066feedde03a4cfe83b29b03", size = 91187, upload-time = "2026-08-30T04:39:55.38Z" }, + { url = "https://files.pythonhosted.org/packages/ad/ec/a7b10705172bdb669b9687a8ff68bbe5f566437d2a49ad6d976af48b6d10/wrapt-2.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:8b464316489fb2fca0669ea0f8f07290054a0f26fc72982d3e4cf95469628ba9", size = 96423, upload-time = "2026-08-30T04:39:56.81Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/e838ac6463a1a1a1817b2f184ee2aa20c54692b80368c5063403c8d2461c/wrapt-2.4.0-cp312-cp312-win_arm64.whl", hash = "sha256:db1285071ea09a7767fac608e7b5c7b03c09833b06186875a359905fbc659d29", size = 93003, upload-time = "2026-08-30T04:39:58.237Z" }, + { url = "https://files.pythonhosted.org/packages/19/86/f9de4e11582ff96ad2199eeeceaa17faa27bbdc599243f520070c4f3de07/wrapt-2.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5c5c4c728cd22a36e4b8bb5df4a7d3bccaa865d27725b36eeb3b6f18fb2e1bc2", size = 96041, upload-time = "2026-08-30T04:39:59.575Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ab/1dbf50802bea3b46192fd0dc39bb0eb2e77a064c813b2bbd88d2888ad49f/wrapt-2.4.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7de5b8d94417e55c02be50cc226e0ae1209bbc73813bf691dff3979c94438115", size = 96269, upload-time = "2026-08-30T04:40:01.182Z" }, + { url = "https://files.pythonhosted.org/packages/cb/a3/a3b5cde1cd06e04b6e95134eb3187a0a7da607a530e7795b221d4e4fa819/wrapt-2.4.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6436e2bda993a3eb69a1b317fc831c8ebcafb5704c390859ebd49f81218c4bbb", size = 225787, upload-time = "2026-08-30T04:40:02.715Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f7/d100f6c348b7669f19119cf890dcd4764623e2233af065586d110e0cd99e/wrapt-2.4.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e084558fbd112d2e1e34b0f5c71e45a3405bdad51a17150368a959bcf6697964", size = 226649, upload-time = "2026-08-30T04:40:04.647Z" }, + { url = "https://files.pythonhosted.org/packages/52/c6/3af8df515d5d7e92306957536f3468c6bdfecbe3659f99dbf09a468c2c4c/wrapt-2.4.0-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e78c947e18fadfd690c9420c30a96d221feeb93fc8f1cc00509b370ac16c3114", size = 206760, upload-time = "2026-08-30T04:40:06.332Z" }, + { url = "https://files.pythonhosted.org/packages/b9/c5/40d355552bd3eb6c5186e26051c19b573d24d7896de42caa7937d6b5ca9f/wrapt-2.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:08d8378c4514ac8dcc0ace76044cf87a873e6a52b5e6109834c8fb9037f4441b", size = 223467, upload-time = "2026-08-30T04:40:07.829Z" }, + { url = "https://files.pythonhosted.org/packages/40/ab/d198eebdb39f0d7e182e771e590a36673489cd58cebdad8aa273dcf28e04/wrapt-2.4.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:93180c2199784dd6a1075b33f9ed636bd0966821edbece6b3d5379b1c4f0bb7d", size = 205358, upload-time = "2026-08-30T04:40:09.344Z" }, + { url = "https://files.pythonhosted.org/packages/1c/0e/974a60672ad507d39a3d8a1c6351ef37fe65b07240d000ceba5d2b83e9e9/wrapt-2.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3d5e5eb76fb87e62752af751d2dcd9d1cd986b12037d2e1363d109ba716029e8", size = 214654, upload-time = "2026-08-30T04:40:10.923Z" }, + { url = "https://files.pythonhosted.org/packages/cc/5a/8b2db70206db0a4246758e0472ce344cb9636217113ef70640fc8d2ce874/wrapt-2.4.0-cp313-cp313-win32.whl", hash = "sha256:49bb5a572469e0e18163a8ec2aa972135a0929899ecbe627665f274506e1b5b4", size = 91171, upload-time = "2026-08-30T04:40:12.895Z" }, + { url = "https://files.pythonhosted.org/packages/fe/1e/e782b511c680dbe7369c92e7d981484aacca0cda584da1f28a84cd9a8e1a/wrapt-2.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:b1737f46b1e4a81eb93500a7f2854319e1c7a86e8863fb050b7b4daadd5a4178", size = 96178, upload-time = "2026-08-30T04:40:14.336Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/095ba31123fa5dd482d6183c05200b061314aabbd5442c010aba4b03ff1c/wrapt-2.4.0-cp313-cp313-win_arm64.whl", hash = "sha256:f1e9e088094f4895f84ab043e7d59401df137d663efbf1e80c82144882960830", size = 92949, upload-time = "2026-08-30T04:40:15.935Z" }, + { url = "https://files.pythonhosted.org/packages/1f/dd/1f269e4daf0c992f675e1ca2de6b1683b761c6d0aeb6c7b4b412486823ea/wrapt-2.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:788e473d1a6786d29d577b1e2bd95e214c09cdafde84907c522c31069c9acfac", size = 96386, upload-time = "2026-08-30T04:40:17.584Z" }, + { url = "https://files.pythonhosted.org/packages/be/42/7ecef06d33c0121c68d66a8a695efe67ebaa57218c1c61c585eca2a6117a/wrapt-2.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:947bd4b3438167b3638bf5477cb83a068a586ffb6d331ac427f39839c2b93b3c", size = 96532, upload-time = "2026-08-30T04:40:19.116Z" }, + { url = "https://files.pythonhosted.org/packages/c2/e3/8fdc9eba0e6cbbfe8303e1e807d734691309a27970b2ea458d099f1a46b0/wrapt-2.4.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3a69161cae7f0dca44c89c1d14146b4a0508a0c3cad98b3f2db1f4e9016c94ba", size = 228775, upload-time = "2026-08-30T04:40:20.604Z" }, + { url = "https://files.pythonhosted.org/packages/f4/77/4ac5882abfb29bf9821c5fa5cf9f30241a194e0f47faa2682b9b29765278/wrapt-2.4.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0536f5d85ff6a157ebe7e0fe08c5479943742cf1ce59569075a66159efcbc495", size = 229029, upload-time = "2026-08-30T04:40:22.186Z" }, + { url = "https://files.pythonhosted.org/packages/c9/c5/8a3608311a02faf3e5c072da38d06a7c623150fc258e29f18fe377d91703/wrapt-2.4.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5f041ed6a4d571010944bd6cfad9072db463e1851877b6d3227467a44af37456", size = 210436, upload-time = "2026-08-30T04:40:23.953Z" }, + { url = "https://files.pythonhosted.org/packages/de/90/e0cbc43f435fd39df25460e9f173e7b96f3dac5c7f66be41c7227166f021/wrapt-2.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f7fed45dbadf5d98a52bfff9624d3cca00affeb9543d493c9632b7a53cdd35c9", size = 226586, upload-time = "2026-08-30T04:40:25.507Z" }, + { url = "https://files.pythonhosted.org/packages/81/6c/7e5f2143228635ec139ef6df733dc477049f7d96a0c49deb23944a73ed6a/wrapt-2.4.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5cc2e7c7b6032e11a2b367a9baadaf0c5241feff2d8205260d87f1aa6dbdf84b", size = 208880, upload-time = "2026-08-30T04:40:27.128Z" }, + { url = "https://files.pythonhosted.org/packages/10/16/1de84402bb7a0916e10739bf6586e031244172b299e87c8cff2a04baf9ff/wrapt-2.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:72826910a1cf5a081234720fd43011304b899acfee219af49148155b4d795533", size = 216689, upload-time = "2026-08-30T04:40:28.844Z" }, + { url = "https://files.pythonhosted.org/packages/20/19/cd6bd5050381a541b44be97c4e0994eed60c5f439f4314f95eb5777d6c1a/wrapt-2.4.0-cp314-cp314-win32.whl", hash = "sha256:0eca69c9e93518240abe8801fb9b2726116a6e48172e4564c2651a2e14521747", size = 91581, upload-time = "2026-08-30T04:40:30.592Z" }, + { url = "https://files.pythonhosted.org/packages/a4/f8/b642f3184619adde676ad449030bcbeae6cc78ea07a92f0b5fddeec4c4e6/wrapt-2.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:63b94f401d7ae3a9a3027472fd3a3ff38afd2ed293b2f0b3b84a6d133a9f99a3", size = 96510, upload-time = "2026-08-30T04:40:32.1Z" }, + { url = "https://files.pythonhosted.org/packages/4d/3b/3415a18b91221261eeac85bf8ee23dfb0e2a39d76b9703a797efca177439/wrapt-2.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:6b3e082d43f592fcd381aee46354a11ce887a813ce5bbcedd9766fd681723c09", size = 93648, upload-time = "2026-08-30T04:40:33.563Z" }, + { url = "https://files.pythonhosted.org/packages/ac/90/80cf6a09e9599a11249775928df9bb790b82471e4312b847a861ffb2c2ed/wrapt-2.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:09064c7be688c38c3ff125ce86bc26b69b5d78dd56062c3ddd9c814b2a25f1e1", size = 99615, upload-time = "2026-08-30T04:40:35.134Z" }, + { url = "https://files.pythonhosted.org/packages/b2/da/c1d3245abb911a42584f8f7e9781995bdc41345c7affba75cf7e376c85ac/wrapt-2.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:4f8ddff4bbb75916be36da5169b8b9d475b59a1bd24acdb7551bb2c71be9aaac", size = 100031, upload-time = "2026-08-30T04:40:36.641Z" }, + { url = "https://files.pythonhosted.org/packages/84/46/8ec4941d0abbb010df7caf0a34840ca0128177389843b0f5ef2f9ee48ac5/wrapt-2.4.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:e9f8017443595870aa31f46125553a5c55ce95a26a267b96261baee6ba566d83", size = 269389, upload-time = "2026-08-30T04:40:38.212Z" }, + { url = "https://files.pythonhosted.org/packages/14/b5/a0ae1b431cc1f49a545d32b8b678a5788c50583ecf0ecb85dc0c7f95b4f6/wrapt-2.4.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:328eb2d978ca3a6ae25f8d8fe560bf8f4bc9778b5932e7b142664eef05b92e8f", size = 281081, upload-time = "2026-08-30T04:40:40.045Z" }, + { url = "https://files.pythonhosted.org/packages/c7/24/dfaf53dd3bdb0703524a9367b48e2a64ea86433fcc854b5f14be6a8e0e39/wrapt-2.4.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7a057d376d994da6bd1bbf955ecfda699aa7353826f98847f5605e1801abdfd4", size = 249637, upload-time = "2026-08-30T04:40:41.657Z" }, + { url = "https://files.pythonhosted.org/packages/3e/27/bdd82044d7503c2bfa78afcc89881f82a1b82b5d2013aabab853d339ce2a/wrapt-2.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3367a5212212c9393e0d3ca6ae029b3a8fa40c5896e4a985d43fe8a4b8322f0d", size = 275322, upload-time = "2026-08-30T04:40:43.408Z" }, + { url = "https://files.pythonhosted.org/packages/c4/82/04f4228eb3fb348d660dd1ea7225e53665b1809df2273ff4861d4d33b741/wrapt-2.4.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c4fca1e63af6675af3df7cdfcd5a0c878b5e655c7e48611ced9dc8d62183a11d", size = 247292, upload-time = "2026-08-30T04:40:45.457Z" }, + { url = "https://files.pythonhosted.org/packages/a2/20/67b2968fa9200458446c51b36a435adb6906083428b70fafb4caf92d4dc2/wrapt-2.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:694005fdc3002ade0f21641408c588028abde03c85961f3ba7727d8bead3ed6b", size = 264586, upload-time = "2026-08-30T04:40:47.079Z" }, + { url = "https://files.pythonhosted.org/packages/d7/fd/0db9ba03e08a7663f52455e95520c723f567bc037bffc6699950fcc456c4/wrapt-2.4.0-cp314-cp314t-win32.whl", hash = "sha256:332d9bad7e9b718974bb2a576504c4956f45b4a0fcd7b3bb7827279167550464", size = 93752, upload-time = "2026-08-30T04:40:48.81Z" }, + { url = "https://files.pythonhosted.org/packages/3f/87/ced171220935c696b157207385fa6be5675558a74655479f071d95a00f1d/wrapt-2.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6d57264c9dfcf37d2bf0b0fbec68d0f6184fc5617267619ada04d03e8b0231f3", size = 99890, upload-time = "2026-08-30T04:40:50.407Z" }, + { url = "https://files.pythonhosted.org/packages/a3/af/4a10c9a6d3b7ae41f830978c28d33a59ceb29537bd6875d2abfe78db4b41/wrapt-2.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:f43af38a642c3d6062e9740d8f5cc0feb5dbe0da516702df892147393b8cb14d", size = 96033, upload-time = "2026-08-30T04:40:51.933Z" }, + { url = "https://files.pythonhosted.org/packages/a0/df/3a0b6225ab88bd47090df70391c059a3308057638f8fc0ae32e8ac9d1886/wrapt-2.4.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:430fde1a116df3ceb5c29035de1da6609b70e680d9b8ce3ee624422f3fe0978c", size = 96389, upload-time = "2026-08-30T04:40:53.555Z" }, + { url = "https://files.pythonhosted.org/packages/ea/6f/803b0d0e14de11781f0e938e6f7d6e29e79652139fe70d7513460357ac78/wrapt-2.4.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:7d28f8f35a02d49f75f57fa4e755db4ba33f65841c0de64cd65b253916f5bf06", size = 96557, upload-time = "2026-08-30T04:40:55.033Z" }, + { url = "https://files.pythonhosted.org/packages/aa/e8/46571e1218d0494604a7aadc4c898c738c4b179052327ee1e57e278cebd6/wrapt-2.4.0-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:efd9a4be6785295e471f71efdf5682bd11d5b822b9665e6e1b4844917cf2f7ac", size = 229230, upload-time = "2026-08-30T04:40:56.703Z" }, + { url = "https://files.pythonhosted.org/packages/78/2e/0cab15fcaec56096a5734feace3620bc01edc885653be04bd756f84a6784/wrapt-2.4.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:75529a2fb569a671cf162f762c1b576f569f571b55ec7f3481258ca842ba507f", size = 229444, upload-time = "2026-08-30T04:40:58.51Z" }, + { url = "https://files.pythonhosted.org/packages/e7/9e/a92c049371a2675f98a0381ab2951f984866d1ba4de0e0771d6a31fdaa2b/wrapt-2.4.0-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66e7512c0d324cc37bba1def2be1fc365cbb685d3aa393a8f6f4d2d00202881d", size = 212482, upload-time = "2026-08-30T04:41:00.224Z" }, + { url = "https://files.pythonhosted.org/packages/ee/3b/8b5b57d0ff24edcd3421dbaeb4e94c89be3616824e47708f4e13f25ae3d7/wrapt-2.4.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:5f3bdfc35c83b562fcaebc0f24593045e5ed9f3b633adafd35222718a0ec38fa", size = 227017, upload-time = "2026-08-30T04:41:01.918Z" }, + { url = "https://files.pythonhosted.org/packages/0e/20/124b40bfd9585848db5a5aa6741d0c8dbf378dd995c6c2d95f090d9cf540/wrapt-2.4.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:d5f45bead708e2c0014be5e98531ce7202916b098a208c7be83c6ceb0a2559fa", size = 210498, upload-time = "2026-08-30T04:41:03.617Z" }, + { url = "https://files.pythonhosted.org/packages/4b/bf/89db9d5a80a9f2af52b24bdfdb5392be80bc0f0fd39fc39d1aab72afd0bd/wrapt-2.4.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:d294576fddac636589e4deccfe782e8f429da10f167c1985c4d51071de3672b7", size = 217046, upload-time = "2026-08-30T04:41:05.473Z" }, + { url = "https://files.pythonhosted.org/packages/3b/0b/021c9d6ce64c639894bffdaa7a895ddd4187abfefb2873ce55e536cd9d56/wrapt-2.4.0-cp315-cp315-win32.whl", hash = "sha256:0191d717dfbb8e519e7bfd4775e5b9bd57e359b3a09ab5db1ea47f6025b4d845", size = 91591, upload-time = "2026-08-30T04:41:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6ebd944041cea0ac4a108a4739510ed2dc891a3f3216e4f7bf0650f5b5a6/wrapt-2.4.0-cp315-cp315-win_amd64.whl", hash = "sha256:e8df31a126a0a247c1aa379e30873839de03912dea09ca360c680f3625d815df", size = 96517, upload-time = "2026-08-30T04:41:08.671Z" }, + { url = "https://files.pythonhosted.org/packages/96/84/7c5e52e450f80ba76fd0282dccf7c79cd004ebd8ccabd0903064d3d2c56e/wrapt-2.4.0-cp315-cp315-win_arm64.whl", hash = "sha256:e9e7e94472f0e3f1447caf27e1939eb384d0e87972a35a05f5c2e0968e9c01af", size = 93652, upload-time = "2026-08-30T04:41:10.258Z" }, + { url = "https://files.pythonhosted.org/packages/35/89/f08ff45d7646de29750932805cc3b1e86b6ac3128015b293ed45fa8efe86/wrapt-2.4.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:8828369b7d3e93c547cc8ad931b5a57b4e8d174035c82762fb1091e7d05ac9f5", size = 99610, upload-time = "2026-08-30T04:41:11.933Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c2/f9a3c40901a36c6bb7ecaff8e1e54af78fa7fa0b95a0e54d13d3a24c8a0a/wrapt-2.4.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:413e757dce7a43fcda8bb8441994b1127492ffac6a5803af777d44516df8c6e2", size = 100064, upload-time = "2026-08-30T04:41:13.492Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e1/e2437f17f2a1ec292056e2fcafe1248269ebc39502f2ffe79424bf86f8a6/wrapt-2.4.0-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:75944792cf6b99262d649d55710bf5901f7013fbb212c7a1d736b97a20517607", size = 269421, upload-time = "2026-08-30T04:41:15.238Z" }, + { url = "https://files.pythonhosted.org/packages/c6/d0/c98d6548dc4c7d12ab9baa192234ca1a57e141afd283252b448faddbd9ef/wrapt-2.4.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:648d1d4f94e8a0a1656675c755f40d2f0ee5fe92c449ab45326f4ecc2738cbe8", size = 281452, upload-time = "2026-08-30T04:41:16.939Z" }, + { url = "https://files.pythonhosted.org/packages/a3/57/673168e00aa03725148ce621ed201b75df4e787a57acd48fecefd2725600/wrapt-2.4.0-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a112a1bfdd2621e4344cb0a32dbaab80636b32dac1b055d03fbb2a67d806d1db", size = 250358, upload-time = "2026-08-30T04:41:18.716Z" }, + { url = "https://files.pythonhosted.org/packages/78/0b/f2e576de5bf53ef5b578470104ea93f33e273a704c825131bc1719fffc42/wrapt-2.4.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0972cd025f4c86fa2d8abd953d9f875779935343af58b4ce019ff89573fc65bd", size = 275654, upload-time = "2026-08-30T04:41:20.418Z" }, + { url = "https://files.pythonhosted.org/packages/33/7f/9347b2e236346b1ba4cb28b82b205b8a377bb2da9417cb81bbe3d25816d7/wrapt-2.4.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:c246aaed719dcdb62eeb7b8d9306a6237777226ef3baad35919c4ae134c91ce7", size = 248662, upload-time = "2026-08-30T04:41:22.371Z" }, + { url = "https://files.pythonhosted.org/packages/a5/36/3b84d9e1ac8393bf2c94272760a2d361dc394ac30301e6d6dbd6583ade2d/wrapt-2.4.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:1656de3835f760781c9b974bce07d8c04edb9c9ad7ad67264aee69cd68a1db09", size = 264813, upload-time = "2026-08-30T04:41:24.116Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a2/de7b1de1702667b4a048318e301e26887268c17b07c8b9797cea06b10aee/wrapt-2.4.0-cp315-cp315t-win32.whl", hash = "sha256:d8e6e1e5dc684dfce7c33fc8b67a08ba2af94f3a45cfc70d5c1d6a839d2caf97", size = 93753, upload-time = "2026-08-30T04:41:25.793Z" }, + { url = "https://files.pythonhosted.org/packages/09/50/4e7ef58c4eb058861ceddc0d1f94a6ed87f62e1cb27783c60b2897ef7e58/wrapt-2.4.0-cp315-cp315t-win_amd64.whl", hash = "sha256:85ed3c67fd39e8d9a36c224758cb6f2f4eb277d07ea677930caa0008c18ec002", size = 99888, upload-time = "2026-08-30T04:41:27.305Z" }, + { url = "https://files.pythonhosted.org/packages/68/64/d15740c763dd0ddea2338ad42e3bd4a84f8702e16083e7ff61674c504a13/wrapt-2.4.0-cp315-cp315t-win_arm64.whl", hash = "sha256:36b56a4fba13b34ed8ff307557325fff215de0a58b5dbaef2c50e4d8aa39dbd1", size = 96039, upload-time = "2026-08-30T04:41:29.062Z" }, + { url = "https://files.pythonhosted.org/packages/79/c8/fafe0002f572ced999c792cfe8b05d39269c63d8193d15d25bd828bcad7a/wrapt-2.4.0-py3-none-any.whl", hash = "sha256:18aabd9301d06026f5900538051773d6f87f65ae02cdc60de482df978513dc0a", size = 73713, upload-time = "2026-08-30T04:41:49.805Z" }, +] + [[package]] name = "zstandard" version = "0.25.0" From 5bd1b4a161eca7afa5b4e74a534ef047472d9aeb Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Tue, 1 Sep 2026 15:37:53 -0400 Subject: [PATCH 02/10] rev: default Langfuse observability to disabled --- EXAMPLES.md | 4 +++- README.md | 9 +++++---- TCT/interfaces/observability.py | 10 ++++------ tests/test_observability.py | 6 +++--- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/EXAMPLES.md b/EXAMPLES.md index 1f712eb..1cea81c 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -260,12 +260,14 @@ export LANGFUSE_PUBLIC_KEY=your-public-key export LANGFUSE_SECRET_KEY=your-secret-key export LANGFUSE_BASE_URL=https://cloud.langfuse.com export LANGFUSE_TRACING_ENVIRONMENT=development +export TCT_LANGFUSE_ENABLED=true uv run tct normalize-nodes --query CHEBI:15365 uv run tct-server ``` -Every call made through either adapter is represented as a Langfuse `tool` +Langfuse remains disabled unless `TCT_LANGFUSE_ENABLED` is explicitly true. +When enabled, every call made through either adapter is represented as a Langfuse `tool` observation. The observation includes the interface, tool name, bound input arguments (including defaults), and a JSON-compatible successful result. No per-tool decorator is needed because both adapters call the same invocation diff --git a/README.md b/README.md index 764383d..97eac0b 100644 --- a/README.md +++ b/README.md @@ -201,15 +201,16 @@ uv sync --extra mcp --extra langfuse export LANGFUSE_PUBLIC_KEY=your-public-key export LANGFUSE_SECRET_KEY=your-secret-key export LANGFUSE_BASE_URL=https://cloud.langfuse.com +export TCT_LANGFUSE_ENABLED=true uv run tct name-lookup --query aspirin uv run tct-server ``` -Tracing turns on automatically when both keys are present. Set -`TCT_LANGFUSE_ENABLED=false` to disable it explicitly, or set it to `true` to -enable it when credentials are supplied by another Langfuse-supported -mechanism. `LANGFUSE_TRACING_ENVIRONMENT` can distinguish deployments such as +Tracing is disabled by default, even when Langfuse credentials are present. +Set `TCT_LANGFUSE_ENABLED=true` to opt in. Accepted true values are `1`, +`true`, `yes`, and `on`; accepted false values are `0`, `false`, `no`, and +`off`. `LANGFUSE_TRACING_ENVIRONMENT` can distinguish deployments such as `ci`, `staging`, and `production` in Langfuse; it is independent of `TCT_ENVIRONMENT`, which selects TCT service endpoints. diff --git a/TCT/interfaces/observability.py b/TCT/interfaces/observability.py index 4d91759..0e26a32 100644 --- a/TCT/interfaces/observability.py +++ b/TCT/interfaces/observability.py @@ -26,8 +26,9 @@ class ObservabilityConfigurationError(RuntimeError): def langfuse_enabled(environ: Mapping[str, str] | None = None) -> bool: """Return whether Langfuse tracing is enabled for interface invocations. - ``TCT_LANGFUSE_ENABLED`` takes precedence when set. Otherwise, tracing is - enabled automatically when both standard Langfuse API keys are present. + Tracing is disabled by default and requires ``TCT_LANGFUSE_ENABLED`` to be + set to an accepted true value. Langfuse credentials alone never activate + instrumentation. """ variables = os.environ if environ is None else environ configured = variables.get(_ENABLED_VARIABLE) @@ -41,10 +42,7 @@ def langfuse_enabled(environ: Mapping[str, str] | None = None) -> bool: f"{_ENABLED_VARIABLE} must be one of: " "1, true, yes, on, 0, false, no, off" ) - return bool( - variables.get("LANGFUSE_PUBLIC_KEY") - and variables.get("LANGFUSE_SECRET_KEY") - ) + return False def _get_langfuse_client() -> Any | None: diff --git a/tests/test_observability.py b/tests/test_observability.py index ac147c4..2c29dcc 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -8,14 +8,14 @@ from TCT.interfaces.invocation import ToolInvocationError -def test_langfuse_activation_uses_standard_keys_and_explicit_override(): - """Credentials enable tracing unless the TCT override disables it.""" +def test_langfuse_activation_is_explicit_and_false_by_default(): + """Credentials alone do not trace; the TCT switch is required.""" credentials = { "LANGFUSE_PUBLIC_KEY": "public", "LANGFUSE_SECRET_KEY": "secret", } - assert observability.langfuse_enabled(credentials) is True + assert observability.langfuse_enabled(credentials) is False assert observability.langfuse_enabled( {**credentials, "TCT_LANGFUSE_ENABLED": "false"} ) is False From 60123ab25e8706608b2bf87a832d2755d0bd76dd Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Tue, 1 Sep 2026 15:42:59 -0400 Subject: [PATCH 03/10] rev: document Langfuse interface observability --- TCT/interfaces/LANGFUSE.md | 167 +++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 TCT/interfaces/LANGFUSE.md diff --git a/TCT/interfaces/LANGFUSE.md b/TCT/interfaces/LANGFUSE.md new file mode 100644 index 0000000..1a1d4bd --- /dev/null +++ b/TCT/interfaces/LANGFUSE.md @@ -0,0 +1,167 @@ +# Langfuse observability for TCT interfaces + +TCT can send CLI and MCP tool invocations to Langfuse without decorating the +individual functions in `tools.py`. Instrumentation lives at the shared +interface invocation boundary: + +```text +CLI ─┐ + ├─> invocation.invoke() ─> TCT tool +MCP ─┘ │ + └─> optional Langfuse tool observation +``` + +Direct calls to the Python library do not cross this boundary and are not +observed. This keeps agent-facing observability separate from the core API +used by application and notebook developers. + +## Default behavior + +Langfuse is **disabled by default**. Installing the SDK or setting Langfuse +credentials does not enable it. TCT starts observations only when +`TCT_LANGFUSE_ENABLED` has an accepted true value. + +| Variable | Required | Purpose | +| --- | --- | --- | +| `TCT_LANGFUSE_ENABLED` | Yes | Explicitly enables TCT instrumentation. Accepts `1`, `true`, `yes`, or `on`; matching false values disable it. | +| `LANGFUSE_PUBLIC_KEY` | Yes for normal SDK authentication | Langfuse project public key. | +| `LANGFUSE_SECRET_KEY` | Yes for normal SDK authentication | Langfuse project secret key. | +| `LANGFUSE_BASE_URL` | For self-hosted Langfuse | Langfuse API base URL; otherwise the SDK default applies. | +| `LANGFUSE_TRACING_ENVIRONMENT` | No | Labels traces by deployment environment. | + +`LANGFUSE_TRACING_ENVIRONMENT` is separate from `TCT_ENVIRONMENT`. +`TCT_ENVIRONMENT` selects Translator service endpoints; it does not enable or +configure Langfuse. + +## Install + +Install only the capabilities required by the process: + +```bash +# CLI observability +pip install 'TCT[langfuse]' + +# MCP server and observability +pip install 'TCT[mcp,langfuse]' +``` + +From a source checkout with UV: + +```bash +uv sync --extra langfuse +uv sync --extra mcp --extra langfuse +``` + +The Langfuse package is imported lazily. A normal TCT installation does not +need the SDK. If instrumentation is explicitly enabled without the optional +package, the CLI or MCP call reports that the `langfuse` extra must be +installed. + +## Configure and run the CLI + +Set credentials through the process environment and opt in explicitly: + +```bash +export LANGFUSE_PUBLIC_KEY=your-public-key +export LANGFUSE_SECRET_KEY=your-secret-key +export LANGFUSE_BASE_URL=https://cloud.langfuse.com +export LANGFUSE_TRACING_ENVIRONMENT=development +export TCT_LANGFUSE_ENABLED=true + +uv run tct name-lookup --query aspirin +``` + +The CLI flushes pending Langfuse events before it exits. To disable tracing +while leaving credentials available: + +```bash +TCT_LANGFUSE_ENABLED=false uv run tct name-lookup --query aspirin +``` + +## Configure and run the MCP server + +The existing MCP entry point does not change: + +```bash +export LANGFUSE_PUBLIC_KEY=your-public-key +export LANGFUSE_SECRET_KEY=your-secret-key +export TCT_LANGFUSE_ENABLED=true + +uv run tct-server +``` + +An MCP client may pass the same variables to the server process. The exact +configuration shape depends on the client; a typical stdio configuration is: + +```json +{ + "mcpServers": { + "tct": { + "command": "uv", + "args": ["run", "tct-server"], + "cwd": "/absolute/path/to/Translator_component_toolkit", + "env": { + "TCT_LANGFUSE_ENABLED": "true", + "LANGFUSE_PUBLIC_KEY": "your-public-key", + "LANGFUSE_SECRET_KEY": "your-secret-key" + } + } + } +} +``` + +Do not commit real credentials in an MCP client configuration. Prefer the +client's secret storage or inherited process environment. The server batches +events while running and flushes them during normal shutdown. + +## Observation contract + +Each observed invocation uses the Langfuse observation type `tool` and the +name `tct.tool.`. For example, `name_lookup` appears as +`tct.tool.name_lookup`. + +TCT attaches the following metadata: + +| Metadata | Value | +| --- | --- | +| `tct.interface` | `cli` or `mcp` | +| `tct.module` | Python module containing the shared callable | +| `tct.tool` | Python callable name | + +Inputs are bound against the Python signature, so the observation includes +applied default values as well as arguments supplied by the caller. Successful +outputs are converted to JSON-compatible values using the same normalization +conventions as CLI results. On failure, the original exception crosses the +Langfuse context before TCT converts it to its stable CLI or MCP error. + +## Test the integration + +Run the isolated tests, which use fakes and do not send data to Langfuse: + +```bash +uv run pytest \ + tests/test_observability.py \ + tests/test_invocation.py \ + tests/test_cli.py \ + tests/test_server.py +``` + +Verify strict opt-in behavior directly: + +```bash +LANGFUSE_PUBLIC_KEY=present \ +LANGFUSE_SECRET_KEY=present \ +uv run python -c \ + 'from TCT.interfaces.observability import langfuse_enabled; assert not langfuse_enabled()' +``` + +For an end-to-end check, supply credentials, set +`TCT_LANGFUSE_ENABLED=true`, invoke a CLI command or an MCP tool, and look for +`tct.tool.` in the configured Langfuse project. + +## Data handling + +Observations may contain biomedical queries, identifiers, complete bound +arguments, and service responses. Enable this integration only when the +Langfuse deployment and project retention policy meet the data-handling +requirements of the environment. From 47260dd660495f4f7a1bcf2732000ae2cf27d0eb Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Tue, 1 Sep 2026 15:58:11 -0400 Subject: [PATCH 04/10] rev: isolate runtime configuration in tests --- tests/conftest.py | 16 ++++++++++++++++ tests/test_config.py | 13 ++++--------- tests/test_node_annotator.py | 13 ++++++++++++- 3 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a207725 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +"""Shared isolation for tests that use TCT's runtime configuration.""" + +from collections.abc import Iterator + +import pytest + +from TCT.config import reset_config + + +@pytest.fixture(autouse=True) +def isolated_runtime_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """Prevent ambient environment or earlier tests from selecting services.""" + monkeypatch.delenv("TCT_ENVIRONMENT", raising=False) + reset_config() + yield + reset_config() diff --git a/tests/test_config.py b/tests/test_config.py index a29f66a..e5e2866 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,18 +5,9 @@ configure, get_runtime_config, load_config, - reset_config, ) from TCT.translator_kpinfo import _select_provider_url - -@pytest.fixture(autouse=True) -def clean_runtime_config(): - reset_config() - yield - reset_config() - - def test_prod_ci_and_test_endpoint_resolution(): prod = RuntimeConfig(environment="prod") ci = RuntimeConfig(environment="ci") @@ -50,6 +41,10 @@ def test_environment_variable_selects_ci(monkeypatch): assert load_config().environment == "ci" +def test_suite_does_not_inherit_runtime_environment(): + assert load_config().environment == "prod" + + def test_configure_sets_process_configuration(): configured = configure(environment="ci") diff --git a/tests/test_node_annotator.py b/tests/test_node_annotator.py index f81d046..7cfd395 100644 --- a/tests/test_node_annotator.py +++ b/tests/test_node_annotator.py @@ -22,7 +22,18 @@ 'expected': { 'query': 'CHEBI:15377', 'boxed_warning': True, - 'sections': ['aeolus', 'chebi', 'chembl', 'clinical_approval', 'clinical_trials', 'drugbank', 'ndc', 'pubchem', 'unichem', 'unii'], + # Clinical approval/trial sections depend on volatile upstream + # providers and are not part of the stable annotator contract. + 'sections': [ + 'aeolus', + 'chebi', + 'chembl', + 'drugbank', + 'ndc', + 'pubchem', + 'unichem', + 'unii', + ], 'chembl.availability_type': 2, }, }, From c728121bfeea8fca8450942e2247a001af5a49cd Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Tue, 1 Sep 2026 16:02:38 -0400 Subject: [PATCH 05/10] rev: make test runtime configuration explicit --- tests/conftest.py | 9 ++++----- tests/test_config.py | 2 +- tests/test_node_annotator.py | 5 ++--- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index a207725..f9b634c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,13 +4,12 @@ import pytest -from TCT.config import reset_config +from TCT.config import configure, reset_config @pytest.fixture(autouse=True) -def isolated_runtime_config(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: - """Prevent ambient environment or earlier tests from selecting services.""" - monkeypatch.delenv("TCT_ENVIRONMENT", raising=False) - reset_config() +def isolated_runtime_config() -> Iterator[None]: + """Give every test an explicit runtime independent of ambient state.""" + configure(environment="prod") yield reset_config() diff --git a/tests/test_config.py b/tests/test_config.py index e5e2866..23eea9f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -42,7 +42,7 @@ def test_environment_variable_selects_ci(monkeypatch): def test_suite_does_not_inherit_runtime_environment(): - assert load_config().environment == "prod" + assert get_runtime_config().environment == "prod" def test_configure_sets_process_configuration(): diff --git a/tests/test_node_annotator.py b/tests/test_node_annotator.py index 7cfd395..4a7430c 100644 --- a/tests/test_node_annotator.py +++ b/tests/test_node_annotator.py @@ -21,9 +21,8 @@ 'curie': 'CHEBI:15377', 'expected': { 'query': 'CHEBI:15377', - 'boxed_warning': True, - # Clinical approval/trial sections depend on volatile upstream - # providers and are not part of the stable annotator contract. + # Warning and clinical approval/trial fields depend on volatile + # upstream providers and are not part of the stable contract. 'sections': [ 'aeolus', 'chebi', From b15ca3b999b17567ffc811f51f56700adcaa109e Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Tue, 1 Sep 2026 16:48:43 -0400 Subject: [PATCH 06/10] rev: add tool cost opportunity telemetry --- README.md | 7 +- EXAMPLES.md => TCT/interfaces/EXAMPLES.md | 5 +- TCT/interfaces/LANGFUSE.md | 21 +++++ TCT/interfaces/invocation.py | 104 +++++++++++++++++++++- tests/test_observability.py | 68 ++++++++++++-- 5 files changed, 192 insertions(+), 13 deletions(-) rename EXAMPLES.md => TCT/interfaces/EXAMPLES.md (96%) diff --git a/README.md b/README.md index 82d044c..e23bc71 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,8 @@ tct name-lookup --query aspirin tct normalize-nodes --query CHEBI:15365 CHEBI:6801 --no-conflate ``` -See [EXAMPLES.md](EXAMPLES.md) for CLI discovery, structured inputs, finder +See [TCT/interfaces/EXAMPLES.md](TCT/interfaces/EXAMPLES.md) for CLI discovery, +structured inputs, finder commands, Python use, and MCP client configuration. ### Run the MCP server @@ -218,6 +219,10 @@ Set `TCT_LANGFUSE_ENABLED=true` to opt in. Accepted true values are `1`, Observations are named `tct.tool.`, tagged with the `cli` or `mcp` interface, and include normalized arguments, defaults, and successful results. +They also include deterministic input/output hashes, encoded byte counts, +per-argument sizes, provider counts, and TRAPI identifier counts. These fields +make repeated calls, large repeated arguments, and under-batched queries +comparable without adding decorators to individual tools. The original exception crosses the observation boundary on failure before the CLI or MCP adapter converts it to its stable interface error. Because this can record biomedical queries and service responses, configure Langfuse according diff --git a/EXAMPLES.md b/TCT/interfaces/EXAMPLES.md similarity index 96% rename from EXAMPLES.md rename to TCT/interfaces/EXAMPLES.md index 1cea81c..2281eee 100644 --- a/EXAMPLES.md +++ b/TCT/interfaces/EXAMPLES.md @@ -269,7 +269,10 @@ uv run tct-server Langfuse remains disabled unless `TCT_LANGFUSE_ENABLED` is explicitly true. When enabled, every call made through either adapter is represented as a Langfuse `tool` observation. The observation includes the interface, tool name, bound input -arguments (including defaults), and a JSON-compatible successful result. No +arguments (including defaults), and a JSON-compatible successful result. It +also records stable payload hashes, byte sizes, batching counts, provider +counts, and TRAPI identifier counts where applicable. These metrics expose +exact duplicates, repeated large arguments, and under-batched calls. No per-tool decorator is needed because both adapters call the same invocation function. diff --git a/TCT/interfaces/LANGFUSE.md b/TCT/interfaces/LANGFUSE.md index 1a1d4bd..01b600a 100644 --- a/TCT/interfaces/LANGFUSE.md +++ b/TCT/interfaces/LANGFUSE.md @@ -127,6 +127,19 @@ TCT attaches the following metadata: | `tct.interface` | `cli` or `mcp` | | `tct.module` | Python module containing the shared callable | | `tct.tool` | Python callable name | +| `tct.input.bytes` | Canonical UTF-8 JSON size of all bound arguments | +| `tct.input.sha256` | Stable identity for detecting an exact repeated call | +| `tct.input.argument..bytes` | Canonical size of one argument | +| `tct.input.argument..sha256` | Stable identity of one repeated argument | +| `tct.output.bytes` | Canonical UTF-8 JSON size of the returned value | +| `tct.output.sha256` | Stable identity for detecting repeated results | + +When applicable, observations also include `tct.provider.name`, +`tct.provider.count`, `tct.batch.item_count`, `tct.batch.argument`, +`tct.query.node_count`, `tct.query.identifier_count`, and +`tct.query.identifier_node_count`. These fields are derived from ordinary +arguments at the shared invocation boundary; individual tools do not require +observability decorators. Inputs are bound against the Python signature, so the observation includes applied default values as well as arguments supplied by the caller. Successful @@ -134,6 +147,14 @@ outputs are converted to JSON-compatible values using the same normalization conventions as CLI results. On failure, the original exception crosses the Langfuse context before TCT converts it to its stable CLI or MCP error. +The hashes identify equal canonical payloads; they are not cache keys exposed +to callers and do not change invocation behavior. Input and output byte counts +measure TCT's normalized logical values, not model tokens or MCP wire framing. +An agent's Langfuse integration remains responsible for generation model, +token usage, and price. When agent and MCP observations share distributed +trace context, those generation costs and these tool metrics can be analyzed +within the same turn. + ## Test the integration Run the isolated tests, which use fakes and do not send data to Langfuse: diff --git a/TCT/interfaces/invocation.py b/TCT/interfaces/invocation.py index fc880d3..6499163 100644 --- a/TCT/interfaces/invocation.py +++ b/TCT/interfaces/invocation.py @@ -3,6 +3,7 @@ from __future__ import annotations import inspect +import hashlib import json import math from collections.abc import Callable, Mapping @@ -48,6 +49,95 @@ def _trace_input( return _trace_value(dict(bound.arguments)) +def _canonical_payload(value: Any) -> bytes: + """Encode normalized trace data deterministically for size and identity.""" + return json.dumps( + value, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + + +def _payload_metadata(prefix: str, value: Any) -> dict[str, Any]: + """Describe payload cost and identity without depending on its contents.""" + payload = _canonical_payload(value) + metadata: dict[str, Any] = { + f"{prefix}.bytes": len(payload), + f"{prefix}.sha256": hashlib.sha256(payload).hexdigest(), + f"{prefix}.type": type(value).__name__, + } + if isinstance(value, Mapping): + metadata[f"{prefix}.item_count"] = len(value) + elif isinstance(value, list): + metadata[f"{prefix}.item_count"] = len(value) + return metadata + + +def _trapi_query_metadata(query: Any) -> dict[str, Any]: + """Return small batching indicators from a TRAPI query, when present.""" + if not isinstance(query, Mapping): + return {} + message = query.get("message") + if not isinstance(message, Mapping): + return {} + query_graph = message.get("query_graph") + if not isinstance(query_graph, Mapping): + return {} + nodes = query_graph.get("nodes") + if not isinstance(nodes, Mapping): + return {} + + identifier_count = 0 + identifier_nodes = 0 + for node in nodes.values(): + if not isinstance(node, Mapping): + continue + identifiers = node.get("ids") + if isinstance(identifiers, list): + identifier_count += len(identifiers) + identifier_nodes += 1 + + return { + "tct.query.node_count": len(nodes), + "tct.query.identifier_count": identifier_count, + "tct.query.identifier_node_count": identifier_nodes, + } + + +def _input_metadata(value: Any) -> dict[str, Any]: + """Build generic and TCT-specific input metrics for opportunity analysis.""" + metadata = _payload_metadata("tct.input", value) + if not isinstance(value, Mapping): + return metadata + + for name, argument in value.items(): + metadata.update(_payload_metadata(f"tct.input.argument.{name}", argument)) + + api_name = value.get("api_name") + if isinstance(api_name, str): + metadata["tct.provider.name"] = api_name + + selected_apis = value.get("selected_apis") + if isinstance(selected_apis, list): + metadata["tct.provider.count"] = len(selected_apis) + + for candidate in ("strings", "node"): + items = value.get(candidate) + if isinstance(items, list): + metadata["tct.batch.item_count"] = len(items) + metadata["tct.batch.argument"] = candidate + break + + query = value.get("query") + if isinstance(query, list): + metadata["tct.batch.item_count"] = len(query) + metadata["tct.batch.argument"] = "query" + + metadata.update(_trapi_query_metadata(value.get("query_json"))) + return metadata + + def invoke( tool: Callable[..., Any], /, @@ -62,14 +152,24 @@ def invoke( "tct.module": tool.__module__, "tct.tool": tool.__name__, } + + def trace_input() -> Any: + value = _trace_input(tool, args, kwargs) + metadata.update(_input_metadata(value)) + return value + with observe_tool( name=f"tct.tool.{tool.__name__}", - input_factory=lambda: _trace_input(tool, args, kwargs), + input_factory=trace_input, metadata=metadata, ) as observation: result = tool(*args, **kwargs) if observation is not None: - observation.update(output=_trace_value(result)) + output = _trace_value(result) + observation.update( + output=output, + metadata=_payload_metadata("tct.output", output), + ) return result except ToolInvocationError: raise diff --git a/tests/test_observability.py b/tests/test_observability.py index 2c29dcc..919eb47 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -78,16 +78,66 @@ def combine(left: str, right: str = "default") -> tuple[str, str]: result = invocation.invoke(combine, "value", _interface="mcp") assert result == ("value", "default") - assert captured == { - "name": "tct.tool.combine", - "input": {"left": "value", "right": "default"}, - "metadata": { - "tct.interface": "mcp", - "tct.module": __name__, - "tct.tool": "combine", - }, - "update": {"output": ["value", "default"]}, + assert captured["name"] == "tct.tool.combine" + assert captured["input"] == {"left": "value", "right": "default"} + assert captured["metadata"] == { + "tct.interface": "mcp", + "tct.module": __name__, + "tct.tool": "combine", + **invocation._input_metadata(captured["input"]), } + assert captured["update"] == { + "output": ["value", "default"], + "metadata": invocation._payload_metadata( + "tct.output", ["value", "default"] + ), + } + + +def test_tool_telemetry_identifies_duplicates_and_batching_opportunities(monkeypatch): + """Stable hashes and query counts expose repeated and under-batched calls.""" + observations = [] + + class Observation: + def update(self, **values): + observations[-1]["update"] = values + + @contextmanager + def fake_observe_tool(*, name, input_factory, metadata): + observations.append({"name": name, "input": input_factory(), "metadata": metadata}) + yield Observation() + + monkeypatch.setattr(invocation, "observe_tool", fake_observe_tool) + + def query_provider(api_name: str, query_json: dict) -> dict: + return {"results": [1, 2]} + + query = { + "message": { + "query_graph": { + "nodes": { + "genes": {"ids": ["NCBIGene:1", "NCBIGene:2"]}, + "disease": {"ids": ["MONDO:1"]}, + } + } + } + } + + invocation.invoke(query_provider, "RTX KG2", query, _interface="mcp") + invocation.invoke(query_provider, "RTX KG2", query, _interface="mcp") + + first = observations[0]["metadata"] + second = observations[1]["metadata"] + assert first["tct.provider.name"] == "RTX KG2" + assert first["tct.query.node_count"] == 2 + assert first["tct.query.identifier_count"] == 3 + assert first["tct.query.identifier_node_count"] == 2 + assert first["tct.input.sha256"] == second["tct.input.sha256"] + assert ( + first["tct.input.argument.query_json.sha256"] + == second["tct.input.argument.query_json.sha256"] + ) + assert observations[0]["update"]["metadata"]["tct.output.bytes"] > 0 def test_tool_errors_cross_the_observation_before_normalization(monkeypatch): From 7baa5bc0cea4cc2db87649ad00ad45d3abd06e21 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Tue, 1 Sep 2026 16:53:41 -0400 Subject: [PATCH 07/10] rev: link agent turns to MCP tool traces --- README.md | 6 +++ TCT/interfaces/LANGFUSE.md | 46 +++++++++++++++++++++ TCT/interfaces/invocation.py | 3 +- TCT/interfaces/mcp.py | 45 ++++++++++++++++++++- TCT/interfaces/observability.py | 43 ++++++++++++++++++++ tests/test_observability.py | 59 +++++++++++++++++++++++++++ tests/test_server.py | 72 +++++++++++++++++++++++++++++++++ 7 files changed, 271 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e23bc71..f98e04d 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,12 @@ CLI or MCP adapter converts it to its stable interface error. Because this can record biomedical queries and service responses, configure Langfuse according to the data-handling requirements of the deployment. +MCP clients can link these tool observations to an instrumented agent turn by +injecting W3C trace context into request `_meta`. TCT restores the context in +MCP middleware without adding trace parameters to the published tool schema. +See [TCT/interfaces/LANGFUSE.md](TCT/interfaces/LANGFUSE.md) for the request +shape, compatibility fallback, and telemetry field contract. + ### Shared tool capabilities The table uses CLI kebab-case spellings; MCP publishes the corresponding diff --git a/TCT/interfaces/LANGFUSE.md b/TCT/interfaces/LANGFUSE.md index 01b600a..d4b183d 100644 --- a/TCT/interfaces/LANGFUSE.md +++ b/TCT/interfaces/LANGFUSE.md @@ -127,6 +127,7 @@ TCT attaches the following metadata: | `tct.interface` | `cli` or `mcp` | | `tct.module` | Python module containing the shared callable | | `tct.tool` | Python callable name | +| `tct.trace.propagated` | Whether an MCP client supplied parent trace context | | `tct.input.bytes` | Canonical UTF-8 JSON size of all bound arguments | | `tct.input.sha256` | Stable identity for detecting an exact repeated call | | `tct.input.argument..bytes` | Canonical size of one argument | @@ -155,6 +156,51 @@ token usage, and price. When agent and MCP observations share distributed trace context, those generation costs and these tool metrics can be analyzed within the same turn. +## Link agent turns to MCP tools + +TCT accepts W3C `traceparent`, `tracestate`, and `baggage` fields in an MCP +tool request's `_meta`. The MCP adapter restores that context for the duration +of dispatch, so its `tct.tool.` observation becomes a child of the +agent's current turn. Trace metadata never becomes part of the shared callable +signature or the discovered tool input schema. + +Clients with direct protocol metadata support should send a request shaped +like this: + +```json +{ + "name": "name_lookup", + "arguments": {"query": "aspirin"}, + "_meta": { + "traceparent": "00---01", + "baggage": "" + } +} +``` + +Some MCP client libraries do not yet expose request-level `_meta`. TCT also +accepts `_meta` temporarily alongside tool arguments and removes it before +FastMCP validates or invokes the tool: + +```python +from langfuse import propagate_attributes +from opentelemetry.propagate import inject + +with propagate_attributes(session_id="conversation-123", as_baggage=True): + carrier = {} + inject(carrier) + await session.call_tool( + "name_lookup", + {"query": "aspirin", "_meta": carrier}, + ) +``` + +The client must inject while the agent turn observation is current. TCT does +not create LLM generation observations and therefore cannot infer model token +usage or price. Instrument the agent's model provider with Langfuse; linked +TCT observations then appear in the same turn trace. Clients that send no +trace context continue to work and receive an independent TCT trace. + ## Test the integration Run the isolated tests, which use fakes and do not send data to Langfuse: diff --git a/TCT/interfaces/invocation.py b/TCT/interfaces/invocation.py index 6499163..250a05b 100644 --- a/TCT/interfaces/invocation.py +++ b/TCT/interfaces/invocation.py @@ -11,7 +11,7 @@ from enum import Enum from typing import Any -from .observability import observe_tool +from .observability import observe_tool, trace_context_was_propagated class ToolInvocationError(RuntimeError): @@ -151,6 +151,7 @@ def invoke( "tct.interface": _interface or "shared", "tct.module": tool.__module__, "tct.tool": tool.__name__, + "tct.trace.propagated": trace_context_was_propagated(), } def trace_input() -> Any: diff --git a/TCT/interfaces/mcp.py b/TCT/interfaces/mcp.py index d9716f5..7d2875a 100644 --- a/TCT/interfaces/mcp.py +++ b/TCT/interfaces/mcp.py @@ -8,21 +8,62 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Mapping from functools import wraps from typing import Any from fastmcp import FastMCP +from fastmcp.server.middleware import Middleware, MiddlewareContext +from fastmcp.tools.tool import ToolResult +from mcp import types as mcp_types from mcp.shared.exceptions import McpError from mcp.types import INTERNAL_ERROR, ErrorData from . import tools as shared_tools from .invocation import ToolInvocationError, invoke as invoke_tool -from .observability import flush_observability +from .observability import flush_observability, use_incoming_trace_context mcp = FastMCP("TCT") + +def _metadata_mapping(value: Any) -> dict[str, Any]: + """Convert protocol metadata to an ordinary mapping, preserving extras.""" + if isinstance(value, Mapping): + return dict(value) + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump(by_alias=True, exclude_none=True) + return {} + + +class _TraceContextMiddleware(Middleware): + """Restore client trace context without publishing it as a tool argument.""" + + async def on_call_tool( + self, + context: MiddlewareContext[mcp_types.CallToolRequestParams], + call_next: Any, + ) -> ToolResult: + message = context.message + metadata = _metadata_mapping(message.meta) + arguments = dict(message.arguments or {}) + + # Some agent MCP wrappers currently place protocol metadata alongside + # tool arguments. Accept that convention without leaking it into the + # callable contract or failing FastMCP argument validation. + argument_metadata = _metadata_mapping(arguments.pop("_meta", None)) + metadata = {**argument_metadata, **metadata} + if arguments != (message.arguments or {}): + message = message.model_copy(update={"arguments": arguments}) + context = context.copy(message=message) + + with use_incoming_trace_context(metadata): + return await call_next(context) + + +mcp.add_middleware(_TraceContextMiddleware()) + _ERROR_PREFIXES = { "get_translator_resources": "Get translator resources error", "name_lookup": "Name lookup error", diff --git a/TCT/interfaces/observability.py b/TCT/interfaces/observability.py index 0e26a32..600e28c 100644 --- a/TCT/interfaces/observability.py +++ b/TCT/interfaces/observability.py @@ -11,12 +11,18 @@ import os from collections.abc import Callable, Generator, Mapping from contextlib import contextmanager +from contextvars import ContextVar from typing import Any _ENABLED_VARIABLE = "TCT_LANGFUSE_ENABLED" _TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) _FALSE_VALUES = frozenset({"0", "false", "no", "off"}) +_TRACE_CONTEXT_FIELDS = frozenset({"traceparent", "tracestate", "baggage"}) +_PROPAGATED_TRACE_CONTEXT: ContextVar[bool] = ContextVar( + "tct_propagated_trace_context", + default=False, +) class ObservabilityConfigurationError(RuntimeError): @@ -60,6 +66,41 @@ def _get_langfuse_client() -> Any | None: return langfuse.get_client() +@contextmanager +def use_incoming_trace_context( + metadata: Mapping[str, Any] | None, +) -> Generator[None, None, None]: + """Restore W3C trace context supplied by an MCP client when tracing. + + Imports remain lazy so MCP and core installations do not acquire a hard + OpenTelemetry dependency. Unknown MCP metadata is deliberately ignored. + """ + carrier = { + key: value + for key, value in (metadata or {}).items() + if key in _TRACE_CONTEXT_FIELDS and isinstance(value, str) + } + if not carrier or not langfuse_enabled(): + yield + return + + otel_context = importlib.import_module("opentelemetry.context") + otel_propagate = importlib.import_module("opentelemetry.propagate") + extracted = otel_propagate.extract(carrier) + otel_token = otel_context.attach(extracted) + propagated_token = _PROPAGATED_TRACE_CONTEXT.set(True) + try: + yield + finally: + _PROPAGATED_TRACE_CONTEXT.reset(propagated_token) + otel_context.detach(otel_token) + + +def trace_context_was_propagated() -> bool: + """Return whether the current invocation inherited client trace context.""" + return _PROPAGATED_TRACE_CONTEXT.get() + + @contextmanager def observe_tool( *, @@ -99,4 +140,6 @@ def flush_observability() -> None: "flush_observability", "langfuse_enabled", "observe_tool", + "trace_context_was_propagated", + "use_incoming_trace_context", ] diff --git a/tests/test_observability.py b/tests/test_observability.py index 919eb47..396d191 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -84,6 +84,7 @@ def combine(left: str, right: str = "default") -> tuple[str, str]: "tct.interface": "mcp", "tct.module": __name__, "tct.tool": "combine", + "tct.trace.propagated": False, **invocation._input_metadata(captured["input"]), } assert captured["update"] == { @@ -180,3 +181,61 @@ def identity(value): value = Value() assert invocation.invoke(identity, value) is value + + +def test_incoming_w3c_context_is_scoped_and_filters_unrelated_metadata(monkeypatch): + """Only standard propagation fields are attached for one MCP dispatch.""" + calls = [] + + class FakeContext: + @staticmethod + def attach(value): + calls.append(("attach", value)) + return "otel-token" + + @staticmethod + def detach(value): + calls.append(("detach", value)) + + class FakePropagate: + @staticmethod + def extract(carrier): + calls.append(("extract", carrier)) + return "extracted-context" + + modules = { + "opentelemetry.context": FakeContext, + "opentelemetry.propagate": FakePropagate, + } + monkeypatch.setattr(observability, "langfuse_enabled", lambda: True) + monkeypatch.setattr( + observability.importlib, + "import_module", + lambda name: modules[name], + ) + + assert observability.trace_context_was_propagated() is False + with observability.use_incoming_trace_context( + { + "traceparent": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + "baggage": "session.id=conversation-123", + "private": "ignored", + } + ): + assert observability.trace_context_was_propagated() is True + + assert observability.trace_context_was_propagated() is False + assert calls == [ + ( + "extract", + { + "traceparent": ( + "00-0123456789abcdef0123456789abcdef-" + "0123456789abcdef-01" + ), + "baggage": "session.id=conversation-123", + }, + ), + ("attach", "extracted-context"), + ("detach", "otel-token"), + ] diff --git a/tests/test_server.py b/tests/test_server.py index 836dafa..fd04953 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1,6 +1,7 @@ """Simple tests for TCT MCP Server functionality.""" import asyncio +from contextlib import contextmanager import pytest @@ -89,3 +90,74 @@ def fake_invoke(tool, *args, **kwargs): }, ) ] + + +def test_mcp_middleware_restores_protocol_context_without_schema_arguments( + monkeypatch, +): + """Trace metadata surrounds dispatch and never reaches the shared tool.""" + from fastmcp.server.middleware import MiddlewareContext + from mcp.types import CallToolRequestParams + + from TCT.interfaces import mcp as adapter + + calls = [] + + @contextmanager + def fake_trace_context(metadata): + calls.append(("enter", metadata)) + try: + yield + finally: + calls.append(("exit", metadata)) + + async def call_next(context): + calls.append(("arguments", context.message.arguments)) + return "result" + + monkeypatch.setattr(adapter, "use_incoming_trace_context", fake_trace_context) + params = CallToolRequestParams( + name="name_lookup", + arguments={ + "query": "aspirin", + "_meta": { + "traceparent": ( + "00-0123456789abcdef0123456789abcdef-" + "0123456789abcdef-01" + ) + }, + }, + _meta={"baggage": "session.id=conversation-123"}, + ) + + result = asyncio.run( + adapter._TraceContextMiddleware().on_call_tool( + MiddlewareContext(message=params), + call_next, + ) + ) + + assert result == "result" + assert calls == [ + ( + "enter", + { + "traceparent": ( + "00-0123456789abcdef0123456789abcdef-" + "0123456789abcdef-01" + ), + "baggage": "session.id=conversation-123", + }, + ), + ("arguments", {"query": "aspirin"}), + ( + "exit", + { + "traceparent": ( + "00-0123456789abcdef0123456789abcdef-" + "0123456789abcdef-01" + ), + "baggage": "session.id=conversation-123", + }, + ), + ] From 32013015f2354b7c096abe8d07d094c7bdeefb70 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Wed, 2 Sep 2026 12:44:47 -0400 Subject: [PATCH 08/10] bench: add Langfuse per-turn payload baseline --- TCT/interfaces/LANGFUSE.md | 5 + TCT/interfaces/observability.py | 3 +- benchmarks/LANGFUSE_TURN_BENCHMARKS.md | 32 +++++ benchmarks/__init__.py | 1 + benchmarks/langfuse_turns.py | 165 ++++++++++++++++++++++++ benchmarks/langfuse_turns_baseline.json | 48 +++++++ tests/test_config.py | 2 + tests/test_langfuse_turn_benchmark.py | 26 ++++ 8 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 benchmarks/LANGFUSE_TURN_BENCHMARKS.md create mode 100644 benchmarks/__init__.py create mode 100644 benchmarks/langfuse_turns.py create mode 100644 benchmarks/langfuse_turns_baseline.json create mode 100644 tests/test_langfuse_turn_benchmark.py diff --git a/TCT/interfaces/LANGFUSE.md b/TCT/interfaces/LANGFUSE.md index d4b183d..b1663a7 100644 --- a/TCT/interfaces/LANGFUSE.md +++ b/TCT/interfaces/LANGFUSE.md @@ -156,6 +156,11 @@ token usage, and price. When agent and MCP observations share distributed trace context, those generation costs and these tool metrics can be analyzed within the same turn. +A deterministic offline baseline is available in +[`benchmarks/LANGFUSE_TURN_BENCHMARKS.md`](../../benchmarks/LANGFUSE_TURN_BENCHMARKS.md). +It compares repeated single-identifier calls, one batched call, and duplicate +batched calls using this metadata contract without contacting Langfuse. + ## Link agent turns to MCP tools TCT accepts W3C `traceparent`, `tracestate`, and `baggage` fields in an MCP diff --git a/TCT/interfaces/observability.py b/TCT/interfaces/observability.py index 600e28c..3088a28 100644 --- a/TCT/interfaces/observability.py +++ b/TCT/interfaces/observability.py @@ -45,8 +45,7 @@ def langfuse_enabled(environ: Mapping[str, str] | None = None) -> bool: if normalized in _FALSE_VALUES: return False raise ObservabilityConfigurationError( - f"{_ENABLED_VARIABLE} must be one of: " - "1, true, yes, on, 0, false, no, off" + f"{_ENABLED_VARIABLE} must be one of: 1, true, yes, on, 0, false, no, off" ) return False diff --git a/benchmarks/LANGFUSE_TURN_BENCHMARKS.md b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md new file mode 100644 index 0000000..5f8d5ac --- /dev/null +++ b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md @@ -0,0 +1,32 @@ +# Langfuse per-turn payload baseline + +The `langfuse-turns-v1` benchmark exercises the same metadata generation used +by TCT's Langfuse tool observations. It groups tool calls into representative +agent turns and compares one-call-per-identifier behavior with a batched call. + +The fixture is deterministic, makes no network requests, and sends nothing to +Langfuse. Regenerate it with: + +```bash +python -m benchmarks.langfuse_turns +``` + +| Metric | One by one | Batched | Duplicate batch | +| --- | ---: | ---: | ---: | +| Tool calls | 7 | 1 | 2 | +| Unique inputs | 7 | 1 | 1 | +| Repeated input calls | 0 | 0 | 1 | +| Input bytes | 10,269 | 1,563 | 3,126 | +| Output bytes | 679 | 457 | 914 | +| Total payload bytes | 10,948 | 2,020 | 4,040 | +| Repeated provider metadata bytes | 8,372 | 1,196 | 2,392 | + +For seven identifiers, batching avoids six tool calls, 8,706 input bytes +(84.8%), and 8,928 total payload bytes (81.5%). It also avoids 7,176 bytes of +provider metadata that would otherwise be repeated within the turn. + +These are tool-boundary payload measurements, not model token counts. Model +input/output tokens and price belong to the parent generation observation. +When the MCP client propagates the parent W3C trace context, Langfuse can join +those generation costs with TCT's tool-call counts, hashes, and payload sizes +for a complete per-turn view. diff --git a/benchmarks/__init__.py b/benchmarks/__init__.py new file mode 100644 index 0000000..d76d7c8 --- /dev/null +++ b/benchmarks/__init__.py @@ -0,0 +1 @@ +"""Deterministic benchmark fixtures for TCT.""" diff --git a/benchmarks/langfuse_turns.py b/benchmarks/langfuse_turns.py new file mode 100644 index 0000000..dd665c4 --- /dev/null +++ b/benchmarks/langfuse_turns.py @@ -0,0 +1,165 @@ +"""Deterministic per-turn benchmarks for TCT's Langfuse telemetry contract. + +The scenarios use in-memory tool functions and observations. They make no +network requests and send no data to Langfuse. +""" + +from __future__ import annotations + +import json +from collections import Counter +from contextlib import contextmanager +from typing import Any + +from TCT.interfaces import invocation + + +IDENTIFIERS = [f"NCBIGene:{1000 + index}" for index in range(7)] +API_NAMES = { + f"Fixture KP {index}": f"https://kp-{index}.example/query" for index in range(12) +} +API_PREDICATES = {name: ["biolink:related_to", "biolink:treats"] for name in API_NAMES} + + +def _query(identifier_values: list[str]) -> dict[str, Any]: + return { + "message": { + "query_graph": { + "nodes": { + "genes": {"ids": identifier_values}, + "disease": {"ids": ["MONDO:0005148"]}, + }, + "edges": { + "e0": { + "subject": "disease", + "object": "genes", + "predicates": ["biolink:related_to"], + } + }, + } + } + } + + +def _fixture_query_provider( + api_name: str, + query_json: dict[str, Any], + api_names: dict[str, str], + api_predicates: dict[str, list[str]], +) -> dict[str, Any]: + identifiers = query_json["message"]["query_graph"]["nodes"]["genes"]["ids"] + return { + "provider": api_name, + "nodes": { + identifier: {"name": f"fixture result for {identifier}"} + for identifier in identifiers + }, + } + + +def _collect_turn(calls: list[list[str]]) -> dict[str, Any]: + records: list[dict[str, Any]] = [] + + class Observation: + def __init__(self, record: dict[str, Any]) -> None: + self.record = record + + def update(self, **values: Any) -> None: + self.record["update"] = values + + @contextmanager + def collect_observation(*, name, input_factory, metadata): + input_value = input_factory() + record = { + "name": name, + "input": input_value, + "metadata": dict(metadata), + } + records.append(record) + yield Observation(record) + + original_observer = invocation.observe_tool + invocation.observe_tool = collect_observation + try: + for identifiers in calls: + invocation.invoke( + _fixture_query_provider, + "Fixture KP 0", + _query(identifiers), + API_NAMES, + API_PREDICATES, + _interface="mcp", + ) + finally: + invocation.observe_tool = original_observer + + hashes = Counter(record["metadata"]["tct.input.sha256"] for record in records) + input_bytes = sum(record["metadata"]["tct.input.bytes"] for record in records) + output_bytes = sum( + record["update"]["metadata"]["tct.output.bytes"] for record in records + ) + return { + "tool_calls": len(records), + "unique_inputs": len(hashes), + "repeated_input_calls": sum(count - 1 for count in hashes.values()), + "input_bytes": input_bytes, + "output_bytes": output_bytes, + "total_payload_bytes": input_bytes + output_bytes, + "query_identifiers": sum( + record["metadata"]["tct.query.identifier_count"] for record in records + ), + "provider_metadata_bytes": sum( + record["metadata"]["tct.input.argument.api_names.bytes"] + + record["metadata"]["tct.input.argument.api_predicates.bytes"] + for record in records + ), + } + + +def benchmark_report() -> dict[str, Any]: + """Compare representative agent-turn tool-call shapes.""" + one_by_one = _collect_turn([[identifier] for identifier in IDENTIFIERS]) + batched = _collect_turn([IDENTIFIERS]) + duplicate = _collect_turn([IDENTIFIERS, IDENTIFIERS]) + input_bytes_avoided = one_by_one["input_bytes"] - batched["input_bytes"] + total_bytes_avoided = ( + one_by_one["total_payload_bytes"] - batched["total_payload_bytes"] + ) + + return { + "fixture": "langfuse-turns-v1", + "notes": { + "network_requests": 0, + "langfuse_uploads": 0, + "token_counts": ( + "Model tokens and price belong to the parent Langfuse generation; " + "this report measures TCT tool payloads within each turn." + ), + }, + "turns": { + "one_call_per_identifier": one_by_one, + "batched_identifiers": batched, + "duplicate_batched_calls": duplicate, + }, + "batching_comparison": { + "tool_calls_avoided": one_by_one["tool_calls"] - batched["tool_calls"], + "input_bytes_avoided": input_bytes_avoided, + "input_bytes_reduction_percent": round( + input_bytes_avoided / one_by_one["input_bytes"] * 100, + 1, + ), + "total_payload_bytes_avoided": total_bytes_avoided, + "total_payload_bytes_reduction_percent": round( + total_bytes_avoided / one_by_one["total_payload_bytes"] * 100, + 1, + ), + "provider_metadata_bytes_avoided": ( + one_by_one["provider_metadata_bytes"] + - batched["provider_metadata_bytes"] + ), + }, + } + + +if __name__ == "__main__": + print(json.dumps(benchmark_report(), indent=2, sort_keys=True)) diff --git a/benchmarks/langfuse_turns_baseline.json b/benchmarks/langfuse_turns_baseline.json new file mode 100644 index 0000000..2e45280 --- /dev/null +++ b/benchmarks/langfuse_turns_baseline.json @@ -0,0 +1,48 @@ +{ + "batching_comparison": { + "input_bytes_avoided": 8706, + "input_bytes_reduction_percent": 84.8, + "provider_metadata_bytes_avoided": 7176, + "tool_calls_avoided": 6, + "total_payload_bytes_avoided": 8928, + "total_payload_bytes_reduction_percent": 81.5 + }, + "fixture": "langfuse-turns-v1", + "notes": { + "langfuse_uploads": 0, + "network_requests": 0, + "token_counts": "Model tokens and price belong to the parent Langfuse generation; this report measures TCT tool payloads within each turn." + }, + "turns": { + "batched_identifiers": { + "input_bytes": 1563, + "output_bytes": 457, + "provider_metadata_bytes": 1196, + "query_identifiers": 8, + "repeated_input_calls": 0, + "tool_calls": 1, + "total_payload_bytes": 2020, + "unique_inputs": 1 + }, + "duplicate_batched_calls": { + "input_bytes": 3126, + "output_bytes": 914, + "provider_metadata_bytes": 2392, + "query_identifiers": 16, + "repeated_input_calls": 1, + "tool_calls": 2, + "total_payload_bytes": 4040, + "unique_inputs": 1 + }, + "one_call_per_identifier": { + "input_bytes": 10269, + "output_bytes": 679, + "provider_metadata_bytes": 8372, + "query_identifiers": 14, + "repeated_input_calls": 0, + "tool_calls": 7, + "total_payload_bytes": 10948, + "unique_inputs": 7 + } + } +} diff --git a/tests/test_config.py b/tests/test_config.py index 65f6547..fd26c1c 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -5,6 +5,7 @@ configure, get_runtime_config, load_config, + reset_config, ) from TCT.translator_kpinfo import _select_provider_url @@ -43,6 +44,7 @@ def test_environment_variable_selects_ci(monkeypatch): def test_ci_is_the_default_environment(monkeypatch): monkeypatch.delenv("TCT_ENVIRONMENT", raising=False) + reset_config() assert RuntimeConfig().environment == "ci" assert load_config().environment == "ci" diff --git a/tests/test_langfuse_turn_benchmark.py b/tests/test_langfuse_turn_benchmark.py new file mode 100644 index 0000000..cc035c5 --- /dev/null +++ b/tests/test_langfuse_turn_benchmark.py @@ -0,0 +1,26 @@ +"""Regression coverage for deterministic per-turn telemetry benchmarks.""" + +import json +from pathlib import Path + +from benchmarks.langfuse_turns import benchmark_report + + +def test_turn_benchmark_exposes_batching_and_duplicate_calls(): + report = benchmark_report() + turns = report["turns"] + + assert turns["one_call_per_identifier"]["tool_calls"] == 7 + assert turns["batched_identifiers"]["tool_calls"] == 1 + assert turns["duplicate_batched_calls"]["repeated_input_calls"] == 1 + assert report["batching_comparison"]["tool_calls_avoided"] == 6 + assert report["batching_comparison"]["input_bytes_avoided"] > 0 + assert report["batching_comparison"]["provider_metadata_bytes_avoided"] > 0 + + +def test_checked_in_turn_baseline_matches_the_benchmark(): + baseline_path = ( + Path(__file__).parents[1] / "benchmarks" / "langfuse_turns_baseline.json" + ) + + assert json.loads(baseline_path.read_text()) == benchmark_report() From 55751467435ad882a5bf3f4829a954ca2cf25197 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Wed, 2 Sep 2026 12:54:25 -0400 Subject: [PATCH 09/10] Add live Langfuse turn acceptance probe --- benchmarks/LANGFUSE_TURN_BENCHMARKS.md | 22 ++ .../langfuse_conversation_acceptance.py | 258 ++++++++++++++++++ ...ngfuse_conversation_acceptance_result.json | 35 +++ .../test_langfuse_conversation_acceptance.py | 89 ++++++ 4 files changed, 404 insertions(+) create mode 100644 benchmarks/langfuse_conversation_acceptance.py create mode 100644 benchmarks/langfuse_conversation_acceptance_result.json create mode 100644 tests/test_langfuse_conversation_acceptance.py diff --git a/benchmarks/LANGFUSE_TURN_BENCHMARKS.md b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md index 5f8d5ac..e127d04 100644 --- a/benchmarks/LANGFUSE_TURN_BENCHMARKS.md +++ b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md @@ -30,3 +30,25 @@ input/output tokens and price belong to the parent generation observation. When the MCP client propagates the parent W3C trace context, Langfuse can join those generation costs with TCT's tool-call counts, hashes, and payload sizes for a complete per-turn view. + +## Live acceptance: conversational turns returned by Langfuse + +The offline benchmark is not proof that a Langfuse deployment accepted the +observations. The live acceptance probe creates two parent `agent` +observations, invokes the TCT fixture tools beneath them, flushes the SDK, and +then queries those trace IDs through the Langfuse observations API. Its JSON +output is computed only from observations returned by Langfuse. + +Set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY`; set +`LANGFUSE_BASE_URL` as well for a self-hosted deployment. Then run: + +```bash +uv run --extra langfuse python -m benchmarks.langfuse_conversation_acceptance +``` + +Exit status zero and `"acceptance": "passed"` mean that both conversational +turns and their TCT child observations were returned with metrics. Credentials +are required but are never included in the report or configuration errors. +The latest sanitized live result is checked in as +`langfuse_conversation_acceptance_result.json` so reviewers can inspect the +server-returned trace IDs and metrics directly in the pull request. diff --git a/benchmarks/langfuse_conversation_acceptance.py b/benchmarks/langfuse_conversation_acceptance.py new file mode 100644 index 0000000..af69ee0 --- /dev/null +++ b/benchmarks/langfuse_conversation_acceptance.py @@ -0,0 +1,258 @@ +"""Live Langfuse acceptance probe for conversational-turn metrics. + +Unlike ``langfuse_turns.py``, this uploads two agent turns to a real Langfuse +project and builds its report from observations read back through the API. +""" + +from __future__ import annotations + +import argparse +import json +import os +import time +import uuid +from collections import Counter +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Any + +from benchmarks.langfuse_turns import ( + API_NAMES, + API_PREDICATES, + IDENTIFIERS, + _fixture_query_provider, + _query, +) + + +TURN_NAME = "TCT acceptance conversational turn" +REQUIRED_ENVIRONMENT = ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY") +EXPECTED_METRICS = { + "one_call_per_identifier": { + "agent_turns_returned": 1, + "tool_observations_returned": 7, + "query_identifiers": 14, + }, + "batched_identifiers": { + "agent_turns_returned": 1, + "tool_observations_returned": 1, + "query_identifiers": 8, + }, +} + + +class AcceptanceConfigurationError(RuntimeError): + """Report missing configuration needed for a live acceptance run.""" + + +def validate_environment(environ: Mapping[str, str]) -> None: + """Require credentials without ever including their values in errors.""" + missing = [name for name in REQUIRED_ENVIRONMENT if not environ.get(name)] + if missing: + raise AcceptanceConfigurationError( + "Live Langfuse acceptance requires: " + ", ".join(missing) + ) + + +def _as_dict(value: Any) -> dict[str, Any]: + if isinstance(value, Mapping): + return dict(value) + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump(mode="json") + to_dict = getattr(value, "dict", None) + if callable(to_dict): + return to_dict() + raise TypeError(f"Cannot normalize Langfuse response type {type(value).__name__}") + + +def _observation_data(response: Any) -> list[dict[str, Any]]: + data = response.get("data", []) if isinstance(response, Mapping) else response.data + return [_as_dict(item) for item in data] + + +def summarize_returned_turn( + observations: list[dict[str, Any]], + *, + trace_id: str, + scenario: str, +) -> dict[str, Any]: + """Derive acceptance metrics exclusively from API-returned observations.""" + roots = [item for item in observations if item.get("name") == TURN_NAME] + tools = [ + item + for item in observations + if item.get("name") == "tct.tool._fixture_query_provider" + ] + hashes = Counter( + item.get("metadata", {}).get("tct.input.sha256") + for item in tools + if item.get("metadata", {}).get("tct.input.sha256") + ) + return { + "scenario": scenario, + "trace_id": trace_id, + "agent_turns_returned": len(roots), + "tool_observations_returned": len(tools), + "input_bytes": sum( + item.get("metadata", {}).get("tct.input.bytes", 0) for item in tools + ), + "output_bytes": sum( + item.get("metadata", {}).get("tct.output.bytes", 0) for item in tools + ), + "query_identifiers": sum( + item.get("metadata", {}).get("tct.query.identifier_count", 0) + for item in tools + ), + "unique_tool_inputs": len(hashes), + "repeated_tool_inputs": sum(count - 1 for count in hashes.values()), + } + + +def _run_turn(client: Any, run_id: str, scenario: str, calls: list[list[str]]) -> str: + from TCT.interfaces import invocation + + with client.start_as_current_observation( + as_type="agent", + name=TURN_NAME, + input={"scenario": scenario, "identifier_count": len(IDENTIFIERS)}, + metadata={ + "tct.acceptance": True, + "tct.acceptance.run_id": run_id, + "tct.acceptance.scenario": scenario, + }, + ) as turn: + trace_id = turn.trace_id + results = [] + for identifiers in calls: + results.append( + invocation.invoke( + _fixture_query_provider, + "Fixture KP 0", + _query(identifiers), + API_NAMES, + API_PREDICATES, + _interface="acceptance", + ) + ) + turn.update(output={"tool_calls": len(results), "status": "complete"}) + return trace_id + + +def _read_back( + client: Any, + trace_ids: Mapping[str, str], + *, + timeout_seconds: float, +) -> dict[str, list[dict[str, Any]]]: + deadline = time.monotonic() + timeout_seconds + pending = dict(trace_ids) + returned: dict[str, list[dict[str, Any]]] = {} + while pending and time.monotonic() < deadline: + for scenario, trace_id in list(pending.items()): + response = client.api.observations.get_many( + trace_id=trace_id, + fields="basic,metadata,metrics", + limit=100, + ) + observations = _observation_data(response) + has_turn = any(item.get("name") == TURN_NAME for item in observations) + has_tool = any( + item.get("name") == "tct.tool._fixture_query_provider" + for item in observations + ) + if has_turn and has_tool: + returned[scenario] = observations + del pending[scenario] + if pending: + time.sleep(1) + if pending: + names = ", ".join(sorted(pending)) + raise TimeoutError( + f"Langfuse did not return complete observations within " + f"{timeout_seconds:g}s for: {names}" + ) + return returned + + +def validate_returned_metrics(turns: Mapping[str, Mapping[str, Any]]) -> None: + """Reject partial read-backs and observations missing payload metrics.""" + problems = [] + for scenario, expected in EXPECTED_METRICS.items(): + actual = turns[scenario] + for metric, expected_value in expected.items(): + if actual[metric] != expected_value: + problems.append( + f"{scenario}.{metric}={actual[metric]!r} " + f"(expected {expected_value!r})" + ) + for metric in ("input_bytes", "output_bytes", "unique_tool_inputs"): + if actual[metric] <= 0: + problems.append(f"{scenario}.{metric} is not positive") + if problems: + raise RuntimeError("Invalid Langfuse metric read-back: " + "; ".join(problems)) + + +def run_acceptance(timeout_seconds: float = 30) -> dict[str, Any]: + """Upload conversational turns, read them back, and return their metrics.""" + validate_environment(os.environ) + os.environ["TCT_LANGFUSE_ENABLED"] = "true" + + from langfuse import get_client + + client = get_client() + run_id = str(uuid.uuid4()) + scenarios = { + "one_call_per_identifier": [[identifier] for identifier in IDENTIFIERS], + "batched_identifiers": [IDENTIFIERS], + } + trace_ids = { + scenario: _run_turn(client, run_id, scenario, calls) + for scenario, calls in scenarios.items() + } + client.flush() + returned = _read_back(client, trace_ids, timeout_seconds=timeout_seconds) + turns = { + scenario: summarize_returned_turn( + returned[scenario], trace_id=trace_id, scenario=scenario + ) + for scenario, trace_id in trace_ids.items() + } + validate_returned_metrics(turns) + one_by_one = turns["one_call_per_identifier"] + batched = turns["batched_identifiers"] + return { + "acceptance": "passed", + "source": "Langfuse observations API", + "run_id": run_id, + "read_back_at": datetime.now(timezone.utc).isoformat(), + "turns": turns, + "comparison": { + "tool_calls_avoided": ( + one_by_one["tool_observations_returned"] + - batched["tool_observations_returned"] + ), + "input_bytes_avoided": ( + one_by_one["input_bytes"] - batched["input_bytes"] + ), + "output_bytes_avoided": ( + one_by_one["output_bytes"] - batched["output_bytes"] + ), + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--timeout", type=float, default=30) + arguments = parser.parse_args() + try: + report = run_acceptance(arguments.timeout) + except (AcceptanceConfigurationError, RuntimeError, TimeoutError) as error: + parser.exit(2, f"acceptance failed: {error}\n") + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/langfuse_conversation_acceptance_result.json b/benchmarks/langfuse_conversation_acceptance_result.json new file mode 100644 index 0000000..3f2dcb8 --- /dev/null +++ b/benchmarks/langfuse_conversation_acceptance_result.json @@ -0,0 +1,35 @@ +{ + "acceptance": "passed", + "comparison": { + "input_bytes_avoided": 8706, + "output_bytes_avoided": 222, + "tool_calls_avoided": 6 + }, + "read_back_at": "2026-09-02T16:54:05.941518+00:00", + "run_id": "3ec4fa9e-fa5a-44f6-bf48-9e284553818c", + "source": "Langfuse observations API", + "turns": { + "batched_identifiers": { + "agent_turns_returned": 1, + "input_bytes": 1563, + "output_bytes": 457, + "query_identifiers": 8, + "repeated_tool_inputs": 0, + "scenario": "batched_identifiers", + "tool_observations_returned": 1, + "trace_id": "461d85e03201fd38f80d9afd031d32c9", + "unique_tool_inputs": 1 + }, + "one_call_per_identifier": { + "agent_turns_returned": 1, + "input_bytes": 10269, + "output_bytes": 679, + "query_identifiers": 14, + "repeated_tool_inputs": 0, + "scenario": "one_call_per_identifier", + "tool_observations_returned": 7, + "trace_id": "811c2a8882a494120e1edb978e91ab29", + "unique_tool_inputs": 7 + } + } +} diff --git a/tests/test_langfuse_conversation_acceptance.py b/tests/test_langfuse_conversation_acceptance.py new file mode 100644 index 0000000..1b0e6aa --- /dev/null +++ b/tests/test_langfuse_conversation_acceptance.py @@ -0,0 +1,89 @@ +from benchmarks.langfuse_conversation_acceptance import ( + AcceptanceConfigurationError, + TURN_NAME, + summarize_returned_turn, + validate_environment, + validate_returned_metrics, +) + + +def test_live_acceptance_requires_credentials_without_disclosing_values(): + try: + validate_environment({"LANGFUSE_PUBLIC_KEY": "pk-secret-value"}) + except AcceptanceConfigurationError as error: + message = str(error) + else: + raise AssertionError("Expected missing secret key to fail") + + assert "LANGFUSE_SECRET_KEY" in message + assert "pk-secret-value" not in message + + +def test_summarizes_metrics_returned_by_langfuse(): + observations = [ + {"name": TURN_NAME, "metadata": {"tct.acceptance": True}}, + { + "name": "tct.tool._fixture_query_provider", + "metadata": { + "tct.input.bytes": 100, + "tct.output.bytes": 20, + "tct.query.identifier_count": 3, + "tct.input.sha256": "same", + }, + }, + { + "name": "tct.tool._fixture_query_provider", + "metadata": { + "tct.input.bytes": 100, + "tct.output.bytes": 20, + "tct.query.identifier_count": 3, + "tct.input.sha256": "same", + }, + }, + ] + + metrics = summarize_returned_turn( + observations, + trace_id="abc123", + scenario="duplicate", + ) + + assert metrics == { + "scenario": "duplicate", + "trace_id": "abc123", + "agent_turns_returned": 1, + "tool_observations_returned": 2, + "input_bytes": 200, + "output_bytes": 40, + "query_identifiers": 6, + "unique_tool_inputs": 1, + "repeated_tool_inputs": 1, + } + + +def test_rejects_a_read_back_without_payload_metrics(): + turns = { + "one_call_per_identifier": { + "agent_turns_returned": 1, + "tool_observations_returned": 7, + "query_identifiers": 14, + "input_bytes": 0, + "output_bytes": 0, + "unique_tool_inputs": 0, + }, + "batched_identifiers": { + "agent_turns_returned": 1, + "tool_observations_returned": 1, + "query_identifiers": 8, + "input_bytes": 0, + "output_bytes": 0, + "unique_tool_inputs": 0, + }, + } + + try: + validate_returned_metrics(turns) + except RuntimeError as error: + assert "input_bytes is not positive" in str(error) + else: + raise AssertionError("Expected missing returned metrics to fail acceptance") From 9258b091fa3cc06beb94a24c3f4aff482bb401e0 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Wed, 2 Sep 2026 13:01:44 -0400 Subject: [PATCH 10/10] Configure rich Codex turn tracing --- TCT/interfaces/LANGFUSE.md | 14 +++++++ benchmarks/LANGFUSE_TURN_BENCHMARKS.md | 5 +++ benchmarks/langfuse_codex_turn_result.json | 18 +++++++++ scripts/setup-langfuse-codex.sh | 46 ++++++++++++++++++++++ 4 files changed, 83 insertions(+) create mode 100644 benchmarks/langfuse_codex_turn_result.json create mode 100755 scripts/setup-langfuse-codex.sh diff --git a/TCT/interfaces/LANGFUSE.md b/TCT/interfaces/LANGFUSE.md index b1663a7..5a97d96 100644 --- a/TCT/interfaces/LANGFUSE.md +++ b/TCT/interfaces/LANGFUSE.md @@ -33,6 +33,20 @@ credentials does not enable it. TCT starts observations only when `TCT_ENVIRONMENT` selects Translator service endpoints; it does not enable or configure Langfuse. +### Codex conversational turns + +The Langfuse Codex tracing plugin does not load the repository `.env` file by +itself. Generate its local, git-ignored configuration with: + +```bash +sh scripts/setup-langfuse-codex.sh +``` + +The generated `.codex/langfuse.json` has mode `600`. After configuration, +completed Codex turns are uploaded with a parent agent observation, child LLM +generations carrying token/cost data, and child tool observations carrying +their input, output, status, and latency. + ## Install Install only the capabilities required by the process: diff --git a/benchmarks/LANGFUSE_TURN_BENCHMARKS.md b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md index e127d04..ce8898a 100644 --- a/benchmarks/LANGFUSE_TURN_BENCHMARKS.md +++ b/benchmarks/LANGFUSE_TURN_BENCHMARKS.md @@ -52,3 +52,8 @@ are required but are never included in the report or configuration errors. The latest sanitized live result is checked in as `langfuse_conversation_acceptance_result.json` so reviewers can inspect the server-returned trace IDs and metrics directly in the pull request. + +The complementary `langfuse_codex_turn_result.json` records a real Codex turn +that successfully called the TCT MCP server. It includes the parent turn's +actual model token, cost, latency, generation, and tool-call rollups returned +by Langfuse. diff --git a/benchmarks/langfuse_codex_turn_result.json b/benchmarks/langfuse_codex_turn_result.json new file mode 100644 index 0000000..74e819b --- /dev/null +++ b/benchmarks/langfuse_codex_turn_result.json @@ -0,0 +1,18 @@ +{ + "acceptance": "passed", + "dashboard_url": "https://us.cloud.langfuse.com/project/cmthc70qs00tmad0dr7hd1ail/traces/94ac7fb1c9a153df10cca766e4d50a66", + "known_plugin_limitation": "Codex 0.152 records MCP calls through the unified exec wrapper, so tracing plugin 0.1.0 displays the tool observation as exec instead of translator-component-toolkit.optimize_query_for_api.", + "source": "Langfuse trace and observations APIs", + "trace": { + "latency_seconds": 13.463, + "llm_calls": 2, + "model": "gpt-5.6-sol", + "name": "Codex Turn", + "prompt_tokens": 34354, + "completion_tokens": 403, + "total_cost_usd": 0.0450216, + "total_tokens": 34757, + "tool_calls": 1 + }, + "trace_id": "94ac7fb1c9a153df10cca766e4d50a66" +} diff --git a/scripts/setup-langfuse-codex.sh b/scripts/setup-langfuse-codex.sh new file mode 100755 index 0000000..bb49b06 --- /dev/null +++ b/scripts/setup-langfuse-codex.sh @@ -0,0 +1,46 @@ +#!/bin/sh +set -eu + +env_file=${1:-./.env} +case "$env_file" in + */*) ;; + *) env_file="./$env_file" ;; +esac + +if [ ! -f "$env_file" ]; then + echo "Langfuse environment file not found: $env_file" >&2 + exit 2 +fi + +set -a +. "$env_file" +set +a + +: "${LANGFUSE_PUBLIC_KEY:?LANGFUSE_PUBLIC_KEY is required}" +: "${LANGFUSE_SECRET_KEY:?LANGFUSE_SECRET_KEY is required}" +: "${LANGFUSE_BASE_URL:?LANGFUSE_BASE_URL is required}" + +mkdir -p .codex +umask 077 +temporary_file=$(mktemp .codex/langfuse.json.XXXXXX) +trap 'rm -f "$temporary_file"' EXIT HUP INT TERM + +jq -n \ + --arg public_key "$LANGFUSE_PUBLIC_KEY" \ + --arg secret_key "$LANGFUSE_SECRET_KEY" \ + --arg base_url "$LANGFUSE_BASE_URL" \ + '{ + enabled: true, + public_key: $public_key, + secret_key: $secret_key, + base_url: $base_url, + environment: "development", + tags: ["tct", "codex"] + }' > "$temporary_file" + +mv "$temporary_file" .codex/langfuse.json +chmod 600 .codex/langfuse.json +trap - EXIT HUP INT TERM + +echo "Configured Codex Langfuse tracing in .codex/langfuse.json (mode 600)." +echo "The file is ignored by git; credential values were not printed."