diff --git a/.gitignore b/.gitignore index 8f6198c..e941443 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,15 @@ __pycache__/ # never source). data/ +# Frontend build outputs: the source of truth is ui/; built assets are copied +# into the hflow-server wheel at packaging time, never committed. +ui/node_modules/ +ui/dist/ +packages/hflow-server/src/hflow_server/static/ + # Local maintainer tooling (agent skills, settings); not part of the public repo. .claude/ .agents/ + +# Transient: `pnpm gen:api` dumps the schema here on its way to src/apiSchema.ts. +.openapi.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d9d027b..6ac8278 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,6 +114,26 @@ repository root, and name the observable result. Keep examples on public APIs; tests belong to business logic and boundary behavior, not to checking that a documentation snippet copied a third-party SDK correctly. +## Changing how episodes are processed + +Identities in HFlow are content hashes, and one of them -- `pipeline_version` +-- is stamped inside the canonical bytes that `episode_id` hashes. A release +number deliberately does **not** feed any of them: a CLI fix or a docs bump +must never invalidate somebody's corpus. What does feed them is +`TRANSFORM_BEHAVIOR_VERSION` in [`src/hflow/behavior.py`](./src/hflow/behavior.py). + +**Bump it in the same commit whenever your change makes the transform write +different bytes for the same input** -- encoder settings or defaults, +chunking and grouping, timestamp handling, the provenance record's shape, or +a bugfix to any of those. Bumping re-versions every existing corpus exactly +once, which is the honest cost; not bumping when behavior changed silently +mixes two behaviors under one version, which is worse. When in doubt, bump, +and say so in the pull request. + +Changes to checks, enrichments, or anything a step merely calls do not need a +bump: a step's own content hash already covers its source and captured +configuration. `tests/test_identity_stability.py` pins these rules. + ## Quality checks Run the Python quality gate and fix every reported issue: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b943634..d0d84d4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -44,7 +44,7 @@ that is actually scheduled. ## Design tenets 1. **Evidence, not verdicts.** Quality checks record measurements, intervals, and tags. Pass/fail policy belongs to the consumer at curation time, never hardcoded into the corpus. -2. **Standard formats at every boundary; no new UIs.** Episodes are standard MCAP (Foxglove/Rerun open them), runs are standard Airflow DAGs (Airflow's UI shows them), the catalog and manifests are Parquet (DuckDB/pandas/anything reads them). We ship no UI and hide nothing; the system is extensible without touching our code. +2. **Standard formats at every boundary; no captive UIs.** Episodes are standard MCAP (Foxglove/Rerun open them), runs are standard Airflow DAGs (Airflow's UI shows them), the catalog and manifests are Parquet (DuckDB/pandas/anything reads them). The optional workspace UI (`hflow serve`, shipped separately as `hflow-server`) is a strict client of these same open surfaces through a documented JSON API -- it hides nothing, gates nothing, and everything it shows stays reachable without it. The system is extensible without touching our code. 3. **Your code stays your code.** Transformations, checks, and enrichments are plain Python functions in the user's own environment. Existing processing code plugs in through small adapters rather than being rebuilt inside a framework. 4. **Ship code only where it earns its place.** Either the canonical format forces bridging (video lives in-band; nothing can read it without our accessors) or the code encodes a painfully-rediscoverable pitfall. We ship no client wrappers around things users already know (`openai`, `subprocess.run(["ffmpeg", ...])`); the examples are the documentation. 5. **Coarse-grained steps.** One task processes one episode or one batch and runs for seconds to minutes. Hot loops live inside a task, never across tasks. diff --git a/docs/FORMAT.md b/docs/FORMAT.md index d0ceb04..787f410 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -16,7 +16,7 @@ The single overriding rule: **a canonical episode is spec-conforming MCAP.** Eve - MCAP magic, `Header`, data section, `DataEnd`, summary section, `Footer`, closing magic, per the [MCAP spec](https://mcap.dev/spec). - `Header.profile` is the empty string `""` (the file mixes protobuf video channels with pass-through channels of arbitrary encoding, so no single profile applies). -- `Header.library` is informational only (e.g. `hflow/0.2.0 episode-format/1`). No reader may key behavior off it (see [Identifier rules](#identifier-rules)). +- `Header.library` is informational only (e.g. `hflow episode-format/1 transform-behavior/1`). It deliberately carries no release number: the header is inside the bytes the content episode id hashes, so a release would otherwise give a byte-identical input a new identity. No reader may key behavior off it (see [Identifier rules](#identifier-rules)). - Chunks are compressed with **zstd** by default (`"none"` is permitted). Each `Chunk` record carries `uncompressed_crc`; the `Footer` carries a summary CRC. - The summary section repeats all `Schema` and `Channel` records and contains `Statistics`, all `ChunkIndex` records, `AttachmentIndex`/`MetadataIndex` records, and `SummaryOffset` records. A canonical episode always has a complete summary; unindexed files are not canonical. diff --git a/docs/HOSTING.md b/docs/HOSTING.md index 002a0b9..2f9380e 100644 --- a/docs/HOSTING.md +++ b/docs/HOSTING.md @@ -154,6 +154,13 @@ deployment against facts: store. - **No tenant-facing log or metrics API.** Observability is Airflow's own UI and task logs on the workspace. +- **The workspace UI (`hflow serve`) authenticates nobody.** It is a local + developer tool bound to `127.0.0.1`, deliberately without a login; it is + not a tenant-facing surface, and serving it to anyone but the workspace's + own operator means putting an authenticating proxy in front of it. Signing + people in and scoping them to a workspace is the control plane's job -- + per-user identity and revocable sessions, which no shared launch secret + could stand in for. - **Task processes share the runtime's environment**, including the workspace's storage credentials, and the venv build runs as root at provision time -- isolation between principals must come from your @@ -161,11 +168,18 @@ deployment against facts: - **A workspace's Airflow stack idles at several GB of RAM** across five long-running services (the compose file defines seven; two are one-shot init containers). -- **Engine upgrades re-version steps.** Step versions content-hash captured - globals, including referenced modules with their versions, so a step that - touches `hflow.*` gets a new version on every hflow release: `hflow - stale` will list its episodes, and curation pins keep working because the - corpus is designed to be permanently mixed-version. +- **Engine upgrades re-version a corpus only when processing changed.** An + hflow release no longer moves any identity by itself: `pipeline_version` + folds in `hflow.behavior.TRANSFORM_BEHAVIOR_VERSION` (bumped deliberately, + only when the transform would write different bytes) instead of the release + number, the canonical file's header carries no release number, and step + versions record the modules they reference by name rather than by version. + A byte-identical input therefore keeps its `episode_id` across upgrades, so + content-addressed dedupe holds. The flip side is a real one: an engine + change that alters processing without a behavior bump is invisible to + `hflow stale`, so operators upgrading across a behavior bump should expect + exactly one corpus-wide re-version and plan reprocessing then. The corpus is + designed to be permanently mixed-version, so curation pins keep working. - **ffmpeg licensing**: the pinned build is BtbN's **GPL** variant (it carries the H.264 encoder the canonical transform needs). GPL source obligations attach to **redistribution** -- shipping worker images or diff --git a/docs/README.md b/docs/README.md index b03b5dc..678c7c5 100644 --- a/docs/README.md +++ b/docs/README.md @@ -44,6 +44,7 @@ the whole workflow before adapting it. Use these when you already know the outcome you need. - [Port existing processing and quality-check code](./PORTING.md) +- [Serve a workspace over HTTP](./SERVE.md) - [Call an OpenAI vision endpoint from a step](./how-to/call-openai-vision.md) - [Run and operate the local Airflow runtime](./RUNTIME.md) - [Deploy into an existing Airflow environment](./RUNTIME.md#bring-your-own-airflow-hflow-deploy) diff --git a/docs/SERVE.md b/docs/SERVE.md new file mode 100644 index 0000000..d168826 --- /dev/null +++ b/docs/SERVE.md @@ -0,0 +1,139 @@ +# Serve a workspace over HTTP: `hflow serve` + +`hflow serve` is a local read-mostly server over one data root: it answers +questions about episodes and their quality evidence, compiles curation SQL, +pins manifests, monitors and triggers ingest runs, and describes the +registered pipeline. The server never rewrites or deletes an episode -- the +only files it writes are the manifests you pin and its own small state file. +It can trigger an ingest run, though, which the runtime then writes through +the normal pipeline; `--read-only` refuses that along with the other writes. + +**The JSON API is the product surface, not an implementation detail of some +frontend.** Every fact a browser could show is reachable from `/api/v1`, and +the OpenAPI schema at `/api/openapi.json` describes all of it -- so a +workspace UI is a *client*, and you can build or swap one without touching +this package. The server ships no frontend of its own; point +`HFLOW_UI_ASSETS` at a directory containing an `index.html` to serve one, or +install a wheel that packages assets under `hflow_server/static/`. + +It ships as a separate package, `hflow-server`, on purpose: pipeline workers +install the `hflow` wheel into every task venv, and they should never carry a +web server. **It is not published to PyPI yet** -- until the first release, +run it from a clone: + +```bash +git clone https://github.com/Hebbian-Robotics/hflow.git +cd hflow +uv sync --all-extras # installs hflow and hflow-server +uv run hflow serve # browses $HFLOW_DATA_ROOT, else ./data +uv run hflow serve --data-root ./data --no-browser +``` + +Starting the server prints its URL (`http://127.0.0.1:4356/`) and opens your +browser. With no assets installed, that URL serves a page pointing at the +API. There is no login: see [Trust posture](#trust-posture) for what that +means and when it stops being appropriate. + +## What it shows + +- **Episodes** -- the corpus as a faceted, sortable table over the catalog's + wide `episodes` view (task, operator, status, quality measurements as + columns). Every filter you click compiles to DuckDB SQL server-side, and + the exact SQL is always visible and copyable at the bottom of the screen -- + ready to paste into `hflow curate`. +- **Episode** -- one recording's dossier: status and quarantine tags, contact + sheets, every check run with its content-hash version, measurements with + their producing step, intervals, tags, append history, and a canonical-MCAP + download. +- **Curate** -- a SQL studio over the catalog views: schema sidebar with + per-table profiles, editor with run-selection, result preview with + per-column statistics, and the coverage report (which checks ran over how + much of the corpus) before you pin. **Pin manifest** freezes a query's + result as an immutable Parquet manifest under `/manifests/`, + recorded with its SQL, row count, and coverage in the Manifests registry. +- **Runs** -- the ingest runtime's health, recent runs with their trigger + configuration, per-stage activity, and a trigger form (`hflow ingest`'s + wire shape, as a form). It addresses a rendered local bundle or a remote + runtime (`HFLOW_AIRFLOW_URL` and friends); when neither is reachable the + page says which it looked for and why it failed, rather than disappearing. +- **Pipeline** -- the generated DAG plus the registered steps by stage, with + content-hash versions, critical flags, and endpoint aliases, and the + versions actually observed in the catalog. Each stage's steps are drawn + *inside* its `process_batch` node, which is where they run: they have no + dependency edges on each other, so the graph nests them instead of + inventing a chain between them. Requires `--pipeline path/to/pipeline.py[:app]`, + which imports (executes) the pipeline file exactly like `hflow manifest` does. + +## Flags + +| flag | meaning | +|---|---| +| `--data-root` | workspace to browse (default `$HFLOW_DATA_ROOT`, else `./data`) | +| `--host` | bind address (default `127.0.0.1`; widening past loopback exposes your corpus) | +| `--port` | default `4356`, auto-retries upward when taken | +| `--no-browser` | do not open a browser (headless machines, tunnels) | +| `--read-only` | viewer mode: hides and refuses manifest pinning, saved-query edits, and run triggering | +| `--pipeline` | pipeline file for the Pipeline page (imported once at startup) | + +## Nothing is UI-only + +The UI is a strict client of a documented JSON API (`/api/v1/...`; a running +server publishes its OpenAPI schema at `/api/openapi.json`, ready for a client +generator or any local OpenAPI viewer). Curation, the runs monitor and the +pipeline page are thin calls into the same library functions the CLI uses; the +episode listing, facets, stats and timeline endpoints compile their own +presentation-shaped SQL over the same [catalog views](./CATALOG.md) that +`hflow curate` reads. Either way, everything the UI can show or do is +reachable with `curl`, scriptable, and buildable-upon. If you want a different +frontend over your workspace, the API is the contract; the shipped UI is the +reference client. + +## Trust posture + +**The server is unauthenticated.** There is no login, no token, and no +session: anyone who can reach the bound address can read your whole workspace +and trigger ingest runs. What protects it is the address it binds -- +`127.0.0.1` by default, reachable only from your own machine. This is the +posture of every local developer tool that browses a working directory +(`mlflow ui`, TensorBoard, `dagster dev`, the DuckDB UI): a credential in +front of a single-user machine buys nothing but friction. + +Passing `--host` past loopback is therefore a deliberate exposure, and it is +the only flag that changes who can reach the data. If you need the UI from +another machine, forward the port over SSH (`ssh -L 4356:127.0.0.1:4356 +host`) rather than binding a network interface; if you must bind one, put a +reverse proxy that authenticates in front of it and firewall the port itself. +`--read-only` narrows what a reacher can *do* (no pins, no saved-query edits, +no triggering) but not what they can *read* -- it is a safety catch, not +access control. + +Hosted, multi-user HFlow is a different problem and is solved elsewhere: the +control plane authenticates people and scopes them to workspaces +([HOSTING.md](./HOSTING.md)). That needs per-user identity and revocable +sessions, which one shared launch secret could never provide -- which is why +this server does not pretend to have a piece of it. + +The rest of the posture is real and holds regardless. The UI runs fully +local: all assets ship in the wheel (no CDN, no fonts, no outbound requests), +and your data never leaves your machine. That is why the server publishes the +schema JSON and no interactive Swagger page -- FastAPI's built-in one fetches +its JavaScript and CSS from a public CDN, which would break the promise and +run third-party script same-origin with your workspace's API. The browser +never sees filesystem paths of its choosing (media is addressed by episode and +artifact name, and the server refuses anything outside the data root), Airflow +credentials stay server-side behind a proxy, and curation SQL runs on a +[constrained DuckDB connection](./CATALOG.md) that cannot reach the catalog's +files or the network. What this server writes: `/curation/state.json` +(saved queries and the manifest registry) and your pinned manifests -- nothing +else. Episodes, media and catalog rows are written by the ingest runtime, on +runs you trigger from the Runs page. + +## See also + +- [Catalog tables and curation API](./CATALOG.md) -- the views and SQL idioms + the Episodes and Curate screens are built on +- [Runtime guide](./RUNTIME.md) -- the Airflow runtime the Runs screen fronts +- [Hosting HFlow](./HOSTING.md) -- the data-plane contract for operating + workspaces for other people, whose seams (bucket data roots, scoped + credentials, constrained SQL, remote runtime addressing) are the ones this + UI reads through diff --git a/packages/hflow-server/README.md b/packages/hflow-server/README.md new file mode 100644 index 0000000..3fba1b7 --- /dev/null +++ b/packages/hflow-server/README.md @@ -0,0 +1,41 @@ +# hflow-server + +The HFlow workspace UI: a local web app over one HFlow data root — browse +episodes, quality evidence, and the Parquet catalog in a browser. It writes +nothing but your pinned manifests and its own small state file. + +```bash +hflow serve --data-root ./data +``` + +It binds `127.0.0.1` and authenticates nobody, like other local developer +tools that browse a working directory: anyone who can reach the bound address +can read the workspace and trigger runs, so binding past loopback is a +deliberate exposure. `docs/SERVE.md` ("Trust posture") has the details. + +This package is not on PyPI yet. Until the first release, run it from a clone +of the [repository](https://github.com/Hebbian-Robotics/hflow); `docs/SERVE.md` +there has the exact steps, including the frontend build. + +The UI is a strict client of the same surfaces the `hflow` CLI uses (the +DuckDB-queryable catalog, episode files, and manifests): everything it shows +is reachable with `curl` against its documented JSON API, and nothing is +UI-only. It runs fully offline — all assets ship in this wheel, and your data +never leaves your machine. There is deliberately no Swagger page: FastAPI's +built-in one would load its JavaScript and CSS from a CDN. + +Every endpoint publishes a typed response schema, so `/api/openapi.json` — the +schema the running server serves — is a usable contract to generate a client +from rather than a list of paths returning "object". One module — +`hflow_server/_contract.py` — owns those payload shapes; the routes construct its +models instead of hand-building dicts. + +This package is deliberately separate from the `hflow` SDK wheel so that +pipeline worker environments (which install `hflow` into every task venv) +never carry a web server. + +It ships no frontend. A UI is a client of the schema above: point +`HFLOW_UI_ASSETS` at a directory containing an `index.html` to serve one, or +package assets under `hflow_server/static/` in a wheel and they are picked up +automatically. Nothing here is reachable only from a browser, so more than +one UI can exist against the same server without forking it. diff --git a/packages/hflow-server/pyproject.toml b/packages/hflow-server/pyproject.toml new file mode 100644 index 0000000..4154597 --- /dev/null +++ b/packages/hflow-server/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "hflow-server" +version = "0.1.0" +description = "Local web UI for HFlow: browse episodes, quality evidence, and the catalog" +readme = "README.md" +license = "Apache-2.0" +authors = [{ name = "Hebbian Robotics" }] +requires-python = ">=3.11" +dependencies = [ + # The floor is a real one, not a formality: this package imports + # hflow.workspace, hflow.import_pipeline_application, + # hflow.runtime.ingest_dag_topology and hflow.app's artifact-key constant + # at module scope, and none of them exist in hflow 0.2.0. Without the + # floor, `pip install hflow-server` beside an older hflow resolves happily and + # then dies with ImportError on `import hflow_server`. [tool.uv.sources] below + # only steers resolution inside this repo -- it is stripped from the built + # wheel's metadata -- and uv ignores this specifier for the workspace + # member, so in-repo development is unaffected by the number. + "hflow>=0.3.0", + "fastapi>=0.115", + "uvicorn>=0.32", +] + +[project.urls] +Repository = "https://github.com/Hebbian-Robotics/hflow" + +[tool.uv.sources] +hflow = { workspace = true } + +[build-system] +requires = ["uv_build>=0.11.33,<0.12"] +build-backend = "uv_build" diff --git a/packages/hflow-server/src/hflow_server/__init__.py b/packages/hflow-server/src/hflow_server/__init__.py new file mode 100644 index 0000000..62bc299 --- /dev/null +++ b/packages/hflow-server/src/hflow_server/__init__.py @@ -0,0 +1,13 @@ +"""HFlow workspace server: a read-mostly HTTP API over one data root. + +The public surface is deliberately tiny: :class:`ServerSettings` (parsed launch +configuration) and :func:`serve` (runs the server). The CLI's ``hflow serve`` +subcommand is a thin caller of exactly these two names. +""" + +from hflow_server._settings import ServerSettings +from hflow_server.server import create_app, serve + +__version__ = "0.1.0" + +__all__ = ["ServerSettings", "__version__", "create_app", "serve"] diff --git a/packages/hflow-server/src/hflow_server/_catalog.py b/packages/hflow-server/src/hflow_server/_catalog.py new file mode 100644 index 0000000..14e997c --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_catalog.py @@ -0,0 +1,904 @@ +"""The query layer: per-request DuckDB connections over one workspace catalog. + +Every request opens (and closes) a FRESH connection: the wide ``episodes`` +view binds one column per measurement key present at open time (see +``hflow.curation``), so a held connection would never show keys recorded +after startup. Opening is cheap -- the views read local Parquet directly. + +Two boundary rules hold everywhere here: + +- Filter VALUES travel as DuckDB bind parameters, never string-interpolated. + The only identifiers ever interpolated are validated against the live + view's DESCRIBE output (``order_by``) or are literal constants (facet + columns), then double-quoted. +- ``recorded_at`` leaves DuckDB as ISO-8601 TEXT: materializing a + TIMESTAMPTZ into Python requires pytz, which hflow deliberately does not + depend on. The connection is pinned to UTC so the rendering is stable + across host timezones. +""" + +import json +import math +from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal +from typing import Literal, TypeVar +from urllib.parse import quote + +import duckdb +from pydantic import BaseModel + +from hflow.app import ARTIFACT_MEASUREMENT_KEY_PREFIX, MEDIA_CONTACT_SHEET_STEP_NAME +from hflow.curation import open_catalog_connection +from hflow.workspace import Workspace +from hflow_server._contract import ( + CategoricalColumnStats, + ColumnDescriptor, + DossierEpisode, + EpisodeCheckRunRecord, + EpisodeColumnStats, + EpisodeDossierResponse, + EpisodeFacetsResponse, + EpisodeIntervalRecord, + EpisodeMeasurementRecord, + EpisodeMediaArtifact, + EpisodePageResponse, + EpisodeStatsResponse, + EpisodeStatus, + EpisodeTagRecord, + EpisodeTimelineResponse, + NumericColumnStats, + NumericHistogramBucket, + SuccessFilterValue, + TimelineInterval, + TimelineMeasurement, + ValueCount, +) +from hflow_server._media import is_uri_servable + +# The faceted columns, owned by the response model itself so the served keys +# and the columns actually counted can never diverge. +_FACET_COLUMN_NAMES = tuple(EpisodeFacetsResponse.model_fields) +_SEARCHED_COLUMN_NAMES = ("episode_id", "task", "operator") + +# %z renders the locked-UTC offset as "+00" on DuckDB 1.5.5, which JS +# Date.parse rejects and the frontend's offset-stripping regex misses; the +# connection is pinned to UTC, so render the wall time and append the offset +# literally -- matching _curation._timestamp_replace_clause's "+00:00". +_ISO_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S.%f" +_ISO_UTC_OFFSET_SUFFIX = "+00:00" + + +class UnknownOrderColumnError(ValueError): + """``order_by`` named a column the live episodes view does not have.""" + + +def open_workspace_connection(data_root: str) -> duckdb.DuckDBPyConnection: + """One fresh connection over ``/catalog`` (see the module note).""" + connection = open_catalog_connection(Workspace.parse(data_root).catalog_root) + connection.execute("SET TimeZone = 'UTC'") + return connection + + +def utc_iso_text(timestamp_expression: str, alias: str) -> str: + """SQL rendering a (UTC-pinned) timestamp expression as ISO-8601 text. + + ``timestamp_expression`` and ``alias`` are code-owned constants, never + user input. + """ + return ( + f"strftime({timestamp_expression}, '{_ISO_TIMESTAMP_FORMAT}') " + f"|| '{_ISO_UTC_OFFSET_SUFFIX}' AS {alias}" + ) + + +def _recorded_at_as_iso_text(qualified_column: str = "recorded_at") -> str: + return utc_iso_text(qualified_column, "recorded_at") + + +def json_safe_value(value: object) -> object: + """One DuckDB cell as a JSON-legal value. + + Datetimes become ISO-8601 strings (a safety net -- timestamp columns are + already rendered to TEXT in SQL) and NaN/inf doubles become null: both + are illegal in JSON and would otherwise poison the whole payload. The + remaining branches exist for the curation studio, where arbitrary user + SELECTs can materialize types JSON cannot carry (DECIMAL literals, + BLOBs, INTERVALs, nested LISTs/STRUCTs): containers are converted + element-wise and anything else is rendered as text -- a legal query must + never 500 over its result types. + """ + if value is None or isinstance(value, bool | int | str): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, datetime | date | time): + return value.isoformat() + if isinstance(value, Decimal): + return json_safe_value(float(value)) + if isinstance(value, list | tuple): + return [json_safe_value(element) for element in value] + if isinstance(value, dict): + return {str(key): json_safe_value(element) for key, element in value.items()} + if isinstance(value, bytes | bytearray): + return value.decode("utf-8", errors="replace") + return str(value) + + +def fetched_json_safe_rows(executed_query: duckdb.DuckDBPyConnection) -> list[dict[str, object]]: + column_names = [str(column[0]) for column in executed_query.description or []] + return [ + {name: json_safe_value(cell) for name, cell in zip(column_names, row, strict=True)} + for row in executed_query.fetchall() + ] + + +_ContractRecord = TypeVar("_ContractRecord", bound=BaseModel) + + +def _validated_records( + record_model: type[_ContractRecord], executed_query: duckdb.DuckDBPyConnection +) -> list[_ContractRecord]: + """A fixed-column query's rows as contract records. + + Rows pass through :func:`json_safe_value` first, so a NaN double is + already null by the time the model sees it. + """ + return [record_model.model_validate(row) for row in fetched_json_safe_rows(executed_query)] + + +@dataclass(frozen=True) +class EpisodeListFilters: + """Parsed /api/v1/episodes filter params -- values only, never SQL. + + ``status`` and ``success`` keep the refined types the HTTP boundary + already parsed them into: this layer never re-checks them, and a caller + cannot hand it a spelling the SQL below would silently match nothing for. + """ + + tasks: tuple[str, ...] = () + operators: tuple[str, ...] = () + embodiments: tuple[str, ...] = () + status: EpisodeStatus | None = None + success: SuccessFilterValue | None = None + search: str | None = None + + +def episode_status_for_quarantine_flag(quarantined: object) -> EpisodeStatus: + """One episode's status derived from its stored ``quarantined`` flag. + + hflow.curation owns the CANONICAL rule as SQL -- the wide ``episodes`` + view's ``CASE WHEN quarantined THEN 'quarantined' ELSE 'ok' END``. The + dossier reads ``episodes_latest``, which carries the raw flag instead of + that derived column, so this is the server's single Python restatement of the + rule; every path that needs a status from a flag calls here. + """ + return "quarantined" if quarantined else "ok" + + +def quoted_identifier(column_name: str) -> str: + return '"' + column_name.replace('"', '""') + '"' + + +def _escaped_like_fragment(raw_value: str) -> str: + """User text made literal inside a LIKE pattern (backslash-escaped).""" + return raw_value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + + +def _quoted_sql_literal(value: str) -> str: + """One string value as a single-quoted SQL literal (internal quotes doubled).""" + return "'" + value.replace("'", "''") + "'" + + +@dataclass(frozen=True) +class _CompiledFilters: + """The WHERE conditions in two parallel forms plus the bind values. + + ``executed_conditions`` carry ``?`` placeholders bound by ``parameters``; + ``display_conditions`` inline the same values as quoted literals. Building + the display form here (rather than by splitting rendered SQL on '?') means + an order_by identifier that itself contains '?' can never be miscounted as + a placeholder. + """ + + executed_conditions: list[str] + display_conditions: list[str] + parameters: list[str] + + def executed_where(self) -> str: + return ( + (" WHERE " + " AND ".join(self.executed_conditions)) if self.executed_conditions else "" + ) + + def display_where(self) -> str: + return ( + (" WHERE " + " AND ".join(self.display_conditions)) if self.display_conditions else "" + ) + + +def _compiled_conditions(filters: EpisodeListFilters) -> _CompiledFilters: + executed_conditions: list[str] = [] + display_conditions: list[str] = [] + parameters: list[str] = [] + exact_match_columns = ( + ("task", filters.tasks), + ("operator", filters.operators), + ("embodiment", filters.embodiments), + ) + for column_name, values in exact_match_columns: + if values: + quoted_column = quoted_identifier(column_name) + placeholders = ", ".join("?" for _ in values) + executed_conditions.append(f"{quoted_column} IN ({placeholders})") + inlined = ", ".join(_quoted_sql_literal(value) for value in values) + display_conditions.append(f"{quoted_column} IN ({inlined})") + parameters.extend(values) + if filters.status is not None: + executed_conditions.append('"status" = ?') + display_conditions.append(f'"status" = {_quoted_sql_literal(filters.status)}') + parameters.append(filters.status) + if filters.success is not None: + # Stored success is a stringified boolean whose casing varies by the + # recording producer; the filter accepts "true"/"false" regardless. + executed_conditions.append('lower("success") = ?') + display_conditions.append(f'lower("success") = {_quoted_sql_literal(filters.success)}') + parameters.append(filters.success) + if filters.search: + like_pattern = "%" + _escaped_like_fragment(filters.search) + "%" + pattern_literal = _quoted_sql_literal(like_pattern) + executed_disjuncts = " OR ".join( + f"{quoted_identifier(name)} ILIKE ? ESCAPE '\\'" for name in _SEARCHED_COLUMN_NAMES + ) + display_disjuncts = " OR ".join( + f"{quoted_identifier(name)} ILIKE {pattern_literal} ESCAPE '\\'" + for name in _SEARCHED_COLUMN_NAMES + ) + executed_conditions.append("(" + executed_disjuncts + ")") + display_conditions.append("(" + display_disjuncts + ")") + parameters.extend([like_pattern] * len(_SEARCHED_COLUMN_NAMES)) + return _CompiledFilters( + executed_conditions=executed_conditions, + display_conditions=display_conditions, + parameters=parameters, + ) + + +def described_episode_columns(connection: duckdb.DuckDBPyConnection) -> list[ColumnDescriptor]: + """The wide view's live columns.""" + return [ + ColumnDescriptor(name=str(row[0]), type=str(row[1])) + for row in connection.execute("DESCRIBE episodes").fetchall() + ] + + +def query_episode_page( + connection: duckdb.DuckDBPyConnection, + filters: EpisodeListFilters, + *, + order_by: str, + descending: bool, + limit: int, + offset: int, +) -> EpisodePageResponse: + """One filtered, ordered page plus the total over the SAME filters.""" + columns = described_episode_columns(connection) + live_column_names = {column.name for column in columns} + if order_by not in live_column_names: + raise UnknownOrderColumnError( + f"unknown order_by column {order_by!r}; order by one of the episodes view's " + "columns (the 'columns' field of this endpoint lists them)" + ) + compiled = _compiled_conditions(filters) + direction = "DESC" if descending else "ASC" + # episode_id is unique in the wide view, so it is a deterministic + # tiebreaker: without it, ordering by any column with duplicate values + # (task, status, ...) leaves ties unstable across DuckDB's per-query + # parallel sort, so successive OFFSET pages could overlap or drop rows. + order_clause = f'ORDER BY {quoted_identifier(order_by)} {direction}, "episode_id" ASC' + executed_tail = ( + f"FROM episodes{compiled.executed_where()} {order_clause} LIMIT {limit} OFFSET {offset}" + ) + display_tail = ( + f"FROM episodes{compiled.display_where()} {order_clause} LIMIT {limit} OFFSET {offset}" + ) + # The executed form renders recorded_at to ISO text in SQL (see the module + # note); the displayed form stays the logical query a user would write. + executed_sql = f"SELECT * REPLACE ({_recorded_at_as_iso_text()}) {executed_tail}" + display_sql = f"SELECT * {display_tail}" + rows = fetched_json_safe_rows(connection.execute(executed_sql, compiled.parameters)) + count_row = connection.execute( + f"SELECT count(*) FROM episodes{compiled.executed_where()}", compiled.parameters + ).fetchone() + total = int(count_row[0]) if count_row is not None else 0 + return EpisodePageResponse(rows=rows, total=total, columns=columns, sql=display_sql) + + +def query_episode_facets(connection: duckdb.DuckDBPyConnection) -> EpisodeFacetsResponse: + """Facet value counts over the wide episodes view; NULL buckets skipped.""" + facets: dict[str, list[ValueCount]] = {} + for facet_column_name in _FACET_COLUMN_NAMES: + quoted_column = quoted_identifier(facet_column_name) + value_counts = connection.execute( + f"SELECT {quoted_column} AS value, count(*) AS value_count FROM episodes " + f"WHERE {quoted_column} IS NOT NULL " + "GROUP BY 1 ORDER BY value_count DESC, value ASC" + ).fetchall() + facets[facet_column_name] = [ + ValueCount(value=str(value), count=int(count)) for value, count in value_counts + ] + return EpisodeFacetsResponse.model_validate(facets) + + +# /api/v1/episodes/stats shape knobs: ~12 histogram buckets per numeric +# column, top 8 values per categorical column (the remainder is other_count), +# and "low-cardinality" capped so id-like columns never masquerade as facets. +HISTOGRAM_BUCKET_COUNT = 12 +TOP_VALUE_LIMIT = 8 +LOW_CARDINALITY_LIMIT = 32 + +_NUMERIC_STAT_TYPES = frozenset( + { + "TINYINT", + "SMALLINT", + "INTEGER", + "BIGINT", + "HUGEINT", + "UTINYINT", + "USMALLINT", + "UINTEGER", + "UBIGINT", + "FLOAT", + "DOUBLE", + } +) +_CATEGORICAL_STAT_TYPES = frozenset({"VARCHAR", "BOOLEAN"}) + +# Which mini-distribution a column earns: the same two words the served +# models discriminate on (_contract.NumericColumnStats.kind / +# CategoricalColumnStats.kind), so the two dispatches below are checked +# against the closed set rather than against a bare string. +_StatKind = Literal["numeric", "categorical"] + + +def _stat_kind(duckdb_type: str) -> _StatKind | None: + """ "numeric"/"categorical" for distributable column types, else ``None`` + (timestamps, JSON blobs, and nested types have no mini-distribution).""" + normalized_type = duckdb_type.upper() + if normalized_type in _NUMERIC_STAT_TYPES or normalized_type.startswith("DECIMAL"): + return "numeric" + if normalized_type in _CATEGORICAL_STAT_TYPES: + return "categorical" + return None + + +@dataclass(frozen=True) +class _NumericColumnPlan: + """One numeric column that earned a histogram, with its bucket geometry.""" + + name: str + minimum: float + maximum: float + + @property + def bucket_width(self) -> float: + return (self.maximum - self.minimum) / HISTOGRAM_BUCKET_COUNT + + +@dataclass(frozen=True) +class _CategoricalColumnPlan: + """One low-cardinality column that earned a top-values breakdown.""" + + name: str + non_null_count: int + + +def query_episode_stats( + connection: duckdb.DuckDBPyConnection, filters: EpisodeListFilters +) -> EpisodeStatsResponse: + """Per-column mini-distributions over the CURRENT filter set. + + Reuses the episode list's filter compilation (one source of truth), so + the sparkbars always describe exactly the rows the table shows. Two + scans total: one aggregate pass classifying every candidate column + (skipping degenerate ones -- all NULL, a single value, NaN/inf-poisoned + numerics, id-like all-unique or over-the-cap categoricals), then one + UNION ALL query computing every surviving column's histogram buckets or + top values against a shared filtered CTE. + """ + compiled = _compiled_conditions(filters) + parameters = compiled.parameters + where_sql = compiled.executed_where() + candidate_columns = [ + (column.name, kind) + for column in described_episode_columns(connection) + if (kind := _stat_kind(column.type)) is not None + ] + if not candidate_columns: + return EpisodeStatsResponse(columns=[]) + + aggregate_expressions: list[str] = [] + for column_name, kind in candidate_columns: + quoted_column = quoted_identifier(column_name) + if kind == "numeric": + aggregate_expressions.extend( + ( + f"count({quoted_column})", + f"min(CAST({quoted_column} AS DOUBLE))", + f"max(CAST({quoted_column} AS DOUBLE))", + ) + ) + else: + aggregate_expressions.extend( + (f"count({quoted_column})", f"count(DISTINCT {quoted_column})") + ) + aggregate_row = connection.execute( + f"SELECT {', '.join(aggregate_expressions)} FROM episodes{where_sql}", parameters + ).fetchone() + if aggregate_row is None: + return EpisodeStatsResponse(columns=[]) + + plans: list[_NumericColumnPlan | _CategoricalColumnPlan] = [] + value_index = 0 + for column_name, kind in candidate_columns: + if kind == "numeric": + non_null_count, minimum, maximum = aggregate_row[value_index : value_index + 3] + value_index += 3 + if int(non_null_count or 0) == 0 or minimum is None or maximum is None: + continue + minimum, maximum = float(minimum), float(maximum) + # NaN/inf values poison min/max (NaN sorts above everything in + # DuckDB), so a non-finite bound marks the whole column degenerate. + # The span (max - min) can itself overflow to inf even when both + # bounds are finite (e.g. -1.7e308 and 1.7e308); an inf bucket + # width would be interpolated as the bare token "inf" into the + # histogram SQL, so require a finite span too. + if ( + not (math.isfinite(minimum) and math.isfinite(maximum)) + or not math.isfinite(maximum - minimum) + or minimum >= maximum + ): + continue + plans.append(_NumericColumnPlan(name=column_name, minimum=minimum, maximum=maximum)) + else: + non_null_count, distinct_count = aggregate_row[value_index : value_index + 2] + value_index += 2 + non_null_count, distinct_count = int(non_null_count or 0), int(distinct_count or 0) + if distinct_count < 2 or distinct_count > LOW_CARDINALITY_LIMIT: + continue + if distinct_count == non_null_count and distinct_count > 2: + # Every value unique: an identifier, not a distribution. + continue + plans.append(_CategoricalColumnPlan(name=column_name, non_null_count=non_null_count)) + if not plans: + return EpisodeStatsResponse(columns=[]) + + union_branches: list[str] = [] + for plan in plans: + quoted_column = quoted_identifier(plan.name) + name_literal = _quoted_sql_literal(plan.name) + if isinstance(plan, _NumericColumnPlan): + # Bounds are data-derived finite floats (never user input), so + # their repr()s are safe SQL literals. + union_branches.append( + f"SELECT {name_literal} AS column_name, " + f"least(CAST(floor((CAST({quoted_column} AS DOUBLE) - {plan.minimum!r}) " + f"/ {plan.bucket_width!r}) AS BIGINT), {HISTOGRAM_BUCKET_COUNT - 1}) " + "AS bucket_index, " + "CAST(NULL AS VARCHAR) AS value, count(*) AS bucket_count " + f"FROM filtered WHERE {quoted_column} IS NOT NULL GROUP BY 2" + ) + else: + union_branches.append( + "SELECT * FROM (" + f"SELECT {name_literal} AS column_name, CAST(NULL AS BIGINT) AS bucket_index, " + f"CAST({quoted_column} AS VARCHAR) AS value, count(*) AS bucket_count " + f"FROM filtered WHERE {quoted_column} IS NOT NULL " + f"GROUP BY 3 ORDER BY bucket_count DESC, value ASC LIMIT {TOP_VALUE_LIMIT})" + ) + # One query for every column: the shared CTE binds the filter parameters + # exactly once and each branch aggregates the same filtered rows. + distribution_rows = connection.execute( + f"WITH filtered AS (SELECT * FROM episodes{where_sql})\n" + + "\nUNION ALL\n".join(union_branches), + parameters, + ).fetchall() + + bucket_counts_by_column: dict[str, dict[int, int]] = {} + value_counts_by_column: dict[str, list[tuple[str, int]]] = {} + for column_name, bucket_index, value, count in distribution_rows: + if bucket_index is not None: + bucket_counts_by_column.setdefault(str(column_name), {})[int(bucket_index)] = int(count) + else: + value_counts_by_column.setdefault(str(column_name), []).append((str(value), int(count))) + + stat_columns: list[EpisodeColumnStats] = [] + for plan in plans: + if isinstance(plan, _NumericColumnPlan): + bucket_counts = bucket_counts_by_column.get(plan.name, {}) + stat_columns.append( + NumericColumnStats( + name=plan.name, + buckets=[ + NumericHistogramBucket( + lo=plan.minimum + index * plan.bucket_width, + hi=( + plan.maximum + if index == HISTOGRAM_BUCKET_COUNT - 1 + else plan.minimum + (index + 1) * plan.bucket_width + ), + count=bucket_counts.get(index, 0), + ) + for index in range(HISTOGRAM_BUCKET_COUNT) + ], + ) + ) + else: + # UNION ALL guarantees no cross-branch order; re-rank here. + top_values = sorted( + value_counts_by_column.get(plan.name, ()), + key=lambda entry: (-entry[1], entry[0]), + ) + stat_columns.append( + CategoricalColumnStats( + name=plan.name, + values=[ValueCount(value=value, count=count) for value, count in top_values], + other_count=plan.non_null_count - sum(count for _value, count in top_values), + ) + ) + return EpisodeStatsResponse(columns=stat_columns) + + +def find_media_uri( + connection: duckdb.DuckDBPyConnection, episode_id: str, artifact_name: str +) -> str | None: + """The cataloged URI behind one (episode, artifact name), if recorded.""" + row = connection.execute( + "SELECT value_text FROM measurements_latest " + "WHERE episode_id = ? AND check_name = ? AND key = ? AND value_text IS NOT NULL", + [ + episode_id, + MEDIA_CONTACT_SHEET_STEP_NAME, + ARTIFACT_MEASUREMENT_KEY_PREFIX + artifact_name, + ], + ).fetchone() + return str(row[0]) if row is not None and row[0] is not None else None + + +def find_canonical_uri(connection: duckdb.DuckDBPyConnection, episode_id: str) -> str | None: + """The latest cataloged canonical-file URI for one episode, if known.""" + row = connection.execute( + "SELECT uri FROM episodes_latest WHERE episode_id = ?", [episode_id] + ).fetchone() + return str(row[0]) if row is not None and row[0] is not None else None + + +def query_latest_run_intervals( + connection: duckdb.DuckDBPyConnection, episode_id: str +) -> list[EpisodeIntervalRecord]: + """One episode's intervals from its LATEST run -- the current evidence. + + ``check_version`` rides in from that run's ``check_runs`` row because the + intervals table does not carry one itself. One owner for this join: the + dossier and the timeline must never disagree about which run's intervals + an episode "has". + """ + return _validated_records( + EpisodeIntervalRecord, + connection.execute( + """ + SELECT i.label, i.start_ns, i.end_ns, i.check_name, r.check_version + FROM intervals AS i + JOIN episodes_latest AS e + ON i.episode_id = e.episode_id AND i.run_fingerprint = e.run_fingerprint + LEFT JOIN check_runs AS r + ON r.episode_id = i.episode_id AND r.run_fingerprint = i.run_fingerprint + AND r.check_name = i.check_name + WHERE i.episode_id = ? + ORDER BY i.start_ns, i.label + """, + [episode_id], + ), + ) + + +def query_episode_dossier( + connection: duckdb.DuckDBPyConnection, episode_id: str, *, data_root: str +) -> EpisodeDossierResponse | None: + """Everything the episode page shows, or ``None`` when the id is unknown.""" + episode_rows = fetched_json_safe_rows( + connection.execute( + f"SELECT * REPLACE ({_recorded_at_as_iso_text()}) " + "FROM episodes_latest WHERE episode_id = ?", + [episode_id], + ) + ) + if not episode_rows: + return None + episode_row = episode_rows[0] + raw_quarantine_tags = episode_row.get("quarantine_tags_json") + quarantine_tags = ( + [str(tag) for tag in json.loads(str(raw_quarantine_tags))] if raw_quarantine_tags else [] + ) + episode = DossierEpisode.model_validate( + { + **episode_row, + "status": episode_status_for_quarantine_flag(episode_row.get("quarantined")), + "quarantine_tags": quarantine_tags, + } + ) + + measurements = _validated_records( + EpisodeMeasurementRecord, + connection.execute( + "SELECT key, value_double, value_text, value_bool, check_name, check_version, " + f"{_recorded_at_as_iso_text()} " + "FROM measurements_latest WHERE episode_id = ? ORDER BY key", + [episode_id], + ), + ) + check_runs = _validated_records( + EpisodeCheckRunRecord, + connection.execute( + "SELECT check_name, check_version, critical, status, duration_s, error, " + f"{_recorded_at_as_iso_text()}, run_fingerprint " + "FROM check_runs WHERE episode_id = ? ORDER BY recorded_at DESC, check_name ASC", + [episode_id], + ), + ) + # Intervals and tags are the episode's LATEST run only -- the current + # evidence. + intervals = query_latest_run_intervals(connection, episode_id) + tags = _validated_records( + EpisodeTagRecord, + connection.execute( + f"SELECT t.tag, t.check_name, {_recorded_at_as_iso_text('t.recorded_at')} " + "FROM tags AS t " + "JOIN episodes_latest AS e " + " ON t.episode_id = e.episode_id AND t.run_fingerprint = e.run_fingerprint " + "WHERE t.episode_id = ? ORDER BY t.tag", + [episode_id], + ), + ) + history = fetched_json_safe_rows( + connection.execute( + f"SELECT * REPLACE ({_recorded_at_as_iso_text()}) " + "FROM episodes_raw WHERE episode_id = ? " + "ORDER BY recorded_at DESC, run_fingerprint DESC", + [episode_id], + ) + ) + + media_rows = connection.execute( + "SELECT key, value_text FROM measurements_latest " + "WHERE episode_id = ? AND check_name = ? AND key LIKE ? AND value_text IS NOT NULL " + "ORDER BY key", + [episode_id, MEDIA_CONTACT_SHEET_STEP_NAME, ARTIFACT_MEASUREMENT_KEY_PREFIX + "%"], + ).fetchall() + quoted_episode_id = quote(episode_id, safe="") + media: list[EpisodeMediaArtifact] = [] + for key, artifact_uri in media_rows: + artifact_name = str(key).removeprefix(ARTIFACT_MEASUREMENT_KEY_PREFIX) + served_url = ( + f"/api/v1/episodes/{quoted_episode_id}/media/{quote(artifact_name, safe='/')}" + if is_uri_servable(str(artifact_uri), data_root=data_root) + else None + ) + media.append( + EpisodeMediaArtifact(name=artifact_name, uri=str(artifact_uri), url=served_url) + ) + + canonical_uri = episode_row.get("uri") + canonical_url = ( + f"/api/v1/episodes/{quoted_episode_id}/canonical" + if isinstance(canonical_uri, str) and is_uri_servable(canonical_uri, data_root=data_root) + else None + ) + return EpisodeDossierResponse( + episode=episode, + measurements=measurements, + check_runs=check_runs, + intervals=intervals, + tags=tags, + history=history, + media=media, + canonical_url=canonical_url, + ) + + +NANOSECONDS_PER_SECOND = 1_000_000_000 + +# Timeline span derivation. Interval times are nanoseconds of LOG time, so an +# episode with intervals carries its own axis; an episode without them can +# still have a length if some check measured one. A measurement key naming a +# duration supplies that length: the token after the key's last '_' picks the +# unit, and a duration key with NO recognized suffix at all +# (``episode_duration``) is read as SECONDS -- the convention every hflow +# example follows. +# +# The two tables below are one fact split in two, and neither may be read +# alone: _UNIT_BY_KEY_SUFFIX owns which suffixes name a dimension at all (and +# what to call it), _NANOSECONDS_PER_DURATION_UNIT owns which of those +# dimensions are TIMES and how long one is. Every key of the second is a key +# of the first. A suffix the first knows and the second does not is a +# NON-time dimension (hz, pct, count, bytes, deg), so a key like +# ``duty_cycle_duration_pct`` measures no length -- reading it as seconds +# would both contradict the "45 %" its own bar is labelled with and stretch +# the episode's axis by 1e9. +_DURATION_KEY_TOKEN = "duration" +_NANOSECONDS_PER_DURATION_UNIT: dict[str, float] = { + "ns": 1.0, + "us": 1e3, + "ms": 1e6, + "s": 1e9, + "sec": 1e9, + "secs": 1e9, + "second": 1e9, + "seconds": 1e9, + "min": 6e10, + "mins": 6e10, + "minute": 6e10, + "minutes": 6e10, +} +_DEFAULT_DURATION_UNIT_NANOSECONDS = 1e9 + +# Units the measurement bars label themselves with, by the same key suffix. +# Absent from this table means "no unit known" -- the bar shows the bare +# number rather than inventing a dimension. +_UNIT_BY_KEY_SUFFIX: dict[str, str] = { + "ns": "ns", + "us": "us", + "ms": "ms", + "s": "s", + "sec": "s", + "secs": "s", + "second": "s", + "seconds": "s", + "min": "min", + "mins": "min", + "minute": "min", + "minutes": "min", + "hz": "Hz", + "pct": "%", + "percent": "%", + "ratio": "ratio", + "count": "count", + "bytes": "bytes", + "mb": "MB", + "gb": "GB", + "m": "m", + "mm": "mm", + "cm": "cm", + "km": "km", + "deg": "deg", + "rad": "rad", + "kg": "kg", + "n": "N", +} + + +def _measurement_key_suffix(key: str) -> str: + """The unit-bearing tail of a measurement key (``max_gap_ms`` -> ``ms``).""" + return key.rsplit("_", 1)[-1].lower() if "_" in key else "" + + +def _duration_nanoseconds(key: str, value: float) -> float | None: + """A duration-naming measurement converted to nanoseconds, if it is one. + + ``None`` for anything that is not a length, INCLUDING a key that says + "duration" but carries a suffix naming another dimension (see the note + above the tables): a measurement the bars label "45 %" must not also + claim the episode ran for 45 seconds. + """ + if _DURATION_KEY_TOKEN not in key.lower() or not math.isfinite(value) or value <= 0: + return None + key_suffix = _measurement_key_suffix(key) + unit_scale = _NANOSECONDS_PER_DURATION_UNIT.get(key_suffix) + if unit_scale is not None: + return value * unit_scale + if key_suffix in _UNIT_BY_KEY_SUFFIX: + return None + return value * _DEFAULT_DURATION_UNIT_NANOSECONDS + + +def _interval_kind(label: str | None, check_name: str | None) -> str: + """The colour group for one interval label. + + Labels are conventionally ``:`` (``gap:/imu``, + ``joint_discontinuity:/joint_states``), so the prefix is the group. A + label with no prefix groups by itself; an empty label falls back to the + check that produced it, which is the only honest grouping left. + """ + text = label.strip() if label is not None else "" + if not text: + return check_name if check_name else "interval" + prefix = text.split(":", 1)[0].strip() + return prefix or text + + +def _relative_seconds(absolute_ns: int | None, span_start_ns: int | None) -> float | None: + if span_start_ns is None or absolute_ns is None: + return None + return (absolute_ns - span_start_ns) / NANOSECONDS_PER_SECOND + + +def query_episode_timeline( + connection: duckdb.DuckDBPyConnection, episode_id: str +) -> EpisodeTimelineResponse | None: + """One episode's time axis, computed server-side (``None`` when unknown). + + The span comes from the latest run's intervals, extended by any duration + measurement that claims a longer episode; an episode with no intervals but + a duration measurement gets a zero-based axis; an episode with neither + gets nulls, and a client says the span is unknown rather than drawing a + fabricated axis. + """ + if ( + connection.execute( + "SELECT 1 FROM episodes_latest WHERE episode_id = ?", [episode_id] + ).fetchone() + is None + ): + return None + + interval_rows = query_latest_run_intervals(connection, episode_id) + measurement_rows = connection.execute( + "SELECT key, value_double FROM measurements_latest " + "WHERE episode_id = ? AND value_double IS NOT NULL ORDER BY key", + [episode_id], + ).fetchall() + numeric_measurements = [ + (str(key), float(value)) + for key, value in measurement_rows + # NaN/inf poison a bar chart exactly as they poison JSON: drop them. + if isinstance(value, int | float) and math.isfinite(float(value)) + ] + + interval_starts = [row.start_ns for row in interval_rows if row.start_ns is not None] + interval_ends = [row.end_ns for row in interval_rows if row.end_ns is not None] + # Several duration-ish measurements: the largest wins, because the span + # must contain every interval AND every claimed duration. + claimed_durations_ns = [ + duration_ns + for key, value in numeric_measurements + if (duration_ns := _duration_nanoseconds(key, value)) is not None + ] + longest_claimed_duration_ns = max(claimed_durations_ns) if claimed_durations_ns else None + + start_ns: int | None = None + end_ns: int | None = None + if interval_starts: + start_ns = min(interval_starts) + end_ns = max([*interval_ends, start_ns]) + if longest_claimed_duration_ns is not None: + end_ns = max(end_ns, start_ns + int(longest_claimed_duration_ns)) + elif longest_claimed_duration_ns is not None: + start_ns, end_ns = 0, int(longest_claimed_duration_ns) + + duration_s = ( + (end_ns - start_ns) / NANOSECONDS_PER_SECOND + if start_ns is not None and end_ns is not None + else None + ) + return EpisodeTimelineResponse( + start_ns=start_ns, + end_ns=end_ns, + duration_s=duration_s, + intervals=[ + TimelineInterval( + label=row.label, + start_ns=row.start_ns, + end_ns=row.end_ns, + start_s=_relative_seconds(row.start_ns, start_ns), + end_s=_relative_seconds(row.end_ns, start_ns), + check_name=row.check_name, + kind=_interval_kind(row.label, row.check_name), + ) + for row in interval_rows + ], + measurements=[ + TimelineMeasurement( + key=key, value=value, unit=_UNIT_BY_KEY_SUFFIX.get(_measurement_key_suffix(key)) + ) + for key, value in numeric_measurements + ], + ) diff --git a/packages/hflow-server/src/hflow_server/_connections.py b/packages/hflow-server/src/hflow_server/_connections.py new file mode 100644 index 0000000..5a83eda --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_connections.py @@ -0,0 +1,90 @@ +"""Opening a catalog connection for one request -- and refusing as one voice. + +Every request that reads the catalog opens a FRESH connection (see +``_catalog``'s module note for why) and must close it again, and every one of +them owes the caller the same answer when the workspace cannot serve it. Both +facts live here so no route restates either: + +- a data root with no catalog is a MISSING RESOURCE (404); +- a catalog present but written in a format version this build cannot read is + a STATE CONFLICT (409) -- the workspace is there, this build just cannot + speak to it. + +The context managers are the only supported way to open a connection inside a +request: they own the ``open -> use -> close`` shape too, so no endpoint +hand-writes another ``try/finally``. +""" + +from collections.abc import Iterator +from contextlib import contextmanager + +import duckdb +from fastapi import HTTPException + +from hflow.curation import open_catalog_connection +from hflow.workspace import Workspace +from hflow_server import _catalog + + +def catalog_unavailable_refusal(error: FileNotFoundError | ValueError) -> HTTPException: + """The HTTP refusal one unusable catalog maps to (see the module note).""" + if isinstance(error, FileNotFoundError): + return HTTPException(status_code=404, detail=str(error)) + return HTTPException(status_code=409, detail=str(error)) + + +@contextmanager +def opened_workspace_connection_or_refuse(data_root: str) -> Iterator[duckdb.DuckDBPyConnection]: + """A live (UTC-pinned) connection for the server's OWN queries.""" + try: + connection = _catalog.open_workspace_connection(data_root) + except (FileNotFoundError, ValueError) as error: + raise catalog_unavailable_refusal(error) from error + try: + yield connection + finally: + connection.close() + + +@contextmanager +def opened_workspace_connection_or_none( + data_root: str, +) -> Iterator[duckdb.DuckDBPyConnection | None]: + """The same connection, but a workspace with NO catalog yields ``None``. + + For the endpoints where "nothing has been recorded yet" is an answer + rather than a 404. A catalog this build cannot read still refuses: that is + a conflict either way. + """ + try: + connection = _catalog.open_workspace_connection(data_root) + except FileNotFoundError: + yield None + return + except ValueError as error: + raise catalog_unavailable_refusal(error) from error + try: + yield connection + finally: + connection.close() + + +@contextmanager +def opened_constrained_connection_or_refuse(data_root: str) -> Iterator[duckdb.DuckDBPyConnection]: + """The connection USER SQL runs on: catalog materialized in memory, file + access and extension loading locked out. + + Its configuration is locked at open, so the ``SET TimeZone`` pin the live + connection uses cannot apply here; timestamp columns are instead rendered + to UTC ISO text in SQL (``_curation._timestamp_replace_clause``). + """ + try: + connection = open_catalog_connection( + Workspace.parse(data_root).catalog_root, constrained=True + ) + except (FileNotFoundError, ValueError) as error: + raise catalog_unavailable_refusal(error) from error + try: + yield connection + finally: + connection.close() diff --git a/packages/hflow-server/src/hflow_server/_contract.py b/packages/hflow-server/src/hflow_server/_contract.py new file mode 100644 index 0000000..9b719ac --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_contract.py @@ -0,0 +1,769 @@ +"""The published JSON contract: one model per payload this API serves. + +The API is the product surface -- the shipped SPA is only its reference +client, and third parties (increasingly coding agents) build against the +schema at ``/api/openapi.json`` (the schema JSON is the whole published docs +surface -- FastAPI's Swagger page is disabled because it loads from a CDN). +So every route declares a model from this module as its response type instead +of hand-building a dict: the model is the ONE owner of that payload's field +names and types, and the generated OpenAPI describes what actually goes over +the wire. + +Four shapes stay deliberately open, each because another module owns it and +a mirror here could only drift: + +- rows of the wide ``episodes`` view and of a user's own SELECT -- their + columns ARE data, described alongside the rows by :class:`ColumnDescriptor`; +- DuckDB ``SUMMARIZE`` rows, whose key set varies by DuckDB version; +- the pipeline manifest, owned and version-stamped by ``hflow.manifest``; +- a dag run's ``conf`` (:class:`RuntimeRunSummary`), which is whatever the + trigger sent -- ``hflow.runtime.AirflowClient.ingest`` owns the shape of + the ones this API mints, but a run started from Airflow's own UI can carry + anything, so no model here could describe it honestly. + +Nullability follows the catalog's DDL (``hflow.catalog.TABLE_COLUMN_DDL``), +which declares no NOT NULL: a field a stored row could carry as NULL is typed +nullable, so odd data is served honestly instead of turning into a 500 from +response validation. + +Two of these models are also the sidecar's ON-DISK shape +(:class:`SavedQueryEntry`, :class:`PinnedManifestEntry`) -- and so is +everything they nest (:class:`CheckCoverageEntry`): ``_sidecar`` stores +exactly what the API serves, so the registry a user can read with ``jq`` and +the payload the API returns can never disagree. Changing any of them +therefore changes the stored format, which ``_sidecar.STATE_VERSION`` guards. +""" + +from typing import Annotated, Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +from hflow.manifest import StepKind, StepManifest +from hflow.steps import Stage + +# --- shared vocabularies ----------------------------------------------------- + +# Vocabularies the SDK already owns as closed enums (``hflow.steps.Stage``, +# ``hflow.manifest.StepKind``) are annotated with the enum itself rather than +# restated as a Literal: pydantic serializes a StrEnum to its value, so the +# wire bytes are unchanged while the schema publishes the closed set and the +# adapters below stop unwrapping with ``.value``. The aliases here are for +# vocabularies no module owns as a type. +# +# hflow.curation owns the canonical ok/quarantined rule in SQL (the wide +# ``episodes`` view's ``CASE WHEN quarantined THEN 'quarantined' ELSE 'ok' +# END``). This alias is this API's one restatement of that vocabulary: the +# status filter, the facet values, and the dossier's derived status all use +# it, and _catalog.episode_status_for_quarantine_flag is the one place that +# derives a value of it from a raw flag. +EpisodeStatus = Literal["ok", "quarantined"] + +# Stored ``success`` is a stringified boolean whose casing varies by producer, +# so the filter matches case-insensitively on these two spellings. +SuccessFilterValue = Literal["true", "false"] + +ListingOrder = Literal["asc", "desc"] + +RuntimeSource = Literal["bundle", "remote"] + +# How a stage sub-DAG run was attributed to a master run. "heuristic" is the +# only honest answer available: Airflow stores no parent-run link, so the +# attribution is by time window alone (see _graph._matched_stage_run). +StageRunMatch = Literal["heuristic"] + +# Which cheap-first tier a registered step runs in (hflow.App._ordered_checks). +StepTier = Literal[1, 2] + +# What a browsable catalog relation is, as information_schema reports it. +CatalogTableKind = Literal["view", "table"] + + +class ColumnDescriptor(BaseModel): + """One result column as DuckDB's ``DESCRIBE`` reports it.""" + + name: str + type: str + + +class ValueCount(BaseModel): + """One value and how many episodes carry it.""" + + value: str + count: int + + +# --- /api/v1/health, /api/v1/config ------------------------------------------- + + +class HealthResponse(BaseModel): + """The liveness answer: the cheapest endpoint a probe can poll.""" + + ok: bool + + +class WorkspaceCapabilities(BaseModel): + """What this launch can actually do over this data root. + + ``runtime`` means ADDRESSED (a rendered bundle or an exported remote URL), + not reachable -- /runtime/status owns liveness. + """ + + catalog: bool + media: bool + curation: bool = Field( + description="Whether the curation studio's durable state can be written at " + "all: saved queries, the pinned-manifest registry, and the manifest files " + "need a LOCAL data root, so a bucket-backed workspace answers 501 for every " + "one of them and the frontend should not offer them." + ) + runtime: bool + pipeline: bool + + +class WorkspaceConfigResponse(BaseModel): + """What this server is serving, and what the frontend may offer. + + Deliberately carries no Airflow deep-link base: /runtime/status is the one + owner of the runtime's addressing facts, including its web URL. + """ + + mode: Literal["local"] + read_only: bool + hflow_version: str + hflow_server_version: str + data_root: str + workspace_id: str | None + capabilities: WorkspaceCapabilities + run_profiles: list[str] = Field( + description="Live run-profile names from hflow.steps.RUN_PROFILES, " + "served so the frontend never hardcodes them." + ) + ingest_modes: list[str] = Field( + description="Live ingest modes from hflow.steps.IngestMode; same contract as run_profiles." + ) + + +# --- /api/v1/episodes --------------------------------------------------------- + + +class EpisodePageResponse(BaseModel): + """One filtered, ordered page of the wide ``episodes`` view.""" + + rows: list[dict[str, Any]] = Field( + description="Rows of the wide episodes view. Its columns are data (one per " + "measurement key present at open time), so they are described by " + "'columns' rather than enumerated here." + ) + total: int = Field(description="Rows matching the SAME filters, ignoring limit/offset.") + columns: list[ColumnDescriptor] + sql: str = Field( + description="The SELECT compiled for exactly these filters, with values inlined " + "so it is copy-pastable and runs against the same catalog." + ) + + +class EpisodeFacetsResponse(BaseModel): + """Facet value counts over the wide episodes view; NULL buckets skipped. + + This model is the one owner of WHICH columns are faceted: ``_catalog`` + reads the column list off these fields rather than restating it. + """ + + task: list[ValueCount] + operator: list[ValueCount] + embodiment: list[ValueCount] + status: list[ValueCount] + pipeline_version: list[ValueCount] + + +class NumericHistogramBucket(BaseModel): + """One histogram bucket: ``lo`` inclusive, ``hi`` inclusive on the last.""" + + lo: float + hi: float + count: int + + +class NumericColumnStats(BaseModel): + """A numeric column's mini-distribution under the current filters.""" + + name: str + kind: Literal["numeric"] = "numeric" + buckets: list[NumericHistogramBucket] + + +class CategoricalColumnStats(BaseModel): + """A low-cardinality column's top values under the current filters.""" + + name: str + kind: Literal["categorical"] = "categorical" + values: list[ValueCount] + other_count: int = Field(description="Non-null rows beyond the served top values.") + + +EpisodeColumnStats = Annotated[ + NumericColumnStats | CategoricalColumnStats, Field(discriminator="kind") +] + + +class EpisodeStatsResponse(BaseModel): + """Per-column mini-distributions; degenerate columns are omitted entirely.""" + + columns: list[EpisodeColumnStats] + + +class DossierEpisode(BaseModel): + """The episode's own ``episodes_latest`` row plus the two derived fields. + + ``extra="allow"``: every column of that row rides along unchanged, because + the catalog's columns are data this module cannot enumerate. + """ + + model_config = ConfigDict(extra="allow") + + status: EpisodeStatus + quarantine_tags: list[str] = Field( + description="Parsed out of the row's quarantine_tags_json; empty when not quarantined." + ) + + +class EpisodeMeasurementRecord(BaseModel): + """One measurement, latest per key.""" + + key: str | None + value_double: float | None + value_text: str | None + value_bool: bool | None + check_name: str | None + check_version: str | None + recorded_at: str | None + + +class EpisodeCheckRunRecord(BaseModel): + """One recorded check invocation.""" + + check_name: str | None + check_version: str | None + critical: bool | None + status: str | None + duration_s: float | None + error: str | None + recorded_at: str | None + run_fingerprint: str | None + + +class EpisodeIntervalRecord(BaseModel): + """One interval of the episode's LATEST run. + + ``check_version`` rides in from that run's ``check_runs`` row (a LEFT + JOIN -- the intervals table carries no version of its own). + """ + + label: str | None + start_ns: int | None + end_ns: int | None + check_name: str | None + check_version: str | None + + +class EpisodeTagRecord(BaseModel): + """One tag of the episode's LATEST run.""" + + tag: str | None + check_name: str | None + recorded_at: str | None + + +class EpisodeMediaArtifact(BaseModel): + """One cataloged media artifact and, when servable, its byte URL.""" + + name: str + uri: str + url: str | None = Field( + description="Same-origin byte-serving path, or null when the cataloged file " + "is missing or lands outside the workspace data root." + ) + + +class EpisodeDossierResponse(BaseModel): + """Everything the episode page shows for one episode.""" + + episode: DossierEpisode + measurements: list[EpisodeMeasurementRecord] + check_runs: list[EpisodeCheckRunRecord] + intervals: list[EpisodeIntervalRecord] + tags: list[EpisodeTagRecord] + history: list[dict[str, Any]] = Field( + description="Every append of this episode, newest first: raw episodes_raw rows, " + "whose columns are the catalog's (see EpisodePageResponse.rows)." + ) + media: list[EpisodeMediaArtifact] + canonical_url: str | None + + +class TimelineInterval(BaseModel): + """One interval placed on the episode's axis, in absolute ns and in + seconds RELATIVE to the span start (both computed server-side).""" + + label: str | None + start_ns: int | None + end_ns: int | None + start_s: float | None + end_s: float | None + check_name: str | None + kind: str = Field( + description="Colour group: the label's ':' prefix, else the " + "whole label, else the check that produced it." + ) + + +class TimelineMeasurement(BaseModel): + """One numeric measurement, ready to draw as a bar.""" + + key: str + value: float + unit: str | None = Field( + description="Inferred from the key's unit suffix; null when no dimension is known." + ) + + +class EpisodeTimelineResponse(BaseModel): + """One episode's time axis. All-null bounds mean the span is unknown -- + a client must say so rather than draw a fabricated axis.""" + + start_ns: int | None + end_ns: int | None + duration_s: float | None + intervals: list[TimelineInterval] + measurements: list[TimelineMeasurement] + + +# --- /api/v1/curation, /api/v1/queries, /api/v1/manifests --------------------- + + +class CurationPreviewResponse(BaseModel): + """A user SELECT's first rows, its full count, and optional column stats.""" + + columns: list[ColumnDescriptor] + rows: list[dict[str, Any]] = Field( + description="Rows of the user's own SELECT; its columns are described by 'columns'." + ) + row_count: int = Field(description="Rows the SELECT returns in full, independent of limit.") + truncated: bool + column_stats: list[dict[str, Any]] | None = Field( + description="DuckDB SUMMARIZE rows (column_name, column_type, min, max, " + "null_percentage, ...). DuckDB owns that shape and varies it by version, " + "so it is served as-is. Null unless the request asked for stats." + ) + sql: str = Field( + description="The logical wrapped SELECT, copy-pastable as-is. The executed " + "statement adds a '* REPLACE (...)' projection rendering TIMESTAMPTZ columns " + "as UTC ISO text (the locked connection cannot SET TimeZone), which is a " + "rendering detail of these rows rather than part of the query a user wrote." + ) + + +class CheckCoverageEntry(BaseModel): + """One check's coverage denominator over the WHOLE catalog, not the cut. + + Also the sidecar's stored shape, nested inside every stored manifest + entry's ``coverage`` (see the module note). + """ + + check_name: str + episodes_ran: int + total_episodes: int + fraction: float + + +class CurationReportResponse(BaseModel): + """What a cut would contain, and what evidence backs it -- no files written.""" + + row_count: int + total_episodes: int + coverage: list[CheckCoverageEntry] + + +class SavedQueryEntry(BaseModel): + """One saved studio query. + + Also the sidecar's stored shape for a saved query (see the module note). + """ + + model_config = ConfigDict(populate_by_name=True) + + query_id: str = Field(alias="id") + name: str + sql: str + updated_at: str = Field(description="ISO-8601 UTC.") + + +class SavedQueryListResponse(BaseModel): + queries: list[SavedQueryEntry] + + +class PinnedManifestEntry(BaseModel): + """One registry entry for an immutable pinned manifest file. + + Also the sidecar's stored shape for a manifest (see the module note). + """ + + model_config = ConfigDict(populate_by_name=True) + + manifest_id: str = Field(alias="id") + name: str + description: str + sql: str + manifest_path: str = Field( + description="Data-root-relative, e.g. 'manifests/-.parquet'." + ) + row_count: int + total_episodes: int + coverage: list[CheckCoverageEntry] = Field(description="Frozen at pin time.") + created_at: str = Field(description="ISO-8601 UTC.") + + +class PinnedManifestListResponse(BaseModel): + manifests: list[PinnedManifestEntry] + + +class CatalogTableDescription(BaseModel): + """One browsable catalog relation and its live columns.""" + + name: str + kind: CatalogTableKind + columns: list[ColumnDescriptor] + + +class CatalogTablesResponse(BaseModel): + tables: list[CatalogTableDescription] + + +class CatalogTableSummaryResponse(BaseModel): + """One relation's row count and DuckDB's own column profile.""" + + row_count: int + columns: list[dict[str, Any]] = Field( + description="DuckDB SUMMARIZE rows; see CurationPreviewResponse.column_stats." + ) + + +# --- /api/v1/runtime ---------------------------------------------------------- + + +class RuntimeHealthComponents(BaseModel): + """Airflow's per-component health. + + This model is the one owner of WHICH components /runtime/status reports: + ``_runtime`` reads the names off these fields. A component absent from the + deployment (a minimal stack runs no triggerer) reports null. + """ + + metadatabase: str | None + scheduler: str | None + triggerer: str | None + dag_processor: str | None + + +class RuntimeStatusResponse(BaseModel): + """Whether this workspace's ingest runtime is addressed AND answering. + + Every field except ``available`` defaults to "not known", so an + unavailable answer states only the facts it actually has -- there is no + second hand-written shape for the unavailable case to drift from. + """ + + available: bool + detail: str | None = Field( + default=None, description="Why the runtime is unavailable; null when it is available." + ) + source: RuntimeSource | None = None + airflow_web_url: str | None = Field( + default=None, + description="Deep-link base for the Airflow web UI, AS ADDRESSED FROM THE " + "WORKSPACE HOST. Only a local bundle records its own address; a remote " + "endpoint's is unknown, never guessed.", + ) + airflow_web_url_host_only: bool = Field( + default=False, + description="True when airflow_web_url is a loopback address, so it resolves " + "only on the workspace host: a browser on another machine cannot follow it, " + "and the runtime is reachable there only through a tunnel or a wider " + "`hflow up --api-bind-host`.", + ) + dag_id: str | None = None + registered: bool | None = Field( + default=None, + description="Whether the master DAG is registered. Null means unknown (an auth " + "or transient failure), which is not the same as false.", + ) + health: RuntimeHealthComponents | None = None + + +class RuntimeRunSummary(BaseModel): + """One master DAG run, reduced to what the Runs page shows.""" + + dag_run_id: str | None + state: str | None + logical_date: str | None + start_date: str | None + end_date: str | None + conf: dict[str, Any] = Field(description="The trigger's own input, forwarded verbatim.") + + +class StageRunSummary(BaseModel): + """One stage sub-DAG run in a stage's recent strip.""" + + dag_run_id: str | None + state: str | None + start_date: str | None + end_date: str | None + + +class StageRecentRuns(BaseModel): + """One stage's most recent runs. NOT correlated with any master run.""" + + stage: Stage + dag_id: str + recent: list[StageRunSummary] + + +class RuntimeRunsResponse(BaseModel): + runs: list[RuntimeRunSummary] + stages: list[StageRecentRuns] | None = Field( + description="Per-stage recent runs; null for a remote runtime, whose stage " + "sub-DAG ids only a bundle manifest records." + ) + + +class IngestTriggerResponse(BaseModel): + """What Airflow answered when the run was triggered.""" + + dag_run_id: str | None + state: str | None + + +# --- /api/v1/pipeline --------------------------------------------------------- + + +class PipelineStepManifest(BaseModel): + """One registered step, exactly as ``hflow.manifest.StepManifest`` renders it.""" + + name: str + kind: StepKind + version: str = Field(description="Content hash of the live function.") + critical: bool + requires: list[str] + uses: str | None + + @classmethod + def from_step_manifest(cls, step: StepManifest) -> "PipelineStepManifest": + return cls( + name=step.name, + kind=step.kind, + version=step.version, + critical=step.critical, + requires=list(step.requires), + uses=step.uses, + ) + + +class ObservedCheckVersion(BaseModel): + """What the catalog has SEEN of one (check, version) pair.""" + + check_name: str | None + check_version: str | None + first_seen: str | None + last_seen: str | None + run_count: int + + +class StaleSummary(BaseModel): + """How many recorded episodes are stale against the App's current versions.""" + + pipeline_version: str + count: int + + +class PipelineResponse(BaseModel): + """The startup-imported App, described over this workspace's catalog.""" + + manifest: dict[str, Any] = Field( + description="The pipeline manifest exactly as hflow.manifest.PipelineManifest " + "renders it. hflow.manifest owns that shape and stamps it with " + "'manifest_version', so it is forwarded rather than mirrored here." + ) + observed: list[ObservedCheckVersion] + stale: StaleSummary | None = Field( + description="Null when staleness is unknowable (no catalog yet)." + ) + + +# --- /api/v1/pipeline/graph, /api/v1/runtime/runs/{id}/graph ------------------- + + +class DagTaskNodePayload(BaseModel): + """One task of a generated DAG (mirrors ``hflow.runtime.DagTaskNode``).""" + + task_id: str + summary: str + mapped: bool = Field(description="Dynamically mapped: one instance per planned batch.") + deferred: bool = Field(description="Defers instead of holding a worker slot.") + + +class DagTopologyPayload(BaseModel): + """One DAG's real shape: its tasks and their real dependency edges.""" + + dag_id: str + tasks: list[DagTaskNodePayload] + edges: list[tuple[str, str]] = Field( + description="[upstream, downstream] task-id pairs, in declaration order." + ) + + +class PipelineEngineStep(BaseModel): + """Engine work inside one stage that no manifest lists.""" + + name: str + summary: str + + +class PipelineUserStep(PipelineStepManifest): + """A registered step as the graph endpoint serves it. + + ``tier`` mirrors ``hflow.App._ordered_checks``: tier 2 is exactly the steps + declaring ``requires`` or ``uses``. Steps within a tier have NO ordering. + """ + + tier: StepTier + + @classmethod + def from_step_manifest_in_tier(cls, step: StepManifest, tier: StepTier) -> "PipelineUserStep": + return cls(**PipelineStepManifest.from_step_manifest(step).model_dump(), tier=tier) + + +class QuarantineGate(BaseModel): + """The one real cross-step edge, served as its own object rather than as + an edge in either graph.""" + + from_stage: Stage + to_stages: list[Stage] + critical_step_names: list[str] + explanation: str + + +class PipelineGraphStage(BaseModel): + """One stage lane of the pipeline graph: its DAG plus what runs inside it.""" + + stage: Stage + title: str + description: str + gate_task_id: str + trigger_task_id: str + enabling_profiles: list[str] + dag: DagTopologyPayload + engine_steps: list[PipelineEngineStep] + user_steps: list[PipelineUserStep] + + +class PipelineGraphResponse(BaseModel): + """The ingest DAG's shape merged with the pipeline's own steps.""" + + dag_ids_known: bool = Field( + description="False when no runtime is addressed: the dag ids are display-only." + ) + steps_known: bool = Field( + description="False without --pipeline: what runs inside process_batch is unknown." + ) + master: DagTopologyPayload + stages: list[PipelineGraphStage] + quarantine_gate: QuarantineGate | None = Field( + description="Null exactly when steps_known is false." + ) + + +class RunTaskInstance(BaseModel): + """One Airflow task instance, reduced to what the graph draws.""" + + task_id: str | None + state: str | None + start_date: str | None + end_date: str | None + queued_at: str | None = Field( + description="When the scheduler queued the task, so a replay can tell " + "'waiting for a worker' from 'running'. Airflow may omit it." + ) + try_number: int | None + map_index: int = Field(description="-1 means the task is not mapped.") + duration_s: float | None + + +class MappedFanOutSummary(BaseModel): + """The fan-out's live split, counted server-side over EVERY mapped instance. + + Complete on its own: ``by_state`` partitions all ``total`` instances of + ``task_id`` (an instance Airflow has not scheduled yet counts under + ``no_status``), so ``total == sum(by_state.values())`` always holds and a + client never has to recount the raw instances to size or colour the + fan-out. Only a replay at some earlier instant is a different fact, and + that one the server cannot answer. + """ + + task_id: str + total: int = Field( + description="Instances reported for the mapped task. Before the fan-out expands " + "Airflow reports one unexpanded instance, which is counted -- that is the " + "truth at that moment." + ) + by_state: dict[str, int] + + +class RunGraphMaster(BaseModel): + """The master run's own live state.""" + + dag_run_id: str + state: str | None + tasks: list[RunTaskInstance] + + +class RunGraphStage(BaseModel): + """One stage's live state for this master run, or explicit nulls when the + stage never ran for it.""" + + stage: Stage + dag_id: str + dag_run_id: str | None + state: str | None + match: StageRunMatch | None = Field( + description="How this stage run was attributed to the master run. Airflow " + "stores no parent-run link, so the only honest answer is 'heuristic' -- the " + "earliest stage run started inside this master run's own window -- or null " + "(nothing matched). Two master runs OVERLAPPING in time can still be " + "attributed the same stage run." + ) + tasks: list[RunTaskInstance] + mapped_summary: MappedFanOutSummary | None + + +class RunGraphResponse(BaseModel): + """One master run's live state over the ingest topology.""" + + master: RunGraphMaster + stages: list[RunGraphStage] + + +# --- byte-serving routes ------------------------------------------------------ + +# The three routes that answer with FILE BYTES rather than JSON. Declared so +# the schema says "binary" instead of the empty schema FastAPI publishes for a +# bare Response return type; the routes pair this with +# ``response_class=FileResponse``, which is what drops the phantom +# application/json entry beside it. +BINARY_FILE_RESPONSES: dict[int | str, dict[str, Any]] = { + 200: { + "description": "The file's bytes. An allowlisted inert media type (image, audio, " + "video) is served inline under its own content type; anything else -- and every " + "download -- is opaque application/octet-stream.", + "content": {"application/octet-stream": {"schema": {"type": "string", "format": "binary"}}}, + } +} diff --git a/packages/hflow-server/src/hflow_server/_curation.py b/packages/hflow-server/src/hflow_server/_curation.py new file mode 100644 index 0000000..5f049c5 --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_curation.py @@ -0,0 +1,557 @@ +"""The curation studio API: preview/report/pin, manifests, queries, tables. + +User-supplied SQL only ever runs on a CONSTRAINED connection +(``hflow.open_catalog_connection(..., constrained=True)`` / +``hflow.curate(..., constrained=True)``): the catalog is materialized in +memory at open, file access and extension loading are locked out, so the SQL +can read the data but can never touch the catalog's files -- hosted parity, +and defense in depth even locally. The server wraps that SQL as a subquery +(``SELECT ... FROM ()``) for LIMITing, counting, and SUMMARIZE, so a +smuggled second statement is a parser error, and every DuckDB parser/binder +error travels back as a 400 whose detail is DuckDB's own message -- the +useful part -- never a 500. + +Workspace convention: pinned manifests are immutable files at +``/manifests/-.parquet`` -- never the +engine's default ``/manifest.parquet``, which the CLI's curate +silently overwrites. A pin refuses loudly rather than overwrite anything. +The registry describing them lives in the sidecar (see ``_sidecar``). +""" + +import re +import threading +import uuid +from datetime import UTC, datetime +from pathlib import Path + +import duckdb +from fastapi import APIRouter, HTTPException +from fastapi.responses import FileResponse, Response +from pydantic import BaseModel, Field + +from hflow.curation import CurationReport, curate +from hflow.workspace import Workspace +from hflow_server import _catalog, _connections, _media, _sidecar +from hflow_server._contract import ( + BINARY_FILE_RESPONSES, + CatalogTableDescription, + CatalogTableKind, + CatalogTablesResponse, + CatalogTableSummaryResponse, + CheckCoverageEntry, + ColumnDescriptor, + CurationPreviewResponse, + CurationReportResponse, + PinnedManifestEntry, + PinnedManifestListResponse, + SavedQueryEntry, + SavedQueryListResponse, +) +from hflow_server._settings import ServerSettings, refuse_when_read_only + +MANIFESTS_DIRECTORY_NAME = "manifests" + +# BROWSING ORDER only, never membership: which relations exist is +# hflow.open_catalog_connection's fact, read live off information_schema (see +# _browsable_relations), so a view the SDK adds or renames shows up here +# instead of 404ing from the summary route or 500ing on a DESCRIBE. A +# relation missing from this tuple simply sorts after the familiar ones. +CATALOG_TABLE_BROWSING_ORDER = ( + "episodes", + "episodes_latest", + "episodes_raw", + "check_runs", + "measurements", + "measurements_latest", + "tags", + "intervals", +) + +_TIMESTAMPTZ_TYPE = "TIMESTAMP WITH TIME ZONE" + +# What a read-only launch refuses on this router; the sentence around it (and +# the 403) belongs to _settings.refuse_when_read_only. +_STUDIO_WRITE_ACTIONS = "pinning manifests and editing saved queries are" + +# Upper bounds on everything that can be persisted into the sidecar (which is +# fully re-read and re-serialized on every list request): a name, a +# description, one SQL body, and the number of stored entries. Generous for +# real use, but they make the one file this server writes outside manifests/ +# bounded instead of unbounded. +_MAX_NAME_LENGTH = 200 +_MAX_DESCRIPTION_LENGTH = 2000 +_MAX_SQL_LENGTH = 100_000 +_MAX_SAVED_QUERIES = 1000 +_MAX_PINNED_MANIFESTS = 1000 + + +class PreviewRequest(BaseModel): + sql: str = Field(max_length=_MAX_SQL_LENGTH) + limit: int = Field(default=100, ge=1, le=1000) + stats: bool = False + + +class ReportRequest(BaseModel): + sql: str = Field(max_length=_MAX_SQL_LENGTH) + + +class PinRequest(BaseModel): + sql: str = Field(max_length=_MAX_SQL_LENGTH) + name: str = Field(min_length=1, max_length=_MAX_NAME_LENGTH) + description: str = Field(default="", max_length=_MAX_DESCRIPTION_LENGTH) + + +class SavedQueryCreateRequest(BaseModel): + name: str = Field(min_length=1, max_length=_MAX_NAME_LENGTH) + sql: str = Field(max_length=_MAX_SQL_LENGTH) + + +class SavedQueryUpdateRequest(BaseModel): + name: str | None = Field(default=None, max_length=_MAX_NAME_LENGTH) + sql: str | None = Field(default=None, max_length=_MAX_SQL_LENGTH) + + +# Fallback filename slug when a name has no ASCII alphanumerics (a name in a +# non-Latin script, or symbols only). The full Unicode name is still stored on +# the registry entry; only the on-disk filename uses the slug, and the +# timestamp suffix keeps every filename unique regardless. +_FALLBACK_MANIFEST_SLUG = "manifest" + + +def slugified_manifest_name(raw_name: str) -> str: + """The user-given name as a filename slug: lowercase, [a-z0-9-], dashes + collapsed. Names with no ASCII alphanumerics (e.g. ``数据集``, ``!!!``) + slug to the fallback rather than being refused.""" + slug = re.sub(r"[^a-z0-9]+", "-", raw_name.lower()).strip("-") + return slug or _FALLBACK_MANIFEST_SLUG + + +def _utc_now_iso() -> str: + return datetime.now(UTC).isoformat() + + +def _manifest_timestamp() -> str: + # Microsecond precision: pins of the same name in the same second still + # get distinct files (pins never overwrite; collisions are refused). + return datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ") + + +def _stripped_sql_or_refuse(raw_sql: str) -> str: + """The user SQL with trailing semicolons dropped (they break subquerying).""" + stripped_sql = raw_sql.strip().rstrip(";").strip() + if not stripped_sql: + raise HTTPException(status_code=400, detail="sql must be a non-empty SELECT") + return stripped_sql + + +def _bad_sql_refusal(error: duckdb.Error) -> HTTPException: + # DuckDB's parser/binder message IS the useful diagnostic; bad SQL is the + # caller's mistake, never a server fault (so 400, never 500). + return HTTPException(status_code=400, detail=str(error)) + + +def _reject_non_single_select(user_sql: str) -> None: + """Refuse anything that is not exactly one SELECT statement. + + ``execute()`` with no bind parameters runs EVERY statement in the string + and returns only the LAST result, so a smuggled second statement + (``SELECT ...); CREATE TABLE ...; SELECT ... FROM (SELECT ...``) would run + silently -- preview 500s on the resulting shape and report answers over + the wrong statement. ``extract_statements`` parses the text WITHOUT + executing it; require exactly one statement whose type is SELECT. + """ + parser_connection = duckdb.connect() + try: + statements = parser_connection.extract_statements(user_sql) + except duckdb.Error as error: + raise _bad_sql_refusal(error) from error + finally: + parser_connection.close() + if len(statements) != 1 or statements[0].type != duckdb.StatementType.SELECT: + raise HTTPException( + status_code=400, detail="sql must be exactly one read-only SELECT statement" + ) + + +def _sidecar_refusal(error: _sidecar.SidecarError) -> HTTPException: + return HTTPException(status_code=error.status_code, detail=error.detail) + + +def _browsable_relations( + connection: duckdb.DuckDBPyConnection, +) -> dict[str, CatalogTableKind]: + """Every relation this catalog connection registered, in browsing order. + + ``hflow.open_catalog_connection`` owns WHICH relations exist; + ``information_schema`` is that fact as the live connection reports it, so + this endpoint and the summary route below both derive membership from the + connection they already hold rather than from a second list here. + """ + kind_rows = connection.execute( + "SELECT table_name, table_type FROM information_schema.tables" + ).fetchall() + kind_by_name: dict[str, CatalogTableKind] = { + str(table_name): ("view" if str(table_type).upper() == "VIEW" else "table") + for table_name, table_type in kind_rows + } + unfamiliar_position = len(CATALOG_TABLE_BROWSING_ORDER) + return { + name: kind_by_name[name] + for name in sorted( + kind_by_name, + key=lambda name: ( + CATALOG_TABLE_BROWSING_ORDER.index(name) + if name in CATALOG_TABLE_BROWSING_ORDER + else unfamiliar_position, + name, + ), + ) + } + + +def _described_columns( + connection: duckdb.DuckDBPyConnection, user_sql: str +) -> list[ColumnDescriptor]: + described_rows = connection.execute(f"DESCRIBE SELECT * FROM ({user_sql})").fetchall() + return [ColumnDescriptor(name=str(row[0]), type=str(row[1])) for row in described_rows] + + +def _timestamp_replace_clause(columns: list[ColumnDescriptor]) -> str: + """A ``* REPLACE (...)`` clause rendering TIMESTAMPTZ results as ISO UTC text. + + Materializing a TIMESTAMPTZ into Python requires pytz (deliberately not a + dependency), and the locked connection cannot ``SET TimeZone`` -- so the + rendering converts to UTC in SQL (``AT TIME ZONE 'UTC'`` yields the naive + UTC wall time) and appends the offset literally. A TIMESTAMPTZ nested + inside a LIST/STRUCT is rendered whole via CAST to text. + """ + replacements: list[str] = [] + for column in columns: + quoted_name = _catalog.quoted_identifier(column.name) + if column.type == _TIMESTAMPTZ_TYPE: + replacements.append( + f"strftime({quoted_name} AT TIME ZONE 'UTC', '%Y-%m-%dT%H:%M:%S.%f') " + f"|| '+00:00' AS {quoted_name}" + ) + elif _TIMESTAMPTZ_TYPE in column.type: + replacements.append(f"CAST({quoted_name} AS VARCHAR) AS {quoted_name}") + return f"REPLACE ({', '.join(replacements)})" if replacements else "" + + +def run_preview( + connection: duckdb.DuckDBPyConnection, user_sql: str, *, limit: int, include_stats: bool +) -> CurationPreviewResponse: + """Preview rows, the full count, and (optionally) SUMMARIZE column stats.""" + columns = _described_columns(connection, user_sql) + replace_clause = _timestamp_replace_clause(columns) + select_head = f"SELECT * {replace_clause}" if replace_clause else "SELECT *" + rows = _catalog.fetched_json_safe_rows( + connection.execute(f"{select_head} FROM ({user_sql}) LIMIT ?", [limit]) + ) + count_row = connection.execute(f"SELECT count(*) FROM ({user_sql})").fetchone() + row_count = int(count_row[0]) if count_row is not None else 0 + # SUMMARIZE over the SAME timestamp-replaced projection the rows use: the + # constrained connection cannot SET TimeZone (locked at open), so a bare + # SUMMARIZE would stringify TIMESTAMPTZ min/max/quartiles in the host's + # timezone -- inconsistent with (and a different calendar day from) the + # UTC ISO text the preview rows already carry. + column_stats = ( + _catalog.fetched_json_safe_rows( + connection.execute(f"SUMMARIZE {select_head} FROM ({user_sql})") + ) + if include_stats + else None + ) + return CurationPreviewResponse( + columns=columns, + rows=rows, + row_count=row_count, + truncated=row_count > len(rows), + column_stats=column_stats, + # The LOGICAL wrapper, deliberately without select_head's REPLACE: the + # timestamp rendering is how these rows are transported, not part of + # the query a user wrote, so what is served stays copy-pastable -- the + # same split (and the same wording) _catalog's episode listing makes. + sql=f"SELECT * FROM ({user_sql}) LIMIT {limit}", + ) + + +def _curated_or_refused(data_root: str, user_sql: str, *, output: Path | None) -> CurationReport: + try: + return curate( + Workspace.parse(data_root).catalog_root, user_sql, output=output, constrained=True + ) + except (FileNotFoundError, ValueError) as error: + raise _connections.catalog_unavailable_refusal(error) from error + except duckdb.Error as error: + raise _bad_sql_refusal(error) from error + + +def _coverage_entries(report: CurationReport) -> list[CheckCoverageEntry]: + """One curation report's coverage as the served (and pinned) entries.""" + return [ + CheckCoverageEntry( + check_name=entry.check_name, + episodes_ran=entry.episodes_ran, + total_episodes=entry.total_episodes, + fraction=entry.fraction, + ) + for entry in report.coverage + ] + + +def create_curation_router(settings: ServerSettings) -> APIRouter: + """Every curation-studio route, closed over one launch's settings.""" + router = APIRouter(prefix="/api/v1") + # FastAPI runs these sync endpoints on a threadpool, so two overlapping + # writes (double-submit, two tabs) would both read the same base sidecar + # and the later store would silently drop the earlier's entry. One + # process-wide lock serializes the whole load->modify->store of every + # mutating route -- sufficient for the single-server design. + sidecar_write_lock = threading.Lock() + + def loaded_sidecar_state() -> _sidecar.SidecarState: + try: + return _sidecar.load_sidecar_state(settings.data_root) + except _sidecar.SidecarError as error: + raise _sidecar_refusal(error) from error + + def stored_sidecar_state(state: _sidecar.SidecarState) -> None: + try: + _sidecar.store_sidecar_state(settings.data_root, state) + except _sidecar.SidecarError as error: + raise _sidecar_refusal(error) from error + + def local_data_root_or_refuse() -> Path: + try: + return _sidecar.local_data_root(settings.data_root) + except _sidecar.SidecarError as error: + raise _sidecar_refusal(error) from error + + @router.post("/curation/preview") + def run_curation_preview(request: PreviewRequest) -> CurationPreviewResponse: + user_sql = _stripped_sql_or_refuse(request.sql) + _reject_non_single_select(user_sql) + with _connections.opened_constrained_connection_or_refuse(settings.data_root) as connection: + try: + return run_preview( + connection, user_sql, limit=request.limit, include_stats=request.stats + ) + except duckdb.Error as error: + raise _bad_sql_refusal(error) from error + + @router.post("/curation/report") + def run_curation_report(request: ReportRequest) -> CurationReportResponse: + user_sql = _stripped_sql_or_refuse(request.sql) + _reject_non_single_select(user_sql) + report = _curated_or_refused(settings.data_root, user_sql, output=None) + return CurationReportResponse( + row_count=report.row_count, + total_episodes=report.total_episodes, + coverage=_coverage_entries(report), + ) + + @router.post("/curation/pin") + def pin_manifest(request: PinRequest) -> PinnedManifestEntry: + refuse_when_read_only(settings, disabled_actions=_STUDIO_WRITE_ACTIONS) + user_sql = _stripped_sql_or_refuse(request.sql) + _reject_non_single_select(user_sql) + manifest_slug = slugified_manifest_name(request.name) + with sidecar_write_lock: + # Load (and thereby validate) the sidecar BEFORE writing the + # manifest, so a corrupt registry never strands an unregistered + # manifest file. The whole load->curate->store runs under the lock + # so a concurrent write cannot drop this pin's acknowledged entry. + state = loaded_sidecar_state() + if len(state.manifests) >= _MAX_PINNED_MANIFESTS: + raise HTTPException( + status_code=409, + detail=f"this workspace already has {_MAX_PINNED_MANIFESTS} pinned " + "manifests (the registry cap); remove some before pinning more", + ) + manifests_directory = local_data_root_or_refuse() / MANIFESTS_DIRECTORY_NAME + manifest_file = manifests_directory / ( + f"{manifest_slug}-{_manifest_timestamp()}.parquet" + ) + if manifest_file.exists(): + raise HTTPException( + status_code=409, + detail=f"manifest file {manifest_file.name} already exists; " + "pinned manifests are immutable and never overwritten -- retry the pin", + ) + report = _curated_or_refused(settings.data_root, user_sql, output=manifest_file) + entry = PinnedManifestEntry( + manifest_id=uuid.uuid4().hex, + name=request.name, + description=request.description, + sql=user_sql, + manifest_path=f"{MANIFESTS_DIRECTORY_NAME}/{manifest_file.name}", + row_count=report.row_count, + total_episodes=report.total_episodes, + coverage=_coverage_entries(report), + created_at=_utc_now_iso(), + ) + stored_sidecar_state( + _sidecar.SidecarState( + saved_queries=state.saved_queries, manifests=(*state.manifests, entry) + ) + ) + return entry + + @router.get("/manifests") + def list_manifests() -> PinnedManifestListResponse: + state = loaded_sidecar_state() + newest_first = sorted(state.manifests, key=lambda entry: entry.created_at, reverse=True) + return PinnedManifestListResponse(manifests=newest_first) + + @router.get( + "/manifests/{manifest_id}/download", + response_class=FileResponse, + responses=BINARY_FILE_RESPONSES, + ) + def download_manifest(manifest_id: str) -> FileResponse: + state = loaded_sidecar_state() + entry = next( + (manifest for manifest in state.manifests if manifest.manifest_id == manifest_id), + None, + ) + if entry is None: + raise HTTPException( + status_code=404, detail=f"no pinned manifest with id {manifest_id!r}" + ) + manifest_file = local_data_root_or_refuse() / entry.manifest_path + try: + # The same strict-resolve + containment check media serving uses: + # even a hand-edited registry path can only serve workspace files. + resolved_file = _media.resolve_served_file( + str(manifest_file), data_root=settings.data_root + ) + except _media.MediaResolutionError as error: + raise _media.media_refusal(error) from error + return _media.served_file_response( + resolved_file, attachment_filename=Path(entry.manifest_path).name + ) + + @router.get("/queries") + def list_saved_queries() -> SavedQueryListResponse: + state = loaded_sidecar_state() + return SavedQueryListResponse(queries=list(state.saved_queries)) + + @router.post("/queries") + def create_saved_query(request: SavedQueryCreateRequest) -> SavedQueryEntry: + refuse_when_read_only(settings, disabled_actions=_STUDIO_WRITE_ACTIONS) + query_name = request.name.strip() + if not query_name: + raise HTTPException(status_code=400, detail="name must be non-empty") + entry = SavedQueryEntry( + query_id=uuid.uuid4().hex, + name=query_name, + sql=_stripped_sql_or_refuse(request.sql), + updated_at=_utc_now_iso(), + ) + with sidecar_write_lock: + state = loaded_sidecar_state() + if len(state.saved_queries) >= _MAX_SAVED_QUERIES: + raise HTTPException( + status_code=409, + detail=f"this workspace already has {_MAX_SAVED_QUERIES} saved queries " + "(the sidecar cap); remove some before saving more", + ) + stored_sidecar_state( + _sidecar.SidecarState( + saved_queries=(*state.saved_queries, entry), manifests=state.manifests + ) + ) + return entry + + @router.put("/queries/{query_id}") + def update_saved_query(query_id: str, request: SavedQueryUpdateRequest) -> SavedQueryEntry: + refuse_when_read_only(settings, disabled_actions=_STUDIO_WRITE_ACTIONS) + with sidecar_write_lock: + state = loaded_sidecar_state() + existing = next( + (entry for entry in state.saved_queries if entry.query_id == query_id), None + ) + if existing is None: + raise HTTPException(status_code=404, detail=f"no saved query with id {query_id!r}") + updated_name = existing.name + if request.name is not None: + updated_name = request.name.strip() + if not updated_name: + raise HTTPException(status_code=400, detail="name must be non-empty") + updated_sql = ( + _stripped_sql_or_refuse(request.sql) if request.sql is not None else existing.sql + ) + updated_entry = SavedQueryEntry( + query_id=query_id, name=updated_name, sql=updated_sql, updated_at=_utc_now_iso() + ) + stored_sidecar_state( + _sidecar.SidecarState( + saved_queries=tuple( + updated_entry if entry.query_id == query_id else entry + for entry in state.saved_queries + ), + manifests=state.manifests, + ) + ) + return updated_entry + + @router.delete("/queries/{query_id}", status_code=204) + def delete_saved_query(query_id: str) -> Response: + refuse_when_read_only(settings, disabled_actions=_STUDIO_WRITE_ACTIONS) + with sidecar_write_lock: + state = loaded_sidecar_state() + if all(entry.query_id != query_id for entry in state.saved_queries): + raise HTTPException(status_code=404, detail=f"no saved query with id {query_id!r}") + stored_sidecar_state( + _sidecar.SidecarState( + saved_queries=tuple( + entry for entry in state.saved_queries if entry.query_id != query_id + ), + manifests=state.manifests, + ) + ) + return Response(status_code=204) + + @router.get("/catalog/tables") + def list_catalog_tables() -> CatalogTablesResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + return CatalogTablesResponse( + tables=[ + CatalogTableDescription( + name=table_name, + kind=kind, + columns=[ + ColumnDescriptor(name=str(row[0]), type=str(row[1])) + for row in connection.execute( + f"DESCRIBE {_catalog.quoted_identifier(table_name)}" + ).fetchall() + ], + ) + for table_name, kind in _browsable_relations(connection).items() + ] + ) + + @router.get("/catalog/tables/{table_name}/summary") + def read_catalog_table_summary(table_name: str) -> CatalogTableSummaryResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + # Identifier-validated against the relations this connection + # actually registered: anything else -- including SQL-shaped names + # -- is simply an unknown table, and nothing unvalidated ever + # reaches the interpolations below. + browsable = _browsable_relations(connection) + if table_name not in browsable: + raise HTTPException( + status_code=404, + detail=f"unknown catalog table {table_name!r}; one of: {', '.join(browsable)}", + ) + quoted_table = _catalog.quoted_identifier(table_name) + count_row = connection.execute(f"SELECT count(*) FROM {quoted_table}").fetchone() + return CatalogTableSummaryResponse( + row_count=int(count_row[0]) if count_row is not None else 0, + columns=_catalog.fetched_json_safe_rows( + connection.execute(f"SUMMARIZE SELECT * FROM {quoted_table}") + ), + ) + + return router diff --git a/packages/hflow-server/src/hflow_server/_graph.py b/packages/hflow-server/src/hflow_server/_graph.py new file mode 100644 index 0000000..abf52cb --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_graph.py @@ -0,0 +1,564 @@ +"""The visualization API: the ingest DAG's shape, and one run's live state. + +Two nested layers meet on these endpoints, and neither may be drawn as the +other: + +- **Orchestration** -- a real DAG with real edges, served straight from + :func:`hflow.runtime.ingest_dag_topology` (the library's description of the + DAGs ``hflow up`` renders, pinned to the templates by the core suite). The + master resolves the run profile, then walks the stage chain gating and + triggering each sub-DAG; every sub-DAG plans batches, fans ``process_batch`` + out over them, and closes on a budget gate. +- **User steps** -- the registered checks and enrichments of a ``--pipeline`` + App, which have NO dependency edges on each other. They all run INSIDE one + ``process_batch`` task of the stage that owns their kind, ordered only by + the engine's two-tier cheap-first policy (:meth:`hflow.App._ordered_checks`: + a step declaring ``requires`` or ``uses`` runs in the second tier). Drawing + arrows between them would be a lie; the payload states the tiers instead. + +The one real cross-step edge is the quarantine gate, and it is served as its +own object rather than as an edge in either graph. + +Both endpoints degrade instead of failing: the pipeline graph answers with +``dag_ids_known``/``steps_known`` flags when no runtime or no pipeline is +addressed, and the run graph refuses with the runs monitor's 409/502 idiom. +""" + +import re +from collections import Counter +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from math import isfinite +from typing import Any + +from fastapi import APIRouter, HTTPException + +from hflow.app import MEDIA_CONTACT_SHEET_STEP_NAME +from hflow.manifest import PipelineManifest +from hflow.runtime import ( + AirflowClient, + AirflowClientError, + DagTaskNode, + DagTopology, + IngestTopology, + StageTopology, + ingest_dag_topology, +) +from hflow.steps import Stage +from hflow_server._contract import ( + DagTaskNodePayload, + DagTopologyPayload, + MappedFanOutSummary, + PipelineEngineStep, + PipelineGraphResponse, + PipelineGraphStage, + PipelineUserStep, + QuarantineGate, + RunGraphMaster, + RunGraphResponse, + RunGraphStage, + RunTaskInstance, + StageRunMatch, +) +from hflow_server._pipeline import PipelineLoaded, PipelineState, registered_steps_by_stage +from hflow_server._runtime import ( + ResolvedRuntime, + RuntimeResolver, + airflow_failure_refusal, + optional_string, + resolved_runtime_or_refuse, +) + +# The display copy for the four stages. Restated here (rather than imported +# from hflow.runtime._bundle's STAGE_TITLES/STAGE_DESCRIPTIONS, which are +# private and worded for Airflow's own UI) so the browser never hardcodes it: +# the thin-client rule applies to prose too. +_STAGE_TITLES: dict[Stage, str] = { + Stage.SYNC: "Transform & sync", + Stage.META: "Metadata", + Stage.LABELS: "Labels & artifacts", + Stage.MEDIA: "Media", +} +_STAGE_DESCRIPTIONS: dict[Stage, str] = { + Stage.SYNC: "canonical transform + derived channels (critical path)", + Stage.META: "checks + catalog registration", + Stage.LABELS: "enrichments (non-critical)", + Stage.MEDIA: "derived media artifacts", +} + +# The master id shown when no runtime is addressed: the DAGs do not exist +# yet, so the graph is drawn under a display-only name (the pipeline's own +# name when one is imported, else this) and ``dag_ids_known`` is false. +DISPLAY_ONLY_MASTER_DAG_ID = "ingest" + +_DAG_ID_UNSAFE_CHARACTERS = re.compile(r"[^A-Za-z0-9_.-]+") + +# How many of a stage sub-DAG's runs the run-graph heuristic looks at. +_STAGE_RUN_SEARCH_LIMIT = 25 + +# How long after a master run ENDED a stage run may still start and count as +# its own. Normally zero is enough (the master defers until each stage run +# finishes), but a master that fails, times out, or is cleared the moment +# after firing a trigger ends before the run it just caused appears -- and the +# two timestamps come from different components' clocks. Generous enough to +# cover that, far short of the gap between two ingests. +_STAGE_RUN_START_GRACE_AFTER_MASTER_END = timedelta(minutes=5) + +# Airflow reports a task instance that has not been scheduled yet with a null +# state; the mapped fan-out summary needs a key for those. +_UNSET_TASK_STATE = "no_status" + + +def _dag_task_node_payload(node: DagTaskNode) -> DagTaskNodePayload: + return DagTaskNodePayload( + task_id=node.task_id, + summary=node.summary, + mapped=node.mapped, + deferred=node.deferred, + ) + + +def _dag_topology_payload(topology: DagTopology) -> DagTopologyPayload: + return DagTopologyPayload( + dag_id=topology.dag_id, + tasks=[_dag_task_node_payload(node) for node in topology.tasks], + edges=[(upstream, downstream) for upstream, downstream in topology.edges], + ) + + +def _display_master_dag_id(pipeline_name: str | None) -> str: + """A stand-in master id for a workspace with no rendered bundle. + + Never presented as real: the response's ``dag_ids_known`` is false, and + the sub-DAG ids derived from it are display-only too. The real id is + ``_ingest``, which only a rendered bundle knows. + """ + if pipeline_name is None: + return DISPLAY_ONLY_MASTER_DAG_ID + sanitized = _DAG_ID_UNSAFE_CHARACTERS.sub("-", pipeline_name).strip("-") + return sanitized or DISPLAY_ONLY_MASTER_DAG_ID + + +def _user_steps(stage: Stage, manifest: PipelineManifest | None) -> list[PipelineUserStep]: + """The registered steps running inside this stage's ``process_batch``. + + Which stage owns which steps, and the order they run in, both come from + :func:`hflow_server._pipeline.registered_steps_by_stage` -- the package's one + owner of that mapping -- so this lane and the pipeline page's lane are + the same steps in the same order, and only ``tier`` is served here. + """ + if manifest is None: + return [] + return [ + PipelineUserStep.from_step_manifest_in_tier(step, tier) + for step, tier in registered_steps_by_stage(manifest)[stage] + ] + + +def _engine_steps(stage: Stage, manifest: PipelineManifest | None) -> list[PipelineEngineStep]: + """The engine's own work inside this stage's ``process_batch``. + + Not registrations -- these are what ``App.process`` does around the user's + steps, and no manifest lists them: the canonical transform (sync), the + catalog append (meta), and the contact-sheet renderer (media). + """ + if stage is Stage.SYNC: + overridden = manifest is not None and manifest.has_transform_override + derived_channel_count = len(manifest.derived_channels) if manifest is not None else 0 + summary = ( + "rewrite the source recording into a canonical MCAP and publish it" + if not overridden + else "rewrite the source recording with this pipeline's transform override " + "and publish it" + ) + if derived_channel_count: + summary += ( + f"; computes {derived_channel_count} registered derived " + f"channel{'s' if derived_channel_count != 1 else ''} over the source" + ) + return [PipelineEngineStep(name="canonical transform", summary=summary)] + if stage is Stage.META: + return [ + PipelineEngineStep( + name="catalog registration", + summary="append this run's episode row and every step's evidence " + "(check runs, measurements, intervals, tags) to the catalog", + ) + ] + if stage is Stage.MEDIA: + return [ + PipelineEngineStep( + name=MEDIA_CONTACT_SHEET_STEP_NAME, + summary="render one contact sheet per camera and record it as a " + "catalog artifact; absent when the episode has no cameras", + ) + ] + return [] + + +# What a failed critical check actually does in App.process: the episode is +# tagged (never deleted), the meta stage skips its REMAINING checks, and every +# enrichment in labels and media is recorded as skipped. +_QUARANTINE_GATE_EXPLANATION = ( + "a False verdict from a critical check quarantines the episode: meta skips its " + "remaining checks, and every enrichment in the labels and media stages is recorded " + "as skipped. Quarantine is a tag, never a deletion." +) +_NO_CRITICAL_CHECKS_EXPLANATION = ( + "this pipeline registers no critical checks, so no check can quarantine an episode. " + "A critical check's False verdict would make meta skip its remaining checks and every " + "enrichment in the labels and media stages." +) + + +def _quarantine_gate(manifest: PipelineManifest | None) -> QuarantineGate | None: + """The one real edge between user steps, or null when no pipeline is known.""" + if manifest is None: + return None + critical_step_names = [step.name for step in manifest.checks if step.critical] + return QuarantineGate( + from_stage=Stage.META, + to_stages=[Stage.LABELS, Stage.MEDIA], + critical_step_names=critical_step_names, + explanation=( + _QUARANTINE_GATE_EXPLANATION if critical_step_names else _NO_CRITICAL_CHECKS_EXPLANATION + ), + ) + + +def _stage_graph( + stage_topology: StageTopology, manifest: PipelineManifest | None +) -> PipelineGraphStage: + stage = stage_topology.stage + return PipelineGraphStage( + stage=stage, + title=_STAGE_TITLES[stage], + description=_STAGE_DESCRIPTIONS[stage], + gate_task_id=stage_topology.gate_task_id, + trigger_task_id=stage_topology.trigger_task_id, + enabling_profiles=list(stage_topology.enabling_profiles), + dag=_dag_topology_payload(stage_topology.dag), + engine_steps=_engine_steps(stage, manifest), + user_steps=_user_steps(stage, manifest), + ) + + +def _parsed_timestamp(value: object) -> datetime | None: + """One Airflow ISO-8601 timestamp as an aware datetime, or None. + + Airflow renders UTC as a trailing ``Z``; it is normalized here rather than + left to ``fromisoformat``'s version-dependent tolerance, and a naive value + is read as UTC so a comparison against another timestamp never raises. + Anything unparseable is None -- a timestamp this build cannot read must + not fail the request. + """ + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC) + + +def _duration_seconds(instance: dict[str, Any]) -> float | None: + """One task instance's wall duration, computed here rather than trusted. + + Airflow's own ``duration`` field is the fallback for an instance whose + timestamps this build cannot parse. + """ + started_at = _parsed_timestamp(instance.get("start_date")) + ended_at = _parsed_timestamp(instance.get("end_date")) + if started_at is not None and ended_at is not None: + return (ended_at - started_at).total_seconds() + reported_duration = instance.get("duration") + if isinstance(reported_duration, int | float) and not isinstance(reported_duration, bool): + return float(reported_duration) if isfinite(float(reported_duration)) else None + return None + + +def _task_instance(instance: dict[str, Any]) -> RunTaskInstance: + """One Airflow task instance reduced to what the graph draws.""" + try_number = instance.get("try_number") + map_index = instance.get("map_index") + return RunTaskInstance( + task_id=optional_string(instance.get("task_id")), + state=optional_string(instance.get("state")), + start_date=optional_string(instance.get("start_date")), + end_date=optional_string(instance.get("end_date")), + # Airflow has spelled the queued timestamp both ways across versions + # and may omit it; absent is fine. + queued_at=optional_string(instance.get("queued_when") or instance.get("queued_at")), + try_number=int(try_number) if isinstance(try_number, int) else None, + # -1 is Airflow's "not a mapped instance"; an absent value means the + # same thing. + map_index=int(map_index) if isinstance(map_index, int) else -1, + duration_s=_duration_seconds(instance), + ) + + +def _sorted_task_instances( + instances: list[dict[str, Any]], topology: DagTopology +) -> list[RunTaskInstance]: + """Task instances in TOPOLOGY order (then by map index), not API order.""" + topology_positions = {node.task_id: index for index, node in enumerate(topology.tasks)} + unknown_task_position = len(topology_positions) + return sorted( + (_task_instance(instance) for instance in instances), + key=lambda task: ( + topology_positions.get(task.task_id or "", unknown_task_position), + task.task_id or "", + task.map_index, + ), + ) + + +def _mapped_summary( + tasks: list[RunTaskInstance], stage_topology: StageTopology +) -> MappedFanOutSummary | None: + """The fan-out's counts: how many mapped instances are in which state. + + The mapped task id comes from the topology (the node flagged ``mapped``), + so this never restates a task name the library owns. Every instance of + that task lands in exactly one ``by_state`` bucket -- an unscheduled one + under ``no_status`` -- which is what makes the served summary complete + enough that a client never has to recount the raw instances. + """ + mapped_task_ids = [node.task_id for node in stage_topology.dag.tasks if node.mapped] + if not mapped_task_ids: + return None + # Every generated stage sub-DAG has exactly one mapped node + # (``process_batch``); a topology that grows a second one needs a summary + # per mapped task, not a silently truncated one. + mapped_task_id = mapped_task_ids[0] + mapped_instances = [task for task in tasks if task.task_id == mapped_task_id] + if not mapped_instances: + return None + state_counts = Counter(task.state or _UNSET_TASK_STATE for task in mapped_instances) + return MappedFanOutSummary( + task_id=mapped_task_id, + # Before the fan-out expands, Airflow reports ONE instance with + # map_index -1; it is counted, because "1 unexpanded instance" is the + # truth at that moment. + total=len(mapped_instances), + by_state=dict(sorted(state_counts.items())), + ) + + +@dataclass(frozen=True) +class _MatchedStageRun: + """The stage run a master run most plausibly triggered, and how it matched.""" + + run: dict[str, Any] + match: StageRunMatch + + +@dataclass(frozen=True) +class _MasterRunWindow: + """When a master run was live: the interval its stage runs must start in. + + ``ended_at`` is None while the run is still going, which leaves the window + open-ended on the right -- the only case where "no upper bound" is true. + """ + + started_at: datetime + ended_at: datetime | None + + def contains_stage_run_start(self, started_at: datetime) -> bool: + if started_at < self.started_at: + return False + if self.ended_at is None: + return True + return started_at <= self.ended_at + _STAGE_RUN_START_GRACE_AFTER_MASTER_END + + +def _master_run_window(master_run: dict[str, Any]) -> _MasterRunWindow | None: + """One master run's live interval, or None when it has not started yet.""" + started_at = _parsed_timestamp(master_run.get("start_date")) + if started_at is None: + return None + return _MasterRunWindow( + started_at=started_at, ended_at=_parsed_timestamp(master_run.get("end_date")) + ) + + +def _matched_stage_run( + stage_runs: list[dict[str, Any]], window: _MasterRunWindow | None +) -> _MatchedStageRun | None: + """The EARLIEST run of one stage sub-DAG that started inside the master's window. + + The master triggers each stage with a deferring + ``TriggerDagRunOperator(wait_for_completion=True)`` and chains the stages + in order (``hflow.runtime`` renders them that way), so a stage run the + master caused always STARTS while the master run is still live. Bounding + the search by the master's own end is therefore not a guess, and it is + what stops an old master run from adopting an unrelated stage run that + happens to be newer -- the stage lanes only ever look back + ``_STAGE_RUN_SEARCH_LIMIT`` runs, so without the bound every candidate + qualified and the newest won. + + Earliest-in-window, not newest: when two master runs overlap, this + master's own stage run is the first one after its start, while the newest + is biased toward the other master's. The cost is that a stage triggered + twice inside ONE master run (a retried trigger task) shows the first + attempt -- accepted, because preferring the newest is exactly what let an + unrelated run be adopted. + + HONEST LIMITATION, restated in the payload as ``"match": "heuristic"``: + the master lets Airflow mint the sub-DAG's run id and forwards a conf that + carries no back-reference, so the API offers nothing that ties a stage run + to the master run that triggered it. Two master runs whose windows OVERLAP + can still be attributed the same stage run. A master run that has not + started yet (no ``start_date``) matches nothing rather than guessing. + """ + if window is None: + return None + earliest_run: dict[str, Any] | None = None + earliest_started_at: datetime | None = None + for run in stage_runs: + started_at = _parsed_timestamp(run.get("start_date")) + if started_at is None or not window.contains_stage_run_start(started_at): + continue + if earliest_started_at is None or started_at < earliest_started_at: + earliest_run, earliest_started_at = run, started_at + if earliest_run is None: + return None + return _MatchedStageRun(run=earliest_run, match="heuristic") + + +def _empty_stage_graph(stage_topology: StageTopology) -> RunGraphStage: + """A stage that never ran for this master run: explicit nulls, not omissions.""" + return RunGraphStage( + stage=stage_topology.stage, + dag_id=stage_topology.dag.dag_id, + dag_run_id=None, + state=None, + match=None, + tasks=[], + mapped_summary=None, + ) + + +def create_graph_router(pipeline_state: PipelineState, resolver: RuntimeResolver) -> APIRouter: + """The visualization routes, closed over one launch's pipeline and runtime. + + Read-only throughout, so unlike the other routers these need no settings: + the pipeline comes from the one startup import and the runtime from the + shared resolver. + """ + router = APIRouter(prefix="/api/v1") + + def stage_task_instances( + client: AirflowClient, dag_id: str, dag_run_id: str + ) -> list[dict[str, Any]]: + try: + return client.task_instances(dag_id, dag_run_id) + except AirflowClientError: + # A stage sub-DAG that vanished (or a run Airflow expired) leaves + # that lane without task detail; the master's own state -- the + # page's point -- is already in hand, so this is a thinner + # drawing, not a failed request. + return [] + + @router.get("/pipeline/graph") + def read_pipeline_graph() -> PipelineGraphResponse: + """The merged picture: the DAG topology plus the pipeline's user steps. + + Three degraded states, each explicit rather than an error: no runtime + addressed (``dag_ids_known: false``, display-only ids), no + ``--pipeline`` (``steps_known: false``, no user steps and no + quarantine gate), and both at once -- the common first-run case. + """ + resolution = resolver.resolve() + dag_ids_known = isinstance(resolution, ResolvedRuntime) + application = ( + pipeline_state.application if isinstance(pipeline_state, PipelineLoaded) else None + ) + master_dag_id = ( + resolution.dag_id + if isinstance(resolution, ResolvedRuntime) + else _display_master_dag_id(application.name if application is not None else None) + ) + manifest = application.manifest() if application is not None else None + topology: IngestTopology = ingest_dag_topology(master_dag_id) + return PipelineGraphResponse( + dag_ids_known=dag_ids_known, + steps_known=manifest is not None, + master=_dag_topology_payload(topology.master), + stages=[_stage_graph(stage_topology, manifest) for stage_topology in topology.stages], + quarantine_gate=_quarantine_gate(manifest), + ) + + @router.get("/runtime/runs/{dag_run_id}/graph") + def read_run_graph(dag_run_id: str) -> RunGraphResponse: + """One master run's live state over the same topology. + + The master run is addressed directly; each stage's sub-DAG run is + resolved by the documented heuristic in :func:`_matched_stage_run`. + """ + runtime = resolved_runtime_or_refuse(resolver) + topology = ingest_dag_topology(runtime.dag_id) + try: + master_run = runtime.client.dag_run(runtime.dag_id, dag_run_id) + except AirflowClientError as error: + if error.status == 404: + # A definitively unknown run is a missing resource, not an + # upstream failure -- and the detail names only ids the + # caller already sent. + raise HTTPException( + status_code=404, + detail=f"no run {dag_run_id!r} of dag {runtime.dag_id!r}", + ) from error + raise airflow_failure_refusal(error, source=runtime.source) from error + try: + master_instances = runtime.client.task_instances(runtime.dag_id, dag_run_id) + except AirflowClientError as error: + raise airflow_failure_refusal(error, source=runtime.source) from error + master_window = _master_run_window(master_run) + + stages: list[RunGraphStage] = [] + for stage_topology in topology.stages: + stage_dag_id = stage_topology.dag.dag_id + try: + stage_runs = runtime.client.dag_runs( + stage_dag_id, limit=_STAGE_RUN_SEARCH_LIMIT, order_by="-id" + ) + except AirflowClientError: + # An unregistered stage sub-DAG (a partial profile, or a + # bundle mid-render) is a stage that never ran here. + stage_runs = [] + matched = _matched_stage_run(stage_runs, master_window) + if matched is None: + stages.append(_empty_stage_graph(stage_topology)) + continue + stage_run_id = optional_string(matched.run.get("dag_run_id")) + stage_tasks = ( + _sorted_task_instances( + stage_task_instances(runtime.client, stage_dag_id, stage_run_id), + stage_topology.dag, + ) + if stage_run_id is not None + else [] + ) + stages.append( + RunGraphStage( + stage=stage_topology.stage, + dag_id=stage_dag_id, + dag_run_id=stage_run_id, + state=optional_string(matched.run.get("state")), + match=matched.match, + tasks=stage_tasks, + mapped_summary=_mapped_summary(stage_tasks, stage_topology), + ) + ) + + return RunGraphResponse( + master=RunGraphMaster( + dag_run_id=optional_string(master_run.get("dag_run_id")) or dag_run_id, + state=optional_string(master_run.get("state")), + tasks=_sorted_task_instances(master_instances, topology.master), + ), + stages=stages, + ) + + return router diff --git a/packages/hflow-server/src/hflow_server/_media.py b/packages/hflow-server/src/hflow_server/_media.py new file mode 100644 index 0000000..3969938 --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_media.py @@ -0,0 +1,207 @@ +"""Media byte-serving: catalog URIs resolved and contained under the data root. + +The browser never chooses a filesystem path. It addresses bytes as +(episode_id, artifact name); the URI comes out of the catalog, and the +strictly-resolved file must land inside the strictly-resolved local data +root -- anything else is refused, and a refusal never echoes the offending +path (only the containment fact appears in errors). +""" + +import mimetypes +from pathlib import Path + +from fastapi import HTTPException +from starlette.responses import FileResponse + +from hflow.storage import is_bucket_url +from hflow.workspace import ( + CATALOG_DIRECTORY_NAME, + EPISODES_DIRECTORY_NAME, + TEST_RUNS_DIRECTORY_NAME, +) +from hflow_server._settings import local_data_root_or_none + +# The layout directories a workspace's own files live under, owned by +# hflow.workspace. Used to recognise a path recorded from another vantage of +# this workspace (a container mount) and re-anchor it here. +_WORKSPACE_LAYOUT_DIRECTORY_NAMES = frozenset( + {EPISODES_DIRECTORY_NAME, CATALOG_DIRECTORY_NAME, TEST_RUNS_DIRECTORY_NAME} +) + +# Media types inert enough to render inline in the browser: raster images and +# common audio/video containers. Deliberately excludes text/html, +# image/svg+xml, and application/xhtml+xml -- an active document served from +# the UI's own origin runs script same-origin with this workspace's API and +# could drive every endpoint it exposes (read the catalog, pin manifests, +# trigger runs), so anything not on this list is forced to download. +_INLINE_SERVABLE_MEDIA_TYPES = frozenset( + { + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + "image/bmp", + "image/apng", + "image/avif", + "image/x-icon", + "video/mp4", + "video/webm", + "video/ogg", + "video/quicktime", + "audio/mpeg", + "audio/ogg", + "audio/wav", + "audio/x-wav", + "audio/webm", + "audio/aac", + "audio/mp4", + "audio/flac", + } +) + +# Every byte-serving response carries these: no sniffing an octet-stream back +# into an active type, and a policy that denies script/resource loads even if +# a viewer opens the bytes directly. +_HARDENING_HEADERS = { + "X-Content-Type-Options": "nosniff", + "Content-Security-Policy": "default-src 'none'; sandbox", +} + + +class MediaResolutionError(Exception): + """One refusal to serve a catalog URI, carrying its HTTP mapping.""" + + def __init__(self, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +def media_refusal(error: MediaResolutionError) -> HTTPException: + """The HTTP refusal one unservable URI maps to. + + Lives beside the error it converts (as ``_connections`` and ``_runtime`` + do for theirs), so the two routes that serve catalog bytes -- episode + media and manifest downloads -- share one mapping instead of each copying + the two field reads. + """ + return HTTPException(status_code=error.status_code, detail=error.detail) + + +def _resolved_local_data_root(data_root: str) -> Path: + local_root = local_data_root_or_none(data_root) + if local_root is None: + raise MediaResolutionError( + 501, + "media serving requires a local data root; bucket-backed workspaces are not served yet", + ) + return local_root.resolve() + + +def _strictly_resolved(candidate: Path) -> Path | None: + """The real file behind a path, or ``None`` when it cannot be reached.""" + try: + return candidate.resolve(strict=True) + except (FileNotFoundError, OSError): + return None + + +def _rebased_onto_this_workspace(recorded_path: Path, resolved_data_root: Path) -> Path | None: + """The same workspace-relative file, as THIS host addresses it. + + One workspace is reachable from several vantages: the Compose runtime + mounts the data root inside its containers, so a run executed there + catalogs ``/opt/airflow/data/episodes//media/x.jpg`` while the very + same bytes sit at ``/episodes//media/x.jpg`` here. A path + recorded from another vantage is not foreign data -- it is this + workspace, named differently -- so it is re-anchored at the first + workspace layout directory in it and re-checked exactly like any other + candidate. Containment is still enforced afterwards, so this only ever + resolves to files already inside the data root; it never widens what may + be served. + """ + for index, component in enumerate(recorded_path.parts): + if component in _WORKSPACE_LAYOUT_DIRECTORY_NAMES: + return resolved_data_root.joinpath(*recorded_path.parts[index:]) + return None + + +def resolve_served_file(uri: str, *, data_root: str) -> Path: + """The real file a catalog URI may be served from, or a typed refusal. + + Resolution is strict (symlinks followed, missing components refused), and + the result must be contained in the resolved data root -- a symlink that + points out of the workspace is refused exactly like a foreign path. A URI + recorded from another vantage of this same workspace (see + :func:`_rebased_onto_this_workspace`) is retried against this host's data + root under the identical containment rule. + """ + if is_bucket_url(uri): + raise MediaResolutionError( + 501, "this file lives in an object store; bucket media serving is not implemented yet" + ) + resolved_data_root = _resolved_local_data_root(data_root) + recorded_path = Path(uri.removeprefix("file://")) + + resolved_file = _strictly_resolved(recorded_path) + escapes_workspace = resolved_file is not None and not resolved_file.is_relative_to( + resolved_data_root + ) + if resolved_file is None or escapes_workspace: + rebased_path = _rebased_onto_this_workspace(recorded_path, resolved_data_root) + rebased_file = None if rebased_path is None else _strictly_resolved(rebased_path) + if rebased_file is not None and rebased_file.is_relative_to(resolved_data_root): + resolved_file = rebased_file + elif escapes_workspace: + raise MediaResolutionError( + 403, "the cataloged URI resolves outside the workspace data root" + ) + else: + raise MediaResolutionError(404, "the cataloged file does not exist on this machine") + + if not resolved_file.is_file(): + raise MediaResolutionError(404, "the cataloged URI does not name a regular file") + return resolved_file + + +def is_uri_servable(uri: str, *, data_root: str) -> bool: + """Whether a GET for this URI would serve bytes (containment + existence).""" + try: + resolve_served_file(uri, data_root=data_root) + except MediaResolutionError: + return False + return True + + +def served_file_response( + resolved_file: Path, *, attachment_filename: str | None = None +) -> FileResponse: + """Bytes served safely: an allowlisted inert media type renders inline; + anything else (or an explicit ``attachment_filename``) is downloaded as + opaque ``application/octet-stream``. Every response carries ``nosniff`` + and a locked-down CSP, so a workspace file whose name ends in .html/.svg + can never execute as an active document on the UI's own origin. + + Starlette's FileResponse handles Range requests where it can, and a plain + GET always works. Passing ``attachment_filename`` names the download + (manifest exports).""" + if attachment_filename is not None: + return FileResponse( + resolved_file, + media_type="application/octet-stream", + filename=attachment_filename, + headers=dict(_HARDENING_HEADERS), + ) + guessed_type, _ = mimetypes.guess_type(resolved_file.name) + if guessed_type in _INLINE_SERVABLE_MEDIA_TYPES: + return FileResponse( + resolved_file, media_type=guessed_type, headers=dict(_HARDENING_HEADERS) + ) + # Not a known-inert type: force a download so an .html/.svg/unknown file + # is never rendered as an active document on this origin. + return FileResponse( + resolved_file, + media_type="application/octet-stream", + filename=resolved_file.name, + headers=dict(_HARDENING_HEADERS), + ) diff --git a/packages/hflow-server/src/hflow_server/_pipeline.py b/packages/hflow-server/src/hflow_server/_pipeline.py new file mode 100644 index 0000000..e8d6f5d --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_pipeline.py @@ -0,0 +1,175 @@ +"""The pipeline page API: the startup-imported App described over the catalog. + +``--pipeline path/to/pipeline.py[:app]`` names a Python file this server +imports -- EXECUTES -- exactly once at startup via the shared +:func:`hflow.import_pipeline_application` seam (the one owner of the "address +a pipeline by file" contract, used by the CLI too; producing a manifest +requires the live functions, because step versions are content hashes of +them). The operator opts into running their own pipeline code by passing the +flag; an import failure never crashes the server -- the error string is +remembered, the config capability reports false, and /api/v1/pipeline answers +409 with the stored reason. +""" + +from dataclasses import dataclass + +from fastapi import APIRouter, HTTPException + +from hflow import App, import_pipeline_application +from hflow.curation import stale_episodes +from hflow.format import EPISODE_FORMAT_VERSION +from hflow.manifest import PipelineManifest, StepManifest +from hflow.steps import Stage +from hflow.workspace import Workspace +from hflow_server import _catalog, _connections +from hflow_server._contract import ( + ObservedCheckVersion, + PipelineResponse, + StaleSummary, + StepTier, +) +from hflow_server._settings import ServerSettings + + +@dataclass(frozen=True) +class PipelineLoaded: + """The one startup import produced a live App.""" + + application: App + + +@dataclass(frozen=True) +class PipelineUnavailable: + """No App for this launch, and exactly why.""" + + detail: str + + +# Two states, never both and never neither -- the same sum ``_runtime`` uses +# for its resolution, so the two capabilities behind the same 409 refusal are +# modelled the same way and the refusal's detail cannot be null. +PipelineState = PipelineLoaded | PipelineUnavailable + + +def load_pipeline_state(pipeline_spec: str | None) -> PipelineState: + """Run the one startup import and remember its outcome, whatever it is.""" + if pipeline_spec is None: + return PipelineUnavailable( + detail=( + "no --pipeline configured: relaunch `hflow serve` with " + "--pipeline path/to/pipeline.py[:app] to serve the pipeline page" + ) + ) + try: + return PipelineLoaded(application=import_pipeline_application(pipeline_spec)) + except ValueError as error: + return PipelineUnavailable(detail=str(error)) + + +def registered_step_tier(step: StepManifest) -> StepTier: + """Which cheap-first tier this step runs in (1 first, 2 second). + + Mirrors :meth:`hflow.App._ordered_checks` and ``_ordered_enrichments`` + EXACTLY: both sort on ``bool(requires) or uses is not None``, so tier 2 is + precisely the steps declaring a required channel or an endpoint alias. + Within a tier there is no ordering at all -- registration order is what + the stable sort preserves, not a dependency. + + The rule ideally belongs in the SDK -- a ``tier`` on + ``hflow.manifest.StepManifest`` that ``App`` sorts on and ``hflow + manifest`` renders, so the CLI could answer "in what order do my steps + run?" too. Until it lives there, this is this package's ONE copy: both + endpoints project from :func:`registered_steps_by_stage` rather than + restating the expression a second time. + """ + return 2 if (bool(step.requires) or step.uses is not None) else 1 + + +def _in_execution_order( + steps: tuple[StepManifest, ...], +) -> tuple[tuple[StepManifest, StepTier], ...]: + # Stable sort on the tier alone: the same sort App._ordered_checks makes, + # so the served order IS the execution order. + return tuple( + (step, registered_step_tier(step)) for step in sorted(steps, key=registered_step_tier) + ) + + +def registered_steps_by_stage( + manifest: PipelineManifest, +) -> dict[Stage, tuple[tuple[StepManifest, StepTier], ...]]: + """Which registered steps run in which stage, in the order they run. + + The ONE owner of that mapping for this package: the pipeline page's lanes + and the graph's per-stage user steps are the same steps in the same order, + differing only in whether the payload carries the tier -- so the two pages + can never show one pipeline as two. + + Stage ownership is the engine's (``hflow.steps``/``App.process``): + registered checks run in META ("checks + catalog registration"), user + enrichments in LABELS ("Labels & artifacts"), while SYNC (the canonical + transform plus derived channels) and MEDIA (the engine's contact-sheet + step) are engine-owned lanes carrying no user-registered steps. + """ + steps_by_stage: dict[Stage, tuple[tuple[StepManifest, StepTier], ...]] = dict.fromkeys( + Stage, () + ) + steps_by_stage[Stage.META] = _in_execution_order(manifest.checks) + steps_by_stage[Stage.LABELS] = _in_execution_order(manifest.enrichments) + return steps_by_stage + + +def _observed_versions_and_stale( + data_root: str, application: App +) -> tuple[list[ObservedCheckVersion], StaleSummary | None]: + """What the catalog has SEEN of this pipeline: per-(check, version) + first/last-seen aggregates, plus the stale count against the App's + current versions. A workspace with no catalog yet has observed nothing + and its staleness is unknowable -- ([], None), not an error.""" + with _connections.opened_workspace_connection_or_none(data_root) as connection: + if connection is None: + return [], None + observed = [ + ObservedCheckVersion.model_validate(row) + for row in _catalog.fetched_json_safe_rows( + connection.execute( + "SELECT check_name, check_version, " + f"{_catalog.utc_iso_text('min(recorded_at)', 'first_seen')}, " + f"{_catalog.utc_iso_text('max(recorded_at)', 'last_seen')}, " + "count(*) AS run_count " + "FROM check_runs GROUP BY check_name, check_version " + "ORDER BY check_name, check_version" + ) + ) + ] + current_pipeline_version = application.pipeline_version + try: + # A pipeline defines the whole current target, format version + # included -- the same pairing the CLI's `hflow stale --pipeline` uses. + stale = stale_episodes( + Workspace.parse(data_root).catalog_root, + pipeline_version=current_pipeline_version, + schema_version=EPISODE_FORMAT_VERSION, + ) + except (FileNotFoundError, ValueError): + return observed, None + return observed, StaleSummary(pipeline_version=current_pipeline_version, count=len(stale)) + + +def create_pipeline_router(settings: ServerSettings, state: PipelineState) -> APIRouter: + """The pipeline route, closed over the one startup import's outcome.""" + router = APIRouter(prefix="/api/v1") + + @router.get("/pipeline") + def read_pipeline() -> PipelineResponse: + if isinstance(state, PipelineUnavailable): + raise HTTPException(status_code=409, detail=state.detail) + manifest = state.application.manifest() + observed, stale = _observed_versions_and_stale(settings.data_root, state.application) + return PipelineResponse( + manifest=manifest.to_json_dict(), + observed=observed, + stale=stale, + ) + + return router diff --git a/packages/hflow-server/src/hflow_server/_runtime.py b/packages/hflow-server/src/hflow_server/_runtime.py new file mode 100644 index 0000000..30bba7a --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_runtime.py @@ -0,0 +1,475 @@ +"""The runs monitor API: address the workspace's ingest runtime, proxy Airflow. + +Addressing mirrors the CLI's ``_resolve_bundle_dir``: a local Compose bundle +at ``/runtime`` (skipped for bucket data roots), then the +``./runtime`` fallback, else a remote endpoint resolved from the +``HFLOW_AIRFLOW_*`` environment via :func:`hflow.runtime.resolve_remote_endpoint`. +Resolution happens lazily per request -- the stack may come up (or go away) +after the server started -- and is cached briefly per launch. + +Two rules hold at this boundary: + +- The browser NEVER receives credentials: the bundle's admin password and any + remote token stay inside :class:`hflow.runtime.AirflowClient`; this server + proxies every Airflow call. The only URL it exposes is the deep-link base + the operator already knows (the bundle's own recorded api-server address). +- A missing or unreachable runtime is an ANSWER, never a traceback: + ``/runtime/status`` reports ``available: false`` with the reason, and the + other endpoints refuse with a clear 4xx/502 detail. +""" + +import ipaddress +import logging +import time +import urllib.parse +from dataclasses import dataclass +from pathlib import Path +from posixpath import normpath +from typing import Any + +from fastapi import APIRouter, HTTPException, Query +from pydantic import BaseModel, Field + +from hflow.runtime import ( + AirflowClient, + AirflowClientError, + client_for_bundle, + client_for_endpoint, + load_bundle, + resolve_remote_endpoint, + sub_dag_id_for_stage, +) +from hflow.steps import RUN_PROFILES, IngestMode, Stage +from hflow.storage import is_bucket_url +from hflow.workspace import RUNTIME_BUNDLE_DIRECTORY_NAME +from hflow_server._contract import ( + IngestTriggerResponse, + RuntimeHealthComponents, + RuntimeRunsResponse, + RuntimeRunSummary, + RuntimeSource, + RuntimeStatusResponse, + StageRecentRuns, + StageRunSummary, +) +from hflow_server._settings import ServerSettings, refuse_when_read_only + +# Mirrors hflow.runtime._endpoint's variable name (a documented public +# contract); restated here rather than imported from that private module. +AIRFLOW_URL_ENVIRONMENT_VARIABLE = "HFLOW_AIRFLOW_URL" + +# How long one resolution (bundle files read, client built) is reused before +# the next request re-probes -- long enough to spare a busy Runs page the +# filesystem walk, short enough that `hflow up` shows up within seconds. +RESOLUTION_CACHE_TTL_S = 5.0 + +# The health components /runtime/status reports, owned by the response model +# so the served keys and the components actually read can never diverge. +_HEALTH_COMPONENT_NAMES = tuple(RuntimeHealthComponents.model_fields) + +_RECENT_STAGE_RUN_LIMIT = 5 + +_LOGGER = logging.getLogger("hflow_server.runtime") + + +def _client_error_reason(error: AirflowClientError) -> str: + """A stable machine-readable classification of an Airflow call failure.""" + if error.status in (401, 403): + return "unauthorized" + if error.status is not None: + return "http_error" + return "unreachable" + + +def is_loopback_web_url(web_url: str | None) -> bool: + """Whether this address resolves only on the machine running this server. + + A rendered bundle records ``http://127.0.0.1:`` because that is + where its api-server binds by default. Handed to a browser on another + machine, that URL points at the VIEWER's own loopback -- their laptop, not + the workspace -- so the Runs page must present it as a fact about the host + rather than as a link to follow. + """ + if web_url is None: + return False + host = urllib.parse.urlparse(web_url).hostname + if host is None: + return False + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return host == "localhost" + + +def client_error_detail(error: AirflowClientError, *, source: RuntimeSource) -> str: + """A browser-safe detail for an Airflow call failure (shared with _graph). + + A local bundle's api-server address is one the operator already has, so + its verbatim message (which embeds that URL) is fine. A REMOTE runtime's + base URL is deliberately withheld on the success path, and the verbatim + message embeds that URL plus an excerpt of the upstream response body -- + so a remote failure returns only a generic detail with a stable reason + code, and the full error is logged server-side for the operator. + + ``source`` is the refined :data:`RuntimeSource`, not a bare string: this + branch decides what a browser is allowed to see, so the type checker -- + not a test -- is what guarantees a third runtime source would have to + state its own disclosure posture here rather than defaulting into one. + """ + if source == "bundle": + # Say WHOSE loopback this is. The message embeds the bundle's own + # address (typically http://127.0.0.1:8080), and a browser reaching + # this server from another machine reads that as its own laptop -- so the + # sentence has to name the workspace host as the one that called. + return f"the workspace host could not reach its own ingest runtime: {error}" + reason = _client_error_reason(error) + _LOGGER.warning("remote Airflow call failed (reason=%s): %s", reason, error) + if error.status is not None: + return ( + f"the remote ingest runtime returned an error (reason: {reason}, status {error.status})" + ) + return f"the remote ingest runtime is not reachable (reason: {reason})" + + +@dataclass(frozen=True) +class ResolvedRuntime: + """One addressable ingest runtime: the client, its DAG, and its shape.""" + + client: AirflowClient + dag_id: str + source: RuntimeSource + airflow_web_url: str | None + # (stage, sub-DAG id) in stage-graph order; None for remote runtimes + # (only the bundle manifest records the stage sub-DAG ids). + stage_dag_ids: tuple[tuple[Stage, str], ...] | None + + +@dataclass(frozen=True) +class RuntimeUnavailable: + """No usable runtime, and exactly why. + + ``addressed`` separates the two reasons: a bundle or a + ``HFLOW_AIRFLOW_URL`` IS pointed at a runtime but the addressing is + half-formed (a bundle mid-render, a URL with no dag id), versus nothing + pointed anywhere at all. /api/v1/config's ``runtime`` capability is that + flag, so "is a runtime addressed?" has one owner -- :func:`resolve_runtime` + -- rather than a second env-var probe beside it. + """ + + detail: str + addressed: bool + + +RuntimeResolution = ResolvedRuntime | RuntimeUnavailable + + +class IngestRequest(BaseModel): + uris: list[str] = Field(min_length=1) + profile: str = "full" + mode: str = IngestMode.BATCH.value + batch_count: int | None = Field(default=None, ge=1) + + +def find_bundle_directory(data_root: str) -> Path | None: + """The rendered local bundle this workspace addresses, if one exists. + + Mirrors the CLI's ``_resolve_bundle_dir`` probing: ``/runtime`` + first (bucket data roots have no local root, so only the fallback + applies), then ``./runtime``; a candidate counts only when its + ``docker-compose.yaml`` exists. Unlike the CLI there is no primary- + candidate fallback -- "no bundle anywhere" is a real answer here. + + The probe ideally belongs in the SDK, beside the renderer that writes the + marker file: a public ``hflow.runtime.find_bundle_directory(data_root)`` + would leave the CLI and this package as one call plus their differing + fallbacks, the way ``hflow.import_pipeline_application`` already does for + "address a pipeline by file". Until it lands, this is the mirror, and the + only thing that differs is the fallback. + """ + candidates = [Path(RUNTIME_BUNDLE_DIRECTORY_NAME)] + if not is_bucket_url(data_root): + candidates.insert(0, Path(data_root) / RUNTIME_BUNDLE_DIRECTORY_NAME) + for candidate in candidates: + if (candidate / "docker-compose.yaml").is_file(): + return candidate + return None + + +def runtime_addressed(resolution: RuntimeResolution) -> bool: + """Whether a runtime is ADDRESSED -- the /api/v1/config capability. + + Addressed, not reachable and not even fully resolvable: a bundle + mid-render or a URL exported without a dag id still means the operator + pointed this workspace at a runtime, and the Runs screen must stay + reachable so /runtime/status can name the variable to set. Derived from + the shared resolution so the capability and the status endpoint can never + tell two different stories. + """ + return isinstance(resolution, ResolvedRuntime) or resolution.addressed + + +def _stage_dag_ids(master_dag_id: str) -> tuple[tuple[Stage, str], ...]: + """(stage, sub-DAG id) pairs in stage-graph order. + + The sub-DAG ids derive from the master's id the same way the renderer + minted them (:func:`hflow.runtime.sub_dag_id_for_stage`), so there is one + owner of that mapping and no second bundle-manifest parser to drift from + the library's version-guarded :func:`hflow.runtime.load_bundle`. + """ + return tuple((stage, sub_dag_id_for_stage(master_dag_id, stage)) for stage in Stage) + + +def resolve_runtime(data_root: str) -> RuntimeResolution: + """One resolution pass: local bundle first, else the remote environment. + + Every failure mode (no bundle anywhere and no URL exported; a half-formed + bundle; a URL exported without dag id or credentials) becomes a + :class:`RuntimeUnavailable` whose detail names the fix -- never an + exception that would surface as a 500. + """ + bundle_directory = find_bundle_directory(data_root) + if bundle_directory is not None: + try: + bundle_paths = load_bundle(bundle_directory) + except (FileNotFoundError, ValueError) as error: + # A bundle directory IS an address, half-formed or not. + return RuntimeUnavailable(detail=str(error), addressed=True) + return ResolvedRuntime( + client=client_for_bundle(bundle_paths), + dag_id=bundle_paths.dag_id, + source="bundle", + airflow_web_url=bundle_paths.api_base_url, + stage_dag_ids=_stage_dag_ids(bundle_paths.dag_id), + ) + try: + endpoint = resolve_remote_endpoint() + except ValueError as error: + # A URL is exported but the resolution is incomplete; the message + # names exactly which HFLOW_AIRFLOW_* variable to set. + return RuntimeUnavailable(detail=str(error), addressed=True) + if endpoint is None: + return RuntimeUnavailable( + detail=( + "no ingest runtime addressed: no rendered bundle at " + f"{Path(data_root) / RUNTIME_BUNDLE_DIRECTORY_NAME} or ./runtime " + f"(run `hflow up`), and {AIRFLOW_URL_ENVIRONMENT_VARIABLE} is not set" + ), + addressed=False, + ) + return ResolvedRuntime( + client=client_for_endpoint(endpoint), + dag_id=endpoint.dag_id, + source="remote", + # Only a bundle records its own web address; guessing that a remote + # API base URL also serves the web UI would not be honest. + airflow_web_url=None, + stage_dag_ids=None, + ) + + +class RuntimeResolver: + """Per-launch cache around :func:`resolve_runtime` (see the TTL note).""" + + def __init__(self, data_root: str) -> None: + self._data_root = data_root + self._cached_resolution: RuntimeResolution | None = None + self._expires_at_monotonic = 0.0 + + def resolve(self) -> RuntimeResolution: + now_monotonic = time.monotonic() + if self._cached_resolution is None or now_monotonic >= self._expires_at_monotonic: + self._cached_resolution = resolve_runtime(self._data_root) + self._expires_at_monotonic = now_monotonic + RESOLUTION_CACHE_TTL_S + return self._cached_resolution + + +def optional_string(value: object) -> str | None: + """One Airflow JSON field as text, or None for anything else. + + Airflow's payloads are an OPEN contract: a field can be absent, null, or + (across versions) another type entirely. Parsing here means the response + models below never see a shape they would have to 500 over. + """ + return value if isinstance(value, str) else None + + +def _run_summary(run: dict[str, Any]) -> RuntimeRunSummary: + """One master dag run reduced to the fields the Runs page shows. + + The full ``conf`` rides along (it is the trigger's own input); everything + else Airflow returns stays server-side. + """ + conf = run.get("conf") + return RuntimeRunSummary( + dag_run_id=optional_string(run.get("dag_run_id")), + state=optional_string(run.get("state")), + logical_date=optional_string(run.get("logical_date")), + start_date=optional_string(run.get("start_date")), + end_date=optional_string(run.get("end_date")), + conf=conf if isinstance(conf, dict) else {}, + ) + + +def _stage_run_summary(run: dict[str, Any]) -> StageRunSummary: + return StageRunSummary( + dag_run_id=optional_string(run.get("dag_run_id")), + state=optional_string(run.get("state")), + start_date=optional_string(run.get("start_date")), + end_date=optional_string(run.get("end_date")), + ) + + +def resolved_runtime_or_refuse(resolver: RuntimeResolver) -> ResolvedRuntime: + """The addressed runtime, or the refusal every runtime-backed route owes. + + 409, not 404: an unaddressed (or half-formed) runtime conflicts with the + workspace's state, the same mapping an unconfigured pipeline uses. Shared + with the graph routes so both refuse identically, detail included. + """ + resolution = resolver.resolve() + if isinstance(resolution, RuntimeUnavailable): + raise HTTPException(status_code=409, detail=resolution.detail) + return resolution + + +def airflow_failure_refusal(error: AirflowClientError, *, source: RuntimeSource) -> HTTPException: + """One failed Airflow call as the 502 every proxying route answers with. + + 502, not 500: the fault is upstream, and the detail is the browser-safe + one :func:`client_error_detail` decides on. + """ + return HTTPException(status_code=502, detail=client_error_detail(error, source=source)) + + +def create_runtime_router(settings: ServerSettings, resolver: RuntimeResolver) -> APIRouter: + """Every runs-monitor route, closed over one launch's settings. + + The resolver is passed in (rather than built here) so the run-graph routes + in ``_graph`` share one addressing cache with this router. + """ + router = APIRouter(prefix="/api/v1") + + @router.get("/runtime/status") + def read_runtime_status() -> RuntimeStatusResponse: + resolution = resolver.resolve() + if isinstance(resolution, RuntimeUnavailable): + return RuntimeStatusResponse(available=False, detail=resolution.detail) + try: + health = resolution.client.health() + except AirflowClientError as error: + # Addressed but not answering (typical between `hflow up` runs): + # still an available:false ANSWER, with the addressing facts. + return RuntimeStatusResponse( + available=False, + detail=client_error_detail(error, source=resolution.source), + source=resolution.source, + airflow_web_url=resolution.airflow_web_url, + airflow_web_url_host_only=is_loopback_web_url(resolution.airflow_web_url), + dag_id=resolution.dag_id, + ) + registered: bool | None + try: + resolution.client.dag(resolution.dag_id) + registered = True + except AirflowClientError as error: + # 404 is the definitive "not registered (yet)"; anything else + # (auth, transient) leaves registration unknown, not false. + registered = False if error.status == 404 else None + return RuntimeStatusResponse( + available=True, + source=resolution.source, + airflow_web_url=resolution.airflow_web_url, + airflow_web_url_host_only=is_loopback_web_url(resolution.airflow_web_url), + dag_id=resolution.dag_id, + registered=registered, + health=RuntimeHealthComponents.model_validate( + { + component_name: health.components.get(component_name) + for component_name in _HEALTH_COMPONENT_NAMES + } + ), + ) + + @router.get("/runtime/runs") + def list_runtime_runs( + limit: int = Query(default=25, ge=1, le=100), + ) -> RuntimeRunsResponse: + runtime = resolved_runtime_or_refuse(resolver) + try: + # order_by="-id": Airflow truncates to `limit` in id order, so + # newest-first is the only ordering that shows recent activity. + master_runs = runtime.client.dag_runs(runtime.dag_id, limit=limit, order_by="-id") + except AirflowClientError as error: + raise airflow_failure_refusal(error, source=runtime.source) from error + stages: list[StageRecentRuns] | None = None + if runtime.stage_dag_ids is not None: + stages = [] + for stage_name, stage_dag_id in runtime.stage_dag_ids: + try: + recent_runs = runtime.client.dag_runs( + stage_dag_id, limit=_RECENT_STAGE_RUN_LIMIT, order_by="-id" + ) + except AirflowClientError: + # A stage sub-DAG that has not registered (or errored) is + # an empty strip, not a failed page. + recent_runs = [] + stages.append( + StageRecentRuns( + stage=stage_name, + dag_id=stage_dag_id, + recent=[_stage_run_summary(run) for run in recent_runs], + ) + ) + return RuntimeRunsResponse(runs=[_run_summary(run) for run in master_runs], stages=stages) + + @router.post("/runtime/ingest") + def trigger_ingest(request: IngestRequest) -> IngestTriggerResponse: + refuse_when_read_only(settings, disabled_actions="triggering ingest runs is") + uris = [uri.strip() for uri in request.uris] + if any(not uri for uri in uris): + raise HTTPException(status_code=400, detail="every uri must be a non-empty string") + # URIs resolve against the runtime's data root; absolute host paths and + # ../ escapes cannot work there, so refuse them before triggering -- + # the same guard `hflow ingest` enforces (src/hflow/cli.py). + for uri in uris: + if uri.startswith("/") or normpath(uri).startswith(".."): + raise HTTPException( + status_code=400, + detail=f"{uri!r} is not relative to the data root -- URIs are resolved " + "against the runtime's configured data root (e.g. " + "`episodes-in/run_0001.mcap`)", + ) + if request.profile not in RUN_PROFILES: + raise HTTPException( + status_code=400, + detail=f"unknown run profile {request.profile!r}; " + f"valid profiles: {sorted(RUN_PROFILES)}", + ) + try: + mode = IngestMode(request.mode) + except ValueError as error: + raise HTTPException( + status_code=400, + detail=f"unknown ingest mode {request.mode!r}; " + f"valid modes: {[known_mode.value for known_mode in IngestMode]}", + ) from error + runtime = resolved_runtime_or_refuse(resolver) + try: + # AirflowClient.ingest owns the trigger conf's shape (uris/profile/ + # mode/batch_count) for every caller -- CLI, UI, control plane -- + # so a client never rebuilds the dict itself. + trigger_response = runtime.client.ingest( + runtime.dag_id, + uris, + profile=request.profile, + online=mode is IngestMode.ONLINE, + batch_count=request.batch_count, + ) + except AirflowClientError as error: + raise airflow_failure_refusal(error, source=runtime.source) from error + return IngestTriggerResponse( + dag_run_id=optional_string(trigger_response.get("dag_run_id")), + state=optional_string(trigger_response.get("state")), + ) + + return router diff --git a/packages/hflow-server/src/hflow_server/_settings.py b/packages/hflow-server/src/hflow_server/_settings.py new file mode 100644 index 0000000..41bcf71 --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_settings.py @@ -0,0 +1,93 @@ +"""Launch configuration for the workspace server, and what it refuses. + +The settings own the launch-wide facts the routers keep asking about -- +``read_only``, the port a launch may bind, and whether the data root is a +local directory -- so each fact is derived here once, beside the field, and +the refusal it maps to lives next to it rather than being hand-written per +router. +""" + +from dataclasses import dataclass +from pathlib import Path + +from fastapi import HTTPException + +from hflow.storage import LocalStorageRoot, parse_storage_root + +DEFAULT_HOST = "127.0.0.1" +# "HFLO" on a phone keypad; mirrored by the core CLI's DEFAULT_SERVER_PORT. +DEFAULT_PORT = 4356 + +# The TCP ports a launch may ask for, the same range (and the same reason for +# excluding 0) that ``hflow.runtime``'s RuntimeConfig enforces for the +# bundle's api port: 0 means "any free port" to bind(2), but this value is +# interpolated into the URL `serve` prints and hands to the browser, and +# http://127.0.0.1:0 is not dialable. +MIN_PORT = 1 +MAX_PORT = 65535 + + +@dataclass(frozen=True) +class ServerSettings: + """One ``hflow serve`` launch, fully parsed. + + ``data_root`` stays a string: it may be a local path or a bucket URL, and + ``hflow.workspace.Workspace.parse`` owns that distinction. ``assets_dir`` + overrides where the built SPA is served from (tests and frontend dev). + + Nothing here is a credential: the server authenticates nobody, so ``host`` + is the whole access-control story (see docs/SERVE.md, "Trust posture"). + """ + + data_root: str + host: str = DEFAULT_HOST + port: int = DEFAULT_PORT + assets_dir: Path | None = None + open_browser: bool = True + # When true, every mutating endpoint (manifest pinning, saved-query + # writes, ingest triggering) answers 403 and /api/v1/config reports it + # (CLI flag: --read-only). + read_only: bool = False + # ``path/to/pipeline.py[:app]`` (CLI flag: --pipeline). The server + # imports -- EXECUTES -- this file exactly once at startup to serve + # /api/v1/pipeline; ``None`` leaves that capability off. + pipeline: str | None = None + + def __post_init__(self) -> None: + # A range invariant of the field, checked where the field is set, so a + # library caller building ServerSettings directly gets the same answer as + # the command line. Left to bind(2) instead, an out-of-range port + # surfaces as an OverflowError from inside the port probe, and port 0 + # binds fine while printing a URL nobody can open. + if not MIN_PORT <= self.port <= MAX_PORT: + raise ValueError(f"port {self.port!r} is not in {MIN_PORT}-{MAX_PORT}") + + +def local_data_root_or_none(data_root: str) -> Path | None: + """The data root as a local directory, or ``None`` for a bucket URL. + + The ONE derivation of "this workspace's files are reachable as paths" -- + the precondition media serving, the sidecar, and pinned manifest files all + share. Each caller decides what to do without one (``_media`` and + ``_sidecar`` refuse 501 in their own error type; /api/v1/config turns it + into capability flags the frontend can hide affordances behind), but none + of them re-derives the predicate. + """ + parsed_root = parse_storage_root(data_root) + return parsed_root.path if isinstance(parsed_root, LocalStorageRoot) else None + + +def refuse_when_read_only(settings: ServerSettings, *, disabled_actions: str) -> None: + """The 403 every mutating route owes a read-only launch. + + One owner for the status and the sentence, shared by the curation studio + and the runs monitor; only the named actions differ, so a third mutating + router cannot invent a third wording or a different code. + ``disabled_actions`` carries its own agreeing verb ("... is" / "... are") + because the routes name one action or several. + """ + if settings.read_only: + raise HTTPException( + status_code=403, + detail=f"this workspace UI is running read-only; {disabled_actions} disabled", + ) diff --git a/packages/hflow-server/src/hflow_server/_sidecar.py b/packages/hflow-server/src/hflow_server/_sidecar.py new file mode 100644 index 0000000..19c23e2 --- /dev/null +++ b/packages/hflow-server/src/hflow_server/_sidecar.py @@ -0,0 +1,217 @@ +"""Curation sidecar state: ``/curation/state.json``, owned here. + +Workspace convention: curation persists exactly two kinds of durable state -- +saved queries and the pinned-manifest registry -- in ONE JSON sidecar file, +``/curation/state.json``. Together with the manifest files under +``/manifests/``, that sidecar is the ONLY thing this server ever +writes into a workspace. + +It sits under ``curation/`` rather than under any client's name because the +content is the operator's, not a browser's: saved queries and pinned +manifests belong to the workspace, and a second client (another UI, a +script) reads the same file. + +Two rules hold at this boundary: + +- Every write is atomic: the payload lands in a temp file beside the target + and is moved into place with ``os.replace`` (via ``Path.replace``), so a + crash never leaves a torn file. Concurrent writers are last-writer-wins, + which a single-operator local tool accepts. +- Every read parses loudly: a payload that is not JSON, carries a + ``state_version`` this build does not speak, or holds a malformed entry is + refused with an error NAMING THE FILE -- never silently coerced, dropped, + or rewritten (the state is the user's curation record). + +The stored entries ARE the published contract models +(:class:`hflow_server._contract.SavedQueryEntry` and +:class:`~hflow_server._contract.PinnedManifestEntry`): the file a user can read +with ``jq`` and the payload the API serves are one shape with one owner, so +they cannot drift apart. Changing either therefore changes this file's +format, which ``STATE_VERSION`` guards. +""" + +import json +import uuid +from dataclasses import dataclass +from pathlib import Path + +from hflow_server._contract import CheckCoverageEntry, PinnedManifestEntry, SavedQueryEntry +from hflow_server._settings import local_data_root_or_none + +STATE_VERSION = 1 +SIDECAR_DIRECTORY_NAME = "curation" +SIDECAR_FILE_NAME = "state.json" + + +class SidecarError(Exception): + """One refusal to read or write the sidecar, carrying its HTTP mapping.""" + + def __init__(self, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +@dataclass(frozen=True) +class SidecarState: + """The whole parsed sidecar; a missing file reads as this default.""" + + saved_queries: tuple[SavedQueryEntry, ...] = () + manifests: tuple[PinnedManifestEntry, ...] = () + + +def local_data_root(data_root: str) -> Path: + """The data root as a local directory; sidecar and manifest writes need one.""" + local_root = local_data_root_or_none(data_root) + if local_root is None: + raise SidecarError( + 501, + "saved queries and pinned manifests need a local data root; " + "bucket-backed workspaces are not supported by the curation studio yet", + ) + return local_root + + +def sidecar_state_file(data_root: str) -> Path: + return local_data_root(data_root) / SIDECAR_DIRECTORY_NAME / SIDECAR_FILE_NAME + + +def load_sidecar_state(data_root: str) -> SidecarState: + """The parsed sidecar; empty when never written, loud on anything corrupt.""" + state_file = sidecar_state_file(data_root) + try: + raw_payload = state_file.read_text(encoding="utf-8") + except FileNotFoundError: + return SidecarState() + except OSError as error: + raise SidecarError(500, f"cannot read curation state file {state_file}: {error}") from error + return _parsed_state(raw_payload, state_file) + + +def store_sidecar_state(data_root: str, state: SidecarState) -> None: + """Atomically replace the sidecar with ``state`` (temp file + os.replace).""" + state_file = sidecar_state_file(data_root) + state_file.parent.mkdir(parents=True, exist_ok=True) + # by_alias: the stored keys are the published ones ("id", not "query_id"). + payload = json.dumps( + { + "state_version": STATE_VERSION, + "saved_queries": [entry.model_dump(by_alias=True) for entry in state.saved_queries], + "manifests": [entry.model_dump(by_alias=True) for entry in state.manifests], + }, + indent=2, + ) + temporary_file = state_file.parent / f".{SIDECAR_FILE_NAME}.{uuid.uuid4().hex}.tmp" + try: + temporary_file.write_text(payload + "\n", encoding="utf-8") + # Path.replace is os.replace: atomic on one filesystem, so a reader + # (or a crash) sees the old complete state or the new one, never a mix. + temporary_file.replace(state_file) + except OSError as error: + temporary_file.unlink(missing_ok=True) + raise SidecarError( + 500, f"cannot write curation state file {state_file}: {error}" + ) from error + + +def _refused(state_file: Path, problem: str) -> SidecarError: + return SidecarError( + 500, f"corrupt curation state file {state_file}: {problem}; fix or remove the file" + ) + + +def _parsed_state(raw_payload: str, state_file: Path) -> SidecarState: + try: + parsed = json.loads(raw_payload) + except json.JSONDecodeError as error: + raise _refused(state_file, f"not valid JSON ({error})") from error + if not isinstance(parsed, dict): + raise _refused(state_file, "expected a JSON object") + found_version = parsed.get("state_version") + if found_version != STATE_VERSION: + # 409, not 500: the same mapping _connections gives a catalog written + # in a format version this build cannot read -- the state is there and + # intact, this build just cannot speak to it, which is a conflict with + # the workspace rather than a fault of this server. A file that is + # corrupt (rather than merely newer) keeps the 500 _refused gives it. + raise SidecarError( + 409, + f"curation state file {state_file} has state_version {found_version!r}; " + f"this build reads version {STATE_VERSION!r}", + ) + saved_queries = tuple( + _parsed_saved_query(entry, state_file) + for entry in _entry_list(parsed, "saved_queries", state_file) + ) + manifests = tuple( + _parsed_manifest(entry, state_file) + for entry in _entry_list(parsed, "manifests", state_file) + ) + return SidecarState(saved_queries=saved_queries, manifests=manifests) + + +def _entry_list(parsed: dict[str, object], key: str, state_file: Path) -> list[dict[str, object]]: + entries = parsed.get(key, []) + if not isinstance(entries, list) or not all(isinstance(entry, dict) for entry in entries): + raise _refused(state_file, f"{key!r} must be a list of objects") + return entries + + +def _string_field(entry: dict[str, object], key: str, state_file: Path) -> str: + value = entry.get(key) + if not isinstance(value, str): + raise _refused(state_file, f"entry field {key!r} must be a string, got {value!r}") + return value + + +def _int_field(entry: dict[str, object], key: str, state_file: Path) -> int: + value = entry.get(key) + if isinstance(value, bool) or not isinstance(value, int): + raise _refused(state_file, f"entry field {key!r} must be an integer, got {value!r}") + return value + + +def _float_field(entry: dict[str, object], key: str, state_file: Path) -> float: + value = entry.get(key) + if isinstance(value, bool) or not isinstance(value, int | float): + raise _refused(state_file, f"entry field {key!r} must be a number, got {value!r}") + return float(value) + + +def _parsed_saved_query(entry: dict[str, object], state_file: Path) -> SavedQueryEntry: + # Field-by-field on purpose: a model_validate refusal would name pydantic's + # own error shape, not this file and the fix for it. + return SavedQueryEntry( + query_id=_string_field(entry, "id", state_file), + name=_string_field(entry, "name", state_file), + sql=_string_field(entry, "sql", state_file), + updated_at=_string_field(entry, "updated_at", state_file), + ) + + +def _parsed_manifest(entry: dict[str, object], state_file: Path) -> PinnedManifestEntry: + raw_coverage = entry.get("coverage", []) + if not isinstance(raw_coverage, list) or not all( + isinstance(coverage_entry, dict) for coverage_entry in raw_coverage + ): + raise _refused(state_file, "'coverage' must be a list of objects") + coverage = [ + CheckCoverageEntry( + check_name=_string_field(coverage_entry, "check_name", state_file), + episodes_ran=_int_field(coverage_entry, "episodes_ran", state_file), + total_episodes=_int_field(coverage_entry, "total_episodes", state_file), + fraction=_float_field(coverage_entry, "fraction", state_file), + ) + for coverage_entry in raw_coverage + ] + return PinnedManifestEntry( + manifest_id=_string_field(entry, "id", state_file), + name=_string_field(entry, "name", state_file), + description=_string_field(entry, "description", state_file), + sql=_string_field(entry, "sql", state_file), + manifest_path=_string_field(entry, "manifest_path", state_file), + row_count=_int_field(entry, "row_count", state_file), + total_episodes=_int_field(entry, "total_episodes", state_file), + coverage=coverage, + created_at=_string_field(entry, "created_at", state_file), + ) diff --git a/packages/hflow-server/src/hflow_server/server.py b/packages/hflow-server/src/hflow_server/server.py new file mode 100644 index 0000000..430e735 --- /dev/null +++ b/packages/hflow-server/src/hflow_server/server.py @@ -0,0 +1,472 @@ +"""The FastAPI app (pure, testable) and the ``hflow serve`` server entry point. + +``create_app`` builds the whole API plus SPA serving from one +:class:`ServerSettings` -- no sockets, no side effects. ``serve`` adds the launch +behavior: pick a free port, print the URL, open the browser, run uvicorn. The +server authenticates nobody: whoever can reach the bound address gets the +whole API (docs/SERVE.md, "Trust posture"). The only workspace files this package +ever writes are the curation studio's: immutable pinned manifests under +``/manifests/`` and the ``/curation/state.json`` sidecar (both +refused when ``settings.read_only``); the server never mints workspace +identity. +""" + +import importlib.resources +import os +import socket +import threading +import webbrowser +from pathlib import Path +from typing import Annotated + +import uvicorn +from fastapi import Depends, FastAPI, HTTPException, Query +from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, Response +from starlette.datastructures import Headers +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +import hflow +from hflow.format import CATALOG_FORMAT_VERSION +from hflow.steps import RUN_PROFILES, IngestMode +from hflow.workspace import Workspace +from hflow_server import _catalog, _connections, _curation, _graph, _media, _pipeline, _runtime +from hflow_server._contract import ( + BINARY_FILE_RESPONSES, + EpisodeDossierResponse, + EpisodeFacetsResponse, + EpisodePageResponse, + EpisodeStatsResponse, + EpisodeStatus, + EpisodeTimelineResponse, + HealthResponse, + ListingOrder, + SuccessFilterValue, + WorkspaceCapabilities, + WorkspaceConfigResponse, +) +from hflow_server._settings import MAX_PORT, ServerSettings, local_data_root_or_none + +ASSETS_ENVIRONMENT_VARIABLE = "HFLOW_UI_ASSETS" + +_PORT_RETRY_ATTEMPTS = 10 + +# A blanket cap on request-body size: comfortably above the curation studio's +# own per-field limits (a 100k SQL body plus JSON overhead), but a hard stop +# on an unbounded POST -- the sidecar is the one file this server writes outside +# manifests/, so nothing it persists should be able to grow without limit. +_MAX_REQUEST_BODY_BYTES = 4 * 1024 * 1024 + + +def _declared_request_body_bytes(scope: Scope) -> int | None: + """The request's Content-Length, or None when it declares none (or lies).""" + declared = Headers(scope=scope).get("content-length") + if declared is None: + return None + try: + return int(declared) + except ValueError: + return None + + +class RequestBodySizeLimitMiddleware: + """Refuses any request body over the size cap with a 413. + + Pure ASGI, and it counts the bytes rather than trusting a header: a + declared ``Content-Length`` is only a claim, and a chunked request makes + none at all, so a header-only check let exactly the thing this middleware + exists to stop -- an unbounded POST buffered whole before any validation + runs -- through by simply omitting the header. A declared length over the + cap is still refused up front, without reading a byte. + + The body is read HERE and replayed downstream rather than counted inside + a wrapped receive channel: an oversized-body error raised from inside the + channel gets rewritten by whatever was reading it (FastAPI's body reader + turns any exception but its own into a generic 400, and an intervening + ``BaseHTTPMiddleware`` collapses even that into an exception group), which + would leave the cap enforced but unsayable. What is buffered is bounded by + the cap itself -- the first chunk that crosses it ends the request -- and + every route on this API reads its whole body anyway, so nothing that would + otherwise have streamed is being held here. + """ + + def __init__(self, app: ASGIApp) -> None: + self._app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self._app(scope, receive, send) + return + declared_body_bytes = _declared_request_body_bytes(scope) + if declared_body_bytes is not None and declared_body_bytes > _MAX_REQUEST_BODY_BYTES: + await _refuse_oversized_body(scope, receive, send) + return + body_messages = await _capped_body_messages(receive) + if body_messages is None: + await _refuse_oversized_body(scope, receive, send) + return + await self._app(scope, _replaying_receive(body_messages, receive), send) + + +async def _capped_body_messages(receive: Receive) -> list[Message] | None: + """One request's body messages, or ``None`` once they exceed the cap.""" + body_messages: list[Message] = [] + received_body_bytes = 0 + while True: + message = await receive() + body_messages.append(message) + if message["type"] != "http.request": + # http.disconnect: no body is coming, and none ever will. + return body_messages + received_body_bytes += len(message.get("body", b"")) + if received_body_bytes > _MAX_REQUEST_BODY_BYTES: + return None + if not message.get("more_body", False): + return body_messages + + +def _replaying_receive(body_messages: list[Message], receive: Receive) -> Receive: + """A receive channel handing back the read body, then the real channel.""" + unread_messages = iter(body_messages) + + async def replaying_receive() -> Message: + unread = next(unread_messages, None) + # Past the buffered body the real channel takes over, so a downstream + # reader still sees the eventual http.disconnect. + return unread if unread is not None else await receive() + + return replaying_receive + + +async def _refuse_oversized_body(scope: Scope, receive: Receive, send: Send) -> None: + await JSONResponse({"detail": "request body too large"}, status_code=413)(scope, receive, send) + + +_FRONTEND_PLACEHOLDER_PAGE = """ + + HFlow workspace API + +

HFlow workspace API

+

No frontend bundle is installed here. The JSON API is live under + /api/v1, and its OpenAPI schema is at + /api/openapi.json — that schema is the product surface: + everything a UI can show is reachable from it, so any client can be built + against it without touching this package.

+

To serve your own build, point the HFLOW_UI_ASSETS + environment variable at a directory containing an + index.html, or pass assets_dir to + ServerSettings. A bundle packaged inside hflow_server + is picked up automatically.

+ + +""" + + +def parse_episode_list_filters( + task: Annotated[list[str] | None, Query()] = None, + operator: Annotated[list[str] | None, Query()] = None, + embodiment: Annotated[list[str] | None, Query()] = None, + status: Annotated[EpisodeStatus | None, Query()] = None, + success: Annotated[SuccessFilterValue | None, Query()] = None, + search: Annotated[str | None, Query()] = None, +) -> _catalog.EpisodeListFilters: + """The filter params /episodes and /episodes/stats BOTH accept. + + One owner for the pair: the two endpoints must describe the same rows, so + a filter added here reaches the listing and its distributions together -- + they cannot drift into accepting different query strings. + """ + return _catalog.EpisodeListFilters( + tasks=tuple(task or ()), + operators=tuple(operator or ()), + embodiments=tuple(embodiment or ()), + status=status, + success=success, + search=search, + ) + + +EpisodeListFilterParams = Annotated[ + _catalog.EpisodeListFilters, Depends(parse_episode_list_filters) +] + + +def create_app(settings: ServerSettings) -> FastAPI: + """The whole workspace server as a plain ASGI app.""" + # Late import: hflow_server/__init__ imports this module, so the package + # attribute exists only once init finished -- which any create_app call is. + from hflow_server import __version__ as hflow_server_version + + application = FastAPI( + title="HFlow workspace API", + version=hflow_server_version, + # No Swagger or ReDoc HTML page. Both of FastAPI's built-in pages load + # their JS and CSS from cdn.jsdelivr.net, which would break the offline + # promise this UI makes (docs/SERVE.md, "Trust posture": no CDN, no + # outbound requests) and would run third-party script same-origin with + # this workspace's API. The generated schema is served as JSON instead + # -- that IS the contract, and any local OpenAPI viewer or client + # generator reads it. test_ui_offline_posture.py pins this. + docs_url=None, + openapi_url="/api/openapi.json", + redoc_url=None, + ) + # Starlette runs middleware outermost-first in REVERSE registration order, + # so the body-size cap being registered last is what makes it outermost: + # an oversized POST is refused before routing touches it. There is no + # request guard here -- this server authenticates nobody (docs/SERVE.md, + # "Trust posture") -- and if one is ever added, this is where it goes, in + # front of the routes and behind the cap. + application.add_middleware(RequestBodySizeLimitMiddleware) + + # --pipeline is imported -- EXECUTED -- exactly once, here at app + # construction; the outcome (the live App, or the remembered failure) is + # what /api/v1/pipeline and the config capability report for this launch. + pipeline_state = _pipeline.load_pipeline_state(settings.pipeline) + # One runtime resolver per launch, shared by the runs monitor and the + # graph routes so both read the same briefly-cached addressing. + runtime_resolver = _runtime.RuntimeResolver(settings.data_root) + + @application.get("/api/v1/health") + def read_health() -> HealthResponse: + return HealthResponse(ok=True) + + @application.get("/api/v1/config") + def read_config() -> WorkspaceConfigResponse: + workspace = Workspace.parse(settings.data_root) + try: + identity = workspace.identity() + except ValueError: + # A corrupt identity marker must not stop the server from booting: the + # id is informational here, and this surface never mints one. + identity = None + workspace_is_local = local_data_root_or_none(settings.data_root) is not None + return WorkspaceConfigResponse( + mode="local", + read_only=settings.read_only, + hflow_version=hflow.__version__, + hflow_server_version=hflow_server_version, + data_root=settings.data_root, + workspace_id=identity.workspace_id if identity is not None else None, + capabilities=WorkspaceCapabilities( + catalog=_catalog_marker_readable(workspace), + # Media bytes and the studio's writes both need the workspace + # reachable as local paths; they are separate flags because + # bucket support will arrive for them separately. + media=workspace_is_local, + curation=workspace_is_local, + # Addressed (bundle dir or HFLOW_AIRFLOW_URL), not necessarily + # reachable -- /runtime/status owns liveness, and it is also + # the one endpoint that serves the Airflow deep-link base. + runtime=_runtime.runtime_addressed(runtime_resolver.resolve()), + pipeline=isinstance(pipeline_state, _pipeline.PipelineLoaded), + ), + # The trigger form's vocabularies, served so the frontend never + # hardcodes them (hflow.steps stays the one owner). + run_profiles=list(RUN_PROFILES), + ingest_modes=[mode.value for mode in IngestMode], + ) + + @application.get("/api/v1/episodes") + def list_episodes( + filters: EpisodeListFilterParams, + order_by: Annotated[str, Query()] = "recorded_at", + order: Annotated[ListingOrder, Query()] = "desc", + limit: Annotated[int, Query(ge=1, le=500)] = 50, + offset: Annotated[int, Query(ge=0)] = 0, + ) -> EpisodePageResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + try: + return _catalog.query_episode_page( + connection, + filters, + order_by=order_by, + descending=order == "desc", + limit=limit, + offset=offset, + ) + except _catalog.UnknownOrderColumnError as error: + raise HTTPException(status_code=400, detail=str(error)) from error + + @application.get("/api/v1/episodes/facets") + def read_episode_facets() -> EpisodeFacetsResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + return _catalog.query_episode_facets(connection) + + # Registered before the {episode_id} route below so the literal path + # segment "stats" can never be read as an episode id. + @application.get("/api/v1/episodes/stats") + def read_episode_stats(filters: EpisodeListFilterParams) -> EpisodeStatsResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + return _catalog.query_episode_stats(connection, filters) + + @application.get("/api/v1/episodes/{episode_id}") + def read_episode(episode_id: str) -> EpisodeDossierResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + dossier = _catalog.query_episode_dossier( + connection, episode_id, data_root=settings.data_root + ) + if dossier is None: + raise HTTPException( + status_code=404, detail=f"no episode {episode_id!r} in this catalog" + ) + return dossier + + @application.get("/api/v1/episodes/{episode_id}/timeline") + def read_episode_timeline(episode_id: str) -> EpisodeTimelineResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + timeline = _catalog.query_episode_timeline(connection, episode_id) + if timeline is None: + raise HTTPException( + status_code=404, detail=f"no episode {episode_id!r} in this catalog" + ) + return timeline + + @application.get( + "/api/v1/episodes/{episode_id}/media/{artifact_name:path}", + response_class=FileResponse, + responses=BINARY_FILE_RESPONSES, + ) + def read_episode_media(episode_id: str, artifact_name: str) -> FileResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + media_uri = _catalog.find_media_uri(connection, episode_id, artifact_name) + if media_uri is None: + raise HTTPException( + status_code=404, + detail=f"episode {episode_id!r} has no media artifact named {artifact_name!r}", + ) + return _served_file_response_or_refuse(media_uri, settings.data_root) + + @application.get( + "/api/v1/episodes/{episode_id}/canonical", + response_class=FileResponse, + responses=BINARY_FILE_RESPONSES, + ) + def read_episode_canonical(episode_id: str) -> FileResponse: + with _connections.opened_workspace_connection_or_refuse(settings.data_root) as connection: + canonical_uri = _catalog.find_canonical_uri(connection, episode_id) + if canonical_uri is None: + raise HTTPException( + status_code=404, detail=f"no episode {episode_id!r} in this catalog" + ) + return _served_file_response_or_refuse(canonical_uri, settings.data_root) + + # The curation studio, runs monitor, pipeline and visualization routes -- + # included BEFORE the SPA catch-all below so they win route matching. + application.include_router(_curation.create_curation_router(settings)) + application.include_router(_runtime.create_runtime_router(settings, runtime_resolver)) + application.include_router(_pipeline.create_pipeline_router(settings, pipeline_state)) + application.include_router(_graph.create_graph_router(pipeline_state, runtime_resolver)) + + @application.get("/{requested_path:path}", include_in_schema=False) + def serve_spa(requested_path: str) -> Response: + return _spa_response(settings, requested_path) + + return application + + +def _catalog_marker_readable(workspace: Workspace) -> bool: + """Whether the catalog's format marker is present and this build reads it.""" + try: + found_version = workspace.catalog_root.read_bytes("format_version").decode().strip() + except (OSError, UnicodeDecodeError): + return False + return found_version == CATALOG_FORMAT_VERSION + + +def _served_file_response_or_refuse(uri: str, data_root: str) -> FileResponse: + try: + resolved_file = _media.resolve_served_file(uri, data_root=data_root) + except _media.MediaResolutionError as error: + raise _media.media_refusal(error) from error + return _media.served_file_response(resolved_file) + + +def _assets_directory(settings: ServerSettings) -> Path | None: + """Where the built SPA lives: explicit setting, env override, then the wheel.""" + if settings.assets_dir is not None: + return settings.assets_dir + environment_override = os.environ.get(ASSETS_ENVIRONMENT_VARIABLE) + if environment_override: + return Path(environment_override) + # This package ships as a plain directory wheel (uv_build, never zipped), + # so the packaged resource is always a real filesystem path. + packaged_static = Path(str(importlib.resources.files("hflow_server").joinpath("static"))) + return packaged_static if packaged_static.is_dir() else None + + +def _spa_response(settings: ServerSettings, requested_path: str) -> Response: + if requested_path == "api" or requested_path.startswith("api/"): + raise HTTPException(status_code=404, detail="unknown API path") + assets_directory = _assets_directory(settings) + if assets_directory is not None and requested_path: + asset_response = _contained_asset_response(assets_directory, requested_path) + if asset_response is not None: + return asset_response + final_segment = requested_path.rsplit("/", 1)[-1] + if "." in final_segment: + # Looks like a file: a missing asset is a 404, never index.html. + raise HTTPException(status_code=404, detail="no such asset") + if assets_directory is not None: + index_file = assets_directory / "index.html" + if index_file.is_file(): + return FileResponse(index_file) + return HTMLResponse(_FRONTEND_PLACEHOLDER_PAGE) + + +def _contained_asset_response(assets_directory: Path, requested_path: str) -> FileResponse | None: + resolved_assets_directory = assets_directory.resolve() + try: + resolved_candidate = (assets_directory / requested_path).resolve(strict=True) + except (OSError, ValueError): + return None + if not resolved_candidate.is_relative_to(resolved_assets_directory): + # Traversal outside the assets tree is answered as if absent. + return None + if not resolved_candidate.is_file(): + return None + return FileResponse(resolved_candidate) + + +def serve(settings: ServerSettings) -> None: + """Run the workspace server: free port, printed URL, browser, uvicorn.""" + application = create_app(settings) + chosen_port = _first_free_port(settings.host, settings.port) + if chosen_port != settings.port: + # flush=True throughout: the URL must reach a piped stdout (tee, a + # supervisor's log) before the blocking uvicorn.run call. + print( + f"hflow serve: port {settings.port} is taken; serving on port {chosen_port} instead", + flush=True, + ) + url_host = "127.0.0.1" if settings.host == "0.0.0.0" else settings.host + workspace_url = f"http://{url_host}:{chosen_port}/" + print(f"hflow serve: serving {settings.data_root} at {workspace_url}", flush=True) + if settings.open_browser: + # uvicorn.run blocks this thread; a short timer opens the browser + # once the server has had time to bind. + browser_timer = threading.Timer(1.0, webbrowser.open, args=[workspace_url]) + browser_timer.daemon = True + browser_timer.start() + # uvicorn's stock logging config: the access line's query string carries + # episode filters and paging, which are useful when debugging a request + # and are not credentials -- this server has none. + uvicorn.run(application, host=settings.host, port=chosen_port, log_level="info") + + +def _first_free_port(host: str, preferred_port: int) -> int: + """The preferred port, or the first free one in the handful above it. + + ``preferred_port`` is already in ``MIN_PORT..MAX_PORT`` (``ServerSettings`` + parses it there), and the retry window is clipped to MAX_PORT so walking + off the top of the range refuses with the same sentence as an occupied + range rather than with bind(2)'s OverflowError. + """ + last_candidate_port = min(preferred_port + _PORT_RETRY_ATTEMPTS - 1, MAX_PORT) + for candidate_port in range(preferred_port, last_candidate_port + 1): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe_socket: + try: + probe_socket.bind((host, candidate_port)) + except OSError: + continue + return candidate_port + raise RuntimeError(f"no free port between {preferred_port} and {last_candidate_port} on {host}") diff --git a/packages/hflow-server/tests/conftest.py b/packages/hflow-server/tests/conftest.py new file mode 100644 index 0000000..76334aa --- /dev/null +++ b/packages/hflow-server/tests/conftest.py @@ -0,0 +1,88 @@ +"""Shared fixtures: real populated workspaces and TestClients over them.""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import PopulatedWorkspace, build_populated_workspace + +from hflow.catalog import Catalog + + +@pytest.fixture(scope="session") +def populated_workspace(tmp_path_factory: pytest.TempPathFactory) -> PopulatedWorkspace: + return build_populated_workspace(tmp_path_factory) + + +@pytest.fixture(scope="session") +def unbuilt_assets_dir(tmp_path_factory: pytest.TempPathFactory) -> Path: + """An empty assets directory, for the clients that assert on served pages. + + The packaged default (hflow_server/static/) holds the built SPA on a machine + that has run the frontend build, and nothing on one that has not, so any + test that asserts on a served PAGE pins assets_dir -- every client fixture + below does, through this fixture. A client built inline inside an API test + needs no pin: it only ever requests /api paths, which never consult the + assets directory. + """ + return tmp_path_factory.mktemp("ui-no-assets") + + +@pytest.fixture(scope="session") +def api(populated_workspace: PopulatedWorkspace, unbuilt_assets_dir: Path) -> TestClient: + """A client over the populated root; the server authenticates nobody.""" + settings = ServerSettings( + data_root=str(populated_workspace.data_root), assets_dir=unbuilt_assets_dir + ) + return TestClient(create_app(settings)) + + +@pytest.fixture(scope="session") +def read_only_api(populated_workspace: PopulatedWorkspace, unbuilt_assets_dir: Path) -> TestClient: + """A client whose server runs read-only: every write endpoint must 403. + + Session-scoped over the shared workspace on purpose -- a read-only server + refuses before touching anything, so it cannot dirty the fixture. + """ + settings = ServerSettings( + data_root=str(populated_workspace.data_root), + assets_dir=unbuilt_assets_dir, + read_only=True, + ) + return TestClient(create_app(settings)) + + +@pytest.fixture() +def writable_workspace(tmp_path_factory: pytest.TempPathFactory) -> PopulatedWorkspace: + """A per-test workspace for tests that WRITE (pins, saved queries).""" + return build_populated_workspace(tmp_path_factory) + + +@pytest.fixture() +def writable_api(writable_workspace: PopulatedWorkspace, unbuilt_assets_dir: Path) -> TestClient: + settings = ServerSettings( + data_root=str(writable_workspace.data_root), assets_dir=unbuilt_assets_dir + ) + return TestClient(create_app(settings)) + + +@pytest.fixture(scope="session") +def empty_workspace_api( + tmp_path_factory: pytest.TempPathFactory, unbuilt_assets_dir: Path +) -> TestClient: + """A client over a data root that has no catalog at all.""" + empty_root = tmp_path_factory.mktemp("ui-empty-root") + settings = ServerSettings(data_root=str(empty_root), assets_dir=unbuilt_assets_dir) + return TestClient(create_app(settings)) + + +@pytest.fixture(scope="session") +def empty_catalog_api( + tmp_path_factory: pytest.TempPathFactory, unbuilt_assets_dir: Path +) -> TestClient: + """A client over a catalog that exists but holds zero episodes.""" + data_root = tmp_path_factory.mktemp("ui-empty-catalog-root") + Catalog(data_root / "catalog") + settings = ServerSettings(data_root=str(data_root), assets_dir=unbuilt_assets_dir) + return TestClient(create_app(settings)) diff --git a/packages/hflow-server/tests/test_server_catalog_edge_cases.py b/packages/hflow-server/tests/test_server_catalog_edge_cases.py new file mode 100644 index 0000000..5d90f18 --- /dev/null +++ b/packages/hflow-server/tests/test_server_catalog_edge_cases.py @@ -0,0 +1,114 @@ +"""Catalog edge cases over purpose-built workspaces: measurement-key columns +whose names contain '?' (sortable header the endpoint advertises) and +numeric columns whose value span overflows to infinity.""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import STAMPS + +import hflow +from hflow.catalog import Catalog, CheckRunRow +from hflow.steps import MeasurementValue + + +def _client_over(data_root: Path) -> TestClient: + return TestClient(create_app(ServerSettings(data_root=str(data_root)))) + + +def _append_with_measurements( + catalog: Catalog, + episodes_dir: Path, + stem: str, + task: str, + measurements: dict[str, MeasurementValue], +) -> str: + canonical = episodes_dir / f"{stem}.canonical.mcap" + canonical.write_bytes(b"canonical " + stem.encode()) + result = catalog.append_episode( + canonical_path=canonical, + stamps=STAMPS, + episode_metadata={"task": task, "operator": "alice", "embodiment": "arm-1"}, + check_rows=[ + CheckRunRow( + check_name="metrics_check", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements=measurements, + ) + ], + ) + return result.episode_id + + +@pytest.fixture() +def question_mark_column_api(tmp_path: Path) -> TestClient: + """A workspace whose measurement key contains '?', so it pivots into an + episodes column named e.g. ``gripper_ok?``.""" + data_root = tmp_path / "data" + episodes_dir = data_root / "episodes" + episodes_dir.mkdir(parents=True) + catalog = Catalog(data_root / "catalog") + _append_with_measurements(catalog, episodes_dir, "a", "fold", {"gripper_ok?": 1.0}) + _append_with_measurements(catalog, episodes_dir, "b", "pour", {"gripper_ok?": 0.0}) + return _client_over(data_root) + + +def test_order_by_a_column_whose_name_contains_a_question_mark( + question_mark_column_api: TestClient, +) -> None: + listing = question_mark_column_api.get("/api/v1/episodes") + assert listing.status_code == 200 + # The endpoint advertises the '?'-named column as sortable. + assert "gripper_ok?" in {column["name"] for column in listing.json()["columns"]} + # Sorting by it must not 500 on the display-SQL renderer. + ordered = question_mark_column_api.get( + "/api/v1/episodes", params={"order_by": "gripper_ok?", "order": "asc"} + ) + assert ordered.status_code == 200, ordered.text + values = [row["gripper_ok?"] for row in ordered.json()["rows"]] + assert values == sorted(values) + # The rendered display SQL still round-trips (its '?' is inside a quoted + # identifier, never miscounted as a bind placeholder). + assert 'ORDER BY "gripper_ok?"' in ordered.json()["sql"] + + +def test_order_by_a_question_mark_column_with_a_filter( + question_mark_column_api: TestClient, +) -> None: + # A filter adds real bind placeholders; the '?' in the order_by identifier + # must still not be counted among them. + filtered = question_mark_column_api.get( + "/api/v1/episodes", params={"order_by": "gripper_ok?", "task": "fold"} + ) + assert filtered.status_code == 200, filtered.text + assert filtered.json()["total"] == 1 + + +@pytest.fixture() +def infinite_span_stats_api(tmp_path: Path) -> TestClient: + """A workspace with a numeric column whose max-min overflows to inf.""" + data_root = tmp_path / "data" + episodes_dir = data_root / "episodes" + episodes_dir.mkdir(parents=True) + catalog = Catalog(data_root / "catalog") + _append_with_measurements(catalog, episodes_dir, "lo", "fold", {"huge": -1.7e308, "ok": 1.0}) + _append_with_measurements(catalog, episodes_dir, "hi", "fold", {"huge": 1.7e308, "ok": 2.0}) + _append_with_measurements(catalog, episodes_dir, "mid", "fold", {"huge": 0.0, "ok": 3.0}) + return _client_over(data_root) + + +def test_stats_do_not_500_when_a_columns_span_overflows_to_infinity( + infinite_span_stats_api: TestClient, +) -> None: + response = infinite_span_stats_api.get("/api/v1/episodes/stats") + assert response.status_code == 200, response.text + profiled = {column["name"] for column in response.json()["columns"]} + # The overflowing column is skipped as degenerate; the well-behaved + # numeric column beside it still earns a histogram. + assert "huge" not in profiled + assert "ok" in profiled diff --git a/packages/hflow-server/tests/test_server_catalog_tables.py b/packages/hflow-server/tests/test_server_catalog_tables.py new file mode 100644 index 0000000..d27a120 --- /dev/null +++ b/packages/hflow-server/tests/test_server_catalog_tables.py @@ -0,0 +1,60 @@ +"""/api/v1/catalog/tables: the browsable schema tree and per-table summaries.""" + +from fastapi.testclient import TestClient + +EXPECTED_TABLE_NAMES = [ + "episodes", + "episodes_latest", + "episodes_raw", + "check_runs", + "measurements", + "measurements_latest", + "tags", + "intervals", +] + + +def test_tables_lists_every_registered_view_with_columns(api: TestClient) -> None: + response = api.get("/api/v1/catalog/tables") + assert response.status_code == 200 + tables = response.json()["tables"] + assert [table["name"] for table in tables] == EXPECTED_TABLE_NAMES + assert all(table["kind"] in ("view", "table") for table in tables) + columns_by_table = {table["name"]: table["columns"] for table in tables} + episode_column_names = {column["name"] for column in columns_by_table["episodes"]} + # The wide view: episode columns, the status column, pivoted measurements. + assert {"episode_id", "task", "status", "max_velocity"} <= episode_column_names + check_run_column_names = {column["name"] for column in columns_by_table["check_runs"]} + assert {"check_name", "status", "duration_s"} <= check_run_column_names + for table in tables: + assert all(set(column) == {"name", "type"} for column in table["columns"]) + + +def test_table_summary_profiles_row_count_and_columns(api: TestClient) -> None: + response = api.get("/api/v1/catalog/tables/episodes/summary") + assert response.status_code == 200 + payload = response.json() + assert payload["row_count"] == 4 + profiled_names = {entry["column_name"] for entry in payload["columns"]} + assert {"episode_id", "task", "status", "max_velocity"} <= profiled_names + velocity_profile = next( + entry for entry in payload["columns"] if entry["column_name"] == "max_velocity" + ) + assert velocity_profile["max"] == "2.0" # SUMMARIZE renders extremes as text + + +def test_unknown_or_hostile_table_names_are_404(api: TestClient) -> None: + assert api.get("/api/v1/catalog/tables/no_such_table/summary").status_code == 404 + hostile = api.get("/api/v1/catalog/tables/episodes; DROP TABLE episodes_raw/summary") + assert hostile.status_code == 404 + + +def test_summary_of_an_empty_catalog_reports_zero_rows(empty_catalog_api: TestClient) -> None: + response = empty_catalog_api.get("/api/v1/catalog/tables/episodes/summary") + assert response.status_code == 200 + assert response.json()["row_count"] == 0 + + +def test_tables_without_a_catalog_is_404(empty_workspace_api: TestClient) -> None: + assert empty_workspace_api.get("/api/v1/catalog/tables").status_code == 404 + assert empty_workspace_api.get("/api/v1/catalog/tables/episodes/summary").status_code == 404 diff --git a/packages/hflow-server/tests/test_server_config.py b/packages/hflow-server/tests/test_server_config.py new file mode 100644 index 0000000..e3a05d7 --- /dev/null +++ b/packages/hflow-server/tests/test_server_config.py @@ -0,0 +1,110 @@ +"""GET /api/v1/config and /api/v1/health.""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import PopulatedWorkspace + +from hflow.runtime import RuntimeConfig, render_bundle +from hflow.workspace import Workspace + + +@pytest.fixture() +def no_ambient_runtime(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """No ./runtime fallback in cwd and no remote environment exported.""" + working_directory = tmp_path / "config-cwd" + working_directory.mkdir() + monkeypatch.chdir(working_directory) + monkeypatch.delenv("HFLOW_AIRFLOW_URL", raising=False) + + +def test_health_reports_ok(api: TestClient) -> None: + response = api.get("/api/v1/health") + assert response.status_code == 200 + assert response.json() == {"ok": True} + + +def test_config_reports_local_mode_and_capabilities( + api: TestClient, populated_workspace: PopulatedWorkspace, no_ambient_runtime: None +) -> None: + payload = api.get("/api/v1/config").json() + assert payload["mode"] == "local" + assert payload["read_only"] is False # the default server accepts writes + assert isinstance(payload["hflow_version"], str) and payload["hflow_version"] + assert payload["hflow_server_version"] == "0.1.0" + assert payload["data_root"] == str(populated_workspace.data_root) + assert payload["capabilities"] == { + "catalog": True, + "media": True, + "curation": True, # a local data root: the studio's writes can land + "runtime": False, # no bundle rendered and no HFLOW_AIRFLOW_URL exported + "pipeline": False, # no --pipeline configured + } + # The trigger form's vocabularies come from the server, never hardcoded. + assert "full" in payload["run_profiles"] + assert payload["ingest_modes"] == ["batch", "online"] + + +def test_config_does_not_restate_the_airflow_deep_link_base( + tmp_path: Path, unbuilt_assets_dir: Path, no_ambient_runtime: None +) -> None: + # /runtime/status is the ONE owner of the runtime's addressing facts, the + # web URL included; config only reports whether a runtime is addressed. + data_root = tmp_path / "data" + pipeline_file = tmp_path / "demo_pipeline.py" + pipeline_file.write_text("import hflow\n\napp = hflow.App('demo', data_root='/tmp/x')\n") + render_bundle( + RuntimeConfig(pipeline_file=pipeline_file, data_root=data_root), data_root / "runtime" + ) + settings = ServerSettings(data_root=str(data_root), assets_dir=unbuilt_assets_dir) + payload = TestClient(create_app(settings)).get("/api/v1/config").json() + # Configured, not necessarily reachable: nothing is running here. + assert payload["capabilities"]["runtime"] is True + assert "airflow_web_url" not in payload + + +def test_config_runtime_capability_from_the_remote_environment( + tmp_path: Path, + unbuilt_assets_dir: Path, + no_ambient_runtime: None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HFLOW_AIRFLOW_URL", "https://workspace.example.com") + data_root = tmp_path / "bare-root" + data_root.mkdir() + settings = ServerSettings(data_root=str(data_root), assets_dir=unbuilt_assets_dir) + payload = TestClient(create_app(settings)).get("/api/v1/config").json() + assert payload["capabilities"]["runtime"] is True + + +def test_config_reports_the_read_only_setting(read_only_api: TestClient) -> None: + assert read_only_api.get("/api/v1/config").json()["read_only"] is True + + +def test_config_never_mints_workspace_identity( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + assert api.get("/api/v1/config").json()["workspace_id"] is None + assert not (populated_workspace.data_root / "workspace.json").exists() + + +def test_config_reports_missing_catalog(empty_workspace_api: TestClient) -> None: + payload = empty_workspace_api.get("/api/v1/config").json() + assert payload["capabilities"]["catalog"] is False + assert payload["capabilities"]["media"] is True # local root: serving is possible + + +def test_config_reports_a_minted_workspace_identity(tmp_path: Path) -> None: + # The TEST mints the identity; the server itself never does. + minted_identity = Workspace.parse(tmp_path).ensure_identity() + client = TestClient(create_app(ServerSettings(data_root=str(tmp_path)))) + payload = client.get("/api/v1/config").json() + assert payload["workspace_id"] == minted_identity.workspace_id + + +def test_mutating_methods_are_rejected(api: TestClient) -> None: + # Read-only surface: no route accepts writes. + assert api.post("/api/v1/episodes").status_code == 405 + assert api.delete("/api/v1/config").status_code == 405 diff --git a/packages/hflow-server/tests/test_server_curation_pin.py b/packages/hflow-server/tests/test_server_curation_pin.py new file mode 100644 index 0000000..f7ac361 --- /dev/null +++ b/packages/hflow-server/tests/test_server_curation_pin.py @@ -0,0 +1,218 @@ +"""POST /api/v1/curation/pin and /api/v1/manifests: immutable pinned cuts.""" + +import json +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime + +import duckdb +import pytest +from fastapi.testclient import TestClient +from hflow_server import _curation +from ui_test_fixtures import PopulatedWorkspace + +OK_CUT_SQL = "SELECT episode_id FROM episodes WHERE status = 'ok'" + + +def _pin(api: TestClient, name: str, sql: str = OK_CUT_SQL, description: str = "") -> dict: + response = api.post( + "/api/v1/curation/pin", json={"sql": sql, "name": name, "description": description} + ) + assert response.status_code == 200, response.text + return response.json() + + +def test_pin_writes_the_manifest_and_a_registry_entry( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + entry = _pin(writable_api, "Clean Fold Cut!", description="ok episodes only") + assert entry["name"] == "Clean Fold Cut!" + assert entry["description"] == "ok episodes only" + assert entry["sql"] == OK_CUT_SQL + assert entry["row_count"] == 3 + assert entry["total_episodes"] == 4 + assert len(entry["id"]) == 32 # uuid hex + assert datetime.fromisoformat(entry["created_at"]).tzinfo is not None + assert entry["manifest_path"].startswith("manifests/clean-fold-cut-") + assert entry["manifest_path"].endswith(".parquet") + assert {coverage["check_name"] for coverage in entry["coverage"]} == { + "joint_check", + "media/contact_sheet", + "camera_blackout", + } + + manifest_file = writable_workspace.data_root / entry["manifest_path"] + assert manifest_file.is_file() + # The pinned Parquet really is the cut: readable, with the cut's rows. + (row_count,) = duckdb.connect().execute( + "SELECT count(*) FROM read_parquet(?)", [str(manifest_file)] + ).fetchone() or (0,) + assert int(row_count) == 3 + + listed = writable_api.get("/api/v1/manifests").json()["manifests"] + assert [manifest["id"] for manifest in listed] == [entry["id"]] + + +def test_second_pin_with_the_same_name_gets_a_distinct_file( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + first_entry = _pin(writable_api, "nightly cut") + second_entry = _pin(writable_api, "nightly cut") + assert first_entry["manifest_path"] != second_entry["manifest_path"] + assert (writable_workspace.data_root / first_entry["manifest_path"]).is_file() + assert (writable_workspace.data_root / second_entry["manifest_path"]).is_file() + # The registry lists newest first. + listed_ids = [ + manifest["id"] for manifest in writable_api.get("/api/v1/manifests").json()["manifests"] + ] + assert listed_ids == [second_entry["id"], first_entry["id"]] + + +def test_a_filename_collision_is_refused_never_overwritten( + writable_api: TestClient, + writable_workspace: PopulatedWorkspace, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(_curation, "_manifest_timestamp", lambda: "20260821T000000000000Z") + entry = _pin(writable_api, "pinned once") + collision = writable_api.post( + "/api/v1/curation/pin", json={"sql": OK_CUT_SQL, "name": "pinned once"} + ) + assert collision.status_code == 409 + assert "never overwritten" in collision.json()["detail"] + manifest_files = list((writable_workspace.data_root / "manifests").glob("*.parquet")) + assert [file.name for file in manifest_files] == [entry["manifest_path"].split("/")[-1]] + + +def test_pin_requires_a_nonempty_name(writable_api: TestClient) -> None: + assert ( + writable_api.post("/api/v1/curation/pin", json={"sql": OK_CUT_SQL, "name": ""}).status_code + == 422 + ) + + +def test_pin_names_without_ascii_alphanumerics_fall_back_to_a_slug( + writable_api: TestClient, +) -> None: + # Symbols-only and non-Latin-script names are valid: the full Unicode name + # is stored, and the on-disk filename uses the fallback slug (the + # timestamp suffix keeps it unique) rather than being refused. + symbols_only = _pin(writable_api, "!!!") + assert symbols_only["name"] == "!!!" + assert symbols_only["manifest_path"].startswith("manifests/manifest-") + + non_latin = _pin(writable_api, "数据集") + assert non_latin["name"] == "数据集" + assert non_latin["manifest_path"].startswith("manifests/manifest-") + + +def test_concurrent_pins_all_land_in_the_registry( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + # FastAPI runs these sync endpoints on a threadpool; without a lock around + # the sidecar read-modify-write, overlapping pins would drop each other's + # acknowledged registry entry and strand parquet files. Fire several at + # once and assert every acknowledged pin is registered. + names = [f"cut-{index}" for index in range(6)] + with ThreadPoolExecutor(max_workers=len(names)) as pool: + responses = list( + pool.map( + lambda name: writable_api.post( + "/api/v1/curation/pin", json={"sql": OK_CUT_SQL, "name": name} + ), + names, + ) + ) + acknowledged_ids = set() + for response in responses: + assert response.status_code == 200, response.text + acknowledged_ids.add(response.json()["id"]) + assert len(acknowledged_ids) == len(names) + registered_ids = { + manifest["id"] for manifest in writable_api.get("/api/v1/manifests").json()["manifests"] + } + # No acknowledged pin was silently dropped. + assert acknowledged_ids <= registered_ids + # And no parquet is stranded (every file on disk has a registry entry). + manifest_files = list((writable_workspace.data_root / "manifests").glob("*.parquet")) + assert len(manifest_files) == len(names) + + +def test_pin_with_bad_sql_is_400_and_registers_nothing( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + response = writable_api.post( + "/api/v1/curation/pin", json={"sql": "SELECT * FROM nope", "name": "broken"} + ) + assert response.status_code == 400 + manifests_directory = writable_workspace.data_root / "manifests" + if manifests_directory.exists(): + assert list(manifests_directory.glob("*.parquet")) == [] + assert writable_api.get("/api/v1/manifests").json()["manifests"] == [] + + +def test_pin_is_403_when_read_only(read_only_api: TestClient) -> None: + response = read_only_api.post( + "/api/v1/curation/pin", json={"sql": OK_CUT_SQL, "name": "should not land"} + ) + assert response.status_code == 403 + assert "read-only" in response.json()["detail"] + + +def test_manifest_download_streams_the_parquet_as_an_attachment( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + entry = _pin(writable_api, "download me") + response = writable_api.get(f"/api/v1/manifests/{entry['id']}/download") + assert response.status_code == 200 + manifest_file = writable_workspace.data_root / entry["manifest_path"] + assert response.content == manifest_file.read_bytes() + content_disposition = response.headers["content-disposition"] + assert "attachment" in content_disposition + assert manifest_file.name in content_disposition + + +def test_manifest_download_of_an_unknown_id_is_404(writable_api: TestClient) -> None: + assert writable_api.get("/api/v1/manifests/no-such-id/download").status_code == 404 + + +def test_manifest_download_refuses_a_registry_path_outside_the_root( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + # A hand-edited registry pointing outside the data root is contained + # exactly like hostile media URIs -- refused, path never echoed. + outside_file = writable_workspace.outside_media_file + state_file = writable_workspace.data_root / "curation" / "state.json" + state_file.parent.mkdir(parents=True, exist_ok=True) + escaping_relative_path = f"../{outside_file.parent.name}/{outside_file.name}" + state_file.write_text( + json.dumps( + { + "state_version": 1, + "saved_queries": [], + "manifests": [ + { + "id": "0" * 32, + "name": "escape", + "description": "", + "sql": "SELECT 1", + "manifest_path": escaping_relative_path, + "row_count": 1, + "total_episodes": 1, + "coverage": [], + "created_at": "2026-08-21T00:00:00+00:00", + } + ], + } + ) + ) + response = writable_api.get(f"/api/v1/manifests/{'0' * 32}/download") + assert response.status_code == 403 + assert str(outside_file) not in response.text + + +def test_manifest_download_whose_file_vanished_is_404( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + entry = _pin(writable_api, "soon gone") + (writable_workspace.data_root / entry["manifest_path"]).unlink() + assert writable_api.get(f"/api/v1/manifests/{entry['id']}/download").status_code == 404 diff --git a/packages/hflow-server/tests/test_server_curation_preview.py b/packages/hflow-server/tests/test_server_curation_preview.py new file mode 100644 index 0000000..ded92a4 --- /dev/null +++ b/packages/hflow-server/tests/test_server_curation_preview.py @@ -0,0 +1,211 @@ +"""POST /api/v1/curation/preview: wrapped user SQL on a constrained connection.""" + +import os +import time +from datetime import UTC, datetime + +import pytest +from fastapi.testclient import TestClient + + +def _preview(api: TestClient, **body: object) -> dict: + response = api.post("/api/v1/curation/preview", json=body) + assert response.status_code == 200, response.text + return response.json() + + +def test_preview_returns_rows_columns_count_and_sql(api: TestClient) -> None: + payload = _preview(api, sql="SELECT episode_id, task, status FROM episodes") + assert payload["row_count"] == 4 + assert len(payload["rows"]) == 4 + assert payload["truncated"] is False + assert payload["column_stats"] is None + assert [column["name"] for column in payload["columns"]] == ["episode_id", "task", "status"] + assert all(set(column) == {"name", "type"} for column in payload["columns"]) + assert "SELECT episode_id, task, status FROM episodes" in payload["sql"] + assert "LIMIT" in payload["sql"] + + +def test_preview_truncation_flag_reflects_the_full_count(api: TestClient) -> None: + payload = _preview(api, sql="SELECT episode_id FROM episodes", limit=2) + assert len(payload["rows"]) == 2 + assert payload["row_count"] == 4 + assert payload["truncated"] is True + + +def test_preview_timestamps_are_iso_8601_utc_text(api: TestClient) -> None: + payload = _preview(api, sql="SELECT episode_id, recorded_at FROM episodes") + for row in payload["rows"]: + assert isinstance(row["recorded_at"], str) + assert row["recorded_at"].endswith("+00:00") + parsed = datetime.fromisoformat(row["recorded_at"]) + assert parsed.tzinfo is not None + assert parsed.utcoffset() == datetime.now(UTC).utcoffset() + + +def test_preview_null_timestamps_stay_null(api: TestClient) -> None: + payload = _preview(api, sql="SELECT NULL::TIMESTAMPTZ AS ts") + assert payload["rows"] == [{"ts": None}] + + +@pytest.fixture() +def _non_utc_host_timezone() -> object: + """Run the body with the process pinned to a non-UTC timezone. + + The constrained connection cannot ``SET TimeZone`` (locked at open), so it + inherits the host's -- this fixture makes that host non-UTC so a + timezone-leaking render is observable. + """ + previous_tz = os.environ.get("TZ") + os.environ["TZ"] = "America/New_York" + time.tzset() + try: + yield + finally: + if previous_tz is None: + os.environ.pop("TZ", None) + else: + os.environ["TZ"] = previous_tz + time.tzset() + + +def test_preview_stats_render_timestamps_in_utc_on_a_non_utc_host( + api: TestClient, _non_utc_host_timezone: object +) -> None: + payload = _preview(api, sql="SELECT recorded_at FROM episodes", stats=True) + # Rows are UTC ISO text... + for row in payload["rows"]: + assert row["recorded_at"].endswith("+00:00") + # ...and the column stats agree (same UTC rendering, not the host's -04), + # so the stats panel never shows a different offset or calendar day. + stats_by_column = {entry["column_name"]: entry for entry in payload["column_stats"]} + recorded_at_stats = stats_by_column["recorded_at"] + for bound_key in ("min", "max"): + bound_value = recorded_at_stats[bound_key] + assert bound_value.endswith("+00:00"), bound_value + assert "-04" not in bound_value + + +def test_preview_stats_returns_summarize_rows(api: TestClient) -> None: + payload = _preview(api, sql="SELECT task, max_velocity FROM episodes", stats=True) + assert isinstance(payload["column_stats"], list) + profiled_names = {entry["column_name"] for entry in payload["column_stats"]} + assert profiled_names == {"task", "max_velocity"} + for entry in payload["column_stats"]: + assert "column_type" in entry + assert "null_percentage" in entry + + +def test_preview_handles_json_hostile_result_types(api: TestClient) -> None: + # DECIMAL literals and nested lists come back JSON-legal, never a 500. + payload = _preview(api, sql="SELECT 1.5 AS a_decimal, [1, 2] AS a_list") + assert payload["rows"] == [{"a_decimal": 1.5, "a_list": [1, 2]}] + + +def test_preview_bad_sql_is_400_with_the_duckdb_message(api: TestClient) -> None: + response = api.post("/api/v1/curation/preview", json={"sql": "SELEC 1"}) + assert response.status_code == 400 + assert "Parser Error" in response.json()["detail"] + unknown_table = api.post( + "/api/v1/curation/preview", json={"sql": "SELECT * FROM no_such_table"} + ) + assert unknown_table.status_code == 400 + assert "no_such_table" in unknown_table.json()["detail"] + + +def test_preview_refuses_multiple_statements(api: TestClient) -> None: + response = api.post( + "/api/v1/curation/preview", + json={"sql": "SELECT 1; DROP TABLE episodes_raw"}, + ) + assert response.status_code == 400 + + +def test_preview_refuses_a_paren_closing_smuggle_as_400_not_500(api: TestClient) -> None: + # This shape closes the subquery wrapper's paren and smuggles a second + # statement; it used to 500 (an unhandled IndexError), and its CREATE + # would run. It must be a clean 400 with nothing executed. + response = api.post( + "/api/v1/curation/preview", + json={ + "sql": "SELECT 1 AS a); CREATE TABLE pwned AS SELECT 1; " + "SELECT count(*) FROM (SELECT 1 AS a" + }, + ) + assert response.status_code == 400 + # A DROP smuggle in the same shape is likewise refused, not run. + drop_smuggle = api.post( + "/api/v1/curation/preview", + json={"sql": "SELECT 1 AS a); DROP VIEW episodes; SELECT 1 AS a FROM (SELECT 1 AS a"}, + ) + assert drop_smuggle.status_code == 400 + # ...and episodes is still queryable afterward. + assert ( + api.post("/api/v1/curation/preview", json={"sql": "SELECT * FROM episodes"}).status_code + == 200 + ) + + +def test_preview_refuses_a_non_select_single_statement(api: TestClient) -> None: + # A single well-formed but non-SELECT statement is refused too. + response = api.post("/api/v1/curation/preview", json={"sql": "CREATE TABLE t AS SELECT 1"}) + assert response.status_code == 400 + + +def test_preview_refuses_an_oversized_sql_body(api: TestClient) -> None: + # The per-field max_length rejects a multi-megabyte SQL body (422) before + # it can be executed or persisted. + huge_sql = "SELECT 1 -- " + "A" * 2_000_000 + response = api.post("/api/v1/curation/preview", json={"sql": huge_sql}) + assert response.status_code == 422 + + +def test_preview_cannot_touch_the_filesystem(api: TestClient) -> None: + # The constrained connection refuses file functions: a 400, never a leak. + response = api.post( + "/api/v1/curation/preview", json={"sql": "SELECT * FROM read_csv('/etc/passwd')"} + ) + assert response.status_code == 400 + + +def test_preview_limit_bounds_are_enforced(api: TestClient) -> None: + assert ( + api.post("/api/v1/curation/preview", json={"sql": "SELECT 1", "limit": 0}).status_code + == 422 + ) + assert ( + api.post("/api/v1/curation/preview", json={"sql": "SELECT 1", "limit": 1001}).status_code + == 422 + ) + + +def test_preview_blank_sql_is_400(api: TestClient) -> None: + response = api.post("/api/v1/curation/preview", json={"sql": " ; "}) + assert response.status_code == 400 + assert "non-empty" in response.json()["detail"] + + +def test_preview_tolerates_a_trailing_semicolon(api: TestClient) -> None: + payload = _preview(api, sql="SELECT episode_id FROM episodes;") + assert payload["row_count"] == 4 + + +def test_preview_still_works_when_read_only(read_only_api: TestClient) -> None: + response = read_only_api.post( + "/api/v1/curation/preview", json={"sql": "SELECT count(*) AS n FROM episodes"} + ) + assert response.status_code == 200 + assert response.json()["rows"] == [{"n": 4}] + + +def test_preview_without_a_catalog_is_404(empty_workspace_api: TestClient) -> None: + response = empty_workspace_api.post("/api/v1/curation/preview", json={"sql": "SELECT 1"}) + assert response.status_code == 404 + assert "catalog" in response.json()["detail"] + + +def test_preview_over_an_empty_catalog_returns_zero_rows(empty_catalog_api: TestClient) -> None: + payload = _preview(empty_catalog_api, sql="SELECT * FROM episodes") + assert payload["rows"] == [] + assert payload["row_count"] == 0 + assert payload["truncated"] is False diff --git a/packages/hflow-server/tests/test_server_curation_report.py b/packages/hflow-server/tests/test_server_curation_report.py new file mode 100644 index 0000000..7c7564b --- /dev/null +++ b/packages/hflow-server/tests/test_server_curation_report.py @@ -0,0 +1,75 @@ +"""POST /api/v1/curation/report: row count + coverage denominators, no writes.""" + +from fastapi.testclient import TestClient + + +def test_report_counts_rows_and_coverage_denominators(api: TestClient) -> None: + response = api.post( + "/api/v1/curation/report", + json={"sql": "SELECT episode_id FROM episodes WHERE status = 'ok'"}, + ) + assert response.status_code == 200, response.text + payload = response.json() + assert payload["row_count"] == 3 + assert payload["total_episodes"] == 4 + coverage_by_check = {entry["check_name"]: entry for entry in payload["coverage"]} + # Coverage is over the WHOLE catalog (denominators), not the cut. + assert set(coverage_by_check) == {"joint_check", "media/contact_sheet", "camera_blackout"} + contact_sheet = coverage_by_check["media/contact_sheet"] + assert contact_sheet == { + "check_name": "media/contact_sheet", + "episodes_ran": 2, + "total_episodes": 4, + "fraction": 0.5, + } + assert coverage_by_check["joint_check"]["episodes_ran"] == 1 + assert coverage_by_check["camera_blackout"]["episodes_ran"] == 1 + + +def test_report_bad_sql_is_400_with_the_duckdb_message(api: TestClient) -> None: + response = api.post("/api/v1/curation/report", json={"sql": "SELECT * FROM nope"}) + assert response.status_code == 400 + assert "nope" in response.json()["detail"] + + +def test_report_refuses_smuggled_second_statement_and_reports_honestly(api: TestClient) -> None: + # A paren-closing smuggle used to slip past the subquery wrapper: the + # extra CREATE ran and the reported row_count came from a trailing SELECT. + smuggle = api.post( + "/api/v1/curation/report", + json={ + "sql": "SELECT episode_id FROM episodes); CREATE TABLE pwn AS SELECT 1; SELECT 999 --" + }, + ) + assert smuggle.status_code == 400 + # The honest single-statement query still reports the real row count. + honest = api.post("/api/v1/curation/report", json={"sql": "SELECT episode_id FROM episodes"}) + assert honest.status_code == 200 + assert honest.json()["row_count"] == 4 + + +def test_report_refuses_a_plain_multi_statement(api: TestClient) -> None: + response = api.post( + "/api/v1/curation/report", + json={"sql": "SELECT 1; DROP TABLE episodes_raw"}, + ) + assert response.status_code == 400 + + +def test_report_over_an_empty_catalog(empty_catalog_api: TestClient) -> None: + response = empty_catalog_api.post( + "/api/v1/curation/report", json={"sql": "SELECT * FROM episodes"} + ) + assert response.status_code == 200 + assert response.json() == {"row_count": 0, "total_episodes": 0, "coverage": []} + + +def test_report_still_works_when_read_only(read_only_api: TestClient) -> None: + response = read_only_api.post("/api/v1/curation/report", json={"sql": "SELECT * FROM episodes"}) + assert response.status_code == 200 + assert response.json()["row_count"] == 4 + + +def test_report_without_a_catalog_is_404(empty_workspace_api: TestClient) -> None: + response = empty_workspace_api.post("/api/v1/curation/report", json={"sql": "SELECT 1"}) + assert response.status_code == 404 diff --git a/packages/hflow-server/tests/test_server_episode_dossier.py b/packages/hflow-server/tests/test_server_episode_dossier.py new file mode 100644 index 0000000..41b569e --- /dev/null +++ b/packages/hflow-server/tests/test_server_episode_dossier.py @@ -0,0 +1,146 @@ +"""GET /api/v1/episodes/{episode_id}: the dossier shape.""" + +from datetime import datetime + +from fastapi.testclient import TestClient +from ui_test_fixtures import PopulatedWorkspace + + +def _dossier(api: TestClient, episode_id: str) -> dict: + response = api.get(f"/api/v1/episodes/{episode_id}") + assert response.status_code == 200, response.text + return response.json() + + +def test_dossier_has_every_contract_section( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + dossier = _dossier(api, populated_workspace.ok_episode_id) + assert set(dossier) == { + "episode", + "measurements", + "check_runs", + "intervals", + "tags", + "history", + "media", + "canonical_url", + } + + +def test_ok_episode_status_and_identity( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + episode = _dossier(api, populated_workspace.ok_episode_id)["episode"] + assert episode["episode_id"] == populated_workspace.ok_episode_id + assert episode["status"] == "ok" + assert episode["quarantine_tags"] == [] + assert episode["task"] == "fold_napkin" + assert episode["operator"] == "alice" + assert episode["embodiment"] == "arm-1" + assert datetime.fromisoformat(episode["recorded_at"]).tzinfo is not None + # Full "+00:00" offset, not DuckDB's bare "+00" (see _catalog). + assert episode["recorded_at"].endswith("+00:00") + + +def test_quarantined_episode_carries_parsed_tags( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + dossier = _dossier(api, populated_workspace.quarantined_episode_id) + assert dossier["episode"]["status"] == "quarantined" + assert dossier["episode"]["quarantine_tags"] == ["failed:camera_blackout"] + check_statuses = {run["check_name"]: run["status"] for run in dossier["check_runs"]} + assert check_statuses["camera_blackout"] == "failed" + + +def test_measurements_are_the_latest_per_key( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + measurements = _dossier(api, populated_workspace.ok_episode_id)["measurements"] + by_key = {entry["key"]: entry for entry in measurements} + assert by_key["max_velocity"]["value_double"] == 2.0 # the second run's value + assert by_key["max_velocity"]["check_name"] == "joint_check" + assert by_key["max_velocity"]["check_version"] == "v1" + assert by_key["nan_metric"]["value_double"] is None # NaN is not JSON + assert by_key["artifact/wrist_cam"]["value_text"] == str(populated_workspace.contact_sheet_file) + + +def test_check_runs_cover_every_recorded_run( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + check_runs = _dossier(api, populated_workspace.ok_episode_id)["check_runs"] + assert len(check_runs) == 4 # two runs x two checks + assert all(run["run_fingerprint"] for run in check_runs) + assert {run["status"] for run in check_runs} == {"measured"} + newest_first = [run["recorded_at"] for run in check_runs] + assert newest_first == sorted(newest_first, reverse=True) + + +def test_intervals_come_from_the_latest_run_with_check_version( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + intervals = _dossier(api, populated_workspace.ok_episode_id)["intervals"] + assert intervals == [ + { + "label": "span", + "start_ns": 0, + "end_ns": 100, + "check_name": "joint_check", + "check_version": "v1", + } + ] + + +def test_tags_come_from_the_latest_run( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + tags = _dossier(api, populated_workspace.ok_episode_id)["tags"] + assert len(tags) == 1 + assert tags[0]["tag"] == "seen" + assert tags[0]["check_name"] == "joint_check" + + +def test_history_lists_every_append_newest_first( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + history = _dossier(api, populated_workspace.ok_episode_id)["history"] + assert len(history) == 2 + assert history[0]["recorded_at"] >= history[1]["recorded_at"] + assert history[0]["run_fingerprint"] != history[1]["run_fingerprint"] + + +def test_media_entries_carry_serving_urls( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + episode_id = populated_workspace.ok_episode_id + media = _dossier(api, episode_id)["media"] + assert media == [ + { + "name": "wrist_cam", + "uri": str(populated_workspace.contact_sheet_file), + "url": f"/api/v1/episodes/{episode_id}/media/wrist_cam", + } + ] + + +def test_unservable_media_urls_are_null( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + dossier = _dossier(api, populated_workspace.escaping_episode_id) + urls_by_name = {entry["name"]: entry["url"] for entry in dossier["media"]} + assert urls_by_name == {"outside": None, "missing": None} + assert dossier["canonical_url"] is None # the canonical file escapes the root too + + +def test_canonical_url_present_for_contained_files( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + episode_id = populated_workspace.ok_episode_id + dossier = _dossier(api, episode_id) + assert dossier["canonical_url"] == f"/api/v1/episodes/{episode_id}/canonical" + + +def test_unknown_episode_is_a_404_with_detail(api: TestClient) -> None: + response = api.get("/api/v1/episodes/definitely-not-an-id") + assert response.status_code == 404 + assert "definitely-not-an-id" in response.json()["detail"] diff --git a/packages/hflow-server/tests/test_server_episode_stats.py b/packages/hflow-server/tests/test_server_episode_stats.py new file mode 100644 index 0000000..c6ad1e6 --- /dev/null +++ b/packages/hflow-server/tests/test_server_episode_stats.py @@ -0,0 +1,143 @@ +"""GET /api/v1/episodes/stats: per-column mini-distributions under filters.""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import STAMPS, PopulatedWorkspace + +import hflow +from hflow.catalog import Catalog, CheckRunRow + +# (task, operator, max_velocity) for six episodes: two tasks, three +# operators, six distinct velocities -- plus constant columns (success, +# pipeline_version, ...) and all-unique ones (episode_id, uri) that the +# degenerate-skip rules must drop. +EPISODE_SPECIFICATIONS = ( + ("fold", "alice", 1.0), + ("fold", "alice", 2.0), + ("fold", "bob", 3.0), + ("fold", "bob", 4.0), + ("pour", "carol", 10.0), + ("pour", "carol", 20.0), +) + + +def _build_stats_workspace(tmp_path: Path) -> Path: + data_root = tmp_path / "stats-root" + episodes_directory = data_root / "episodes" + episodes_directory.mkdir(parents=True) + catalog = Catalog(data_root / "catalog") + for index, (task, operator, velocity) in enumerate(EPISODE_SPECIFICATIONS): + canonical_file = episodes_directory / f"episode_{index}.canonical.mcap" + canonical_file.write_bytes(f"canonical body {index}".encode()) + appended = catalog.append_episode( + canonical_path=canonical_file, + stamps=STAMPS, + episode_metadata={"task": task, "operator": operator, "success": "true"}, + check_rows=[ + CheckRunRow( + check_name="velocity_check", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={"max_velocity": velocity}, + ) + ], + ) + assert appended.written + return data_root + + +@pytest.fixture() +def stats_api(tmp_path: Path, unbuilt_assets_dir: Path) -> TestClient: + data_root = _build_stats_workspace(tmp_path) + settings = ServerSettings(data_root=str(data_root), assets_dir=unbuilt_assets_dir) + return TestClient(create_app(settings)) + + +def _columns_by_name(payload: dict) -> dict[str, dict]: + return {column["name"]: column for column in payload["columns"]} + + +def test_stats_shapes_numeric_histograms_and_categorical_top_values( + stats_api: TestClient, +) -> None: + response = stats_api.get("/api/v1/episodes/stats") + assert response.status_code == 200 + columns = _columns_by_name(response.json()) + + velocity = columns["max_velocity"] + assert velocity["kind"] == "numeric" + assert len(velocity["buckets"]) == 12 + assert sum(bucket["count"] for bucket in velocity["buckets"]) == 6 + assert velocity["buckets"][0]["lo"] == 1.0 + assert velocity["buckets"][-1]["hi"] == 20.0 + # The maximum value lands in the LAST bucket, never off the end. + assert velocity["buckets"][-1]["count"] >= 1 + + task = columns["task"] + assert task["kind"] == "categorical" + assert task["values"] == [ + {"value": "fold", "count": 4}, + {"value": "pour", "count": 2}, + ] + assert task["other_count"] == 0 + + operator = columns["operator"] + assert {entry["value"]: entry["count"] for entry in operator["values"]} == { + "alice": 2, + "bob": 2, + "carol": 2, + } + + +def test_stats_skips_degenerate_columns(stats_api: TestClient) -> None: + column_names = set(_columns_by_name(stats_api.get("/api/v1/episodes/stats").json())) + # All-unique (id-like) columns are not distributions. + assert "episode_id" not in column_names + assert "uri" not in column_names + # Single-valued columns carry no information under these episodes. + assert "success" not in column_names + assert "pipeline_version" not in column_names + assert "schema_version" not in column_names + assert "status" not in column_names + + +def test_stats_respects_the_active_filters(stats_api: TestClient) -> None: + columns = _columns_by_name( + stats_api.get("/api/v1/episodes/stats", params={"task": "fold"}).json() + ) + velocity = columns["max_velocity"] + assert sum(bucket["count"] for bucket in velocity["buckets"]) == 4 + assert velocity["buckets"][-1]["hi"] == 4.0 # pour's 10.0/20.0 filtered away + assert {entry["value"] for entry in columns["operator"]["values"]} == {"alice", "bob"} + # Under the filter every row is task=fold: now degenerate, so dropped. + assert "task" not in columns + + +def test_stats_filter_values_are_bound_not_interpolated(stats_api: TestClient) -> None: + response = stats_api.get("/api/v1/episodes/stats", params={"task": "x' OR '1'='1"}) + assert response.status_code == 200 + assert response.json() == {"columns": []} # matches nothing, injects nothing + + +def test_stats_over_the_populated_workspace_keeps_the_status_split( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + columns = _columns_by_name(api.get("/api/v1/episodes/stats").json()) + status = columns["status"] + assert status["kind"] == "categorical" + assert {entry["value"]: entry["count"] for entry in status["values"]} == { + "ok": 3, + "quarantined": 1, + } + # NaN/inf-poisoned measurement columns are degenerate, never a crash. + assert "nan_metric" not in columns + assert "inf_metric" not in columns + + +def test_stats_without_a_catalog_is_a_404(empty_workspace_api: TestClient) -> None: + assert empty_workspace_api.get("/api/v1/episodes/stats").status_code == 404 diff --git a/packages/hflow-server/tests/test_server_episode_timeline.py b/packages/hflow-server/tests/test_server_episode_timeline.py new file mode 100644 index 0000000..58c96c7 --- /dev/null +++ b/packages/hflow-server/tests/test_server_episode_timeline.py @@ -0,0 +1,265 @@ +"""GET /api/v1/episodes/{id}/timeline: the episode's time axis, server-side. + +The span derivation is the point: intervals give the axis when they exist, a +duration-naming measurement gives (or extends) it otherwise, and an episode +that offers neither returns nulls so the UI can say "unknown" instead of +drawing a fabricated axis. +""" + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import STAMPS, PopulatedWorkspace + +import hflow +from hflow.catalog import Catalog, CheckRunRow + +NANOSECONDS_PER_SECOND = 1_000_000_000 + + +@pytest.fixture(scope="module") +def timeline_workspace(tmp_path_factory: pytest.TempPathFactory) -> dict[str, str]: + """A catalog whose episodes exercise every branch of the span derivation.""" + data_root = tmp_path_factory.mktemp("ui-timeline-root") + episodes_directory = data_root / "episodes" + episodes_directory.mkdir() + catalog = Catalog(data_root / "catalog") + + def appended(name: str, check_rows: list[CheckRunRow]) -> str: + canonical_file = episodes_directory / f"{name}.canonical.mcap" + canonical_file.write_bytes(f"canonical {name}".encode()) + return catalog.append_episode( + canonical_path=canonical_file, + stamps=STAMPS, + episode_metadata={"task": name}, + check_rows=check_rows, + ).episode_id + + intervals_and_duration = appended( + "intervals_and_duration", + [ + CheckRunRow( + check_name="gap_check", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={ + "episode_duration_s": 12.5, + "max_gap_ms": 220.0, + "black_pct": 3.5, + "max_velocity": 1.25, + }, + intervals=[ + hflow.Interval( + start_ns=1 * NANOSECONDS_PER_SECOND, + end_ns=2 * NANOSECONDS_PER_SECOND, + label="gap:/imu", + ), + hflow.Interval( + start_ns=3 * NANOSECONDS_PER_SECOND, + end_ns=4 * NANOSECONDS_PER_SECOND, + label="joint_discontinuity:/joint_states", + ), + hflow.Interval( + start_ns=5 * NANOSECONDS_PER_SECOND, + end_ns=6 * NANOSECONDS_PER_SECOND, + label="", + ), + ], + ) + ], + ) + duration_only = appended( + "duration_only", + [ + CheckRunRow( + check_name="length_check", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={"episode_duration": 30.0}, + ) + ], + ) + duration_named_percentage = appended( + "duration_named_percentage", + [ + CheckRunRow( + check_name="duty_cycle_check", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + # Says "duration", measures a percentage: the suffix names a + # dimension that is not a time. + measurements={"duty_cycle_duration_pct": 45.0}, + ) + ], + ) + no_span = appended( + "no_span", + [ + CheckRunRow( + check_name="counting_check", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={"frame_count": 900.0, "note": "text is not a bar"}, + ) + ], + ) + return { + "data_root": str(data_root), + "intervals_and_duration": intervals_and_duration, + "duration_only": duration_only, + "duration_named_percentage": duration_named_percentage, + "no_span": no_span, + } + + +@pytest.fixture(scope="module") +def timeline_api( + timeline_workspace: dict[str, str], tmp_path_factory: pytest.TempPathFactory +) -> TestClient: + assets_directory = tmp_path_factory.mktemp("ui-timeline-assets") + settings = ServerSettings( + data_root=timeline_workspace["data_root"], assets_dir=assets_directory + ) + return TestClient(create_app(settings)) + + +def test_timeline_spans_the_intervals_and_the_duration_measurement( + timeline_api: TestClient, timeline_workspace: dict[str, str] +) -> None: + payload = timeline_api.get( + f"/api/v1/episodes/{timeline_workspace['intervals_and_duration']}/timeline" + ).json() + # The axis starts at the first interval; the 12.5s duration measurement + # claims a longer episode than the intervals do, so the span extends. + assert payload["start_ns"] == 1 * NANOSECONDS_PER_SECOND + assert payload["end_ns"] == 1 * NANOSECONDS_PER_SECOND + int(12.5 * NANOSECONDS_PER_SECOND) + assert payload["duration_s"] == pytest.approx(12.5) + + intervals = payload["intervals"] + assert [interval["kind"] for interval in intervals] == [ + "gap", + "joint_discontinuity", + # An empty label groups under the check that produced it. + "gap_check", + ] + assert intervals[0]["label"] == "gap:/imu" + assert intervals[0]["check_name"] == "gap_check" + # Seconds are RELATIVE to the span start, computed server-side. + assert intervals[0]["start_s"] == pytest.approx(0.0) + assert intervals[0]["end_s"] == pytest.approx(1.0) + assert intervals[1]["start_s"] == pytest.approx(2.0) + assert intervals[1]["end_s"] == pytest.approx(3.0) + + +def test_timeline_measurements_are_numeric_bars_with_inferred_units( + timeline_api: TestClient, timeline_workspace: dict[str, str] +) -> None: + payload = timeline_api.get( + f"/api/v1/episodes/{timeline_workspace['intervals_and_duration']}/timeline" + ).json() + measurements_by_key = {entry["key"]: entry for entry in payload["measurements"]} + assert measurements_by_key["max_gap_ms"] == { + "key": "max_gap_ms", + "value": 220.0, + "unit": "ms", + } + assert measurements_by_key["black_pct"]["unit"] == "%" + assert measurements_by_key["episode_duration_s"]["unit"] == "s" + # A key with no recognized unit suffix gets no invented dimension. + assert measurements_by_key["max_velocity"]["unit"] is None + assert [entry["key"] for entry in payload["measurements"]] == sorted(measurements_by_key) + + +def test_timeline_from_a_duration_measurement_alone_is_zero_based( + timeline_api: TestClient, timeline_workspace: dict[str, str] +) -> None: + payload = timeline_api.get( + f"/api/v1/episodes/{timeline_workspace['duration_only']}/timeline" + ).json() + assert payload["intervals"] == [] + # No intervals to anchor the axis: an unsuffixed duration key reads as + # seconds and the axis starts at zero. + assert payload["start_ns"] == 0 + assert payload["end_ns"] == 30 * NANOSECONDS_PER_SECOND + assert payload["duration_s"] == pytest.approx(30.0) + + +def test_a_duration_key_measuring_a_percentage_does_not_become_the_axis( + timeline_api: TestClient, timeline_workspace: dict[str, str] +) -> None: + """The bar's unit and the axis must read the same key the same way. + + ``duty_cycle_duration_pct`` is labelled "45 %"; reading its suffix as a + time as well would claim a 45-second episode -- a fabricated axis 1e9 + times the number's own dimension. + """ + payload = timeline_api.get( + f"/api/v1/episodes/{timeline_workspace['duration_named_percentage']}/timeline" + ).json() + assert payload["measurements"] == [ + {"key": "duty_cycle_duration_pct", "value": 45.0, "unit": "%"} + ] + assert payload["start_ns"] is None + assert payload["end_ns"] is None + assert payload["duration_s"] is None + + +def test_timeline_without_any_span_returns_nulls_not_a_guess( + timeline_api: TestClient, timeline_workspace: dict[str, str] +) -> None: + payload = timeline_api.get(f"/api/v1/episodes/{timeline_workspace['no_span']}/timeline").json() + assert payload["start_ns"] is None + assert payload["end_ns"] is None + assert payload["duration_s"] is None + assert payload["intervals"] == [] + # The bars still work without an axis -- and text measurements are not bars. + assert payload["measurements"] == [{"key": "frame_count", "value": 900.0, "unit": "count"}] + + +def test_timeline_over_the_shared_fixture_drops_illegal_doubles( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + payload = api.get(f"/api/v1/episodes/{populated_workspace.ok_episode_id}/timeline").json() + assert payload["start_ns"] == 0 + assert payload["end_ns"] == 100 + assert payload["duration_s"] == pytest.approx(100 / NANOSECONDS_PER_SECOND) + assert [interval["kind"] for interval in payload["intervals"]] == ["span"] + # NaN/inf are illegal in JSON and meaningless as bars: both are dropped, + # and the artifact URI measurement is text, not a bar. + assert [entry["key"] for entry in payload["measurements"]] == ["max_velocity"] + assert payload["measurements"][0]["value"] == pytest.approx(2.0) + + +def test_timeline_of_an_episode_without_evidence_is_all_nulls( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + payload = api.get(f"/api/v1/episodes/{populated_workspace.minimal_episode_id}/timeline").json() + assert payload == { + "start_ns": None, + "end_ns": None, + "duration_s": None, + "intervals": [], + "measurements": [], + } + + +def test_timeline_of_an_unknown_episode_is_a_404(api: TestClient) -> None: + response = api.get("/api/v1/episodes/does-not-exist/timeline") + assert response.status_code == 404 + assert "does-not-exist" in response.json()["detail"] + + +def test_timeline_without_a_catalog_is_a_404_not_a_500( + empty_workspace_api: TestClient, +) -> None: + response = empty_workspace_api.get("/api/v1/episodes/anything/timeline") + assert response.status_code == 404 + assert "Traceback" not in response.text diff --git a/packages/hflow-server/tests/test_server_episodes.py b/packages/hflow-server/tests/test_server_episodes.py new file mode 100644 index 0000000..4b10baa --- /dev/null +++ b/packages/hflow-server/tests/test_server_episodes.py @@ -0,0 +1,227 @@ +"""GET /api/v1/episodes: filters, ordering, pagination, facets, SQL safety.""" + +from datetime import datetime + +from fastapi.testclient import TestClient +from ui_test_fixtures import PIPELINE_VERSION, PopulatedWorkspace + +import hflow + + +def _episode_rows(api: TestClient, **query_params: str | int | list[str]) -> dict: + response = api.get("/api/v1/episodes", params=query_params) + assert response.status_code == 200, response.text + return response.json() + + +def test_default_listing_returns_every_episode(api: TestClient) -> None: + payload = _episode_rows(api) + assert payload["total"] == 4 + assert len(payload["rows"]) == 4 + column_names = {column["name"] for column in payload["columns"]} + assert {"episode_id", "task", "status", "recorded_at"} <= column_names + assert all(set(column) == {"name", "type"} for column in payload["columns"]) + + +def test_measurement_keys_become_columns(api: TestClient) -> None: + payload = _episode_rows(api) + column_names = {column["name"] for column in payload["columns"]} + assert "max_velocity" in column_names # pivoted from the measurements table + + +def test_timestamps_are_iso_8601_strings(api: TestClient) -> None: + for row in _episode_rows(api)["rows"]: + assert isinstance(row["recorded_at"], str) + assert datetime.fromisoformat(row["recorded_at"]).tzinfo is not None + + +def test_timestamps_use_a_full_colon_utc_offset(api: TestClient) -> None: + # DuckDB's strftime %z renders the offset as "+00" (which JS Date.parse + # rejects and the frontend's offset regex misses); every timestamp must + # carry the full "+00:00" the rest of the stack assumes. + for row in _episode_rows(api)["rows"]: + assert row["recorded_at"].endswith("+00:00"), row["recorded_at"] + assert not row["recorded_at"].endswith("+00") # sanity: not the bare form + + +def test_pagination_over_a_tied_sort_key_never_overlaps_or_drops(api: TestClient) -> None: + # pipeline_version is identical across all four episodes, so without a + # deterministic tiebreaker successive pages could overlap or drop rows. + total = _episode_rows(api, order_by="pipeline_version")["total"] + assert total == 4 + seen_ids: list[str] = [] + for offset in range(0, total, 2): + page = _episode_rows(api, order_by="pipeline_version", order="asc", limit=2, offset=offset) + seen_ids.extend(row["episode_id"] for row in page["rows"]) + # Every episode appears exactly once across the walked pages. + assert len(seen_ids) == total + assert len(set(seen_ids)) == total + # And the walk is repeatable: the same offsets return the same rows. + repeat_first_page = _episode_rows( + api, order_by="pipeline_version", order="asc", limit=2, offset=0 + ) + assert [row["episode_id"] for row in repeat_first_page["rows"]] == seen_ids[:2] + + +def test_non_finite_doubles_become_null( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + rows_by_id = {row["episode_id"]: row for row in _episode_rows(api)["rows"]} + ok_row = rows_by_id[populated_workspace.ok_episode_id] + assert ok_row["nan_metric"] is None + assert ok_row["inf_metric"] is None + assert ok_row["max_velocity"] == 2.0 # the latest run's value + + +def test_task_filter_matches_exactly(api: TestClient) -> None: + payload = _episode_rows(api, task="fold_napkin") + assert payload["total"] == 2 + assert all(row["task"] == "fold_napkin" for row in payload["rows"]) + + +def test_repeated_filter_values_are_or_combined(api: TestClient) -> None: + payload = _episode_rows(api, task=["fold_napkin", "pour_water"]) + assert payload["total"] == 3 + + +def test_different_filters_are_and_combined(api: TestClient) -> None: + payload = _episode_rows(api, task="fold_napkin", operator="alice") + assert payload["total"] == 1 + assert payload["rows"][0]["operator"] == "alice" + + +def test_status_filter(api: TestClient, populated_workspace: PopulatedWorkspace) -> None: + payload = _episode_rows(api, status="quarantined") + assert payload["total"] == 1 + assert payload["rows"][0]["episode_id"] == populated_workspace.quarantined_episode_id + assert _episode_rows(api, status="ok")["total"] == 3 + + +def test_invalid_status_value_is_a_422(api: TestClient) -> None: + assert api.get("/api/v1/episodes", params={"status": "banana"}).status_code == 422 + + +def test_success_filter(api: TestClient) -> None: + assert _episode_rows(api, success="true")["total"] == 1 + assert _episode_rows(api, success="false")["total"] == 1 + + +def test_search_is_case_insensitive_substring( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + assert _episode_rows(api, search="FOLD")["total"] == 2 # task match + assert _episode_rows(api, search="ALICE")["total"] == 1 # operator match + by_id = _episode_rows(api, search=populated_workspace.ok_episode_id) + assert by_id["total"] == 1 # episode_id match + assert by_id["rows"][0]["episode_id"] == populated_workspace.ok_episode_id + + +def test_search_wildcards_are_literal(api: TestClient) -> None: + # Un-escaped, "%" would match every row and "_" any character. + assert _episode_rows(api, search="%")["total"] == 0 + assert _episode_rows(api, search="fold_napkin")["total"] == 2 + assert _episode_rows(api, search="foldXnapkin")["total"] == 0 + + +def test_filter_values_cannot_inject_sql(api: TestClient) -> None: + hostile_value = "x' OR '1'='1" + assert _episode_rows(api, task=hostile_value)["total"] == 0 + assert _episode_rows(api, search=hostile_value)["total"] == 0 + assert _episode_rows(api, search="'; DROP TABLE episodes; --")["total"] == 0 + + +def test_order_by_direction_flips_the_listing(api: TestClient) -> None: + ascending_ids = [ + row["episode_id"] for row in _episode_rows(api, order_by="recorded_at", order="asc")["rows"] + ] + descending_ids = [ + row["episode_id"] + for row in _episode_rows(api, order_by="recorded_at", order="desc")["rows"] + ] + assert ascending_ids == list(reversed(descending_ids)) + + +def test_order_by_any_view_column_including_measurements(api: TestClient) -> None: + payload = _episode_rows(api, order_by="task", order="asc") + listed_tasks = [row["task"] for row in payload["rows"]] + assert listed_tasks == sorted(listed_tasks) + assert _episode_rows(api, order_by="max_velocity")["total"] == 4 + + +def test_unknown_order_by_column_is_a_400_not_sql(api: TestClient) -> None: + response = api.get("/api/v1/episodes", params={"order_by": "no_such_column"}) + assert response.status_code == 400 + assert "no_such_column" in response.json()["detail"] + hostile = api.get("/api/v1/episodes", params={"order_by": "task; DROP TABLE episodes"}) + assert hostile.status_code == 400 + + +def test_pagination_pages_share_one_total(api: TestClient) -> None: + first_page = _episode_rows(api, limit=2, offset=0, order_by="episode_id", order="asc") + second_page = _episode_rows(api, limit=2, offset=2, order_by="episode_id", order="asc") + beyond_page = _episode_rows(api, limit=2, offset=4, order_by="episode_id", order="asc") + assert first_page["total"] == second_page["total"] == beyond_page["total"] == 4 + assert len(first_page["rows"]) == 2 + assert len(second_page["rows"]) == 2 + assert beyond_page["rows"] == [] + first_ids = {row["episode_id"] for row in first_page["rows"]} + second_ids = {row["episode_id"] for row in second_page["rows"]} + assert first_ids.isdisjoint(second_ids) + + +def test_pagination_bounds_are_enforced(api: TestClient) -> None: + assert api.get("/api/v1/episodes", params={"limit": 501}).status_code == 422 + assert api.get("/api/v1/episodes", params={"limit": 0}).status_code == 422 + assert api.get("/api/v1/episodes", params={"offset": -1}).status_code == 422 + + +def test_compiled_sql_is_returned_and_runnable( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + payload = _episode_rows(api, task="fold_napkin", limit=2) + compiled_sql = payload["sql"] + assert "\"task\" IN ('fold_napkin')" in compiled_sql + assert "LIMIT 2" in compiled_sql + assert "?" not in compiled_sql # fully rendered, copy-pastable + # The affordance is real: the displayed SQL runs against the same catalog. + # (Counted in SQL: materializing a TIMESTAMPTZ into Python needs pytz.) + connection = hflow.open_catalog_connection(populated_workspace.data_root / "catalog") + try: + replayed_count_row = connection.execute(f"SELECT count(*) FROM ({compiled_sql})").fetchone() + finally: + connection.close() + assert replayed_count_row is not None + assert int(replayed_count_row[0]) == len(payload["rows"]) + + +def test_listing_without_a_catalog_is_a_404(empty_workspace_api: TestClient) -> None: + response = empty_workspace_api.get("/api/v1/episodes") + assert response.status_code == 404 + assert "catalog" in response.json()["detail"] + + +def test_listing_an_empty_catalog_returns_zero_rows(empty_catalog_api: TestClient) -> None: + payload = empty_catalog_api.get("/api/v1/episodes").json() + assert payload["rows"] == [] + assert payload["total"] == 0 + assert {column["name"] for column in payload["columns"]} >= {"episode_id", "status"} + + +def test_facets_counts_skip_null_buckets(api: TestClient) -> None: + response = api.get("/api/v1/episodes/facets") + assert response.status_code == 200 + facets = response.json() + assert set(facets) == {"task", "operator", "embodiment", "status", "pipeline_version"} + task_counts = {entry["value"]: entry["count"] for entry in facets["task"]} + assert task_counts == {"fold_napkin": 2, "pour_water": 1, "stack_blocks": 1} + # The no-operator episode contributes no null bucket. + assert {entry["value"] for entry in facets["operator"]} == {"alice", "bob", "carol"} + status_counts = {entry["value"]: entry["count"] for entry in facets["status"]} + assert status_counts == {"ok": 3, "quarantined": 1} + pipeline_counts = {entry["value"]: entry["count"] for entry in facets["pipeline_version"]} + assert pipeline_counts == {PIPELINE_VERSION: 4} + + +def test_facets_of_an_empty_catalog_are_empty_lists(empty_catalog_api: TestClient) -> None: + facets = empty_catalog_api.get("/api/v1/episodes/facets").json() + assert all(entries == [] for entries in facets.values()) diff --git a/packages/hflow-server/tests/test_server_media.py b/packages/hflow-server/tests/test_server_media.py new file mode 100644 index 0000000..73b5649 --- /dev/null +++ b/packages/hflow-server/tests/test_server_media.py @@ -0,0 +1,206 @@ +"""Media and canonical byte-serving: content, Range tolerance, containment.""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import STAMPS, PopulatedWorkspace + +import hflow +from hflow.catalog import Catalog, CheckRunRow + + +def test_media_bytes_are_served_with_content_type( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + response = api.get(f"/api/v1/episodes/{populated_workspace.ok_episode_id}/media/wrist_cam") + assert response.status_code == 200 + assert response.content == populated_workspace.contact_sheet_file.read_bytes() + assert response.headers["content-type"] == "image/jpeg" + # An inert image renders inline, but every media response is still hardened. + assert "attachment" not in response.headers.get("content-disposition", "") + assert response.headers["x-content-type-options"] == "nosniff" + assert "default-src 'none'" in response.headers["content-security-policy"] + + +@pytest.fixture() +def mixed_media_workspace(tmp_path: Path) -> tuple[TestClient, str]: + """A workspace with .jpg, .html, and .svg contact-sheet artifacts, all + inside the data root.""" + data_root = tmp_path / "data" + episodes_dir = data_root / "episodes" + episodes_dir.mkdir(parents=True) + media_dir = data_root / "media" + media_dir.mkdir() + (media_dir / "frame.jpg").write_bytes(b"\xff\xd8\xff\xe0 jpeg \xff\xd9") + # The payload names the real risk of rendering this inline: script from + # the UI's own origin can drive every endpoint the API exposes. + (media_dir / "report.html").write_text("") + (media_dir / "sheet.svg").write_text( + "" + ) + catalog = Catalog(data_root / "catalog") + canonical = episodes_dir / "a.canonical.mcap" + canonical.write_bytes(b"canonical a") + result = catalog.append_episode( + canonical_path=canonical, + stamps=STAMPS, + episode_metadata={"task": "fold", "operator": "alice", "embodiment": "arm-1"}, + check_rows=[ + CheckRunRow( + check_name="media/contact_sheet", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={ + "artifact/frame": str(media_dir / "frame.jpg"), + "artifact/report": str(media_dir / "report.html"), + "artifact/sheet": str(media_dir / "sheet.svg"), + }, + ) + ], + ) + client = TestClient(create_app(ServerSettings(data_root=str(data_root)))) + return client, result.episode_id + + +def test_inert_image_is_served_inline(mixed_media_workspace: tuple[TestClient, str]) -> None: + client, episode_id = mixed_media_workspace + response = client.get(f"/api/v1/episodes/{episode_id}/media/frame") + assert response.status_code == 200 + assert response.headers["content-type"] == "image/jpeg" + assert "attachment" not in response.headers.get("content-disposition", "") + assert response.headers["x-content-type-options"] == "nosniff" + + +def test_html_artifact_is_forced_to_download_not_rendered( + mixed_media_workspace: tuple[TestClient, str], +) -> None: + client, episode_id = mixed_media_workspace + response = client.get(f"/api/v1/episodes/{episode_id}/media/report") + assert response.status_code == 200 + # Never text/html on the UI's own origin: opaque bytes, forced download. + assert response.headers["content-type"] == "application/octet-stream" + assert "attachment" in response.headers["content-disposition"] + assert response.headers["x-content-type-options"] == "nosniff" + assert "default-src 'none'" in response.headers["content-security-policy"] + + +def test_svg_artifact_is_forced_to_download_not_rendered( + mixed_media_workspace: tuple[TestClient, str], +) -> None: + client, episode_id = mixed_media_workspace + response = client.get(f"/api/v1/episodes/{episode_id}/media/sheet") + assert response.status_code == 200 + # image/svg+xml is active content; it must not render inline. + assert response.headers["content-type"] == "application/octet-stream" + assert "attachment" in response.headers["content-disposition"] + assert response.headers["x-content-type-options"] == "nosniff" + + +def test_a_range_request_does_not_500( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + response = api.get( + f"/api/v1/episodes/{populated_workspace.ok_episode_id}/media/wrist_cam", + headers={"Range": "bytes=0-3"}, + ) + assert response.status_code in (200, 206) + if response.status_code == 206: + assert response.content == populated_workspace.contact_sheet_file.read_bytes()[:4] + + +def test_media_outside_the_data_root_is_403_without_the_path( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + response = api.get(f"/api/v1/episodes/{populated_workspace.escaping_episode_id}/media/outside") + assert response.status_code == 403 + assert str(populated_workspace.outside_media_file) not in response.text + assert str(populated_workspace.outside_media_file.parent) not in response.text + + +def test_media_whose_file_vanished_is_404( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + response = api.get(f"/api/v1/episodes/{populated_workspace.escaping_episode_id}/media/missing") + assert response.status_code == 404 + + +def test_unknown_artifact_name_is_404( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + response = api.get( + f"/api/v1/episodes/{populated_workspace.ok_episode_id}/media/no_such_artifact" + ) + assert response.status_code == 404 + + +def test_media_of_an_unknown_episode_is_404(api: TestClient) -> None: + assert api.get("/api/v1/episodes/not-an-id/media/wrist_cam").status_code == 404 + + +def test_canonical_bytes_are_served( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + response = api.get(f"/api/v1/episodes/{populated_workspace.ok_episode_id}/canonical") + assert response.status_code == 200 + assert response.content == populated_workspace.canonical_file.read_bytes() + + +def test_canonical_outside_the_data_root_is_403( + api: TestClient, populated_workspace: PopulatedWorkspace +) -> None: + response = api.get(f"/api/v1/episodes/{populated_workspace.escaping_episode_id}/canonical") + assert response.status_code == 403 + + +def test_canonical_of_an_unknown_episode_is_404(api: TestClient) -> None: + assert api.get("/api/v1/episodes/not-an-id/canonical").status_code == 404 + + +def test_media_recorded_from_a_container_vantage_is_served(tmp_path: Path) -> None: + """The Compose runtime catalogs its own mount path for the same bytes. + + A run executed in the bundle records + ``/opt/airflow/data/episodes//media/x.jpg``; those very bytes are at + ``/episodes//media/x.jpg`` on this host, so the episode + page must show the camera views rather than refusing its own workspace. + """ + from hflow_server._media import MediaResolutionError, resolve_served_file + + data_root = tmp_path / "workspace" + host_media_file = data_root / "episodes" / "run-0001-abcdef" / "media" / "wrist_cam.jpg" + host_media_file.parent.mkdir(parents=True) + host_media_file.write_bytes(b"\xff\xd8\xff\xe0 jpeg bytes") + + container_uri = "/opt/airflow/data/episodes/run-0001-abcdef/media/wrist_cam.jpg" + assert resolve_served_file(container_uri, data_root=str(data_root)) == host_media_file.resolve() + + # Re-anchoring never widens what may be served: a traversal in the + # recorded tail still lands outside the root and is refused. + with pytest.raises(MediaResolutionError): + resolve_served_file( + "/opt/airflow/data/episodes/../../../etc/passwd", data_root=str(data_root) + ) + # A foreign path with no workspace layout component stays refused. + foreign_file = tmp_path / "elsewhere" / "secret.jpg" + foreign_file.parent.mkdir(parents=True) + foreign_file.write_bytes(b"nope") + with pytest.raises(MediaResolutionError) as refusal: + resolve_served_file(str(foreign_file), data_root=str(data_root)) + assert refusal.value.status_code == 403 + + +def test_container_vantage_rebasing_requires_the_file_to_exist_here(tmp_path: Path) -> None: + """A vantage-shaped URI with no counterpart here is still a 404.""" + from hflow_server._media import MediaResolutionError, resolve_served_file + + data_root = tmp_path / "workspace" + (data_root / "episodes").mkdir(parents=True) + with pytest.raises(MediaResolutionError) as refusal: + resolve_served_file( + "/opt/airflow/data/episodes/never-synced/media/x.jpg", data_root=str(data_root) + ) + assert refusal.value.status_code == 404 diff --git a/packages/hflow-server/tests/test_server_offline_posture.py b/packages/hflow-server/tests/test_server_offline_posture.py new file mode 100644 index 0000000..c923155 --- /dev/null +++ b/packages/hflow-server/tests/test_server_offline_posture.py @@ -0,0 +1,55 @@ +"""The offline promise: nothing this server serves points at another host. + +docs/SERVE.md's "Trust posture" tells an operator the UI makes no CDN and no +outbound requests -- a claim they act on when deciding to run it on an +air-gapped host or in front of a colleague. FastAPI's built-in Swagger and +ReDoc pages would quietly falsify it (their JS, CSS and favicon come from +cdn.jsdelivr.net and fastapi.tiangolo.com), so ``create_app`` disables both +and serves only ``/api/openapi.json``. These tests are what keeps that true +if someone re-enables a docs page or pastes a font link into a rendered page. +""" + +import re + +import pytest +from fastapi.testclient import TestClient + +# Every absolute URL, whatever quoting or markup surrounds it. +_ABSOLUTE_URL = re.compile(r"https?://[^\s\"'<>)]+", re.IGNORECASE) + +# The surfaces this package renders itself, plus the schema it publishes. +# /api/docs and /api/redoc are here on purpose: they must stay unserved. +_SERVER_OWNED_PATHS = ( + "/", + "/episodes/some-episode-id", + "/api/docs", + "/api/redoc", + "/api/openapi.json", + "/api/v1/health", + "/api/v1/config", +) + + +def _referenced_hosts(body: str) -> list[str]: + return _ABSOLUTE_URL.findall(body) + + +@pytest.mark.parametrize("path", _SERVER_OWNED_PATHS) +def test_no_served_surface_references_an_external_host(api: TestClient, path: str) -> None: + response = api.get(path) + assert _referenced_hosts(response.text) == [], f"{path} points at another host" + assert "cdn.jsdelivr" not in response.text + + +def test_the_interactive_docs_pages_are_not_served(api: TestClient) -> None: + # Not "they happen to 404": FastAPI serves these by default, so a config + # change that re-enables either one must fail here rather than in a + # customer's browser. + for docs_path in ("/api/docs", "/api/redoc"): + assert api.get(docs_path).status_code == 404 + + +def test_the_openapi_schema_is_the_published_contract(api: TestClient) -> None: + schema = api.get("/api/openapi.json").json() + assert schema["info"]["title"] == "HFlow workspace API" + assert "/api/v1/episodes" in schema["paths"] diff --git a/packages/hflow-server/tests/test_server_pipeline.py b/packages/hflow-server/tests/test_server_pipeline.py new file mode 100644 index 0000000..a8caca1 --- /dev/null +++ b/packages/hflow-server/tests/test_server_pipeline.py @@ -0,0 +1,137 @@ +"""GET /api/v1/pipeline plus the config capability behind it. + +Every test imports a REAL tiny pipeline file written into tmp_path; the apps +inside are constructed WITHOUT data_root (environment resolution), so the +startup import is side-effect-free. +""" + +from pathlib import Path + +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import PopulatedWorkspace + +WORKING_PIPELINE_SOURCE = """import hflow + +app = hflow.App("ui-demo") + + +@app.check(name="joint_check", critical=True) +def joint_check(episode): + return hflow.CheckResult(measurements={"max_velocity": 1.0}, verdict=True) + + +@app.enrich(name="caption") +def caption(episode): + return hflow.EnrichmentResult(labels={"caption": "hello"}) +""" + +RAISING_PIPELINE_SOURCE = 'raise RuntimeError("boom at import")\n' + + +def _client_over(data_root: Path, assets_dir: Path, *, pipeline: str | None) -> TestClient: + settings = ServerSettings(data_root=str(data_root), assets_dir=assets_dir, pipeline=pipeline) + return TestClient(create_app(settings)) + + +def _written_pipeline_file(tmp_path: Path, source: str) -> Path: + pipeline_file = tmp_path / "ui_pipeline.py" + pipeline_file.write_text(source) + return pipeline_file + + +def test_pipeline_page_reports_manifest_lanes_observed_and_stale( + populated_workspace: PopulatedWorkspace, unbuilt_assets_dir: Path, tmp_path: Path +) -> None: + pipeline_file = _written_pipeline_file(tmp_path, WORKING_PIPELINE_SOURCE) + client = _client_over( + populated_workspace.data_root, unbuilt_assets_dir, pipeline=str(pipeline_file) + ) + response = client.get("/api/v1/pipeline") + assert response.status_code == 200 + payload = response.json() + + manifest = payload["manifest"] + assert manifest["pipeline_name"] == "ui-demo" + assert [check["name"] for check in manifest["checks"]] == ["joint_check"] + assert manifest["checks"][0]["critical"] is True + assert manifest["checks"][0]["kind"] == "check" + assert manifest["checks"][0]["version"] + assert [enrichment["name"] for enrichment in manifest["enrichments"]] == ["caption"] + + # No stage lanes here: /pipeline/graph is the one owner of stage grouping, + # so this page cannot disagree with it about which steps run where. + assert "stages" not in payload + + observed_by_identity = { + (row["check_name"], row["check_version"]): row for row in payload["observed"] + } + joint_check_observed = observed_by_identity[("joint_check", "v1")] + assert joint_check_observed["run_count"] == 2 # the ok episode's two appends + assert joint_check_observed["first_seen"] <= joint_check_observed["last_seen"] + assert observed_by_identity[("camera_blackout", "v1")]["run_count"] == 1 + + # The fixture stamps every episode with another pipeline_version, so all + # four source recordings are stale against this App's current versions. + assert payload["stale"] == { + "pipeline_version": manifest["pipeline_version"], + "count": 4, + } + + +def test_pipeline_capability_and_empty_catalog(tmp_path: Path, unbuilt_assets_dir: Path) -> None: + pipeline_file = _written_pipeline_file(tmp_path, WORKING_PIPELINE_SOURCE) + data_root = tmp_path / "no-catalog-root" + data_root.mkdir() + client = _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + assert client.get("/api/v1/config").json()["capabilities"]["pipeline"] is True + payload = client.get("/api/v1/pipeline").json() + assert payload["manifest"]["pipeline_name"] == "ui-demo" + # No catalog: nothing observed, staleness unknowable -- not an error. + assert payload["observed"] == [] + assert payload["stale"] is None + + +def test_pipeline_spec_selects_a_named_app_variable( + tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file( + tmp_path, 'import hflow\n\nmy_app = hflow.App("named-app")\n' + ) + data_root = tmp_path / "root" + data_root.mkdir() + client = _client_over(data_root, unbuilt_assets_dir, pipeline=f"{pipeline_file}:my_app") + payload = client.get("/api/v1/pipeline").json() + assert payload["manifest"]["pipeline_name"] == "named-app" + + +def test_pipeline_import_failure_is_remembered_not_fatal( + tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file(tmp_path, RAISING_PIPELINE_SOURCE) + data_root = tmp_path / "root" + data_root.mkdir() + client = _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + # The server still boots and answers; the capability reports the failure. + assert client.get("/api/v1/config").json()["capabilities"]["pipeline"] is False + response = client.get("/api/v1/pipeline") + assert response.status_code == 409 + assert "boom at import" in response.json()["detail"] + + +def test_pipeline_file_without_the_app_variable_is_a_409( + tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file(tmp_path, "x = 1\n") + data_root = tmp_path / "root" + data_root.mkdir() + client = _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + response = client.get("/api/v1/pipeline") + assert response.status_code == 409 + assert "no hflow.App named 'app'" in response.json()["detail"] + + +def test_pipeline_unconfigured_is_a_409_naming_the_flag(api: TestClient) -> None: + response = api.get("/api/v1/pipeline") + assert response.status_code == 409 + assert "--pipeline" in response.json()["detail"] diff --git a/packages/hflow-server/tests/test_server_pipeline_graph.py b/packages/hflow-server/tests/test_server_pipeline_graph.py new file mode 100644 index 0000000..37e546d --- /dev/null +++ b/packages/hflow-server/tests/test_server_pipeline_graph.py @@ -0,0 +1,393 @@ +"""GET /api/v1/pipeline/graph: the DAG topology merged with the user's steps. + +Nothing here needs Docker: the bundle fixtures render real files with +``render_bundle`` (plain file writing) and no Airflow call is made -- the +pipeline graph is pure description, and its only runtime input is whether a +bundle is ADDRESSED. +""" + +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app + +from hflow.runtime import RuntimeConfig, render_bundle +from hflow.steps import RUN_PROFILES, Stage + +# Mixed tiers on purpose: `cheap_check` declares neither requires nor uses +# (tier 1), `needs_channel` declares requires, `needs_endpoint` declares uses +# -- and both are registered BEFORE the cheap one, so a payload that merely +# echoed registration order would fail the ordering assertions. +TIERED_PIPELINE_SOURCE = """import hflow + +app = hflow.App("tiered-demo", endpoints={"vlm": "http://vlm.invalid"}) + + +@app.check(name="needs_channel", requires=["/camera/wrist"], critical=True) +def needs_channel(episode): + return hflow.CheckResult(measurements={"frames": 1.0}) + + +@app.check(name="needs_endpoint", uses="vlm") +def needs_endpoint(episode): + return hflow.CheckResult(measurements={"score": 1.0}) + + +@app.check(name="cheap_check", critical=True) +def cheap_check(episode): + return hflow.CheckResult(verdict=True) + + +@app.enrich(name="rich_caption", uses="vlm") +def rich_caption(episode): + return hflow.EnrichmentResult(labels={"caption": "x"}) + + +@app.enrich(name="cheap_label") +def cheap_label(episode): + return hflow.EnrichmentResult(labels={"ok": True}) +""" + +NO_CRITICAL_PIPELINE_SOURCE = """import hflow + +app = hflow.App("no-critical-demo") + + +@app.check(name="just_evidence") +def just_evidence(episode): + return hflow.CheckResult(measurements={"value": 1.0}) +""" + +BUNDLE_PIPELINE_SOURCE = "import hflow\n\napp = hflow.App('demo', data_root='/opt/airflow/data')\n" + + +@pytest.fixture() +def runtime_free_cwd(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """A cwd with no ./runtime fallback and no remote environment exported.""" + working_directory = tmp_path / "cwd" + working_directory.mkdir() + monkeypatch.chdir(working_directory) + for variable in ("HFLOW_AIRFLOW_URL", "HFLOW_AIRFLOW_DAG_ID", "HFLOW_AIRFLOW_TOKEN"): + monkeypatch.delenv(variable, raising=False) + return working_directory + + +def _client_over(data_root: Path, assets_dir: Path, *, pipeline: str | None = None) -> TestClient: + settings = ServerSettings(data_root=str(data_root), assets_dir=assets_dir, pipeline=pipeline) + return TestClient(create_app(settings)) + + +def _written_pipeline_file(tmp_path: Path, source: str, name: str = "graph_pipeline.py") -> Path: + pipeline_file = tmp_path / name + pipeline_file.write_text(source) + return pipeline_file + + +def _rendered_bundle_root(tmp_path: Path) -> Path: + data_root = tmp_path / "bundle-data" + bundle_pipeline_file = tmp_path / "demo_pipeline.py" + bundle_pipeline_file.write_text(BUNDLE_PIPELINE_SOURCE) + render_bundle( + RuntimeConfig(pipeline_file=bundle_pipeline_file, data_root=data_root), + data_root / "runtime", + ) + return data_root + + +def test_graph_with_neither_runtime_nor_pipeline_is_explicit_not_an_error( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + response = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/pipeline/graph") + assert response.status_code == 200 + payload = response.json() + assert payload["dag_ids_known"] is False + assert payload["steps_known"] is False + # The shape is still fully drawable under a display-only master id. + assert payload["master"]["dag_id"] == "ingest" + assert [task["task_id"] for task in payload["master"]["tasks"]][:3] == [ + "resolve_profile", + "enabled_sync", + "trigger_sync", + ] + assert [stage["stage"] for stage in payload["stages"]] == ["sync", "meta", "labels", "media"] + assert all(stage["user_steps"] == [] for stage in payload["stages"]) + assert payload["quarantine_gate"] is None + assert "Traceback" not in response.text + + +def test_graph_without_a_runtime_uses_the_pipeline_name_as_a_display_only_id( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file(tmp_path, TIERED_PIPELINE_SOURCE) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = ( + _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + .get("/api/v1/pipeline/graph") + .json() + ) + assert payload["dag_ids_known"] is False + assert payload["steps_known"] is True + assert payload["master"]["dag_id"] == "tiered-demo" + stages_by_name = {stage["stage"]: stage for stage in payload["stages"]} + assert stages_by_name["meta"]["dag"]["dag_id"] == "tiered-demo_meta" + + +def test_graph_over_a_rendered_bundle_serves_the_real_dag_ids( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = _rendered_bundle_root(tmp_path) + payload = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/pipeline/graph").json() + assert payload["dag_ids_known"] is True + assert payload["steps_known"] is False + assert payload["master"]["dag_id"] == "demo_pipeline_ingest" + assert [stage["dag"]["dag_id"] for stage in payload["stages"]] == [ + "demo_pipeline_sync", + "demo_pipeline_meta", + "demo_pipeline_labels", + "demo_pipeline_media", + ] + # No pipeline imported: the steps are unknown, but the engine's own work + # is a fact of the ENGINE, so every stage still describes it. + assert payload["quarantine_gate"] is None + engine_step_names = { + stage["stage"]: [step["name"] for step in stage["engine_steps"]] + for stage in payload["stages"] + } + assert engine_step_names == { + "sync": ["canonical transform"], + "meta": ["catalog registration"], + "labels": [], + "media": ["media/contact_sheet"], + } + + +def test_graph_master_edges_chain_the_stages( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/pipeline/graph").json() + edges = {(edge[0], edge[1]) for edge in payload["master"]["edges"]} + for stage in Stage: + assert ("resolve_profile", f"enabled_{stage.value}") in edges + assert (f"enabled_{stage.value}", f"trigger_{stage.value}") in edges + # The chain: each stage's gate waits for the previous stage's trigger. + assert ("trigger_sync", "enabled_meta") in edges + assert ("trigger_meta", "enabled_labels") in edges + assert ("trigger_labels", "enabled_media") in edges + tasks_by_id = {task["task_id"]: task for task in payload["master"]["tasks"]} + assert tasks_by_id["trigger_meta"]["deferred"] is True + assert tasks_by_id["enabled_meta"]["deferred"] is False + + +def test_graph_sub_dag_shape_is_plan_fan_out_gate( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/pipeline/graph").json() + stages_by_name = {stage["stage"]: stage for stage in payload["stages"]} + meta = stages_by_name["meta"] + assert [task["task_id"] for task in meta["dag"]["tasks"]] == [ + "plan", + "process_batch", + "quarantine_budget_gate", + ] + assert meta["dag"]["edges"] == [ + ["plan", "process_batch"], + ["process_batch", "quarantine_budget_gate"], + ] + mapped_tasks = [task["task_id"] for task in meta["dag"]["tasks"] if task["mapped"]] + assert mapped_tasks == ["process_batch"] + # Only meta gates on the quarantine budget; the others on errors alone. + assert stages_by_name["labels"]["gate_task_id"] == "enabled_labels" + assert [task["task_id"] for task in stages_by_name["labels"]["dag"]["tasks"]][-1] == ( + "error_budget_gate" + ) + # The master's gate/trigger ids are named per stage, and the profiles that + # enable each stage ride along for the lane header. + assert stages_by_name["labels"]["trigger_task_id"] == "trigger_labels" + assert set(stages_by_name["labels"]["enabling_profiles"]) == { + name for name, stages in RUN_PROFILES.items() if Stage.LABELS in stages + } + + +def test_graph_user_step_tiers_match_the_app_ordering( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file(tmp_path, TIERED_PIPELINE_SOURCE) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = ( + _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + .get("/api/v1/pipeline/graph") + .json() + ) + stages_by_name = {stage["stage"]: stage for stage in payload["stages"]} + meta_steps = stages_by_name["meta"]["user_steps"] + # Cheap-first: the tier-1 check leads even though it registered last, and + # tier 2 keeps registration order (requires before uses). + assert [(step["name"], step["tier"]) for step in meta_steps] == [ + ("cheap_check", 1), + ("needs_channel", 2), + ("needs_endpoint", 2), + ] + needs_channel = next(step for step in meta_steps if step["name"] == "needs_channel") + assert needs_channel["requires"] == ["/camera/wrist"] + assert needs_channel["uses"] is None + assert needs_channel["critical"] is True + assert needs_channel["kind"] == "check" + assert needs_channel["version"] + + labels_steps = stages_by_name["labels"]["user_steps"] + assert [(step["name"], step["tier"]) for step in labels_steps] == [ + ("cheap_label", 1), + ("rich_caption", 2), + ] + assert next(step for step in labels_steps if step["name"] == "rich_caption")["uses"] == "vlm" + # Enrichments are never critical (only checks carry the gate flag). + assert all(step["critical"] is False for step in labels_steps) + # Sync and media carry no user-registered steps at all. + assert stages_by_name["sync"]["user_steps"] == [] + assert stages_by_name["media"]["user_steps"] == [] + + +def test_graph_tier_derivation_matches_the_engines_own_ordering( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + """The payload's order IS App._ordered_checks' order, not a lookalike.""" + from hflow import import_pipeline_application + + pipeline_file = _written_pipeline_file(tmp_path, TIERED_PIPELINE_SOURCE) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = ( + _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + .get("/api/v1/pipeline/graph") + .json() + ) + application = import_pipeline_application(str(pipeline_file)) + stages_by_name = {stage["stage"]: stage for stage in payload["stages"]} + assert [step["name"] for step in stages_by_name["meta"]["user_steps"]] == [ + registered.name for registered in application._ordered_checks() + ] + assert [step["name"] for step in stages_by_name["labels"]["user_steps"]] == [ + registered.name for registered in application._ordered_enrichments() + ] + + +def test_the_pipeline_page_and_the_graph_describe_one_pipeline_the_same_way( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + """Steps are served in EXECUTION order, not registration order. + + The graph is the one owner of stage grouping (the pipeline page serves no + lanes), so this pins the property that owner must hold: the cheap tier + runs first. The fixture registers its tier-2 checks FIRST, so an endpoint + that echoed registration order would fail here. + """ + pipeline_file = _written_pipeline_file(tmp_path, TIERED_PIPELINE_SOURCE) + data_root = tmp_path / "bare-root" + data_root.mkdir() + client = _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + graph_steps_by_stage = { + stage["stage"]: stage["user_steps"] + for stage in client.get("/api/v1/pipeline/graph").json()["stages"] + } + meta_steps = graph_steps_by_stage["meta"] + assert [step["name"] for step in meta_steps] == [ + "cheap_check", + "needs_channel", + "needs_endpoint", + ] + assert [step["tier"] for step in meta_steps] == [1, 2, 2] + + +def test_graph_quarantine_gate_lists_exactly_the_critical_checks( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file(tmp_path, TIERED_PIPELINE_SOURCE) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = ( + _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + .get("/api/v1/pipeline/graph") + .json() + ) + gate = payload["quarantine_gate"] + assert gate["from_stage"] == "meta" + assert gate["to_stages"] == ["labels", "media"] + assert sorted(gate["critical_step_names"]) == ["cheap_check", "needs_channel"] + # The explanation must describe what App.process actually does. + assert "quarantines the episode" in gate["explanation"] + assert "skipped" in gate["explanation"] + assert "never a deletion" in gate["explanation"] + + +def test_graph_quarantine_gate_is_honest_when_nothing_is_critical( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file(tmp_path, NO_CRITICAL_PIPELINE_SOURCE) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = ( + _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + .get("/api/v1/pipeline/graph") + .json() + ) + gate = payload["quarantine_gate"] + assert gate["critical_step_names"] == [] + assert "no critical checks" in gate["explanation"] + + +def test_graph_sync_engine_step_reports_a_transform_override( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + pipeline_file = _written_pipeline_file( + tmp_path, + """import hflow + +app = hflow.App("override-demo") + + +@app.transform +def transform(source_path, destination_path, config): + raise NotImplementedError + + +@app.derive("/derived/speed") +def speed(episode): + raise NotImplementedError +""", + ) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = ( + _client_over(data_root, unbuilt_assets_dir, pipeline=str(pipeline_file)) + .get("/api/v1/pipeline/graph") + .json() + ) + sync_stage = next(stage for stage in payload["stages"] if stage["stage"] == "sync") + summary = sync_stage["engine_steps"][0]["summary"] + assert "transform override" in summary + assert "1 registered derived channel" in summary + + +def test_graph_serves_the_stage_display_copy( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/pipeline/graph").json() + titles = {stage["stage"]: stage["title"] for stage in payload["stages"]} + assert titles == { + "sync": "Transform & sync", + "meta": "Metadata", + "labels": "Labels & artifacts", + "media": "Media", + } + assert all(stage["description"] for stage in payload["stages"]) diff --git a/packages/hflow-server/tests/test_server_queries.py b/packages/hflow-server/tests/test_server_queries.py new file mode 100644 index 0000000..af2485c --- /dev/null +++ b/packages/hflow-server/tests/test_server_queries.py @@ -0,0 +1,136 @@ +"""/api/v1/queries: saved-query CRUD over the sidecar state.""" + +from collections.abc import Iterator + +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import PopulatedWorkspace + + +def _created_query(api: TestClient, name: str, sql: str) -> dict: + response = api.post("/api/v1/queries", json={"name": name, "sql": sql}) + assert response.status_code == 200, response.text + return response.json() + + +def test_queries_start_empty(writable_api: TestClient) -> None: + assert writable_api.get("/api/v1/queries").json() == {"queries": []} + + +def test_create_list_update_delete_roundtrip(writable_api: TestClient) -> None: + created = _created_query(writable_api, "ok cut", "SELECT * FROM episodes") + assert set(created) == {"id", "name", "sql", "updated_at"} + assert created["name"] == "ok cut" + + listed = writable_api.get("/api/v1/queries").json()["queries"] + assert listed == [created] + + renamed = writable_api.put(f"/api/v1/queries/{created['id']}", json={"name": "great cut"}) + assert renamed.status_code == 200 + assert renamed.json()["name"] == "great cut" + assert renamed.json()["sql"] == created["sql"] # untouched fields survive + + new_sql = "SELECT episode_id FROM episodes WHERE status = 'ok'" + edited = writable_api.put(f"/api/v1/queries/{created['id']}", json={"sql": new_sql}) + assert edited.status_code == 200 + assert edited.json()["sql"] == new_sql + assert edited.json()["name"] == "great cut" + assert edited.json()["updated_at"] >= created["updated_at"] + + deleted = writable_api.delete(f"/api/v1/queries/{created['id']}") + assert deleted.status_code == 204 + assert writable_api.get("/api/v1/queries").json() == {"queries": []} + + +def test_unknown_query_ids_are_404(writable_api: TestClient) -> None: + assert writable_api.put("/api/v1/queries/missing", json={"name": "x"}).status_code == 404 + assert writable_api.delete("/api/v1/queries/missing").status_code == 404 + + +def test_create_requires_nonempty_name_and_sql(writable_api: TestClient) -> None: + assert ( + writable_api.post("/api/v1/queries", json={"name": "", "sql": "SELECT 1"}).status_code + == 422 + ) + assert ( + writable_api.post("/api/v1/queries", json={"name": " ", "sql": "SELECT 1"}).status_code + == 400 + ) + assert writable_api.post("/api/v1/queries", json={"name": "q", "sql": " ; "}).status_code == 400 + + +def test_update_to_a_blank_name_or_sql_is_refused(writable_api: TestClient) -> None: + created = _created_query(writable_api, "keep me", "SELECT 1") + assert ( + writable_api.put(f"/api/v1/queries/{created['id']}", json={"name": " "}).status_code == 400 + ) + assert ( + writable_api.put(f"/api/v1/queries/{created['id']}", json={"sql": ";"}).status_code == 400 + ) + + +def test_create_refuses_oversized_name_and_sql(writable_api: TestClient) -> None: + over_long_name = writable_api.post( + "/api/v1/queries", json={"name": "n" * 201, "sql": "SELECT 1"} + ) + assert over_long_name.status_code == 422 + over_long_sql = writable_api.post( + "/api/v1/queries", json={"name": "ok", "sql": "SELECT 1 -- " + "A" * 100_001} + ) + assert over_long_sql.status_code == 422 + + +def _oversized_query_body() -> dict[str, str]: + """A body far larger than the request cap, and than any per-field cap.""" + return {"name": "ok", "sql": "SELECT 1 -- " + "A" * (5 * 1024 * 1024)} + + +def test_oversized_request_body_is_refused_at_the_boundary(writable_api: TestClient) -> None: + # Refused before the body is parsed or persisted, not by a per-field cap. + response = writable_api.post("/api/v1/queries", json=_oversized_query_body()) + assert response.status_code == 413 + + +def test_the_body_cap_holds_without_a_declared_content_length(writable_api: TestClient) -> None: + """A streamed request declares no length; the cap cannot trust the claim. + + A chunked POST sends no Content-Length at all, so a cap that reads only + the header lets the whole body be buffered and parsed -- and a 422 over + an oversized SQL string echoes every byte of it back. + """ + + def streamed_oversized_body() -> Iterator[bytes]: + yield b'{"name": "ok", "sql": "SELECT 1 -- ' + for _ in range(6): + yield b"A" * (1024 * 1024) + yield b'"}' + + response = writable_api.post( + "/api/v1/queries", + content=streamed_oversized_body(), + headers={"content-type": "application/json"}, + ) + assert response.status_code == 413 + assert "content-length" not in response.request.headers + + +def test_query_writes_are_403_when_read_only(read_only_api: TestClient) -> None: + assert read_only_api.get("/api/v1/queries").status_code == 200 # reading stays open + refusals = [ + read_only_api.post("/api/v1/queries", json={"name": "x", "sql": "SELECT 1"}), + read_only_api.put("/api/v1/queries/some-id", json={"name": "x"}), + read_only_api.delete("/api/v1/queries/some-id"), + ] + for refusal in refusals: + assert refusal.status_code == 403 + assert "read-only" in refusal.json()["detail"] + + +def test_saved_queries_persist_across_server_restarts( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + created = _created_query(writable_api, "durable", "SELECT 1") + restarted_api = TestClient( + create_app(ServerSettings(data_root=str(writable_workspace.data_root))) + ) + assert restarted_api.get("/api/v1/queries").json()["queries"] == [created] diff --git a/packages/hflow-server/tests/test_server_run_graph.py b/packages/hflow-server/tests/test_server_run_graph.py new file mode 100644 index 0000000..1eec647 --- /dev/null +++ b/packages/hflow-server/tests/test_server_run_graph.py @@ -0,0 +1,513 @@ +"""GET /api/v1/runtime/runs/{dag_run_id}/graph: one run's live state. + +Every Airflow call is stubbed at the AirflowClient method level (the idiom of +``test_ui_runtime.py``): no Docker, no live Airflow, and the bundle under test +is a really rendered one so the dag ids are the real derived ones. +""" + +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app + +from hflow.runtime import RuntimeConfig, render_bundle +from hflow.runtime._client import AirflowClient, AirflowClientError + +PIPELINE_SOURCE = "import hflow\n\napp = hflow.App('demo', data_root='/opt/airflow/data')\n" + +MASTER_DAG_ID = "demo_pipeline_ingest" +MASTER_RUN_ID = "manual__2026-08-21T10:00:00+00:00" +MASTER_STARTED_AT = "2026-08-21T10:00:00+00:00" +YESTERDAYS_MASTER_RUN_ID = "manual__2026-08-20T09:00:00+00:00" + + +@pytest.fixture() +def runtime_free_cwd(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + working_directory = tmp_path / "cwd" + working_directory.mkdir() + monkeypatch.chdir(working_directory) + for variable in ("HFLOW_AIRFLOW_URL", "HFLOW_AIRFLOW_DAG_ID", "HFLOW_AIRFLOW_TOKEN"): + monkeypatch.delenv(variable, raising=False) + return working_directory + + +@pytest.fixture() +def bundle_workspace(tmp_path: Path) -> Path: + data_root = tmp_path / "data" + pipeline_file = tmp_path / "demo_pipeline.py" + pipeline_file.write_text(PIPELINE_SOURCE) + render_bundle( + RuntimeConfig(pipeline_file=pipeline_file, data_root=data_root), data_root / "runtime" + ) + return data_root + + +def _client_over(data_root: Path, assets_dir: Path) -> TestClient: + return TestClient(create_app(ServerSettings(data_root=str(data_root), assets_dir=assets_dir))) + + +@pytest.fixture() +def bundle_api(bundle_workspace: Path, unbuilt_assets_dir: Path) -> TestClient: + return _client_over(bundle_workspace, unbuilt_assets_dir) + + +def _task_instance( + task_id: str, + state: str | None, + *, + map_index: int = -1, + start_date: str | None = None, + end_date: str | None = None, + try_number: int = 1, +) -> dict[str, Any]: + return { + "task_id": task_id, + "state": state, + "map_index": map_index, + "start_date": start_date, + "end_date": end_date, + "try_number": try_number, + "operator": "never surfaced", + } + + +def _stubbed_airflow( + monkeypatch: pytest.MonkeyPatch, + *, + master_run: dict[str, Any], + stage_runs: dict[str, list[dict[str, Any]]], + task_instances: dict[tuple[str, str], list[dict[str, Any]]], +) -> None: + monkeypatch.setattr(AirflowClient, "dag_run", lambda self, dag_id, dag_run_id: dict(master_run)) + monkeypatch.setattr( + AirflowClient, + "dag_runs", + lambda self, dag_id, *, limit=100, order_by=None: list(stage_runs.get(dag_id, [])), + ) + monkeypatch.setattr( + AirflowClient, + "task_instances", + lambda self, dag_id, dag_run_id: list(task_instances.get((dag_id, dag_run_id), [])), + ) + + +def test_run_graph_without_a_runtime_is_a_clear_409( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + response = _client_over(data_root, unbuilt_assets_dir).get( + "/api/v1/runtime/runs/manual__1/graph" + ) + assert response.status_code == 409 + assert "hflow up" in response.json()["detail"] + assert "Traceback" not in response.text + + +def test_run_graph_colours_the_master_and_matches_stage_runs( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _stubbed_airflow( + monkeypatch, + master_run={ + "dag_run_id": MASTER_RUN_ID, + "state": "running", + "start_date": MASTER_STARTED_AT, + }, + stage_runs={ + # Two runs of the sync sub-DAG: one from BEFORE this master run + # (a previous ingest) and the one this run triggered. + "demo_pipeline_sync": [ + { + "dag_run_id": "sync__new", + "state": "success", + "start_date": "2026-08-21T10:00:05+00:00", + }, + { + "dag_run_id": "sync__old", + "state": "failed", + "start_date": "2026-08-20T09:00:00+00:00", + }, + ], + "demo_pipeline_meta": [ + { + "dag_run_id": "meta__new", + "state": "running", + "start_date": "2026-08-21T10:02:00+00:00", + } + ], + }, + task_instances={ + (MASTER_DAG_ID, MASTER_RUN_ID): [ + _task_instance( + "trigger_sync", + "success", + start_date="2026-08-21T10:00:02+00:00", + end_date="2026-08-21T10:01:32+00:00", + ), + _task_instance("resolve_profile", "success"), + _task_instance("enabled_sync", "success"), + _task_instance("trigger_meta", "deferred"), + ], + ("demo_pipeline_sync", "sync__new"): [ + _task_instance("plan", "success"), + _task_instance("process_batch", "success", map_index=0), + _task_instance("process_batch", "failed", map_index=1), + _task_instance("process_batch", "success", map_index=2), + _task_instance("error_budget_gate", "success"), + ], + ("demo_pipeline_meta", "meta__new"): [ + _task_instance("plan", "success"), + _task_instance("process_batch", "running", map_index=0), + _task_instance("process_batch", None, map_index=1), + ], + }, + ) + response = bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph") + assert response.status_code == 200 + payload = response.json() + + assert payload["master"]["dag_run_id"] == MASTER_RUN_ID + assert payload["master"]["state"] == "running" + # Task instances come back in TOPOLOGY order, not the API's order. + assert [task["task_id"] for task in payload["master"]["tasks"]] == [ + "resolve_profile", + "enabled_sync", + "trigger_sync", + "trigger_meta", + ] + trigger_sync = payload["master"]["tasks"][2] + assert trigger_sync["duration_s"] == 90.0 + assert trigger_sync["try_number"] == 1 + assert trigger_sync["map_index"] == -1 + # Airflow's own extra fields never reach the browser. + assert "operator" not in trigger_sync + + stages_by_name = {stage["stage"]: stage for stage in payload["stages"]} + sync_stage = stages_by_name["sync"] + # The heuristic picks the run that started after the master run, never + # the older one, and says out loud that it is a heuristic. + assert sync_stage["dag_run_id"] == "sync__new" + assert sync_stage["state"] == "success" + assert sync_stage["match"] == "heuristic" + assert sync_stage["mapped_summary"] == { + "task_id": "process_batch", + "total": 3, + "by_state": {"failed": 1, "success": 2}, + } + assert stages_by_name["meta"]["mapped_summary"] == { + "task_id": "process_batch", + "total": 2, + # A task instance Airflow has not scheduled yet has a null state. + "by_state": {"no_status": 1, "running": 1}, + } + # Stages with no run of their own are explicit nulls, not omissions. + for never_ran in ("labels", "media"): + assert stages_by_name[never_ran]["dag_run_id"] is None + assert stages_by_name[never_ran]["state"] is None + assert stages_by_name[never_ran]["match"] is None + assert stages_by_name[never_ran]["tasks"] == [] + assert stages_by_name[never_ran]["mapped_summary"] is None + assert stages_by_name[never_ran]["dag_id"] == f"demo_pipeline_{never_ran}" + + +def test_run_graph_passes_the_run_id_through_verbatim( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Airflow run ids carry ':' and '+'; the path must not mangle them.""" + requested: list[tuple[str, str]] = [] + + def capturing_dag_run(self: AirflowClient, dag_id: str, dag_run_id: str) -> dict[str, Any]: + requested.append((dag_id, dag_run_id)) + return {"dag_run_id": dag_run_id, "state": "success", "start_date": None} + + monkeypatch.setattr(AirflowClient, "dag_run", capturing_dag_run) + monkeypatch.setattr(AirflowClient, "task_instances", lambda self, dag_id, dag_run_id: []) + monkeypatch.setattr( + AirflowClient, "dag_runs", lambda self, dag_id, *, limit=100, order_by=None: [] + ) + response = bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph") + assert response.status_code == 200 + assert requested == [(MASTER_DAG_ID, MASTER_RUN_ID)] + assert response.json()["master"]["dag_run_id"] == MASTER_RUN_ID + + +def test_run_graph_mapped_summary_counts_the_unexpanded_placeholder( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + _stubbed_airflow( + monkeypatch, + master_run={ + "dag_run_id": MASTER_RUN_ID, + "state": "running", + "start_date": MASTER_STARTED_AT, + }, + stage_runs={ + "demo_pipeline_sync": [ + { + "dag_run_id": "sync__new", + "state": "running", + "start_date": "2026-08-21T10:00:05+00:00", + } + ] + }, + task_instances={ + ("demo_pipeline_sync", "sync__new"): [ + _task_instance("plan", "running"), + # Before the fan-out expands, Airflow reports ONE instance. + _task_instance("process_batch", None, map_index=-1), + ] + }, + ) + payload = bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph").json() + sync_stage = next(stage for stage in payload["stages"] if stage["stage"] == "sync") + assert sync_stage["mapped_summary"] == { + "task_id": "process_batch", + "total": 1, + "by_state": {"no_status": 1}, + } + + +def test_run_graph_without_a_master_start_matches_no_stage_run( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """A queued master run cannot be attributed stage runs -- and says so.""" + _stubbed_airflow( + monkeypatch, + master_run={"dag_run_id": MASTER_RUN_ID, "state": "queued", "start_date": None}, + stage_runs={ + "demo_pipeline_sync": [ + { + "dag_run_id": "sync__unrelated", + "state": "success", + "start_date": "2026-08-20T10:00:05+00:00", + } + ] + }, + task_instances={}, + ) + payload = bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph").json() + assert payload["master"]["state"] == "queued" + assert all(stage["dag_run_id"] is None for stage in payload["stages"]) + + +def test_an_ended_master_run_never_adopts_a_later_unrelated_stage_run( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """Two ingests over one lane: each run shows the stage runs it triggered. + + The stage lanes only look back a fixed number of runs, so an unbounded + "started at/after the master" filter matched every candidate for any + master run that was not the latest -- and keeping the newest handed an old + run today's unrelated stage run, labelled as an attribution. + """ + master_runs = { + YESTERDAYS_MASTER_RUN_ID: { + "dag_run_id": YESTERDAYS_MASTER_RUN_ID, + "state": "success", + "start_date": "2026-08-20T09:00:00+00:00", + "end_date": "2026-08-20T09:04:00+00:00", + }, + MASTER_RUN_ID: { + "dag_run_id": MASTER_RUN_ID, + "state": "running", + "start_date": MASTER_STARTED_AT, + "end_date": None, + }, + } + # Newest first, exactly as the runs are fetched (order_by="-id"). + interleaved_stage_runs = { + "demo_pipeline_sync": [ + { + "dag_run_id": "sync__today", + "state": "running", + "start_date": "2026-08-21T10:00:05+00:00", + }, + { + "dag_run_id": "sync__yesterday", + "state": "success", + "start_date": "2026-08-20T09:00:05+00:00", + }, + ], + # Today's ingest reached meta; yesterday's never did. + "demo_pipeline_meta": [ + { + "dag_run_id": "meta__today", + "state": "running", + "start_date": "2026-08-21T10:02:00+00:00", + } + ], + } + monkeypatch.setattr( + AirflowClient, "dag_run", lambda self, dag_id, dag_run_id: dict(master_runs[dag_run_id]) + ) + monkeypatch.setattr( + AirflowClient, + "dag_runs", + lambda self, dag_id, *, limit=100, order_by=None: list( + interleaved_stage_runs.get(dag_id, []) + ), + ) + monkeypatch.setattr(AirflowClient, "task_instances", lambda self, dag_id, dag_run_id: []) + + yesterday = { + stage["stage"]: stage + for stage in bundle_api.get( + f"/api/v1/runtime/runs/{YESTERDAYS_MASTER_RUN_ID}/graph" + ).json()["stages"] + } + assert yesterday["sync"]["dag_run_id"] == "sync__yesterday" + assert yesterday["sync"]["state"] == "success" + assert yesterday["sync"]["match"] == "heuristic" + # Nothing ran in that master run's window, so the lane stays empty rather + # than borrowing today's still-running one. + assert yesterday["meta"]["dag_run_id"] is None + assert yesterday["meta"]["match"] is None + + today = { + stage["stage"]: stage + for stage in bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph").json()["stages"] + } + assert today["sync"]["dag_run_id"] == "sync__today" + assert today["meta"]["dag_run_id"] == "meta__today" + + +def test_a_stage_run_starting_just_after_the_master_ended_still_matches( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """A master that dies right after firing a trigger keeps its stage run. + + The two timestamps come from different components, and a failing master + can end before the run it just caused appears, so the window carries a + grace period rather than a hard edge. + """ + _stubbed_airflow( + monkeypatch, + master_run={ + "dag_run_id": MASTER_RUN_ID, + "state": "failed", + "start_date": MASTER_STARTED_AT, + "end_date": "2026-08-21T10:00:04+00:00", + }, + stage_runs={ + "demo_pipeline_sync": [ + { + "dag_run_id": "sync__just_after", + "state": "running", + "start_date": "2026-08-21T10:00:06+00:00", + } + ] + }, + task_instances={}, + ) + payload = bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph").json() + sync_stage = next(stage for stage in payload["stages"] if stage["stage"] == "sync") + assert sync_stage["dag_run_id"] == "sync__just_after" + + +def test_run_graph_tolerates_unregistered_stage_sub_dags( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def stage_runs_404( + self: AirflowClient, dag_id: str, *, limit: int = 100, order_by: str | None = None + ) -> list[dict[str, Any]]: + raise AirflowClientError(f"GET /dags/{dag_id}/dagRuns failed with HTTP 404", status=404) + + monkeypatch.setattr( + AirflowClient, + "dag_run", + lambda self, dag_id, dag_run_id: { + "dag_run_id": dag_run_id, + "state": "success", + "start_date": MASTER_STARTED_AT, + }, + ) + monkeypatch.setattr(AirflowClient, "dag_runs", stage_runs_404) + monkeypatch.setattr(AirflowClient, "task_instances", lambda self, dag_id, dag_run_id: []) + response = bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph") + assert response.status_code == 200 + payload = response.json() + assert payload["master"]["state"] == "success" + assert all(stage["dag_run_id"] is None for stage in payload["stages"]) + + +def test_run_graph_unknown_run_is_a_404( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def missing_run(self: AirflowClient, dag_id: str, dag_run_id: str) -> dict[str, Any]: + raise AirflowClientError( + f"GET /dags/{dag_id}/dagRuns/{dag_run_id} failed with HTTP 404", status=404 + ) + + monkeypatch.setattr(AirflowClient, "dag_run", missing_run) + response = bundle_api.get("/api/v1/runtime/runs/manual__nope/graph") + assert response.status_code == 404 + assert "manual__nope" in response.json()["detail"] + assert "Traceback" not in response.text + + +def test_run_graph_maps_airflow_failures_to_502( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def failing_run(self: AirflowClient, dag_id: str, dag_run_id: str) -> dict[str, Any]: + raise AirflowClientError("GET /dagRuns failed with HTTP 503: scheduler down", status=503) + + monkeypatch.setattr(AirflowClient, "dag_run", failing_run) + response = bundle_api.get(f"/api/v1/runtime/runs/{MASTER_RUN_ID}/graph") + assert response.status_code == 502 + assert "scheduler down" in response.json()["detail"] + + +def test_run_graph_remote_failure_does_not_leak_the_base_url( + runtime_free_cwd: Path, + tmp_path: Path, + unbuilt_assets_dir: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HFLOW_AIRFLOW_URL", "https://airflow.internal.corp:8443") + monkeypatch.setenv("HFLOW_AIRFLOW_DAG_ID", "kitchen_ingest") + monkeypatch.setenv("HFLOW_AIRFLOW_TOKEN", "minted-token") + + def failing_run(self: AirflowClient, dag_id: str, dag_run_id: str) -> dict[str, Any]: + raise AirflowClientError( + "GET https://airflow.internal.corp:8443/api/v2/dags/kitchen_ingest/dagRuns/x " + "failed with HTTP 500: internal" + ) + + monkeypatch.setattr(AirflowClient, "dag_run", failing_run) + data_root = tmp_path / "bare-root" + data_root.mkdir() + response = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/runtime/runs/x/graph") + assert response.status_code == 502 + assert "airflow.internal.corp" not in response.text + + +def test_run_graph_over_a_remote_runtime_derives_the_stage_dag_ids( + runtime_free_cwd: Path, + tmp_path: Path, + unbuilt_assets_dir: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A remote master id derives its sub-DAG ids the same way the renderer did.""" + monkeypatch.setenv("HFLOW_AIRFLOW_URL", "https://workspace.example.com") + monkeypatch.setenv("HFLOW_AIRFLOW_DAG_ID", "kitchen_ingest") + monkeypatch.setenv("HFLOW_AIRFLOW_TOKEN", "minted-token") + _stubbed_airflow( + monkeypatch, + master_run={"dag_run_id": "r1", "state": "success", "start_date": MASTER_STARTED_AT}, + stage_runs={}, + task_instances={}, + ) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = ( + _client_over(data_root, unbuilt_assets_dir).get("/api/v1/runtime/runs/r1/graph").json() + ) + assert [stage["dag_id"] for stage in payload["stages"]] == [ + "kitchen_sync", + "kitchen_meta", + "kitchen_labels", + "kitchen_media", + ] diff --git a/packages/hflow-server/tests/test_server_runtime.py b/packages/hflow-server/tests/test_server_runtime.py new file mode 100644 index 0000000..7ec2266 --- /dev/null +++ b/packages/hflow-server/tests/test_server_runtime.py @@ -0,0 +1,479 @@ +"""/api/v1/runtime/*: bundle/remote addressing with stubbed AirflowClient. + +No Docker and no live Airflow anywhere: bundles are rendered with +``render_bundle`` (plain file writing) and every Airflow API call is stubbed +at the AirflowClient method level, the same idiom as the repository's +``tests/test_runtime_cli.py``. +""" + +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app + +from hflow.runtime import RuntimeConfig, render_bundle +from hflow.runtime._client import AirflowClient, AirflowClientError, AirflowHealth + +HEALTHY = AirflowHealth( + components={ + "metadatabase": "healthy", + "scheduler": "healthy", + "dag_processor": "healthy", + "triggerer": "healthy", + } +) + +PIPELINE_SOURCE = "import hflow\n\napp = hflow.App('demo', data_root='/opt/airflow/data')\n" + + +def _client_over(data_root: Path, assets_dir: Path, *, read_only: bool = False) -> TestClient: + settings = ServerSettings(data_root=str(data_root), assets_dir=assets_dir, read_only=read_only) + return TestClient(create_app(settings)) + + +@pytest.fixture() +def runtime_free_cwd(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """A cwd with no ./runtime fallback and no remote environment exported.""" + working_directory = tmp_path / "cwd" + working_directory.mkdir() + monkeypatch.chdir(working_directory) + for variable in ("HFLOW_AIRFLOW_URL", "HFLOW_AIRFLOW_DAG_ID", "HFLOW_AIRFLOW_TOKEN"): + monkeypatch.delenv(variable, raising=False) + return working_directory + + +@pytest.fixture() +def bundle_workspace(tmp_path: Path) -> Path: + """A data root whose ``runtime/`` holds a really rendered Compose bundle.""" + data_root = tmp_path / "data" + pipeline_file = tmp_path / "demo_pipeline.py" + pipeline_file.write_text(PIPELINE_SOURCE) + render_bundle( + RuntimeConfig(pipeline_file=pipeline_file, data_root=data_root), data_root / "runtime" + ) + return data_root + + +@pytest.fixture() +def bundle_api(bundle_workspace: Path, unbuilt_assets_dir: Path) -> TestClient: + return _client_over(bundle_workspace, unbuilt_assets_dir) + + +def test_status_without_any_runtime_is_available_false_not_a_traceback( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + response = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/runtime/status") + assert response.status_code == 200 + payload = response.json() + assert payload["available"] is False + assert "hflow up" in payload["detail"] + assert "HFLOW_AIRFLOW_URL" in payload["detail"] + assert payload["source"] is None + assert payload["dag_id"] is None + assert payload["health"] is None + assert "Traceback" not in response.text + + +def test_status_reports_a_healthy_bundle_runtime( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(AirflowClient, "health", lambda self: HEALTHY) + monkeypatch.setattr(AirflowClient, "dag", lambda self, dag_id: {"dag_id": dag_id}) + payload = bundle_api.get("/api/v1/runtime/status").json() + assert payload == { + "available": True, + "detail": None, + "source": "bundle", + "airflow_web_url": "http://127.0.0.1:8080", + # A rendered bundle binds its api-server to loopback, so the address + # is only followable on the workspace host -- a browser elsewhere + # would aim it at its own machine. + "airflow_web_url_host_only": True, + "dag_id": "demo_pipeline_ingest", + "registered": True, + "health": { + "metadatabase": "healthy", + "scheduler": "healthy", + "triggerer": "healthy", + "dag_processor": "healthy", + }, + } + + +def test_status_reports_an_unregistered_dag( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def dag_not_found(self: AirflowClient, dag_id: str) -> dict[str, Any]: + raise AirflowClientError(f"GET /dags/{dag_id} failed with HTTP 404", status=404) + + monkeypatch.setattr(AirflowClient, "health", lambda self: HEALTHY) + monkeypatch.setattr(AirflowClient, "dag", dag_not_found) + payload = bundle_api.get("/api/v1/runtime/status").json() + assert payload["available"] is True + assert payload["registered"] is False + + +def test_status_with_unreachable_bundle_runtime_stays_a_calm_answer( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def unreachable_health(self: AirflowClient) -> AirflowHealth: + raise AirflowClientError("GET http://127.0.0.1:8080 unreachable: connection refused") + + monkeypatch.setattr(AirflowClient, "health", unreachable_health) + response = bundle_api.get("/api/v1/runtime/status") + assert response.status_code == 200 + payload = response.json() + assert payload["available"] is False + assert "unreachable" in payload["detail"] + # The addressing facts still ride along: the bundle IS configured. + assert payload["source"] == "bundle" + assert payload["dag_id"] == "demo_pipeline_ingest" + assert payload["airflow_web_url"] == "http://127.0.0.1:8080" + assert "Traceback" not in response.text + + +def test_status_addresses_the_remote_environment( + runtime_free_cwd: Path, + tmp_path: Path, + unbuilt_assets_dir: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HFLOW_AIRFLOW_URL", "https://workspace.example.com") + monkeypatch.setenv("HFLOW_AIRFLOW_DAG_ID", "kitchen_ingest") + monkeypatch.setenv("HFLOW_AIRFLOW_TOKEN", "minted-token") + monkeypatch.setattr(AirflowClient, "health", lambda self: HEALTHY) + monkeypatch.setattr(AirflowClient, "dag", lambda self, dag_id: {"dag_id": dag_id}) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/runtime/status").json() + assert payload["available"] is True + assert payload["source"] == "remote" + assert payload["dag_id"] == "kitchen_ingest" + # Only a bundle records its own web address; a remote endpoint's is unknown. + assert payload["airflow_web_url"] is None + + +def test_status_remote_failure_does_not_leak_the_base_url( + runtime_free_cwd: Path, + tmp_path: Path, + unbuilt_assets_dir: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HFLOW_AIRFLOW_URL", "https://airflow.internal.corp:8443") + monkeypatch.setenv("HFLOW_AIRFLOW_DAG_ID", "kitchen_ingest") + monkeypatch.setenv("HFLOW_AIRFLOW_TOKEN", "minted-token") + + def unreachable_health(self: AirflowClient) -> AirflowHealth: + raise AirflowClientError( + "GET https://airflow.internal.corp:8443/api/v2/monitor/health " + "unreachable: [Errno 111] Connection refused" + ) + + monkeypatch.setattr(AirflowClient, "health", unreachable_health) + data_root = tmp_path / "bare-root" + data_root.mkdir() + payload = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/runtime/status").json() + assert payload["available"] is False + # The remote base URL the success path withholds must not leak on failure. + assert "airflow.internal.corp" not in payload["detail"] + assert payload["airflow_web_url"] is None + # A stable machine-readable reason still rides along. + assert "unreachable" in payload["detail"] + + +def test_status_remote_incomplete_environment_names_the_missing_variable( + runtime_free_cwd: Path, + tmp_path: Path, + unbuilt_assets_dir: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("HFLOW_AIRFLOW_URL", "https://workspace.example.com") + data_root = tmp_path / "bare-root" + data_root.mkdir() + response = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/runtime/status") + assert response.status_code == 200 + payload = response.json() + assert payload["available"] is False + assert "HFLOW_AIRFLOW_DAG_ID" in payload["detail"] + assert "Traceback" not in response.text + + +def test_runs_shape_over_a_bundle_with_stage_strips( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + dag_runs_calls: list[tuple[str, int, str | None]] = [] + + def fake_dag_runs( + self: AirflowClient, dag_id: str, *, limit: int = 100, order_by: str | None = None + ) -> list[dict[str, Any]]: + dag_runs_calls.append((dag_id, limit, order_by)) + if dag_id == "demo_pipeline_ingest": + return [ + { + "dag_run_id": "manual__1", + "state": "success", + "logical_date": None, + "start_date": "2026-08-21T00:00:00Z", + "end_date": "2026-08-21T00:05:00Z", + "conf": {"uris": ["a.mcap"], "profile": "full", "mode": "batch"}, + "internal_field": "never surfaced", + } + ] + return [{"dag_run_id": f"{dag_id}__r1", "state": "running"}] + + monkeypatch.setattr(AirflowClient, "dag_runs", fake_dag_runs) + response = bundle_api.get("/api/v1/runtime/runs", params={"limit": 5}) + assert response.status_code == 200 + payload = response.json() + assert payload["runs"] == [ + { + "dag_run_id": "manual__1", + "state": "success", + "logical_date": None, + "start_date": "2026-08-21T00:00:00Z", + "end_date": "2026-08-21T00:05:00Z", + "conf": {"uris": ["a.mcap"], "profile": "full", "mode": "batch"}, + } + ] + assert [stage["stage"] for stage in payload["stages"]] == ["sync", "meta", "labels", "media"] + assert [stage["dag_id"] for stage in payload["stages"]] == [ + "demo_pipeline_sync", + "demo_pipeline_meta", + "demo_pipeline_labels", + "demo_pipeline_media", + ] + assert payload["stages"][0]["recent"] == [ + { + "dag_run_id": "demo_pipeline_sync__r1", + "state": "running", + "start_date": None, + "end_date": None, + } + ] + # Newest-first ordering is requested explicitly (Airflow truncates in id + # order otherwise), and the master honors the caller's limit. + assert dag_runs_calls[0] == ("demo_pipeline_ingest", 5, "-id") + + +def test_runs_tolerates_an_unregistered_stage_dag( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def fake_dag_runs( + self: AirflowClient, dag_id: str, *, limit: int = 100, order_by: str | None = None + ) -> list[dict[str, Any]]: + if dag_id == "demo_pipeline_ingest": + return [] + raise AirflowClientError(f"GET /dags/{dag_id}/dagRuns failed with HTTP 404", status=404) + + monkeypatch.setattr(AirflowClient, "dag_runs", fake_dag_runs) + payload = bundle_api.get("/api/v1/runtime/runs").json() + assert payload["runs"] == [] + assert all(stage["recent"] == [] for stage in payload["stages"]) + + +def test_runs_without_a_runtime_is_a_clear_409( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + response = _client_over(data_root, unbuilt_assets_dir).get("/api/v1/runtime/runs") + assert response.status_code == 409 + assert "hflow up" in response.json()["detail"] + + +def test_ingest_is_refused_read_only(bundle_workspace: Path, unbuilt_assets_dir: Path) -> None: + client = _client_over(bundle_workspace, unbuilt_assets_dir, read_only=True) + response = client.post( + "/api/v1/runtime/ingest", json={"uris": ["a.mcap"], "profile": "full", "mode": "batch"} + ) + assert response.status_code == 403 + assert "read-only" in response.json()["detail"] + + +def test_ingest_validates_profile_and_mode_before_touching_any_runtime( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + client = _client_over(data_root, unbuilt_assets_dir) + + bad_profile = client.post( + "/api/v1/runtime/ingest", + json={"uris": ["a.mcap"], "profile": "everything", "mode": "batch"}, + ) + assert bad_profile.status_code == 400 + assert "full" in bad_profile.json()["detail"] + + bad_mode = client.post( + "/api/v1/runtime/ingest", + json={"uris": ["a.mcap"], "profile": "full", "mode": "streaming"}, + ) + assert bad_mode.status_code == 400 + assert "online" in bad_mode.json()["detail"] + + blank_uri = client.post( + "/api/v1/runtime/ingest", json={"uris": [" "], "profile": "full", "mode": "batch"} + ) + assert blank_uri.status_code == 400 + + no_uris = client.post( + "/api/v1/runtime/ingest", json={"uris": [], "profile": "full", "mode": "batch"} + ) + assert no_uris.status_code == 422 # pydantic: the list itself must be non-empty + + +def test_ingest_rejects_absolute_and_escaping_uris_before_any_runtime( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + # URIs resolve against the runtime's data root; absolute paths and ../ + # escapes cannot work there, so they are refused with a 400 before the + # runtime is even resolved -- the same guard `hflow ingest` enforces. + data_root = tmp_path / "bare-root" + data_root.mkdir() + client = _client_over(data_root, unbuilt_assets_dir) + for hostile_uri in ("/etc/passwd", "../../etc/shadow", "sub/../../escape.mcap"): + response = client.post( + "/api/v1/runtime/ingest", + json={"uris": [hostile_uri], "profile": "full", "mode": "batch"}, + ) + assert response.status_code == 400, hostile_uri + assert "not relative to the data root" in response.json()["detail"] + + +def test_ingest_without_a_runtime_is_a_clear_409( + runtime_free_cwd: Path, tmp_path: Path, unbuilt_assets_dir: Path +) -> None: + data_root = tmp_path / "bare-root" + data_root.mkdir() + response = _client_over(data_root, unbuilt_assets_dir).post( + "/api/v1/runtime/ingest", json={"uris": ["a.mcap"], "profile": "full", "mode": "batch"} + ) + assert response.status_code == 409 + assert "Traceback" not in response.text + + +def test_ingest_triggers_the_master_dag( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + captured: dict[str, object] = {} + + def fake_ingest( + self: AirflowClient, + dag_id: str, + uris: list[str], + *, + profile: str = "full", + online: bool = False, + batch_count: int | None = None, + dag_run_id: str | None = None, + ) -> dict[str, str]: + captured["dag_id"] = dag_id + captured["uris"] = uris + captured["profile"] = profile + captured["online"] = online + captured["batch_count"] = batch_count + return {"dag_run_id": "manual__ui", "state": "queued"} + + monkeypatch.setattr(AirflowClient, "ingest", fake_ingest) + response = bundle_api.post( + "/api/v1/runtime/ingest", + json={"uris": ["a.mcap", "sub/b.mcap"], "profile": "relabel", "mode": "online"}, + ) + assert response.status_code == 200 + assert response.json() == {"dag_run_id": "manual__ui", "state": "queued"} + assert captured == { + "dag_id": "demo_pipeline_ingest", + "uris": ["a.mcap", "sub/b.mcap"], + "profile": "relabel", + "online": True, + "batch_count": None, + } + + +def test_ingest_batch_count_rides_the_trigger_conf( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + captured: dict[str, object] = {} + + def fake_trigger( + self: AirflowClient, + dag_id: str, + conf: dict[str, Any] | None = None, + *, + dag_run_id: str | None = None, + ) -> dict[str, str]: + captured["dag_id"] = dag_id + captured["conf"] = conf + return {"dag_run_id": "manual__sharded", "state": "queued"} + + monkeypatch.setattr(AirflowClient, "trigger_dag_run", fake_trigger) + response = bundle_api.post( + "/api/v1/runtime/ingest", + json={"uris": ["a.mcap"], "profile": "full", "mode": "batch", "batch_count": 3}, + ) + assert response.status_code == 200 + assert captured["dag_id"] == "demo_pipeline_ingest" + assert captured["conf"] == { + "uris": ["a.mcap"], + "profile": "full", + "mode": "batch", + "batch_count": 3, + } + + +def test_ingest_maps_airflow_errors_to_502( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + def failing_ingest( + self: AirflowClient, + dag_id: str, + uris: list[str], + *, + profile: str = "full", + online: bool = False, + batch_count: int | None = None, + dag_run_id: str | None = None, + ) -> dict[str, str]: + raise AirflowClientError("POST /dagRuns failed with HTTP 503: scheduler down") + + monkeypatch.setattr(AirflowClient, "ingest", failing_ingest) + response = bundle_api.post( + "/api/v1/runtime/ingest", json={"uris": ["a.mcap"], "profile": "full", "mode": "batch"} + ) + assert response.status_code == 502 + assert "scheduler down" in response.json()["detail"] + + +def test_loopback_classification_marks_only_host_local_addresses() -> None: + """The Runs page needs to know when a deep link cannot be followed. + + A viewer on another machine reads http://127.0.0.1:8080 as their OWN + laptop, so the payload states whether the address is host-local rather + than leaving the browser to guess. + """ + from hflow_server._runtime import is_loopback_web_url + + assert is_loopback_web_url("http://127.0.0.1:8080") is True + assert is_loopback_web_url("http://localhost:8080") is True + assert is_loopback_web_url("http://[::1]:8080") is True + assert is_loopback_web_url("http://100.104.216.28:8080") is False + assert is_loopback_web_url("https://airflow.example.com") is False + assert is_loopback_web_url(None) is False + + +def test_an_unreachable_bundle_names_the_workspace_host_as_the_caller( + bundle_api: TestClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """The failure text must not read as the viewer's own machine failing.""" + + def refuse(self: AirflowClient) -> None: + raise AirflowClientError("GET http://127.0.0.1:8080/api/v2/monitor/health unreachable") + + monkeypatch.setattr(AirflowClient, "health", refuse) + payload = bundle_api.get("/api/v1/runtime/status").json() + assert payload["available"] is False + assert "the workspace host could not reach its own ingest runtime" in payload["detail"] diff --git a/packages/hflow-server/tests/test_server_settings.py b/packages/hflow-server/tests/test_server_settings.py new file mode 100644 index 0000000..6b43331 --- /dev/null +++ b/packages/hflow-server/tests/test_server_settings.py @@ -0,0 +1,24 @@ +"""ServerSettings: the launch values parsed where they are set.""" + +import pytest +from hflow_server import ServerSettings + + +def test_a_port_that_cannot_be_served_is_refused_at_the_boundary() -> None: + """Every unusable port answers the same way, before anything is built. + + Left to ``bind(2)``, an out-of-range port surfaces as an OverflowError + from inside the free-port probe, and 0 binds happily while the URL the + launch prints -- ``http://127.0.0.1:0/`` -- is not dialable by the + browser it is handed to. Checking the field where it is set is also what + gives a library caller the same answer as the command line. + """ + for unusable_port in (-1, 0, 65536, 70000): + with pytest.raises(ValueError, match="1-65535"): + ServerSettings(data_root="/tmp/does-not-need-to-exist", port=unusable_port) + + +def test_the_default_launch_is_loopback_and_servable() -> None: + settings = ServerSettings(data_root="/tmp/does-not-need-to-exist") + assert settings.host == "127.0.0.1" + assert 1 <= settings.port <= 65535 diff --git a/packages/hflow-server/tests/test_server_sidecar.py b/packages/hflow-server/tests/test_server_sidecar.py new file mode 100644 index 0000000..4a3a0d9 --- /dev/null +++ b/packages/hflow-server/tests/test_server_sidecar.py @@ -0,0 +1,93 @@ +"""The /curation/state.json sidecar: atomic writes, loud boundary parsing.""" + +import json +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from ui_test_fixtures import PopulatedWorkspace + + +@pytest.fixture() +def bare_root(tmp_path: Path) -> Path: + """A data root with no catalog: the sidecar endpoints need none.""" + return tmp_path + + +@pytest.fixture() +def bare_api(bare_root: Path) -> TestClient: + return TestClient(create_app(ServerSettings(data_root=str(bare_root)))) + + +def _write_state_file(bare_root: Path, payload: str) -> Path: + """Plant a sidecar file and return its path. Named for the write: calling + this in an assertion would overwrite the very bytes under test.""" + state_file = bare_root / "curation" / "state.json" + state_file.parent.mkdir(parents=True, exist_ok=True) + state_file.write_text(payload) + return state_file + + +def test_a_missing_state_file_reads_as_empty_state(bare_api: TestClient) -> None: + assert bare_api.get("/api/v1/queries").json() == {"queries": []} + assert bare_api.get("/api/v1/manifests").json() == {"manifests": []} + + +def test_a_corrupt_state_file_is_a_loud_500_naming_the_file( + bare_api: TestClient, bare_root: Path +) -> None: + state_file = _write_state_file(bare_root, "this is { not json") + response = bare_api.get("/api/v1/queries") + assert response.status_code == 500 + detail = response.json()["detail"] + assert str(state_file) in detail + assert "corrupt" in detail + + +def test_a_wrong_state_version_is_refused_loudly(bare_api: TestClient, bare_root: Path) -> None: + state_file = _write_state_file( + bare_root, json.dumps({"state_version": 2, "saved_queries": [], "manifests": []}) + ) + response = bare_api.get("/api/v1/manifests") + # 409, matching the catalog's format-version refusal: the state is intact, + # this build just cannot read its version -- not a fault of the server. + assert response.status_code == 409 + detail = response.json()["detail"] + assert str(state_file) in detail + assert "state_version" in detail + assert "2" in detail + + +def test_a_malformed_entry_is_refused_loudly(bare_api: TestClient, bare_root: Path) -> None: + _write_state_file( + bare_root, + json.dumps({"state_version": 1, "saved_queries": [{"id": "only-an-id"}], "manifests": []}), + ) + response = bare_api.get("/api/v1/queries") + assert response.status_code == 500 + assert "name" in response.json()["detail"] + + +def test_a_corrupt_sidecar_blocks_writes_too(bare_api: TestClient, bare_root: Path) -> None: + state_file = _write_state_file(bare_root, "garbage") + response = bare_api.post("/api/v1/queries", json={"name": "x", "sql": "SELECT 1"}) + assert response.status_code == 500 + # The refusal must leave the operator's file byte-for-byte intact: a + # rewrite here would silently destroy their saved queries and manifest + # registry, which is precisely what the unreadable state is protecting. + assert state_file.read_text() == "garbage" + assert [file.name for file in state_file.parent.iterdir()] == ["state.json"] + + +def test_writes_land_atomically_in_the_documented_shape( + writable_api: TestClient, writable_workspace: PopulatedWorkspace +) -> None: + created = writable_api.post( + "/api/v1/queries", json={"name": "shape check", "sql": "SELECT 1"} + ).json() + state_file = writable_workspace.data_root / "curation" / "state.json" + stored = json.loads(state_file.read_text()) + assert stored == {"state_version": 1, "saved_queries": [created], "manifests": []} + # No temp debris: the write moved into place, it did not copy. + assert [file.name for file in state_file.parent.iterdir()] == ["state.json"] diff --git a/packages/hflow-server/tests/test_server_spa.py b/packages/hflow-server/tests/test_server_spa.py new file mode 100644 index 0000000..5aa0bf9 --- /dev/null +++ b/packages/hflow-server/tests/test_server_spa.py @@ -0,0 +1,137 @@ +"""SPA serving: built assets, client-route fallback, placeholder, traversal.""" + +from pathlib import Path + +import hflow_server +import pytest +from fastapi.testclient import TestClient +from hflow_server import ServerSettings, create_app +from hflow_server.server import ASSETS_ENVIRONMENT_VARIABLE, _assets_directory +from ui_test_fixtures import PopulatedWorkspace + + +@pytest.fixture() +def built_assets_directory(tmp_path: Path) -> Path: + assets_directory = tmp_path / "dist" + assets_directory.mkdir() + (assets_directory / "index.html").write_text("SPA INDEX") + (assets_directory / "assets").mkdir() + (assets_directory / "assets" / "app.js").write_text("console.log('hflow');") + # A secret OUTSIDE the assets tree, for the traversal test. + (tmp_path / "secret.txt").write_text("do not serve this") + return assets_directory + + +@pytest.fixture() +def spa_api(populated_workspace: PopulatedWorkspace, built_assets_directory: Path) -> TestClient: + settings = ServerSettings( + data_root=str(populated_workspace.data_root), + assets_dir=built_assets_directory, + ) + return TestClient(create_app(settings)) + + +def test_placeholder_page_when_no_frontend_is_installed(api: TestClient) -> None: + """No bundle is not an error: the API is the product, a UI is a client.""" + response = api.get("/") + assert response.status_code == 200 + assert "text/html" in response.headers["content-type"] + assert "No frontend bundle" in response.text + assert "/api/v1" in response.text # the API stays discoverable + assert "/api/openapi.json" in response.text # ...and so does how to build against it + assert "HFLOW_UI_ASSETS" in response.text # how to serve your own + + +def test_client_routes_get_the_placeholder_too(api: TestClient) -> None: + response = api.get("/episodes/some-episode-id") + assert response.status_code == 200 + assert "text/html" in response.headers["content-type"] + + +def test_index_is_served_at_the_root(spa_api: TestClient) -> None: + response = spa_api.get("/") + assert response.status_code == 200 + assert "SPA INDEX" in response.text + + +def test_assets_are_served_by_path(spa_api: TestClient) -> None: + response = spa_api.get("/assets/app.js") + assert response.status_code == 200 + assert "console.log" in response.text + + +def test_extensionless_paths_fall_back_to_index(spa_api: TestClient) -> None: + response = spa_api.get("/episodes/abc123") + assert response.status_code == 200 + assert "SPA INDEX" in response.text + + +def test_missing_asset_files_are_404_not_index(spa_api: TestClient) -> None: + assert spa_api.get("/missing.png").status_code == 404 + + +def test_path_traversal_out_of_the_assets_tree_is_refused(spa_api: TestClient) -> None: + response = spa_api.get("/%2e%2e/secret.txt") + assert response.status_code == 404 + assert "do not serve this" not in response.text + + +def test_the_assets_environment_override_serves_a_local_build( + populated_workspace: PopulatedWorkspace, + built_assets_directory: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The lane the placeholder page tells frontend developers to use: no + # assets_dir pinned, HFLOW_UI_ASSETS pointing at a `pnpm build` output. + monkeypatch.setenv(ASSETS_ENVIRONMENT_VARIABLE, str(built_assets_directory)) + client = TestClient(create_app(ServerSettings(data_root=str(populated_workspace.data_root)))) + assert "SPA INDEX" in client.get("/").text + + +def test_a_pinned_assets_directory_beats_the_environment_override( + populated_workspace: PopulatedWorkspace, + built_assets_directory: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Precedence matters because `hflow serve` never pins: an override left in a + # developer's shell must not silently outrank an explicit setting. + pinned_directory = tmp_path / "pinned" + pinned_directory.mkdir() + (pinned_directory / "index.html").write_text("PINNED INDEX") + monkeypatch.setenv(ASSETS_ENVIRONMENT_VARIABLE, str(built_assets_directory)) + client = TestClient( + create_app( + ServerSettings( + data_root=str(populated_workspace.data_root), + assets_dir=pinned_directory, + ) + ) + ) + assert "PINNED INDEX" in client.get("/").text + + +def test_packaged_assets_are_looked_up_inside_the_installed_package( + populated_workspace: PopulatedWorkspace, monkeypatch: pytest.MonkeyPatch +) -> None: + """The branch every installed-wheel user takes: nothing pinned, no override. + + hflow_server/static/ is a build artifact (gitignored, written by the frontend + build), so this cannot assert a served page in either direction. What it + can pin is the resolution: the importlib.resources anchor must land beside + the installed hflow_server package, and the packaged branch must be taken + exactly when that directory exists. A wrong anchor or a changed wheel + layout degrades every real launch to the placeholder page, silently. + """ + monkeypatch.delenv(ASSETS_ENVIRONMENT_VARIABLE, raising=False) + settings = ServerSettings(data_root=str(populated_workspace.data_root)) + packaged_static = Path(str(hflow_server.__file__)).parent / "static" + + assert _assets_directory(settings) == (packaged_static if packaged_static.is_dir() else None) + + +def test_unknown_api_paths_are_json_404s(api: TestClient) -> None: + response = api.get("/api/v1/definitely-not-a-route") + assert response.status_code == 404 + assert "application/json" in response.headers["content-type"] + assert response.json()["detail"] diff --git a/packages/hflow-server/tests/ui_test_fixtures.py b/packages/hflow-server/tests/ui_test_fixtures.py new file mode 100644 index 0000000..a6214ae --- /dev/null +++ b/packages/hflow-server/tests/ui_test_fixtures.py @@ -0,0 +1,173 @@ +"""Workspace builders shared by the hflow-server suite. + +A uniquely-named module (never ``conftest``) so test modules can import the +types under any pytest import mode without basename collisions against the +repository's root test conftest. +""" + +import time +from dataclasses import dataclass +from pathlib import Path + +import pytest + +import hflow +from hflow.catalog import Catalog, CheckRunRow +from hflow.transform import EpisodeStamps + +PIPELINE_VERSION = "pipeline0000001" + +STAMPS = EpisodeStamps( + schema_version="1", + pipeline_version=PIPELINE_VERSION, + ffmpeg_version="ffmpeg version test", + robot_software_version="sim-0.1.0", +) + + +@dataclass(frozen=True) +class PopulatedWorkspace: + """One data root with four episodes covering the whole API surface.""" + + data_root: Path + ok_episode_id: str # fold_napkin/alice: two runs, media, measurements + quarantined_episode_id: str # pour_water/bob: quarantined with tags + escaping_episode_id: str # fold_napkin/carol: canonical + media OUTSIDE the root + minimal_episode_id: str # stack_blocks: no operator, no checks + contact_sheet_file: Path + outside_media_file: Path + canonical_file: Path + + +def build_populated_workspace(tmp_path_factory: pytest.TempPathFactory) -> PopulatedWorkspace: + data_root = tmp_path_factory.mktemp("ui-data-root") + outside_directory = tmp_path_factory.mktemp("outside-the-root") + episodes_directory = data_root / "episodes" + episodes_directory.mkdir() + media_directory = data_root / "media" + media_directory.mkdir() + catalog = Catalog(data_root / "catalog") + + contact_sheet_file = media_directory / "wrist_cam.jpg" + contact_sheet_file.write_bytes(b"\xff\xd8\xff\xe0 fake jpeg body \xff\xd9") + outside_media_file = outside_directory / "leaked.jpg" + outside_media_file.write_bytes(b"\xff\xd8\xff\xe0 outside the root \xff\xd9") + + def joint_check_row(max_velocity: float) -> CheckRunRow: + return CheckRunRow( + check_name="joint_check", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={ + "max_velocity": max_velocity, + # JSON-illegal doubles: the API must null these, not crash. + "nan_metric": float("nan"), + "inf_metric": float("inf"), + }, + tags=["seen"], + intervals=[hflow.Interval(start_ns=0, end_ns=100, label="span")], + ) + + contact_sheet_row = CheckRunRow( + check_name="media/contact_sheet", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={"artifact/wrist_cam": str(contact_sheet_file)}, + ) + + canonical_file = episodes_directory / "fold_a.canonical.mcap" + canonical_file.write_bytes(b"canonical fold napkin A") + ok_metadata = { + "task": "fold_napkin", + "operator": "alice", + "success": "true", + "embodiment": "arm-1", + } + first_append = catalog.append_episode( + canonical_path=canonical_file, + stamps=STAMPS, + episode_metadata=ok_metadata, + check_rows=[joint_check_row(1.5), contact_sheet_row], + ) + # Distinct recorded_at so "latest run" ordering stays deterministic. + time.sleep(0.01) + second_append = catalog.append_episode( + canonical_path=canonical_file, + stamps=STAMPS, + episode_metadata=ok_metadata, + check_rows=[joint_check_row(2.0), contact_sheet_row], + ) + assert first_append.written and second_append.written + assert first_append.episode_id == second_append.episode_id + time.sleep(0.01) + + quarantined_canonical = episodes_directory / "pour_b.canonical.mcap" + quarantined_canonical.write_bytes(b"canonical pour water B") + quarantined_append = catalog.append_episode( + canonical_path=quarantined_canonical, + stamps=STAMPS, + episode_metadata={ + "task": "pour_water", + "operator": "bob", + "success": "false", + "embodiment": "arm-1", + }, + check_rows=[ + CheckRunRow( + check_name="camera_blackout", + check_version="v1", + critical=True, + status=hflow.CheckStatus.FAILED, + duration_s=0.02, + measurements={"black_pct": 80.0}, + ) + ], + quarantine_tags=["failed:camera_blackout"], + ) + time.sleep(0.01) + + escaping_canonical = outside_directory / "escape_c.canonical.mcap" + escaping_canonical.write_bytes(b"canonical outside the data root C") + escaping_append = catalog.append_episode( + canonical_path=escaping_canonical, + stamps=STAMPS, + episode_metadata={"task": "fold_napkin", "operator": "carol", "embodiment": "arm-2"}, + check_rows=[ + CheckRunRow( + check_name="media/contact_sheet", + check_version="v1", + critical=False, + status=hflow.CheckStatus.MEASURED, + duration_s=0.01, + measurements={ + "artifact/outside": str(outside_media_file), + "artifact/missing": str(media_directory / "never_written.jpg"), + }, + ) + ], + ) + time.sleep(0.01) + + minimal_canonical = episodes_directory / "stack_d.canonical.mcap" + minimal_canonical.write_bytes(b"canonical stack blocks D") + minimal_append = catalog.append_episode( + canonical_path=minimal_canonical, + stamps=STAMPS, + episode_metadata={"task": "stack_blocks"}, + check_rows=[], + ) + + return PopulatedWorkspace( + data_root=data_root, + ok_episode_id=first_append.episode_id, + quarantined_episode_id=quarantined_append.episode_id, + escaping_episode_id=escaping_append.episode_id, + minimal_episode_id=minimal_append.episode_id, + contact_sheet_file=contact_sheet_file, + outside_media_file=outside_media_file, + canonical_file=canonical_file, + ) diff --git a/pyproject.toml b/pyproject.toml index 5ce1788..af3e490 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,10 @@ hflow = "hflow.cli:main" [dependency-groups] dev = [ + "hflow-server", # the workspace UI package, present in dev so the suite tests it + # Starlette's TestClient transport for the hflow-server suite. httpx2, not + # httpx: starlette 1.6 deprecates the httpx transport in its favour. + "httpx2>=2.10", "obstore>=0.10.0", # the bucket extra, present in dev so the suite tests it "pytest>=9.1.1", "pyyaml>=6.0.3", @@ -65,8 +69,14 @@ build-backend = "uv_build" # trusted first-party packages. exclude-newer = "5 days" +[tool.uv.workspace] +members = ["packages/*"] + +[tool.uv.sources] +hflow-server = { workspace = true } + [tool.pytest.ini_options] -testpaths = ["tests"] +testpaths = ["tests", "packages/hflow-server/tests"] [tool.ruff] line-length = 100 diff --git a/src/hflow/__init__.py b/src/hflow/__init__.py index cfb8b33..1a2dc1b 100644 --- a/src/hflow/__init__.py +++ b/src/hflow/__init__.py @@ -13,6 +13,7 @@ CheckStatus, EnrichmentRunReport, TestReport, + import_pipeline_application, ) from hflow.batching import PlannedBatch, plan_batches, plan_batches_from_files from hflow.catalog import AppendResult, Catalog, CheckRunRow @@ -125,6 +126,7 @@ "diagnose", "fetch_uri", "ffmpeg", + "import_pipeline_application", "is_bucket_url", "open_catalog_connection", "open_reader", diff --git a/src/hflow/app.py b/src/hflow/app.py index 8f157d5..070bce1 100644 --- a/src/hflow/app.py +++ b/src/hflow/app.py @@ -37,8 +37,8 @@ AppendResult, Catalog, CheckRunRow, + QuarantineHistory, content_episode_id, - latest_quarantine, ) from hflow.episode import Episode, _sanitize_topic from hflow.ffmpeg import contact_sheet @@ -122,6 +122,9 @@ def _resolve_data_root(data_root: "Path | str | StorageRoot | None") -> "Path | # contact sheet per camera topic, recorded exactly like an enrichment so its # catalog rows flow through CheckRunRow like everything else. MEDIA_CONTACT_SHEET_STEP_NAME = "media/contact_sheet" +# Published artifacts are recorded as measurements under this prefix, so a +# reader can tell "here is where the file went" from an ordinary label. +ARTIFACT_MEASUREMENT_KEY_PREFIX = "artifact/" _MEDIA_CONTACT_SHEET_FPS = 0.5 _SYNC_COMPLETION_MARKER_NAME = ".sync-complete.json" @@ -549,7 +552,10 @@ def summary(self) -> str: artifact_location = enrichment_run.artifact_uris.get( artifact_name, str(artifact_path) ) - lines.append(f" artifact/{artifact_name} = {artifact_location}") + lines.append( + f" {ARTIFACT_MEASUREMENT_KEY_PREFIX}{artifact_name} = " + f"{artifact_location}" + ) return "\n".join(lines) def __str__(self) -> str: @@ -1045,6 +1051,7 @@ def process( verbose: bool = False, record: bool = True, stages: Iterable[Stage] | str | None = None, + quarantine_history: QuarantineHistory | None = None, ) -> TestReport: """Process one episode through the enabled stages of the stage graph: transform to canonical (``sync``), run checks with gate @@ -1063,6 +1070,10 @@ def process( stamps are reconstructed from its own provenance record. Without ``meta``, the quarantine gate for ``labels``/``media`` comes from the episode's latest cataloged state (no catalog = no known quarantine). + + ``quarantine_history`` is that gate's catalog reader, open across a + whole batch so a stage does not re-sync and re-open the catalog once + per episode; omit it and this call opens one for itself. """ enabled_stages = _resolve_stages(stages) source_identifier = _source_identity(episode, self.storage_root) @@ -1242,11 +1253,14 @@ def process( # A cataloged quarantine is carried into this run's tags so a # recorded run without meta never masks the state. if Stage.META not in enabled_stages: - cataloged_quarantine = latest_quarantine( - self.workspace.catalog_root, content_episode_id(canonical_path) - ) - if cataloged_quarantine is not None and cataloged_quarantine.quarantined: - report.quarantine_tags.extend(cataloged_quarantine.tags) + episode_id = content_episode_id(canonical_path) + if quarantine_history is not None: + carried_tags = quarantine_history.quarantine_tags(episode_id) + else: + with QuarantineHistory(self.workspace.catalog_root) as history: + carried_tags = history.quarantine_tags(episode_id) + if carried_tags is not None: + report.quarantine_tags.extend(carried_tags) quarantine_skip_reason = ( f"episode quarantined ({', '.join(report.quarantine_tags)})" if report.quarantined @@ -1349,7 +1363,7 @@ def render_contact_sheets(media_episode: Episode) -> EnrichmentResult: if enrichment_result is not None: labels.update( { - f"artifact/{artifact_name}": artifact_uri + f"{ARTIFACT_MEASUREMENT_KEY_PREFIX}{artifact_name}": artifact_uri for artifact_name, artifact_uri in enrichment_run.artifact_uris.items() } ) @@ -1378,3 +1392,44 @@ def render_contact_sheets(media_episode: Episode) -> EnrichmentResult: if verbose: print(report.summary()) return report + + +def parse_pipeline_spec(pipeline_spec: str) -> tuple[Path, str]: + """Split ``path/to/pipeline.py[:app_variable]`` (default variable: ``app``).""" + path_part, separator, variable_part = pipeline_spec.rpartition(":") + if separator and path_part and variable_part.isidentifier(): + return Path(path_part), variable_part + return Path(pipeline_spec), "app" + + +def import_pipeline_application(pipeline_spec: str) -> "App": + """Import ``path/to/pipeline.py[:app]`` and return its :class:`App`, loudly. + + One owner for the "address a pipeline by file" contract every vantage + needs -- the CLI's ``manifest``/``up``/``deploy``/``stale``, and any other + caller that must hold a user's pipeline (the workspace UI's pipeline + page). The pipeline file is arbitrary user code, so importing EXECUTES + it: any exception it raises is a boundary failure reported as a + ``ValueError`` naming the file, never a crash of the calling program. + """ + import importlib.util + + pipeline_file, app_variable = parse_pipeline_spec(pipeline_spec) + spec = importlib.util.spec_from_file_location("hflow_user_pipeline", pipeline_file) + if spec is None or spec.loader is None: + raise ValueError(f"cannot import pipeline file {pipeline_file}") + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except (Exception, SystemExit) as error: + # SystemExit is a BaseException: a pipeline that guards its config at + # import time (sys.exit("set ROBOT_FLEET"), or a module-scope argparse) + # would otherwise walk past `except Exception` and take the calling + # program's exit status with it -- killing a long-lived UI server at + # startup. KeyboardInterrupt stays uncaught on purpose: that one + # belongs to whoever pressed it, not to the pipeline file. + raise ValueError(f"importing {pipeline_file} failed: {error}") from error + application = getattr(module, app_variable, None) + if not isinstance(application, App): + raise ValueError(f"{pipeline_file} has no hflow.App named {app_variable!r}") + return application diff --git a/src/hflow/behavior.py b/src/hflow/behavior.py new file mode 100644 index 0000000..23af62f --- /dev/null +++ b/src/hflow/behavior.py @@ -0,0 +1,61 @@ +"""Processing-behavior versions: what actually changes a corpus's identity. + +HFlow's identities are content hashes, and until identity epoch 2 they +folded in ``hflow.__version__`` -- the RELEASE number. That made every +release, however unrelated to processing, change three things at once: +``pipeline_version`` (so ``hflow stale`` listed the whole corpus), +``episode_id`` (because the pipeline version is stamped inside the canonical +bytes the id hashes, breaking content-addressed dedupe), and every step +version whose function happened to reference the ``hflow`` module. A CLI fix +or a docs-driven version bump invalidated a petabyte. + +The release number is a bad proxy for "does this build process data +differently". This module holds the honest answer instead: a version that a +maintainer bumps DELIBERATELY when canonicalization semantics change -- when +the same input would now produce different canonical bytes. + +**Bump ``TRANSFORM_BEHAVIOR_VERSION`` when, and only when, a change makes the +transform write different bytes for the same input**: encoder settings or +defaults, chunking and grouping, timestamp handling, the provenance record's +shape, or a fix to any of them. Bumping it re-versions every corpus exactly +once, which is the cost of telling the truth; NOT bumping it when behavior +changed silently mixes two behaviors under one version, which is worse. When +in doubt, bump. + +Deliberately NOT here: an "analysis" behavior version covering what a check +observes. Step versions must not fold in an engine-wide constant, because +``@app.derive`` channel versions flow into ``compute_pipeline_version`` and +therefore into ``episode_id`` -- one engine-wide analysis bump would churn +every derived-channel user's episode identities, reintroducing the defect +this module exists to remove. + +That leaves a known hole, stated rather than papered over: a change to a +built-in check's algorithm is caught in the step's own content hash only if +the step's source references the built-in as a FUNCTION +(``from hflow.checks import timestamp_regularity``), because a step hash +folds in a referenced function's source. The idiom the docs and examples +teach reaches it through the module (``hflow.checks.timestamp_regularity``), +and a module contributes only its NAME to a step hash -- so for the common +spelling the change is not caught at all. Two exceptions to that live in +:func:`hflow.transform.compute_pipeline_version`, which folds in +``RESAMPLE_POLICY_VERSION`` for episodes that have derived channels: the +resample policy decides those samples and no step hash can see it. + +The release number is not lost, only demoted from identity to provenance: +:attr:`hflow.PipelineManifest.hflow_version` and the rendered bundle's +``hflow-bundle.json`` both record which build produced a pipeline. + +Identity epochs live here as prose, not as a constant: no build stamps an +epoch into a corpus, so a module attribute nothing reads would be a comment +wearing a type annotation. Epoch 1 folded ``hflow.__version__`` into +``pipeline_version``, ``episode_id``, and step versions; epoch 2 -- what this +build produces -- folds in only author-owned facts plus +``TRANSFORM_BEHAVIOR_VERSION``. Written down so a reader can explain the +one-time re-version between them rather than guess at it. +""" + +# Canonicalization semantics: bump when the transform would write different +# bytes for the same input. See the module docstring for the rule. Annotated +# as ``str`` rather than inferred as a literal: the whole point is that it +# changes. +TRANSFORM_BEHAVIOR_VERSION: str = "2" diff --git a/src/hflow/catalog.py b/src/hflow/catalog.py index 534d893..4696d33 100644 --- a/src/hflow/catalog.py +++ b/src/hflow/catalog.py @@ -152,68 +152,98 @@ class AppendResult: written: bool # False when this exact run was already recorded -@dataclass(frozen=True) -class LatestQuarantine: - """The quarantine facts of an episode's most recent cataloged run.""" +class QuarantineHistory: + """Which episodes the catalog last recorded as quarantined, read once. + + The labels and media stages gate on this per episode, and the ``episodes`` + table is append-only with one parquet file per append -- so asking it one + episode at a time costs one scan of a file set that grows with the corpus, + per episode, plus one bucket-mirror sync each. Reading every quarantined + episode in a single pass makes a whole stage batch pay that once. + + Scope is deliberately the batch, not the process: the snapshot is taken + when the first lookup arrives, which is what the gate wants. The meta + stage that decides quarantine has already finished by the time labels or + media reads it, and the rows a stage appends for its own episodes as it + runs cannot change another episode's answer. + + Only quarantined episodes are held. A clean episode and an unknown one + lead the gate to the same decision -- proceed -- so carrying the rest of + the corpus in memory would be paying to learn nothing. + """ - quarantined: bool - tags: list[str] + def __init__(self, catalog_root: "Path | str | StorageRoot") -> None: + location = parse_storage_root(catalog_root) + match location: + case LocalStorageRoot(path=local_root): + self._episodes_dir = local_root / "episodes" + case BucketStorageRoot(): + location.sync_into_mirror(("episodes",)) + self._episodes_dir = location.mirror / "episodes" + self._tags_by_episode: dict[str, list[str]] | None = None + def quarantine_tags(self, episode_id: str) -> list[str] | None: + """The tags of the quarantine on ``episode_id``'s latest recorded run, + or ``None`` when that run left it clean -- or when the catalog has + never seen it, which the gate treats identically. -def latest_quarantine( - catalog_root: "Path | str | StorageRoot", episode_id: str -) -> LatestQuarantine | None: - """The latest recorded quarantine facts for ``episode_id``, or ``None`` - when the catalog (or the episode) is unknown. - - "Latest" follows the ``episodes_latest`` view semantics: most recent - ``recorded_at``, ties broken by ``run_fingerprint``. Bucket catalogs sync - the episodes table into their mirror first (only files the mirror lacks - download; table files are append-only and content-named). - """ - location = parse_storage_root(catalog_root) - match location: - case LocalStorageRoot(path=local_root): - episodes_dir = local_root / "episodes" - case BucketStorageRoot(): - location.sync_into_mirror(("episodes",)) - episodes_dir = location.mirror / "episodes" - if not episodes_dir.is_dir() or not any(episodes_dir.glob("*.parquet")): - return None - glob_pattern = str(episodes_dir / "*.parquet").replace("'", "''") - connection = duckdb.connect() - try: - row = connection.execute( - f""" - SELECT quarantined, quarantine_tags_json - FROM read_parquet('{glob_pattern}', union_by_name=true) - WHERE episode_id = ? - ORDER BY recorded_at DESC, run_fingerprint DESC - LIMIT 1 - """, - [episode_id], - ).fetchone() - finally: - connection.close() - if row is None: - return None - quarantined, tags_json = row - return LatestQuarantine( - quarantined=bool(quarantined), - tags=[str(tag) for tag in json.loads(tags_json)] if tags_json else [], - ) + "Latest" follows the ``episodes_latest`` view semantics: most recent + ``recorded_at``, ties broken by ``run_fingerprint``. + """ + if self._tags_by_episode is None: + self._tags_by_episode = self._read_quarantined_episodes() + return self._tags_by_episode.get(episode_id) + + def _read_quarantined_episodes(self) -> dict[str, list[str]]: + if not self._episodes_dir.is_dir() or not any(self._episodes_dir.glob("*.parquet")): + return {} + glob_pattern = str(self._episodes_dir / "*.parquet").replace("'", "''") + connection = duckdb.connect() + try: + # QUALIFY picks each episode's newest row first; the outer WHERE + # then keeps only the ones that row calls quarantined. Filtering + # before the window would instead find the newest QUARANTINED row, + # resurrecting a quarantine that a later clean run had cleared. + rows = connection.execute( + f""" + SELECT episode_id, quarantine_tags_json FROM ( + SELECT episode_id, quarantined, quarantine_tags_json + FROM read_parquet('{glob_pattern}', union_by_name=true) + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY episode_id + ORDER BY recorded_at DESC, run_fingerprint DESC + ) = 1 + ) + WHERE quarantined + """ + ).fetchall() + finally: + connection.close() + return { + str(episode_id): [str(tag) for tag in json.loads(tags_json)] if tags_json else [] + for episode_id, tags_json in rows + } + def close(self) -> None: + """Drop the snapshot, so a reused history re-reads the table.""" + self._tags_by_episode = None -def latest_quarantine_state( - catalog_root: "Path | str | StorageRoot", episode_id: str -) -> bool | None: - """Whether ``episode_id``'s most recent cataloged run left it quarantined. + def __enter__(self) -> "QuarantineHistory": + return self + + def __exit__(self, *_exception: object) -> None: + self.close() - ``None`` when the catalog or the episode is unknown -- no catalog means - no known quarantine, and callers proceed. + +def latest_quarantine_tags( + catalog_root: "Path | str | StorageRoot", episode_id: str +) -> list[str] | None: + """One episode's carried-forward quarantine tags (see + :meth:`QuarantineHistory.quarantine_tags`). Reuse a + :class:`QuarantineHistory` when asking about more than one episode. """ - latest = latest_quarantine(catalog_root, episode_id) - return None if latest is None else latest.quarantined + with QuarantineHistory(catalog_root) as history: + return history.quarantine_tags(episode_id) def content_episode_id(canonical_path: Path) -> str: diff --git a/src/hflow/cli.py b/src/hflow/cli.py index 0909de5..f35a473 100644 --- a/src/hflow/cli.py +++ b/src/hflow/cli.py @@ -1,9 +1,16 @@ """Command-line entry point. Subcommands: ``curate``, ``stale``, ``doctor``, ``manifest``, the Compose -runtime family ``up``/``down``/``ingest``/``status``, and ``deploy`` for -bring-your-own Airflow. Everything the CLI does is a thin call into the -library: no behavior lives only here. +runtime family ``up``/``down``/``ingest``/``status``, ``deploy`` for +bring-your-own Airflow, and ``serve`` for the workspace HTTP server (a +separate ``hflow-server`` package, imported only when invoked). Everything +the CLI does is a thin call into the library: no behavior lives only here. + +Two of these start long-running processes and they are not the same thing: +``up`` brings up the RUNTIME that processes episodes (an Airflow stack in +Docker), while ``serve`` serves the WORKSPACE over HTTP -- one process that +reads the data root and can trigger a run on a runtime, but executes nothing +itself. Either is useful without the other. ``ingest`` and ``status`` address either a LOCAL rendered bundle (the default: ``--bundle-dir`` or its auto-discovery) or a REMOTE runtime by URL @@ -19,7 +26,7 @@ from typing import TYPE_CHECKING from hflow import __version__ -from hflow.app import DATA_ROOT_ENVIRONMENT_VARIABLE, DEFAULT_DATA_ROOT +from hflow.app import DATA_ROOT_ENVIRONMENT_VARIABLE, DEFAULT_DATA_ROOT, parse_pipeline_spec from hflow.curation import curate, stale_episodes from hflow.doctor import diagnose from hflow.runtime._deploy import DEFAULT_DEPLOY_VENV_PYTHON @@ -35,6 +42,9 @@ # Mirrors RuntimeConfig.api_port; kept here so the parser can state it without # importing the runtime package, which `up` defers until it actually runs. DEFAULT_API_PORT = 8080 +# The workspace UI's fixed default port ("HFLO" on a phone keypad); stated +# here so the parser needs no import from the optional hflow-server package. +DEFAULT_SERVER_PORT = 4356 def _environment_data_root() -> str: @@ -341,6 +351,53 @@ def _build_parser() -> argparse.ArgumentParser: ), ) _add_remote_endpoint_arguments(status_parser) + + serve_parser = subparsers.add_parser( + "serve", + help=( + "serve this workspace over HTTP: a JSON API over the catalog, and any " + "UI assets installed (requires the hflow-server package). Distinct from " + "`up`, which starts the runtime that PROCESSES episodes -- this only " + "reads the data root, and can trigger a run on a runtime that exists." + ), + ) + serve_parser.add_argument( + "--data-root", + default=_environment_data_root(), + help=( + f"workspace data root to browse (default: $HFLOW_DATA_ROOT, else {DEFAULT_DATA_ROOT})" + ), + ) + serve_parser.add_argument( + "--host", + default="127.0.0.1", + help="bind address (default 127.0.0.1; widening past loopback exposes your corpus)", + ) + serve_parser.add_argument( + "--port", + type=int, + default=DEFAULT_SERVER_PORT, + help=f"port to serve on (default {DEFAULT_SERVER_PORT}; auto-retries upward when taken)", + ) + serve_parser.add_argument( + "--no-browser", + action="store_true", + help="do not open a browser after starting (headless use)", + ) + serve_parser.add_argument( + "--read-only", + action="store_true", + help="viewer mode: hide and refuse manifest pinning, saved-query edits, and run triggering", + ) + serve_parser.add_argument( + "--pipeline", + default=None, + help=( + "pipeline file for the Pipeline page, optionally with the App variable " + "name: path/to/pipeline.py[:app]; importing EXECUTES the file, exactly " + "like `hflow manifest`" + ), + ) return parser @@ -398,38 +455,16 @@ def _resolve_bundle_dir(bundle_dir_argument: Path | None) -> Path: return candidates[0] -def _parse_pipeline_spec(pipeline_spec: str) -> tuple[Path, str]: - """Split ``path/to/pipeline.py[:app_variable]`` (default variable: ``app``).""" - path_part, separator, variable_part = pipeline_spec.rpartition(":") - if separator and path_part and variable_part.isidentifier(): - return Path(path_part), variable_part - return Path(pipeline_spec), "app" - - def _import_pipeline_app(pipeline_spec: str) -> "App": """Import ``path/to/pipeline.py[:app]`` and return its App, loudly. - The pipeline file is arbitrary user code: any exception it raises is a - boundary failure of the calling command (reported as a ``ValueError`` - naming the file), never a crash. + The library owns the contract (:func:`hflow.app.import_pipeline_application`) + so every vantage that addresses a pipeline by file -- these commands and + the workspace UI -- resolves it identically. """ - import importlib.util - - from hflow.app import App + from hflow.app import import_pipeline_application - pipeline_file, app_variable = _parse_pipeline_spec(pipeline_spec) - spec = importlib.util.spec_from_file_location("hflow_user_pipeline", pipeline_file) - if spec is None or spec.loader is None: - raise ValueError(f"cannot import pipeline file {pipeline_file}") - module = importlib.util.module_from_spec(spec) - try: - spec.loader.exec_module(module) - except Exception as error: - raise ValueError(f"importing {pipeline_file} failed: {error}") from error - app = getattr(module, app_variable, None) - if not isinstance(app, App): - raise ValueError(f"{pipeline_file} has no hflow.App named {app_variable!r}") - return app + return import_pipeline_application(pipeline_spec) def _command_manifest(arguments: argparse.Namespace) -> int: @@ -487,7 +522,7 @@ def _command_up(arguments: argparse.Namespace) -> int: started_summary, ) - pipeline_file, app_variable = _parse_pipeline_spec(arguments.pipeline) + pipeline_file, app_variable = parse_pipeline_spec(arguments.pipeline) hflow_source = ( arguments.hflow_source if arguments.hflow_source is not None else infer_hflow_source() ) @@ -553,7 +588,7 @@ def print_progress_to_stderr(message: str) -> None: def _command_deploy(arguments: argparse.Namespace) -> int: from hflow.runtime._deploy import DeployConfig, render_deploy_bundle - pipeline_file, app_variable = _parse_pipeline_spec(arguments.pipeline) + pipeline_file, app_variable = parse_pipeline_spec(arguments.pipeline) try: config = DeployConfig( pipeline_file=pipeline_file, @@ -716,6 +751,29 @@ def _command_doctor(arguments: argparse.Namespace) -> int: return exit_code +def _command_serve(arguments: argparse.Namespace) -> int: + try: + from hflow_server import ServerSettings, serve + except ImportError: + print( + "serve: the workspace server ships as a separate package so pipeline " + "workers never carry it; install it with `uv add hflow-server` " + "(or `pip install hflow-server`)", + file=sys.stderr, + ) + return 2 + settings = ServerSettings( + data_root=arguments.data_root, + host=arguments.host, + port=arguments.port, + open_browser=not arguments.no_browser, + read_only=arguments.read_only, + pipeline=arguments.pipeline, + ) + serve(settings) + return 0 + + def main(argv: list[str] | None = None) -> int: arguments = _build_parser().parse_args(argv) @@ -743,6 +801,8 @@ def main(argv: list[str] | None = None) -> int: return _command_ingest(arguments) if arguments.command == "status": return _command_status(arguments) + if arguments.command == "serve": + return _command_serve(arguments) raise AssertionError(f"unhandled command {arguments.command!r}") diff --git a/src/hflow/format.py b/src/hflow/format.py index 922519a..1bdb571 100644 --- a/src/hflow/format.py +++ b/src/hflow/format.py @@ -49,7 +49,9 @@ # Multi-rate alignment is where format converters silently diverge # (docs/ARCHITECTURE.md, "Transform"), so the policy is explicit and # versioned: bump this when the grid or selection semantics change. -RESAMPLE_POLICY_VERSION = "1" +# Annotated as ``str`` rather than inferred as a literal, for the same reason +# TRANSFORM_BEHAVIOR_VERSION is: the whole point is that it changes. +RESAMPLE_POLICY_VERSION: str = "1" # ``provenance/v1`` keys written when derived channels are present: # ``derived/`` holds each derived channel's content-hash version, and diff --git a/src/hflow/mcap_writer.py b/src/hflow/mcap_writer.py index fb000a5..57ecddd 100644 --- a/src/hflow/mcap_writer.py +++ b/src/hflow/mcap_writer.py @@ -59,23 +59,27 @@ SummaryOffset, ) +from hflow.behavior import TRANSFORM_BEHAVIOR_VERSION from hflow.format import DEFAULT_CHUNK_SIZE_BYTES, EPISODE_FORMAT_VERSION MCAP0_MAGIC = struct.pack("<8B", 137, 77, 67, 65, 80, 48, 13, 10) def _default_library_identifier() -> str: - """Project name + package version + episode format version. + """Project name + episode format version + transform behavior version. Informational only (MCAP Header ``library`` field): stored-data identifiers stay neutral per ``hflow.format``. """ - # Local import: hflow/__init__.py imports this module, so a top-level - # import would be circular. By the time a writer is constructed the - # package is fully imported. - from hflow import __version__ as package_version - - return f"hflow/{package_version} episode-format/{EPISODE_FORMAT_VERSION}" + # Deliberately free of the release number: this string is written into + # the MCAP header, so it is part of the bytes content_episode_id hashes. + # Embedding hflow.__version__ here gave a byte-identical input a new + # episode identity on every release, breaking content-addressed dedupe. + # The behavior version changes only when canonicalization does. + return ( + f"hflow episode-format/{EPISODE_FORMAT_VERSION} " + f"transform-behavior/{TRANSFORM_BEHAVIOR_VERSION}" + ) class _GroupChunkBuilder: diff --git a/src/hflow/runtime/__init__.py b/src/hflow/runtime/__init__.py index 2156e3b..fa94330 100644 --- a/src/hflow/runtime/__init__.py +++ b/src/hflow/runtime/__init__.py @@ -48,6 +48,13 @@ start_runtime, started_summary, ) +from hflow.runtime._topology import ( + DagTaskNode, + DagTopology, + IngestTopology, + StageTopology, + ingest_dag_topology, +) __all__ = [ "DEFAULT_AIRFLOW_IMAGE", @@ -58,11 +65,15 @@ "BearerToken", "BundlePaths", "ComposeError", + "DagTaskNode", + "DagTopology", "DeployConfig", "DeployPaths", + "IngestTopology", "PasswordCredentials", "RemoteRuntimeEndpoint", "RuntimeConfig", + "StageTopology", "bundle_dag_ids", "client_for_bundle", "client_for_endpoint", @@ -73,6 +84,7 @@ "describe_remote_status", "describe_runtime_status", "infer_hflow_source", + "ingest_dag_topology", "load_bundle", "render_bundle", "render_deploy_bundle", diff --git a/src/hflow/runtime/_client.py b/src/hflow/runtime/_client.py index 431a2dd..7ec8772 100644 --- a/src/hflow/runtime/_client.py +++ b/src/hflow/runtime/_client.py @@ -17,6 +17,7 @@ import json import time import urllib.error +import urllib.parse import urllib.request from collections.abc import Callable from dataclasses import dataclass @@ -75,6 +76,17 @@ def summary(self) -> str: return ", ".join(parts) +def _dag_run_path(dag_id: str, dag_run_id: str) -> str: + """The one owner of run-id-to-URL encoding: every per-run endpoint uses it. + + Run ids carry ':' and '+' (manual__2026-08-22T03:06:55+00:00), and a + caller-supplied idempotency id may carry '#' or '?' -- which urllib reads + as a fragment or a query and silently drops from the path, so two calls + with the same id would address two different runs. + """ + return f"/api/v2/dags/{dag_id}/dagRuns/{urllib.parse.quote(dag_run_id, safe='')}" + + class AirflowClient: """Minimal typed client for the deployment endpoints the SDK needs. @@ -262,7 +274,7 @@ def dag(self, dag_id: str) -> dict[str, Any]: return self._authenticated("GET", f"/api/v2/dags/{dag_id}") def dag_run(self, dag_id: str, dag_run_id: str) -> dict[str, Any]: - return self._authenticated("GET", f"/api/v2/dags/{dag_id}/dagRuns/{dag_run_id}") + return self._authenticated("GET", _dag_run_path(dag_id, dag_run_id)) def dag_runs( self, dag_id: str, *, limit: int = 100, order_by: str | None = None @@ -278,6 +290,20 @@ def dag_runs( runs = response.get("dag_runs") return runs if isinstance(runs, list) else [] + def task_instances(self, dag_id: str, dag_run_id: str) -> list[dict[str, Any]]: + """Every task instance of one run, as the API returns them. + + Includes one entry per dynamically mapped instance (``process_batch`` + fans out over the planned batches), distinguished by ``map_index``; + ``-1`` means the task was not mapped. What a caller reads from each + entry -- state, timings, try number -- is Airflow's vocabulary, not + HFlow's: this is a thin pass-through so a UI can colour the task graph + :func:`hflow.runtime.ingest_dag_topology` describes. + """ + response = self._authenticated("GET", f"{_dag_run_path(dag_id, dag_run_id)}/taskInstances") + task_instances = response.get("task_instances") + return task_instances if isinstance(task_instances, list) else [] + def unpause_dag(self, dag_id: str) -> dict[str, Any]: return self._authenticated("PATCH", f"/api/v2/dags/{dag_id}", {"is_paused": False}) @@ -288,6 +314,7 @@ def ingest( *, profile: str = "full", online: bool = False, + batch_count: int | None = None, dag_run_id: str | None = None, ) -> dict[str, Any]: """Trigger the MASTER ingest DAG over ``uris`` (the SDK/CLI entry point). @@ -297,12 +324,27 @@ def ingest( only the enabled stage sub-DAGs). ``online`` selects the latency-first trigger lane -- the sub-DAGs process the uris as one immediate batch, no bin-packing, no stagger -- instead of the default - staggered batch lane. Supply ``dag_run_id`` when the caller may retry + staggered batch lane. ``batch_count`` overrides the master's own + bin-packing for the batch lane (ignored by the online lane, which is + always one batch). Supply ``dag_run_id`` when the caller may retry (see :meth:`trigger_dag_run` for the idempotency contract). + + This method owns the trigger conf's shape: every caller -- the CLI, + the workspace UI, a control plane -- goes through it rather than + rebuilding the dict. Owning the shape includes refusing a value the + run cannot honour: ``batch_count`` below 1 raises here rather than + reaching the sub-DAG's ``plan`` task, which would fail the run after + it exists and leave it in the operator's history. """ - conf = { + if batch_count is not None and batch_count < 1: + # Same wording as hflow.batching.plan_batches, the task-side owner + # of this invariant, so both entry points say the same thing. + raise ValueError(f"batch_count must be >= 1, got {batch_count}") + conf: dict[str, Any] = { "uris": uris, "profile": profile, "mode": "online" if online else "batch", } + if batch_count is not None: + conf["batch_count"] = batch_count return self.trigger_dag_run(dag_id, conf=conf, dag_run_id=dag_run_id) diff --git a/src/hflow/runtime/_topology.py b/src/hflow/runtime/_topology.py new file mode 100644 index 0000000..00404d0 --- /dev/null +++ b/src/hflow/runtime/_topology.py @@ -0,0 +1,201 @@ +"""The generated ingest DAGs' shape, as data. + +The templates in :mod:`hflow.runtime._templates` render the master DAG and its +four stage sub-DAGs; this module states the same task graph as inspectable +values, so a UI, a control plane, or a doc generator can draw what a bundle +will do without parsing generated Python. The templates remain the +implementation; :func:`ingest_dag_topology` is the description, and +``tests/test_runtime_topology.py`` pins the two together by checking every +task id here against the rendered source. + +Two layers meet in these DAGs, and the distinction matters to anyone +rendering them: + +- **Orchestration** (here): real dependency edges. The master validates the + conf, then walks the stage chain, gating each stage on the run profile and + waiting for its sub-DAG. Each sub-DAG plans batches, fans out over them, + and closes with a budget gate. +- **User steps** (:class:`hflow.PipelineManifest`): registered checks and + enrichments, which have NO dependency edges on each other. They run inside + one ``process_batch`` task of the stage that owns their kind. +""" + +from dataclasses import dataclass, field + +from hflow.runtime._bundle import sub_dag_id_for_stage +from hflow.steps import RUN_PROFILES, Stage + +# The master's first task: validates the trigger conf against the vocabulary +# baked into the bundle and publishes the enabled stage list. +RESOLVE_PROFILE_TASK_ID = "resolve_profile" + +# Per stage, the master renders a skip gate and a deferred trigger. +STAGE_GATE_TASK_PREFIX = "enabled_" +STAGE_TRIGGER_TASK_PREFIX = "trigger_" + +# Every sub-DAG's shape: bin-pack, fan out, then gate on the budget. +PLAN_TASK_ID = "plan" +PROCESS_BATCH_TASK_ID = "process_batch" +QUARANTINE_BUDGET_GATE_TASK_ID = "quarantine_budget_gate" +ERROR_BUDGET_GATE_TASK_ID = "error_budget_gate" + + +def budget_gate_task_id(stage: Stage) -> str: + """The gate task closing one stage's sub-DAG. + + Meta owns the quarantine budget because it is the stage that runs checks + and therefore the only one that can quarantine; the others gate on the + error budget alone. Mirrors the template selection in + ``_bundle.render_sub_dag_source``, exhaustively and for the same reason: + a new stage must make its author choose a gate here too, rather than + inheriting the error budget from an ``else``. + """ + match stage: + case Stage.META: + return QUARANTINE_BUDGET_GATE_TASK_ID + case Stage.SYNC | Stage.LABELS | Stage.MEDIA: + return ERROR_BUDGET_GATE_TASK_ID + + +@dataclass(frozen=True) +class DagTaskNode: + """One task in a generated DAG.""" + + task_id: str + # What the task does, in the vocabulary a reader of the UI has -- not + # Airflow's operator names. + summary: str + # True for the dynamically mapped task: one instance per planned batch, + # so a renderer draws it as a fan-out rather than a single box. + mapped: bool = False + # True when the task defers (releases its worker slot while waiting); + # a renderer should say "waiting", never "stalled". + deferred: bool = False + + +@dataclass(frozen=True) +class DagTopology: + """One generated DAG: its tasks and the edges between them.""" + + dag_id: str + tasks: tuple[DagTaskNode, ...] + # (upstream task id, downstream task id) pairs, declaration order. + edges: tuple[tuple[str, str], ...] + + +@dataclass(frozen=True) +class StageTopology: + """One stage sub-DAG, plus how the master reaches it.""" + + stage: Stage + dag: DagTopology + # The master's tasks that gate and trigger this stage. + gate_task_id: str + trigger_task_id: str + # The run profiles that enable this stage; a profile outside this set + # skips the stage at its gate. + enabling_profiles: tuple[str, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class IngestTopology: + """The whole rendered bundle's task graph: master plus stage sub-DAGs.""" + + master: DagTopology + stages: tuple[StageTopology, ...] + + +def _sub_dag_topology(master_dag_id: str, stage: Stage) -> DagTopology: + gate_task_id = budget_gate_task_id(stage) + return DagTopology( + dag_id=sub_dag_id_for_stage(master_dag_id, stage), + tasks=( + DagTaskNode( + task_id=PLAN_TASK_ID, + summary="bin-pack the trigger's uris into batches (online: one immediate batch)", + ), + DagTaskNode( + task_id=PROCESS_BATCH_TASK_ID, + summary=f"run the {stage.value} stage over every episode in one batch", + mapped=True, + ), + DagTaskNode( + task_id=gate_task_id, + summary=( + "fail the run when quarantines exceed the budget" + if gate_task_id == QUARANTINE_BUDGET_GATE_TASK_ID + else "fail the run when errors exceed the budget" + ), + ), + ), + edges=( + (PLAN_TASK_ID, PROCESS_BATCH_TASK_ID), + (PROCESS_BATCH_TASK_ID, gate_task_id), + ), + ) + + +def _enabling_profiles(stage: Stage) -> tuple[str, ...]: + return tuple(sorted(name for name, stages in RUN_PROFILES.items() if stage in stages)) + + +def ingest_dag_topology(master_dag_id: str) -> IngestTopology: + """The task graph a bundle rendered for ``master_dag_id`` will run. + + Derived from the same facts the renderer uses (the stage vocabulary, the + sub-DAG id derivation, and the per-stage gate choice), so the description + cannot drift from the bundle as long as the pinning test passes. + """ + master_tasks: list[DagTaskNode] = [ + DagTaskNode( + task_id=RESOLVE_PROFILE_TASK_ID, + summary="validate the run profile and mode; publish the enabled stages", + ) + ] + master_edges: list[tuple[str, str]] = [] + stages: list[StageTopology] = [] + + previous_trigger_task_id: str | None = None + for stage in Stage: + gate_task_id = f"{STAGE_GATE_TASK_PREFIX}{stage.value}" + trigger_task_id = f"{STAGE_TRIGGER_TASK_PREFIX}{stage.value}" + master_tasks.append( + DagTaskNode( + task_id=gate_task_id, + summary=f"skip {stage.value} when the run profile disables it", + ) + ) + master_tasks.append( + DagTaskNode( + task_id=trigger_task_id, + summary=f"trigger the {stage.value} sub-DAG and wait for it to finish", + deferred=True, + ) + ) + # resolve_profile feeds every gate its enabled-stage list, and the + # previous stage's trigger must finish before the next gate opens -- + # that pair of edges is what makes the stages a chain, not a fan-out. + master_edges.append((RESOLVE_PROFILE_TASK_ID, gate_task_id)) + master_edges.append((gate_task_id, trigger_task_id)) + if previous_trigger_task_id is not None: + master_edges.append((previous_trigger_task_id, gate_task_id)) + previous_trigger_task_id = trigger_task_id + + stages.append( + StageTopology( + stage=stage, + dag=_sub_dag_topology(master_dag_id, stage), + gate_task_id=gate_task_id, + trigger_task_id=trigger_task_id, + enabling_profiles=_enabling_profiles(stage), + ) + ) + + return IngestTopology( + master=DagTopology( + dag_id=master_dag_id, + tasks=tuple(master_tasks), + edges=tuple(master_edges), + ), + stages=tuple(stages), + ) diff --git a/src/hflow/stage_execution.py b/src/hflow/stage_execution.py index 16aa498..f3cb4c6 100644 --- a/src/hflow/stage_execution.py +++ b/src/hflow/stage_execution.py @@ -20,11 +20,13 @@ import math import os import traceback -from collections.abc import Sequence +from collections.abc import Iterator, Sequence +from contextlib import contextmanager from pathlib import Path from typing import TYPE_CHECKING, TypedDict from hflow.batching import plan_batches +from hflow.catalog import QuarantineHistory from hflow.steps import IngestMode, Stage from hflow.storage import is_bucket_url, parse_storage_root @@ -165,32 +167,56 @@ def process_stage_batch( gates apply the run budget to the tallies, so mass failure stays loud while a stray bad episode never blocks a run. """ + # Stage(stage_name) parses the conf string at this boundary: an unknown + # stage is a loud ValueError before any episode is touched. + stage = Stage(stage_name) data_root = str(application.data_root) counts: StageBatchCounts = {"processed": 0, "quarantined": 0, "errors": 0} - for uri in uris: - try: - episode_reference = resolve_episode_reference(data_root, str(uri)) - # Stage(stage_name) parses the conf string at this boundary: an - # unknown stage is a loud ValueError, counted as that episode's - # error like any other infrastructure failure. - report = application.process(episode_reference, record=True, stages={Stage(stage_name)}) - except Exception: - traceback.print_exc() - counts["errors"] += 1 - continue - if report.has_errors: - # app.process collects per-step diagnostics for the dev loop and - # catalog, so step failures are explicit report outcomes rather - # than escaping exceptions. They still count against the - # runtime's infrastructure-error budget. - counts["errors"] += 1 - elif report.quarantined: - counts["quarantined"] += 1 - else: - counts["processed"] += 1 + # The labels and media gates read the episode's cataloged quarantine + # state; opening that reader once per batch rather than once per episode + # is the difference between one mirror sync and one per episode. Stages + # that decide quarantine themselves never read it. + with _batch_quarantine_history(application, stage) as quarantine_history: + for uri in uris: + try: + episode_reference = resolve_episode_reference(data_root, str(uri)) + report = application.process( + episode_reference, + record=True, + stages={stage}, + quarantine_history=quarantine_history, + ) + except Exception: + traceback.print_exc() + counts["errors"] += 1 + continue + if report.has_errors: + # app.process collects per-step diagnostics for the dev loop + # and catalog, so step failures are explicit report outcomes + # rather than escaping exceptions. They still count against + # the runtime's infrastructure-error budget. + counts["errors"] += 1 + elif report.quarantined: + counts["quarantined"] += 1 + else: + counts["processed"] += 1 return counts +@contextmanager +def _batch_quarantine_history( + application: "App", stage: Stage +) -> Iterator[QuarantineHistory | None]: + """One catalog reader for a whole batch, or ``None`` where the stage + never asks: only stages running without ``meta`` consult the catalog for + quarantine, and ``meta`` itself decides it in memory.""" + if stage is Stage.META: + yield None + return + with QuarantineHistory(application.workspace.catalog_root) as history: + yield history + + def _tally_batch_counts(batch_counts: Sequence[StageBatchCounts]) -> tuple[int, int, int]: """(total, quarantined, errors) across every batch's counts.""" total = sum( diff --git a/src/hflow/steps.py b/src/hflow/steps.py index 241feda..2daaed7 100644 --- a/src/hflow/steps.py +++ b/src/hflow/steps.py @@ -331,11 +331,14 @@ def _stable_version_identity_value(value: object) -> VersionIdentityValue: if isinstance(value, Path): return {"path": str(value)} if isinstance(value, ModuleType): - module_version = getattr(value, "__version__", None) - return { - "module": value.__name__, - "version": module_version if isinstance(module_version, str) else None, - } + # The module's IDENTITY, never its ``__version__``: a version number + # is a poor proxy for "does this library compute differently", and + # folding it in re-versioned every step that merely referenced + # ``hflow`` or ``numpy`` on any release of those packages -- including + # releases that changed nothing a step can observe. What a step + # actually does still lives in its own source and captured values, + # which this hash covers directly. + return {"module": value.__name__} if isinstance(value, type): return {"type": f"{value.__module__}.{value.__qualname__}"} if isinstance(value, CodeType): diff --git a/src/hflow/transform.py b/src/hflow/transform.py index 9077eb4..dc7d7b3 100644 --- a/src/hflow/transform.py +++ b/src/hflow/transform.py @@ -50,6 +50,7 @@ from typing import Any, Literal from hflow import video as video_module +from hflow.behavior import TRANSFORM_BEHAVIOR_VERSION from hflow.ffmpeg import ffmpeg_version from hflow.format import ( CAMERA_SCHEMA_NAMES, @@ -144,14 +145,19 @@ def compute_pipeline_version( ``derived_versions`` (derived topic -> derived-channel version) folds the derived signals into the hash: a passed-in mapping rather than App state, so the transform stays a library function. - """ - # Imported here to avoid a cycle with the package root. - from hflow import __version__ + The engine's contribution is :data:`hflow.behavior.TRANSFORM_BEHAVIOR_VERSION`, + NOT the release number: this hash is stamped into ``provenance/v1``, which + lives inside the bytes ``content_episode_id`` hashes, so folding in + ``hflow.__version__`` gave every release a new ``pipeline_version`` AND a + new ``episode_id`` for byte-identical inputs -- breaking dedupe and making + ``hflow stale`` list the whole corpus after a change that processed + nothing differently. + """ payload = json.dumps( { "format": EPISODE_FORMAT_VERSION, - "sdk": __version__, + "transform_behavior": TRANSFORM_BEHAVIOR_VERSION, "gop_preset": str(config.gop_preset), "gop_seconds": config.gop_seconds, "crf": config.crf, @@ -159,6 +165,13 @@ def compute_pipeline_version( "compression": config.compression, "topic_groups": dict(sorted(config.topic_groups.items())), "derived": dict(sorted(derived_versions.items())) if derived_versions else {}, + # Only when the episode HAS derived channels: the policy decides + # their samples, so a bump must make them stale -- but folding it + # unconditionally would re-version corpora that never resampled + # anything. A step's own version cannot cover this, because the + # taught idiom reaches ``hflow.resample.to_grid`` through the + # module and a module contributes only its name to a step hash. + **({"resample_policy": RESAMPLE_POLICY_VERSION} if derived_versions else {}), }, sort_keys=True, ) @@ -436,7 +449,27 @@ def group_for(info: TopicInfo) -> str: ) ) - outgoing.sort(key=lambda message: message.log_time) + # A TOTAL order, because these bytes are the episode's identity. + # + # log_time alone leaves ties -- on a realistic multi-stream episode + # roughly 40% of messages share a timestamp with another -- and a + # stable sort then settles them by the order this function happened to + # append them in: passthrough during the read, then transcoded video, + # then derived channels. That is a property of the code, not of the + # episode, so reordering any of those appends would silently change + # every content-addressed episode_id in every corpus. + # + # Topic breaks the tie instead: it is semantic and comparable across + # recordings, where a recorder-assigned channel id is neither. Within + # one topic the sort's stability preserves the source's own order for + # that topic, which is already well defined. + def output_topic(message: _OutgoingMessage) -> str: + source_channel = message.source_channel_id + return ( + source_channel if isinstance(source_channel, str) else infos[source_channel].topic + ) + + outgoing.sort(key=lambda message: (message.log_time, output_topic(message))) from mcap_protobuf.schema import build_file_descriptor_set diff --git a/tests/test_cli.py b/tests/test_cli.py index 6f538a1..9105560 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -62,3 +62,22 @@ def test_cli_manifest_reports_a_broken_pipeline_instead_of_crashing( exit_code = main(["manifest", "--pipeline", str(pipeline_file)]) assert exit_code == 2 assert "boom at import" in capsys.readouterr().err + + +def test_cli_manifest_does_not_inherit_a_pipeline_that_exits( + tmp_path: Path, capsys: CaptureFixture +) -> None: + """A config guard at import time is a boundary failure, not our exit code. + + ``sys.exit`` raises SystemExit -- a BaseException -- so importing user code + can walk straight past an ``except Exception`` and take the calling program + with it. Here that would mean the pipeline's own status instead of 2; in the + workspace UI it would kill a long-lived server at startup. + """ + pipeline_file = tmp_path / "guarded.py" + pipeline_file.write_text("import sys\n\nsys.exit('set ROBOT_FLEET')\n") + exit_code = main(["manifest", "--pipeline", str(pipeline_file)]) + assert exit_code == 2 + error_output = capsys.readouterr().err + assert str(pipeline_file) in error_output + assert "set ROBOT_FLEET" in error_output diff --git a/tests/test_identity_stability.py b/tests/test_identity_stability.py new file mode 100644 index 0000000..9f2194a --- /dev/null +++ b/tests/test_identity_stability.py @@ -0,0 +1,208 @@ +"""A release must not re-version a corpus that processed nothing differently. + +Until the identity epoch in :mod:`hflow.behavior`, ``hflow.__version__`` was +folded into ``pipeline_version`` (and thence, via ``provenance/v1``, into the +canonical bytes and so into ``episode_id``), into the MCAP header's library +string (also inside those bytes), and into any step version whose function +referenced a module. A CLI-only patch release therefore invalidated an entire +corpus: ``hflow stale`` listed everything, and re-ingesting byte-identical +sources minted new episode identities instead of deduping. + +These tests fail if any of those couplings comes back. +""" + +from collections import Counter +from collections.abc import Callable +from pathlib import Path + +import numpy +from mcap.reader import make_reader + +import hflow +from hflow.behavior import TRANSFORM_BEHAVIOR_VERSION +from hflow.mcap_writer import _default_library_identifier +from hflow.steps import CheckResult, compute_check_version +from hflow.testing import SyntheticEpisodeSpec, synthesize_episode +from hflow.transform import ( + TransformConfig, + compute_pipeline_version, + write_canonical_episode, +) + +FAKE_RELEASE = "9.9.9" + + +def _step_version(function: Callable[..., object]) -> str: + return compute_check_version( + name="probe", + function=function, + critical=False, + requires=frozenset(), + uses=None, + ) + + +def _with_faked_release(compute: Callable[[], str]) -> tuple[str, str]: + """Return (value now, value under a faked hflow release).""" + before = compute() + original = hflow.__version__ + hflow.__version__ = FAKE_RELEASE + try: + after = compute() + finally: + hflow.__version__ = original + return before, after + + +def test_pipeline_version_survives_a_release() -> None: + before, after = _with_faked_release(lambda: compute_pipeline_version(TransformConfig())) + assert before == after, "a release must not mark every episode stale" + + +def test_pipeline_version_still_tracks_transform_configuration() -> None: + baseline = compute_pipeline_version(TransformConfig()) + assert compute_pipeline_version(TransformConfig(crf=30)) != baseline + assert compute_pipeline_version(TransformConfig(), {"/derived/speed": "abc"}) != baseline + + +def test_pipeline_version_tracks_the_transform_behavior_version() -> None: + """The deliberate lever still works: bumping behavior re-versions.""" + baseline = compute_pipeline_version(TransformConfig()) + import hflow.transform as transform_module + + original = transform_module.TRANSFORM_BEHAVIOR_VERSION + # Any value but the current one; a literal here would silently stop + # testing anything the moment the constant caught up with it. + transform_module.TRANSFORM_BEHAVIOR_VERSION = f"{original}-probe" + try: + assert compute_pipeline_version(TransformConfig()) != baseline + finally: + transform_module.TRANSFORM_BEHAVIOR_VERSION = original + + +def test_pipeline_version_tracks_the_resample_policy_only_when_derived_channels_exist() -> None: + """The resample policy decides derived samples and no step hash sees it. + + A step version folds in a referenced MODULE's name only, so a pipeline + whose derive function calls ``hflow.resample.to_grid`` cannot notice the + policy changing. ``compute_pipeline_version`` folds it in instead -- but + only for episodes that actually have derived channels, so bumping the + policy never churns a corpus that resampled nothing. + """ + import hflow.transform as transform_module + + derived = {"/derived/joint_grid": "abc123"} + original = transform_module.RESAMPLE_POLICY_VERSION + without_derived = compute_pipeline_version(TransformConfig()) + with_derived = compute_pipeline_version(TransformConfig(), derived) + + transform_module.RESAMPLE_POLICY_VERSION = f"{original}-probe" + try: + assert compute_pipeline_version(TransformConfig(), derived) != with_derived, ( + "a resample policy bump must make derived-channel episodes stale" + ) + assert compute_pipeline_version(TransformConfig()) == without_derived, ( + "a corpus with no derived channels must not move" + ) + finally: + transform_module.RESAMPLE_POLICY_VERSION = original + + +def test_canonical_bytes_carry_no_release_number() -> None: + """The MCAP header's library string is inside the hashed bytes.""" + identifier = _default_library_identifier() + assert hflow.__version__ not in identifier + assert TRANSFORM_BEHAVIOR_VERSION in identifier + + before, after = _with_faked_release(_default_library_identifier) + assert before == after, "a release must not change episode_id" + + +def test_step_referencing_the_hflow_module_survives_a_release() -> None: + def check_via_module(episode: object) -> object: + return hflow.CheckResult(measurements={"ok": 1.0}) + + before, after = _with_faked_release(lambda: _step_version(check_via_module)) + assert before == after + + +def test_step_referencing_a_third_party_module_survives_its_upgrade() -> None: + """The defect was never hflow-specific: numpy upgrades churned too.""" + + def check_via_numpy(values: list[float]) -> float: + return float(numpy.mean(values)) + + before = _step_version(check_via_numpy) + original = numpy.__version__ + numpy.__version__ = "99.0.0" + try: + after = _step_version(check_via_numpy) + finally: + numpy.__version__ = original + assert before == after + + +def test_step_version_still_tracks_what_the_author_wrote() -> None: + """Author-owned facts must still re-version -- that is the point.""" + + def original_threshold(episode: object) -> CheckResult: + return CheckResult(measurements={"limit": 1.0}) + + def changed_threshold(episode: object) -> CheckResult: + return CheckResult(measurements={"limit": 2.0}) + + assert _step_version(original_threshold) != _step_version(changed_threshold) + + captured_limit = 1.0 + + def uses_closure(episode: object) -> CheckResult: + return CheckResult(measurements={"limit": captured_limit}) + + with_first_capture = _step_version(uses_closure) + captured_limit = 2.0 + assert _step_version(uses_closure) != with_first_capture + + +def test_canonical_write_order_is_total_so_identity_cannot_ride_on_append_order( + tmp_path: Path, +) -> None: + """Messages sharing a timestamp must be ordered by something in the DATA. + + ``episode_id`` is a hash of the canonical bytes, and those bytes are + written in the order the transform sorted its messages into. Sorting on + ``log_time`` alone leaves ties -- the fixture below has hundreds -- which + a stable sort then settles by the order the transform happened to append + them: passthrough during the read, then transcoded video, then derived + channels. That is a property of the code, so any reordering of those + appends would silently mint new identities for an unchanged corpus. + + Asserting the ORDER rather than a golden digest is deliberate: the + canonical embeds transcoded video, so a pinned hash would fail on a + different ffmpeg build for reasons that have nothing to do with ordering. + """ + source = synthesize_episode( + tmp_path / "episode.mcap", + SyntheticEpisodeSpec( + duration_s=6.0, + cameras=("wrist_cam", "overhead_cam"), + image_hz=30.0, + joint_hz=100.0, + ), + ) + canonical = tmp_path / "episode.canonical.mcap" + write_canonical_episode(source, canonical) + + written: list[tuple[int, str]] = [] + with canonical.open("rb") as stream: + for _schema, channel, message in make_reader(stream).iter_messages(): + written.append((message.log_time, channel.topic)) + + timestamp_counts = Counter(log_time for log_time, _topic in written) + tied_messages = sum(count for count in timestamp_counts.values() if count > 1) + assert tied_messages > 0, "fixture has no simultaneous messages; it proves nothing" + + assert written == sorted(written), ( + "canonical messages must be written in (log_time, topic) order -- " + f"{tied_messages} of {len(written)} share a timestamp, and without a " + "total order their arrangement is decided by append order" + ) diff --git a/tests/test_mcap_writer.py b/tests/test_mcap_writer.py index 291f082..bffee13 100644 --- a/tests/test_mcap_writer.py +++ b/tests/test_mcap_writer.py @@ -392,8 +392,13 @@ def test_default_library_identifier_names_project_and_format() -> None: pass stream.seek(0) header = make_reader(stream).get_header() - assert header.library.startswith("hflow/") + assert header.library.startswith("hflow ") assert "episode-format/1" in header.library + # The header is inside the bytes content_episode_id hashes, so it must + # carry no release number: see tests/test_identity_stability.py. + import hflow + + assert hflow.__version__ not in header.library explicit_stream = BytesIO() with CanonicalMcapWriter(explicit_stream, library="custom-writer/9"): diff --git a/tests/test_runtime_cli.py b/tests/test_runtime_cli.py index 29c605f..46e5264 100644 --- a/tests/test_runtime_cli.py +++ b/tests/test_runtime_cli.py @@ -12,7 +12,8 @@ import pytest import hflow -from hflow.cli import _parse_pipeline_spec, main +from hflow.app import parse_pipeline_spec +from hflow.cli import main from hflow.runtime import AirflowHealth, RuntimeConfig, render_bundle from hflow.runtime._client import AirflowClient, AirflowClientError, PasswordCredentials @@ -71,10 +72,10 @@ def _rendered_bundle(tmp_path: Path, pipeline_file: Path) -> Path: def test_parse_pipeline_spec_variants() -> None: - assert _parse_pipeline_spec("pipe.py") == (Path("pipe.py"), "app") - assert _parse_pipeline_spec("dir/pipe.py:my_app") == (Path("dir/pipe.py"), "my_app") + assert parse_pipeline_spec("pipe.py") == (Path("pipe.py"), "app") + assert parse_pipeline_spec("dir/pipe.py:my_app") == (Path("dir/pipe.py"), "my_app") # A trailing non-identifier is part of the path, not a variable name. - assert _parse_pipeline_spec("dir/pipe.py:not-an-identifier") == ( + assert parse_pipeline_spec("dir/pipe.py:not-an-identifier") == ( Path("dir/pipe.py:not-an-identifier"), "app", ) diff --git a/tests/test_runtime_client.py b/tests/test_runtime_client.py index 189f96d..d3af6b4 100644 --- a/tests/test_runtime_client.py +++ b/tests/test_runtime_client.py @@ -75,6 +75,12 @@ def do_POST(self) -> None: def do_GET(self) -> None: authorization = self._record(None) + if self.path.endswith("/taskInstances"): + if not self._bearer_ok(authorization): + self._respond(401, {"detail": "expired"}) + return + self._respond(200, {"task_instances": [{"task_id": "plan", "map_index": -1}]}) + return if "/dagRuns/" in self.path: if not self._bearer_ok(authorization): self._respond(401, {"detail": "expired"}) @@ -189,6 +195,31 @@ def test_caller_supplied_dag_run_id_makes_retries_idempotent(stub_server: str) - assert existing == {"dag_run_id": "already-exists", "state": "running"} +def test_per_run_endpoints_address_the_same_run(stub_server: str) -> None: + """One run id, one URL rule -- whichever per-run endpoint asks for it. + + Airflow's own ids carry ':' and '+', and a caller-supplied idempotency id + may carry '#', which urllib reads as a fragment and drops from the path. + Left raw, the run detail and its task instances would describe two + different runs (and the 409 retry in trigger_dag_run would 404). + """ + client = AirflowClient(stub_server, "airflow", "right-password") + dag_run_id = "manual__2026-08-22T03:06:55+00:00#retry-1" + client.dag_run("pipeline_ingest", dag_run_id) + client.task_instances("pipeline_ingest", dag_run_id) + + encoded_run = "manual__2026-08-22T03%3A06%3A55%2B00%3A00%23retry-1" + per_run_paths = [ + path + for method, path, _payload, _authorization in _StubAirflowHandler.requests_seen + if method == "GET" and "/dagRuns/" in path + ] + assert per_run_paths == [ + f"/api/v2/dags/pipeline_ingest/dagRuns/{encoded_run}", + f"/api/v2/dags/pipeline_ingest/dagRuns/{encoded_run}/taskInstances", + ] + + def test_conflict_without_a_dag_run_id_still_raises(stub_server: str) -> None: # Without a caller id there is nothing to idempotently return; the 409 # must surface. @@ -198,6 +229,19 @@ def test_conflict_without_a_dag_run_id_still_raises(stub_server: str) -> None: assert error_info.value.status == 409 +def test_ingest_refuses_a_batch_count_the_run_could_not_honour(stub_server: str) -> None: + """The conf's owner refuses it here, before a run exists to fail. + + ``plan_batches`` enforces ``>= 1`` inside the sync sub-DAG's plan task, so + without this the SDK reports a triggered run and the operator's history + collects a failure for a value the client could have named. + """ + client = AirflowClient(stub_server, "airflow", "right-password") + with pytest.raises(ValueError, match="batch_count must be >= 1, got 0"): + client.ingest("pipeline_ingest", ["a.mcap"], batch_count=0) + assert _StubAirflowHandler.requests_seen == [] + + def test_bad_credentials_surface_clearly(stub_server: str) -> None: client = AirflowClient(stub_server, "airflow", "wrong-password") with pytest.raises(AirflowClientError) as error_info: diff --git a/tests/test_runtime_topology.py b/tests/test_runtime_topology.py new file mode 100644 index 0000000..bcfddb7 --- /dev/null +++ b/tests/test_runtime_topology.py @@ -0,0 +1,124 @@ +"""The described topology must match the DAGs the renderer actually writes. + +``hflow.runtime.ingest_dag_topology`` exists so a UI can draw a bundle's task +graph without parsing generated Python. That is only safe while the two agree, +so these tests read the rendered source and check every task id and edge the +description claims. +""" + +from itertools import pairwise +from pathlib import Path + +import pytest + +from hflow.runtime import ( + RuntimeConfig, + ingest_dag_topology, + render_bundle, + sub_dag_id_for_stage, +) +from hflow.runtime._topology import ( + PLAN_TASK_ID, + PROCESS_BATCH_TASK_ID, + RESOLVE_PROFILE_TASK_ID, + budget_gate_task_id, +) +from hflow.steps import Stage + +MASTER_DAG_ID = "kitchen_ingest" + + +@pytest.fixture(scope="module") +def rendered_bundle(tmp_path_factory: pytest.TempPathFactory) -> Path: + bundle_dir = tmp_path_factory.mktemp("topology-bundle") + pipeline_file = bundle_dir / "pipeline.py" + pipeline_file.write_text("import hflow\n\napp = hflow.App('kitchen')\n") + paths = render_bundle( + RuntimeConfig(pipeline_file=pipeline_file, data_root=bundle_dir / "data"), + bundle_dir / "runtime", + ) + return paths.bundle_dir / "dags" + + +def _dag_source(dags_directory: Path, file_name: str) -> str: + return (dags_directory / file_name).read_text() + + +def test_master_task_ids_appear_in_the_rendered_master_dag(rendered_bundle: Path) -> None: + source = _dag_source(rendered_bundle, "ingest.py") + topology = ingest_dag_topology(MASTER_DAG_ID) + + assert f"def {RESOLVE_PROFILE_TASK_ID}(" in source + # The gate and trigger ids are built from f-strings in the template, so the + # rendered source carries one shape covering every stage; what varies per + # stage is the id the description derives from it. + assert 'task_id=f"enabled_{stage_name}"' in source + assert 'task_id=f"trigger_{stage_name}"' in source + for stage_topology in topology.stages: + assert stage_topology.gate_task_id == f"enabled_{stage_topology.stage.value}" + assert stage_topology.trigger_task_id == f"trigger_{stage_topology.stage.value}" + + +def test_master_edges_describe_the_stage_chain() -> None: + topology = ingest_dag_topology(MASTER_DAG_ID) + edges = set(topology.master.edges) + stages = list(Stage) + + for stage in stages: + assert (RESOLVE_PROFILE_TASK_ID, f"enabled_{stage.value}") in edges + assert (f"enabled_{stage.value}", f"trigger_{stage.value}") in edges + for upstream, downstream in pairwise(stages): + assert (f"trigger_{upstream.value}", f"enabled_{downstream.value}") in edges + + # The chain, not a fan-out: nothing runs a later stage before the earlier + # stage's trigger has finished waiting. + assert (f"trigger_{stages[-1].value}", f"enabled_{stages[0].value}") not in edges + + +@pytest.mark.parametrize("stage", list(Stage)) +def test_sub_dag_tasks_match_the_rendered_stage_dag(rendered_bundle: Path, stage: Stage) -> None: + source = _dag_source(rendered_bundle, f"ingest_{stage.value}.py") + stage_topology = next( + candidate + for candidate in ingest_dag_topology(MASTER_DAG_ID).stages + if candidate.stage is stage + ) + + assert stage_topology.dag.dag_id == sub_dag_id_for_stage(MASTER_DAG_ID, stage) + for task in stage_topology.dag.tasks: + assert f"def {task.task_id}(" in source, f"{task.task_id} missing from {stage.value}" + assert stage_topology.dag.edges == ( + (PLAN_TASK_ID, PROCESS_BATCH_TASK_ID), + (PROCESS_BATCH_TASK_ID, budget_gate_task_id(stage)), + ) + assert f"{PROCESS_BATCH_TASK_ID}.expand(" in source # the fan-out the UI draws + + +def test_only_meta_gates_on_the_quarantine_budget() -> None: + assert budget_gate_task_id(Stage.META) == "quarantine_budget_gate" + for stage in (Stage.SYNC, Stage.LABELS, Stage.MEDIA): + assert budget_gate_task_id(stage) == "error_budget_gate" + + +def test_process_batch_is_the_only_mapped_task() -> None: + for stage_topology in ingest_dag_topology(MASTER_DAG_ID).stages: + mapped = [task.task_id for task in stage_topology.dag.tasks if task.mapped] + assert mapped == [PROCESS_BATCH_TASK_ID] + + +def test_enabling_profiles_name_which_profiles_run_each_stage() -> None: + """The profile table as the UI reads it, stated rather than recomputed. + + A profile missing from a stage's tuple is exactly what the master's gate + skips on -- metadata_backfill runs meta and nothing else. + """ + profiles_by_stage = { + stage_topology.stage: stage_topology.enabling_profiles + for stage_topology in ingest_dag_topology(MASTER_DAG_ID).stages + } + assert profiles_by_stage == { + Stage.SYNC: ("full",), + Stage.META: ("full", "metadata_backfill"), + Stage.LABELS: ("full", "relabel"), + Stage.MEDIA: ("full",), + } diff --git a/tests/test_serve_cli.py b/tests/test_serve_cli.py new file mode 100644 index 0000000..2b3e050 --- /dev/null +++ b/tests/test_serve_cli.py @@ -0,0 +1,79 @@ +"""``hflow serve`` flags to :class:`hflow_server.ServerSettings` -- the launch contract. + +``_command_ui`` is the only place that turns the CLI's flags into a launch, so +a flag that stops reaching ``ServerSettings`` (or a default that drifts) is +invisible to every hflow-server test, which builds its settings by hand. +``--host`` is the one flag with a posture consequence: the server +authenticates nobody, so what it binds is the whole access-control story. + +``serve`` is the process boundary and is monkeypatched here: these tests +assert the settings it was handed, never a running server. +""" + +import sys +from pathlib import Path + +import pytest +from hflow_server import ServerSettings + +from hflow.cli import DEFAULT_SERVER_PORT, main + + +@pytest.fixture +def served_settings(monkeypatch: pytest.MonkeyPatch) -> list[ServerSettings]: + """Capture what ``hflow serve`` would launch, instead of launching it.""" + launches: list[ServerSettings] = [] + monkeypatch.setattr("hflow_server.serve", launches.append) + return launches + + +def test_ui_flags_land_in_the_launch_settings( + served_settings: list[ServerSettings], tmp_path: Path +) -> None: + exit_code = main( + [ + "serve", + "--data-root", + str(tmp_path / "workspace"), + "--host", + "0.0.0.0", + "--port", + "9999", + "--no-browser", + "--read-only", + "--pipeline", + "kitchen.py:my_app", + ] + ) + assert exit_code == 0 + (settings,) = served_settings + assert settings.data_root == str(tmp_path / "workspace") + assert settings.host == "0.0.0.0" + assert settings.port == 9999 + assert settings.open_browser is False + assert settings.read_only is True + assert settings.pipeline == "kitchen.py:my_app" + + +def test_a_bare_ui_launch_uses_the_documented_defaults( + served_settings: list[ServerSettings], tmp_path: Path +) -> None: + """What `hflow serve` with no flags promises -- loopback above all.""" + assert main(["serve", "--data-root", str(tmp_path)]) == 0 + (settings,) = served_settings + assert (settings.host, settings.port) == ("127.0.0.1", DEFAULT_SERVER_PORT) + assert settings.open_browser is True + assert settings.read_only is False + assert settings.pipeline is None + + +def test_ui_without_the_package_exits_with_the_install_hint( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """hflow-server is optional, so its absence is an instruction, not a traceback.""" + monkeypatch.setitem(sys.modules, "hflow_server", None) + assert main(["serve", "--data-root", str(tmp_path)]) == 2 + streams = capsys.readouterr() + assert streams.out == "" + assert "uv add hflow-server" in streams.err + assert "Traceback" not in streams.err diff --git a/tests/test_stage_execution.py b/tests/test_stage_execution.py index 07ae5b6..729baff 100644 --- a/tests/test_stage_execution.py +++ b/tests/test_stage_execution.py @@ -4,11 +4,12 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import cast +from typing import ClassVar, cast import pytest import hflow +from hflow import stage_execution from hflow.stage_execution import ( StageBatchCounts, load_pipeline_application, @@ -135,18 +136,34 @@ class _StubReport: quarantined: bool = False +@dataclass +class _StubWorkspace: + catalog_root: Path + + @dataclass class _StubApp: """A processing double at the App boundary: records calls, scripts outcomes.""" data_root: str + workspace: _StubWorkspace + expected_stage: str = "meta" outcomes: dict[str, object] = field(default_factory=dict) processed_references: list[object] = field(default_factory=list) + received_histories: list[object] = field(default_factory=list) - def process(self, episode_reference: object, *, record: bool, stages: set[str]) -> _StubReport: + def process( + self, + episode_reference: object, + *, + record: bool, + stages: set[str], + quarantine_history: object = None, + ) -> _StubReport: assert record is True - assert stages == {"meta"} + assert stages == {self.expected_stage} self.processed_references.append(episode_reference) + self.received_histories.append(quarantine_history) outcome = self.outcomes.get(Path(str(episode_reference)).name, "ok") if outcome == "crash": raise RuntimeError("episode exploded") @@ -156,6 +173,7 @@ def process(self, episode_reference: object, *, record: bool, stages: set[str]) def test_process_stage_batch_counts_every_outcome_kind(tmp_path: Path) -> None: stub_app = _StubApp( data_root=str(tmp_path), + workspace=_StubWorkspace(catalog_root=tmp_path / "catalog"), outcomes={ "good.mcap": "ok", "quarantined.mcap": "quarantined", @@ -171,3 +189,83 @@ def test_process_stage_batch_counts_every_outcome_kind(tmp_path: Path) -> None: # Per-episode crashes are counted, never batch-fatal: all four were tried. assert len(stub_app.processed_references) == 4 assert counts == {"processed": 1, "quarantined": 1, "errors": 2} + + +class _CountingQuarantineHistory: + """Stands in for the real reader so opens can be counted, not timed.""" + + opened: ClassVar[list[object]] = [] + + def __init__(self, catalog_root: object) -> None: + type(self).opened.append(catalog_root) + + def quarantine_tags(self, episode_id: str) -> None: + return None + + def __enter__(self) -> "_CountingQuarantineHistory": + return self + + def __exit__(self, *_exception: object) -> None: + return None + + +@pytest.fixture +def counting_quarantine_history(monkeypatch: pytest.MonkeyPatch) -> list[object]: + _CountingQuarantineHistory.opened = [] + monkeypatch.setattr(stage_execution, "QuarantineHistory", _CountingQuarantineHistory) + return _CountingQuarantineHistory.opened + + +@pytest.mark.parametrize("stage_name", ["sync", "labels", "media"]) +def test_a_gated_stage_opens_one_quarantine_reader_for_the_whole_batch( + tmp_path: Path, counting_quarantine_history: list[object], stage_name: str +) -> None: + """The quarantine gate reads the catalog once per batch, not per episode. + + Every stage but ``meta`` asks the catalog for the episode's recorded + quarantine state. Opening that reader per episode makes a bucket catalog + re-sync its mirror once per episode for one boolean, which is the whole + cost this batching exists to remove -- so count the opens. + """ + catalog_root = tmp_path / "catalog" + stub_app = _StubApp( + data_root=str(tmp_path), + workspace=_StubWorkspace(catalog_root=catalog_root), + expected_stage=stage_name, + ) + counts = process_stage_batch( + cast("hflow.App", stub_app), + [f"episode_{index}.mcap" for index in range(5)], + stage_name, + ) + + assert counts == {"processed": 5, "quarantined": 0, "errors": 0} + assert counting_quarantine_history == [catalog_root] + # ...and every episode was handed that one reader, so none of them falls + # back to opening its own inside App.process. + assert stub_app.received_histories == [stub_app.received_histories[0]] * 5 + assert isinstance(stub_app.received_histories[0], _CountingQuarantineHistory) + + +def test_the_meta_stage_never_opens_the_quarantine_reader( + tmp_path: Path, counting_quarantine_history: list[object] +) -> None: + """Meta decides quarantine in memory, so reading it back would be a lie.""" + stub_app = _StubApp( + data_root=str(tmp_path), + workspace=_StubWorkspace(catalog_root=tmp_path / "catalog"), + expected_stage="meta", + ) + process_stage_batch(cast("hflow.App", stub_app), ["a.mcap", "b.mcap"], "meta") + + assert counting_quarantine_history == [] + assert stub_app.received_histories == [None, None] + + +def test_an_unknown_stage_is_refused_before_any_episode_is_touched(tmp_path: Path) -> None: + stub_app = _StubApp( + data_root=str(tmp_path), workspace=_StubWorkspace(catalog_root=tmp_path / "catalog") + ) + with pytest.raises(ValueError, match="not a valid Stage"): + process_stage_batch(cast("hflow.App", stub_app), ["a.mcap"], "nonsense") + assert stub_app.processed_references == [] diff --git a/uv.lock b/uv.lock index 700c9a3..e079447 100644 --- a/uv.lock +++ b/uv.lock @@ -10,6 +10,21 @@ resolution-markers = [ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P5D" +[manifest] +members = [ + "hflow", + "hflow-server", +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + [[package]] name = "annotated-types" version = "0.8.0" @@ -32,6 +47,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, ] +[[package]] +name = "click" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -86,6 +113,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/2c/95d9216b79e9273689d7ebce125a54503ed0c9bd7da931f0265888e99779/duckdb-1.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:63e48d4b74b15aeacd688976432a7225163df8c226eddeb8536bba2d4d4ff433", size = 14470180, upload-time = "2026-07-22T10:55:14.445Z" }, ] +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + [[package]] name = "foxglove-schemas-protobuf" version = "0.4.0" @@ -135,6 +178,8 @@ openai = [ [package.dev-dependencies] dev = [ + { name = "hflow-server" }, + { name = "httpx2" }, { name = "obstore" }, { name = "pytest" }, { name = "pyyaml" }, @@ -159,6 +204,8 @@ provides-extras = ["arrow", "bucket", "openai"] [package.metadata.requires-dev] dev = [ + { name = "hflow-server", editable = "packages/hflow-server" }, + { name = "httpx2", specifier = ">=2.10" }, { name = "obstore", specifier = ">=0.10.0" }, { name = "pytest", specifier = ">=9.1.1" }, { name = "pyyaml", specifier = ">=6.0.3" }, @@ -166,6 +213,23 @@ dev = [ { name = "ty", specifier = ">=0.0.71" }, ] +[[package]] +name = "hflow-server" +version = "0.1.0" +source = { editable = "packages/hflow-server" } +dependencies = [ + { name = "fastapi" }, + { name = "hflow" }, + { name = "uvicorn" }, +] + +[package.metadata] +requires-dist = [ + { name = "fastapi", specifier = ">=0.115" }, + { name = "hflow", editable = "." }, + { name = "uvicorn", specifier = ">=0.32" }, +] + [[package]] name = "httpcore2" version = "2.10.0" @@ -896,27 +960,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.2" +version = "0.16.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } +sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, - { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, - { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, - { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, - { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, - { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, - { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, - { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, - { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, - { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, - { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, - { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, - { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, - { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, + { url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" }, + { url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" }, + { url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" }, + { url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" }, + { url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" }, + { url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" }, + { url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" }, + { url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" }, + { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, + { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, ] [[package]] @@ -928,6 +992,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "starlette" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/b4/205b0d5241d934e8add0c38aa924c4f9fb7330834ff11e5444db964ec3f9/starlette-1.6.0.tar.gz", hash = "sha256:d4e3ac5e546444960c710297a3c9fc3f7ebae1b7e963f3d36173b49da535be9b", size = 2716969, upload-time = "2026-08-08T18:27:57.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/cb/6a6a47d5b464bd08695d254f3da6e7986cc70c9fa5d778eda57538edfe56/starlette-1.6.0-py3-none-any.whl", hash = "sha256:a86dd39d14bb45f85a3d18525215a9ef0cfd1f192ac793220e72598c90335f0c", size = 75969, upload-time = "2026-08-08T18:27:56.196Z" }, +] + [[package]] name = "tqdm" version = "4.70.0" @@ -951,27 +1028,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.71" +version = "0.0.72" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/dd/e2/f6e716371b5913a31190db1ad250ac2b5c68b3ca2db71afeba3f98f5fe50/ty-0.0.71.tar.gz", hash = "sha256:c2a24f2745294946c27cef8cc012b84fb2db5405ecefddbe845be4162833da01", size = 6624721, upload-time = "2026-08-13T00:39:31.035Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/e4/8d6d17827c5335d0efe54241fb54fed3cb6ca2d2eca62f3f2382be196b91/ty-0.0.71-py3-none-linux_armv6l.whl", hash = "sha256:a309c9a35e69f45d7053e9205c7fe2295fac09e07e3a2fdb791524f30440a9cc", size = 12576435, upload-time = "2026-08-13T00:38:50.617Z" }, - { url = "https://files.pythonhosted.org/packages/d5/cb/d4c48832ee3d162abc6c834ba62242ec27630cb93014a0e1be82e86c3ee3/ty-0.0.71-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2d42e9ae4b754ce1f34dd1b545f1cdcfc8e704d7460b584c898be156f048828f", size = 12177806, upload-time = "2026-08-13T00:38:53.068Z" }, - { url = "https://files.pythonhosted.org/packages/f0/97/e04782c9eaa1a0b634830f61b0e18cd1b3415332143d45a28882802c8ea1/ty-0.0.71-py3-none-macosx_11_0_arm64.whl", hash = "sha256:4d0b1f2002adc03f3a53aeb70b5cafcea634bb48726d82a307b0f15580d2f74b", size = 12015265, upload-time = "2026-08-13T00:38:55.209Z" }, - { url = "https://files.pythonhosted.org/packages/4f/5c/d716b9049c11a7b85b3b74ec3547d2dbab9a261347a6558420dcae37083d/ty-0.0.71-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:165e6086363f5c149ae4acacfbc36eea0093552a60f4153ef48321e4a3a15c4c", size = 12119608, upload-time = "2026-08-13T00:38:57.336Z" }, - { url = "https://files.pythonhosted.org/packages/b8/78/560ce2d874467d50605b7783e19e45b571fea7a89f664bd0ccdf252adf8a/ty-0.0.71-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:067e16f80855afbda024e7ff6fa5ed3ada5290c403badc63800f0534bd617c04", size = 12353762, upload-time = "2026-08-13T00:38:59.729Z" }, - { url = "https://files.pythonhosted.org/packages/a3/7b/f6a22c0bf2c0dbfe20b7ce90b51bc09efa9a9e245f1c35e160e39ed5dea9/ty-0.0.71-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:416149a550bcb678e619b4e2cd1cca9066d28edc52df76ad9d3640b36e6b5bf1", size = 13088120, upload-time = "2026-08-13T00:39:02.34Z" }, - { url = "https://files.pythonhosted.org/packages/a3/95/6f5382781f2e4fa933296d303376c61b8b3475d58ed60125c54de665fd21/ty-0.0.71-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39e04c41e6d0e74f73cf8b3247d4dde3642ec38aeda7f4674110155b3da416a5", size = 13545140, upload-time = "2026-08-13T00:39:04.655Z" }, - { url = "https://files.pythonhosted.org/packages/6d/e7/8f0ad7b6c4804f8682e08fb161eaf710dd1765006b1b6df7c657b9707c95/ty-0.0.71-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7fb696bdea4a3554a0d0a7bbb36e4c5f508acb2bbad435e91c7fc812c1de6bfb", size = 13266730, upload-time = "2026-08-13T00:39:06.858Z" }, - { url = "https://files.pythonhosted.org/packages/ac/06/b83158fdf1473c2486fba0de337a963e9cc21317ccaa3e54ab003d419737/ty-0.0.71-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3ec4ca9d4ba3e11ecc282c3d50d0730a2e181da7142734219d9f213fcbaaa00", size = 12673786, upload-time = "2026-08-13T00:39:09.281Z" }, - { url = "https://files.pythonhosted.org/packages/7c/9a/10f37c0550277722ac1d2c8096e492e0f3fc1aa2e587082439dd066fdb8d/ty-0.0.71-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:48a231253b32639ff4b19f74e476bacdba0150182603011d3792d1f1a335b932", size = 13140294, upload-time = "2026-08-13T00:39:11.659Z" }, - { url = "https://files.pythonhosted.org/packages/3b/e6/5710de1da7eb8aa755d289aa25316691e545b72b4ee2ab14172612eec1c9/ty-0.0.71-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:1a082f57d1fcbe209afdcacff37870defd4cea15ce69e2c6c652a9208462722f", size = 12171216, upload-time = "2026-08-13T00:39:14.382Z" }, - { url = "https://files.pythonhosted.org/packages/80/c5/8d9113e3cf0d4c6c0a9c9e088cee93e41facb278026bea6f6e12533b09dd/ty-0.0.71-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:cbe1c962f6c9e8964180cd171cc6ad35d687f74d51c07f60b670d3f1c5d58ffe", size = 12360626, upload-time = "2026-08-13T00:39:16.747Z" }, - { url = "https://files.pythonhosted.org/packages/7d/a0/0e167507c4c863814d3ea7602bde958baefd03346bd38bae3a430dff1ea9/ty-0.0.71-py3-none-musllinux_1_2_i686.whl", hash = "sha256:25f641b988916b3975e50b2a59cbc5179f2a466d181f207f3032e1e6bc617a88", size = 12637309, upload-time = "2026-08-13T00:39:19.14Z" }, - { url = "https://files.pythonhosted.org/packages/1d/c6/ff6719b91e4916985e9a92f9bddb10d9fe3cb3bdf5abf0f9019a67aebe21/ty-0.0.71-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c5fe916b6e5b152ef4f583324e1efd770ac3316894776dc1b57c354cb767b8ee", size = 12937996, upload-time = "2026-08-13T00:39:21.533Z" }, - { url = "https://files.pythonhosted.org/packages/ee/3e/21a0873da8f1ece28e166fbbaf696cd48c55fd7cd203dba0266a1fe05ceb/ty-0.0.71-py3-none-win32.whl", hash = "sha256:a273fe0dcc453e94cf2e1955076efec8b739e6e1a890862de29d3c5a1c9ddaff", size = 11953321, upload-time = "2026-08-13T00:39:23.804Z" }, - { url = "https://files.pythonhosted.org/packages/69/fa/ce6614c748f7abfc546d12983d4bad1085116b6458b8c84d9372eb713f08/ty-0.0.71-py3-none-win_amd64.whl", hash = "sha256:65f5f980551ed79f68a0f6c0f2fb71b39a8d54ed0625d2529b01f6c919db72c5", size = 12570891, upload-time = "2026-08-13T00:39:26.346Z" }, - { url = "https://files.pythonhosted.org/packages/77/7f/0fb022535c66fd96e7dd2a1f9c14e3a0e39544a4c3f35a451e8835562730/ty-0.0.71-py3-none-win_arm64.whl", hash = "sha256:6d5552078b9934d359bd5f381dbcb160f1fc5addfca59a960021adca38a73741", size = 12338716, upload-time = "2026-08-13T00:39:28.987Z" }, + { url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" }, + { url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" }, + { url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" }, + { url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" }, + { url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" }, + { url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" }, + { url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" }, + { url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" }, + { url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" }, + { url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" }, ] [[package]] @@ -995,6 +1072,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "uvicorn" +version = "0.52.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/28/64ca011edf31c715b4fad359c587ea52391aaffa125065695590241ff617/uvicorn-0.52.3.tar.gz", hash = "sha256:18857b9e6579300be55c91c0a1cfd37d9a2cf0cabea33b88275f199eb73b8b58", size = 100621, upload-time = "2026-08-13T16:50:02.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/2b/ebd108734a8204c6b4b93c681c9a38c5273b3ccd5d129fee4ffc1d97772c/uvicorn-0.52.3-py3-none-any.whl", hash = "sha256:116af2710dbf47c80f463cd20ee4884b6662f4c9f227d797ddc7279d2fcc2c7c", size = 79859, upload-time = "2026-08-13T16:50:01.323Z" }, +] + [[package]] name = "zstandard" version = "0.25.0"