From 03c961414f6e7ad31067cdaf97d2e4e35d6e2f4b Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Thu, 3 Sep 2026 07:38:02 +0200 Subject: [PATCH 1/5] Run the website's optimization demo in the browser behind the setting --- apps/petrinaut-website/.env.example | 2 + apps/petrinaut-website/README.md | 27 ++-- .../app/browser-optimization-provider.tsx | 20 +++ apps/petrinaut-website/src/routes/index.tsx | 15 +- .../@hashintel/petrinaut/docs/optimization.md | 113 ++++++++++++-- .../petrinaut/docs/visual-settings.md | 4 + .../diagrams/browser-optimization-loop.d2 | 25 ++++ .../content/experiments/sweep-surface.mdx | 6 +- .../content/optimizer/browser-runtime.mdx | 141 ++++++++++++++++++ .../optimizer/running-the-loop-locally.mdx | 21 ++- .../content/optimizer/subprocess-boundary.mdx | 16 ++ 11 files changed, 351 insertions(+), 39 deletions(-) create mode 100644 apps/petrinaut-website/src/main/app/browser-optimization-provider.tsx create mode 100644 libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 create mode 100644 libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx diff --git a/apps/petrinaut-website/.env.example b/apps/petrinaut-website/.env.example index 2e3a4e81d8a..75dae3da7b9 100644 --- a/apps/petrinaut-website/.env.example +++ b/apps/petrinaut-website/.env.example @@ -2,3 +2,5 @@ OPENAI_API_KEY=sk-xxxx OPENAI_VOICE_API_KEY= PETRINAUT_OPENAI_VOICE_ENABLED=false VITE_BRUNCH_CHAT_ENDPOINT= +# "service" enables the /optimization route against the Python optimizer service behind the dev proxy; `turbo run dev -- --with-optimizer-service` sets it. Unset hides the route. The main demo runs its optimizer in the browser. +# VITE_PETRINAUT_OPT_PROVIDER=service diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 30cc5d6a7e3..93723bfb0c4 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -47,25 +47,34 @@ cannot include the current example URL. `FullExamplePage` adds the standard Consumers that do not execute JavaScript must call `/api/oembed` directly or use provider-pattern discovery instead. -### Optimization demo with Petrinaut Opt +### Optimization demo -From the repository root, run: +The main demo at [http://localhost:5173](http://localhost:5173) runs the +optimizer in the browser: the Optuna study runs in a Pyodide web worker and +each optimization step runs on Petrinaut's own experiments backend, so no +Python service is involved. The **Optimizations** tab appears once the +experimental **In-browser optimization** setting is on, under **Viewport +controls > Settings > Simulation**. The first optimization in a browser +downloads the Python runtime from jsDelivr and Optuna from PyPI; later runs use +the browser cache. + +The `/optimization` route is the Python-service variant. It returns the +website's not-found page unless `VITE_PETRINAUT_OPT_PROVIDER=service` is set. +To run it, from the repository root: ```sh turbo run dev --filter @apps/petrinaut-website -- --with-optimizer-service ``` The flag builds and starts the local Petrinaut Opt Docker image, waits for its -health endpoint, and starts the website with the real optimization provider. -Open [http://localhost:5173/optimization](http://localhost:5173/optimization). +health endpoint, and starts the website with +`VITE_PETRINAUT_OPT_PROVIDER=service`. Open +[http://localhost:5173/optimization](http://localhost:5173/optimization). Stopping the command also stops and removes its optimizer container. The development server proxies `/api/petrinaut-opt/*` to the optimizer on `127.0.0.1:4004`, avoiding development-only CORS changes to the Python service. -Regular `yarn dev` does not enable optimization; use the dedicated command to -connect the website to the real optimizer service. The `/optimization` route -returns the website's not-found page when the provider is disabled. Storybook -provides a fake optimizer for isolated UI development. +Storybook provides a fake optimizer for isolated UI development. ## Environment variables @@ -77,7 +86,7 @@ provides a fake optimizer for isolated UI development. | `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | | `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | | `VITE_BRUNCH_CHAT_ENDPOINT` | for voice input | website | Full Brunch Petrinaut chat endpoint used by the panel. | -| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the optimization route. | +| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the `/optimization` route. | | `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. diff --git a/apps/petrinaut-website/src/main/app/browser-optimization-provider.tsx b/apps/petrinaut-website/src/main/app/browser-optimization-provider.tsx new file mode 100644 index 00000000000..a5ec056c5bc --- /dev/null +++ b/apps/petrinaut-website/src/main/app/browser-optimization-provider.tsx @@ -0,0 +1,20 @@ +import { createBrowserOptimization } from "@hashintel/petrinaut-core/browser-optimization"; +import { PetrinautOptimizationContext } from "@hashintel/petrinaut/react"; + +import type { FC, PropsWithChildren } from "react"; + +const browserOptimization = createBrowserOptimization(); + +/** + * The in-tab optimizer: the Optuna study runs in a Pyodide worker and each + * step runs on the editor's own experiments backend, so the demo needs no + * optimizer service. Petrinaut connects it only while its experimental + * In-browser optimization setting is on. + */ +export const BrowserOptimizationProvider: FC = ({ + children, +}) => ( + + {children} + +); diff --git a/apps/petrinaut-website/src/routes/index.tsx b/apps/petrinaut-website/src/routes/index.tsx index 4f969a72595..3a52ca4fb51 100644 --- a/apps/petrinaut-website/src/routes/index.tsx +++ b/apps/petrinaut-website/src/routes/index.tsx @@ -5,6 +5,7 @@ import { } from "@tanstack/react-router"; import { validateSharedExampleSearch } from "../examples/example-search"; +import { BrowserOptimizationProvider } from "../main/app/browser-optimization-provider"; import { LocalStorageDemoApp } from "../main/app/local-storage-demo/local-storage-demo-app"; function IndexRoute() { @@ -12,12 +13,14 @@ function IndexRoute() { const search = useSearch({ from: "/" }); return ( - { - void navigate({ replace: history === "replace", search: nextSearch }); - }} - search={search} - /> + + { + void navigate({ replace: history === "replace", search: nextSearch }); + }} + search={search} + /> + ); } diff --git a/libs/@hashintel/petrinaut/docs/optimization.md b/libs/@hashintel/petrinaut/docs/optimization.md index 79284d42d9b..8b7bf28c2b3 100644 --- a/libs/@hashintel/petrinaut/docs/optimization.md +++ b/libs/@hashintel/petrinaut/docs/optimization.md @@ -6,9 +6,12 @@ metric defined only for that optimization. Use it when you know the outcome you want and want Petrinaut to explore a bounded set of scenario inputs. Optimizations live under the **Simulate** global mode. The **Optimizations** tab -is available only when the host application reports that an optimization -service is configured. A temporary service outage does not hide the tab; a run -attempted during the outage instead reports an error in its result drawer. +is available only when the host application provides an optimizer: a remote +optimization service, or an in-browser optimizer once you turn on the +experimental **In-browser optimization** setting (see +[Running in the browser](#running-in-the-browser)). A temporary service outage +does not hide the tab; a run attempted during the outage instead reports an +error in its result drawer. ## Before you start @@ -35,8 +38,17 @@ parameter behind the scenes. Petrinaut never picks a scenario automatically. Selecting another scenario resets the optimization form for that scenario. 4. Give the optimization a name and choose its number of optimization steps - (between 1 and 1,000), time step (default `0.1`), and maximum simulation - time. + (between 1 and 1,000), **runs per step** (between 1 and 100, default `1`), + time step (default `0.1`), and maximum simulation time. A step's objective + is the mean over its runs, so more runs per step give the optimizer a + steadier signal on a stochastic model, at the cost of more simulations per + step. With the in-browser optimizer and **WebGPU** on in the + [settings dialog](visual-settings.md#webgpu-experimental), a **Backend** + switch appears next to these fields. For an optimization it stays greyed + out, with the reason on hover: the objective is an expression metric, which + the GPU backend cannot compute (see + [Compute backend](experiments.md#compute-backend-experimental)), so the + steps run on the CPU. 5. In **Parameters**, leave a parameter at its current **Value** or enable **Optimize** and enter its search range. At least one parameter must be optimized. @@ -48,10 +60,10 @@ parameter behind the scenes. optimization objectives. 7. Click **Run**. -The metric is evaluated on the final frame of each optimization step's -simulation. The current model is reduced to an immutable snapshot containing -the selected scenario and objective metric when the optimization starts. Later -edits do not change an in-flight run. +The metric is evaluated on the final frame of each run, and a step's objective +is the mean over its runs. The current model is reduced to an immutable snapshot +containing the selected scenario and objective metric when the optimization +starts. Later edits do not change an in-flight run. ## Search domains @@ -90,6 +102,28 @@ connection reports how many of the requested trials had completed and includes a diagnostic identifier for support. Trials received before the failure are kept, and a **Retry** action starts a fresh run with the same settings. +A study that runs in the browser (see [Running in the +browser](#running-in-the-browser)) shows more, because the machine computing +it is yours: + +- A badge beside the **Summary** heading says where the steps run. It reads + **CPU**, because the GPU backend cannot compute an expression objective (see + step 4 of [Creating an optimization](#creating-an-optimization)). +- A **Parameters** band with one slider per optimized numeric parameter and a + switch per optimized boolean parameter. While the study runs, **Follow + steps** is on: the controls move to each step's values as it is evaluated and + the status line reads **Following step N**. Move any control and following + stops; the point you picked computes in escalating batches (8, 25, then 100 + runs) while the line reads **N of M runs — refining**. Turn **Follow steps** + back on to rejoin the step in flight. +- The **Surface** section whenever two or more numeric parameters are + optimized, without the Optimization surface setting. The ringed dot is the + Parameters band's position, and clicking or dragging the plot moves it. +- A **Metrics** section with the objective metric's distribution over + simulation time at that position: the step being evaluated while following, + otherwise the point you picked. It streams again whenever the position + changes. + ## The surface view The surface is experimental and off by default. Turn on **Optimization @@ -114,14 +148,63 @@ Log-scale domains slide in log space, and integer domains snap to their step. Local points always reflect the model as it was when the study launched, even if you have edited the net since. +A study that runs in the browser shows the Surface without this setting +whenever it has two or more optimized numeric parameters, and its sliders are +the drawer's **Parameters** band (see [Running in the +browser](#running-in-the-browser)). + +## Running in the browser + +When the host provides the in-browser optimizer, the whole study runs in your +tab: the optimizer runs in a background worker, and each optimization step runs +as a batch of seeded simulations on the same compute backend as your +experiments. + +- Turn it on under **Viewport controls > Settings > Simulation > In-browser + optimization** (Experimental). The setting is off by default, and while it is + off the **Optimizations** tab stays hidden even though the host provides the + in-browser optimizer. Turning the setting off while an in-browser + optimization is running cancels it. A remote optimization service is never + affected by the setting. +- The first optimization in a browser downloads the Python runtime and the + optimizer packages before its first step starts; the run shows as + **Running** with no steps completed while that happens. Later runs reuse the + browser's cache. +- **Runs per step** sets how many seeded simulations each step averages. With + WebGPU on, the form also shows the **Backend** switch, but for an + optimization it stays greyed out and the steps run on the CPU: the objective + is an expression metric, which the GPU backend cannot compute. +- While the study runs, the drawer follows it: the **Parameters** band moves + to each step's values and the **Metrics** section streams the objective + metric over that step's runs as they complete. Move a slider, flip a switch, + or click the surface to look at any other point at any time — its objective + computes in escalating batches on the same backend — and turn **Follow + steps** back on to rejoin the study. +- When a point cannot be computed — the objective metric does not compile, + the backend declines the model, or some of its runs fail — the + **Parameters** band says why under **Could not compute** and the **Metrics** + section stays empty; a step that fails this way is pruned. Moving to another + point, or back to this one, tries again. +- Once the study is finished the controls stay. The point they hold — the last + step, or wherever you moved them — refines up to 100 runs, and every point + you visit is kept for the record's lifetime, so returning to one is instant. + **Cancel** stops the current step and the study; the drawer keeps the + received steps and computes again only when you move. +- Keep the tab open. Closing or reloading the page ends the study, and the + record is gone on the next load. +- Given the same settings, each step of an in-browser optimization runs on the + CPU with the same seeds as the optimization service, so it produces the + service's objective values and the optimizer proposes the same parameter + values, step for step. + ## Connection drops and reloads -An optimization runs on the server, not in your browser tab. If the connection -drops while you watch one, Petrinaut reconnects automatically and resumes from -the last result it received — the status shows **(reconnecting…)** while it -retries, and no trials are lost or double-counted. Only if reconnecting keeps -failing does the run report a connection error, which keeps the received -trials and offers **Retry**. +When the host uses an optimization service, an optimization runs on the server, +not in your browser tab. If the connection drops while you watch one, Petrinaut +reconnects automatically and resumes from the last result it received — the +status shows **(reconnecting…)** while it retries, and no trials are lost or +double-counted. Only if reconnecting keeps failing does the run report a +connection error, which keeps the received trials and offers **Retry**. Reloading or closing the page is different: the page loses its view of a still-running optimization. The run itself continues on the server until it diff --git a/libs/@hashintel/petrinaut/docs/visual-settings.md b/libs/@hashintel/petrinaut/docs/visual-settings.md index 2c09ca8b344..721930692d4 100644 --- a/libs/@hashintel/petrinaut/docs/visual-settings.md +++ b/libs/@hashintel/petrinaut/docs/visual-settings.md @@ -64,6 +64,10 @@ Off by default. Adds a **Sweep** toggle to every numeric scenario parameter in t Off by default. Adds a **Surface** section to an optimization study with two or more optimized numeric parameters, computed locally on your machine. See [The surface view](optimization.md#the-surface-view). +### In-browser optimization (experimental) + +Shown only when the host application provides an optimizer that runs in your browser. Off by default. On, Petrinaut connects that optimizer: the **Optimizations** tab appears under Simulate, each study's steps run on the experiments backend, and the study drawer streams the objective's metrics for the step being evaluated (see [Running in the browser](optimization.md#running-in-the-browser)). Off, the tab stays hidden and any running in-browser optimization is cancelled. A remote optimization service provided by the host is not affected by this setting. + ### Arcs rendering Choose how arcs are drawn between nodes: diff --git a/libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 b/libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 new file mode 100644 index 00000000000..0908eb182b0 --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 @@ -0,0 +1,25 @@ +# Hand-written; rendered by the arch-docs build. +shape: sequence_diagram + +provider: Optimizations provider +capability: Browser capability +worker: Pyodide worker +optuna: Optuna (optimizer-core) +channel: Channel +experiments: Experiments provider + +provider -> capability: createOptimizationRun(manifest) +capability -> worker: "start { runId, description }" +worker -> optuna: run_browser_study +optuna -> worker: ask, suggest values +worker -> capability: "evaluate { requestId, trial, suggestedValues }" +capability -> channel: evaluateTrial(request) +channel -> experiments: "runDetachedObjective (runCount = seeds, seeds pinned on the CPU)" +experiments -> provider: frames stream into the followed selection +experiments -> channel: "outcome: the result, or { reason, cancelled } when refused, failed or cancelled" +channel -> capability: "{ objective, replicates } or { pruned, reason }" +capability -> worker: "evaluated { requestId, outcome }" +worker -> optuna: tell +optuna -> worker: trial event with best +worker -> capability: "trial { event }" +capability -> provider: trial event through attachOptimizationRun diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx index 44ef6be6618..08d25957167 100644 --- a/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx +++ b/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx @@ -112,4 +112,8 @@ its objective as an expression metric, and returns the finished frames. The component keeps its own per-position cache and climbs the same run ladder for the selected point, streaming the objective's mean and median. The study's completed trials draw as projected markers over the locally computed fill — -the first form of the optimizer streaming its evaluations back. +the first form of the optimizer streaming its evaluations back. A study run in +the browser streams its trials into this view as Optuna tells them, and its +ring follows the record's `navigation` rather than component state, so the +drawer's Parameters band and the plot move together +([the browser runtime](doc:optimizer/browser-runtime)). diff --git a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx new file mode 100644 index 00000000000..22e51298eed --- /dev/null +++ b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx @@ -0,0 +1,141 @@ +--- +title: The browser runtime +description: How an optimization runs entirely in the tab — Optuna in a Pyodide worker, trials as detached objective runs on the experiments backend — and what each piece owns. +sidebar_order: 10 +attachTo: core.optimization.browser +--- + +An optimization needs two things the service supplies around the CLI: an +Optuna study that suggests parameter values, and a simulator that scores them. +In the browser both exist already — Optuna runs under Pyodide, and the +editor's experiments backend simulates — so this layer is only the adapter +between them. It implements `PetrinautConnectedOptimization`: a source the +optimizations provider `connect`s with a channel, yielding the same +`PetrinautOptimization` capability a remote service offers. The drawers and +the event schema never learn which one they have. + +The provider reaches the source through `useOptimizationSource()`, which +returns a connected source only while the experimental **In-browser +optimization** user setting is on. Off, the hook returns `null`: the +Optimizations tab hides, the provider disposes the connection and cancels the +study's runs. A remote capability passes through the hook unchanged. + +![One trial through the browser runtime](@diagrams/browser-optimization-loop.svg) + +## The data flow + +1. **Drawer → provider.** The create drawer builds an optimization manifest + (model snapshot, scenario, objective metric, parameter domains, study + settings including `seedsPerTrial`) and the + [optimizations provider](layer:react.optimizations) calls + `createOptimizationRun` on the connected capability. The requested compute + backend travels beside the manifest, on the record. +2. **Capability → worker.** The capability validates the manifest, describes + the study with the same `describeOptimization` the CLI uses, allocates a + run id, appends `started` to the run's event log and posts `start` to a + shared module worker. Runs queue: one study executes at a time. +3. **Worker → Optuna.** On first use the worker loads Pyodide from the CDN + pinned in the runtime lock, installs Optuna with micropip, writes the + [optimizer-core](layer:optimizer-core) sources into the virtual filesystem + and imports them. `run_browser_study` asks the sampler for values and + awaits a JavaScript `evaluate` callback for each trial. +4. **Worker → channel.** Each `evaluate` crosses back to the main thread as a + message; the capability resolves the trial's scenario parameter values and + seeds (`deriveOptimizationTrialSeeds`, the CLI's helper) and calls + `channel.evaluateTrial`. +5. **Channel → experiments.** The [channel](layer:react.optimizations.channel) + turns the trial into one detached objective run: `runDetachedObjective` on + the [experiments provider](layer:react.experiments), keyed by the run id so + a study compiles its snapshot once and its trials queue in order. The + request carries the trial's values, `runCount = seeds.length`, the seeds + pinned per run, and the study's requested backend. The sampler picks the + backend once per study — `selectExperimentBackend` over the GPU-first + registrations with the CPU pool last, at a third of the pool's shards so + the surface walk and the user's own experiments keep workers — and reuses + it for every later batch. Pinned seeds go only to the CPU pool; the GPU + backend refuses them and derives run seeds from the base seed. The trial's + objective is the mean of the per-run finals the CPU reports (`runResults`, + run index ↔ seed, handed to Optuna as replicates) or, where the backend + reports no run axis, the objective metric's last sampled frame reduced to a + mean. A batch that does not complete settles with a reason — the metric's + compile diagnostics, each backend's refusal, the terminal error, or the + count of errored runs — and the channel prunes the trial with that reason; + a cancelled batch is pruned as `cancelled`, and a non-finite objective + names the metric. The channel never throws. +6. **Events → drawer.** Each `tell` produces a `trial` event carrying the + running best; the summary produces `complete`. The provider consumes them + through `attachOptimizationRun`, which replays the run log from a cursor + and tails it, so a re-attach neither misses nor duplicates an event. The + record's `navigation` and `selection` fields carry the local side: where + the drawer points and the objective's live stream there. + +## Following and refinement + +The provider keeps one connected-study object per record +(`react/optimizations/provider/connected-study.ts`). While +`navigation.followTrials` is on — it starts on — `trialStarted` moves the +navigation to the trial's values (`optimizationAxisPositionFor` per axis) and +mirrors the batch's frames and progress stores into `selection` under the key +`trial:`, so the drawer's **Metrics** section streams the step being +evaluated. A user move clears `followTrials` and hands the navigation to point +refinement (`provider/point-refinement.ts`): the experiments' run ladder capped +at 100, one batch per rung with `seed = sweepBatchSeed(studySeed, from)` and +`runCount = target − from`, frames merged with `mergeMetricFramesAcrossCells`, +results cached per navigation key for the record's lifetime. A key change +cancels the batch in flight; returning to a cached key is instant. A rung that +fails stops the ladder at that key and records the batch's reason on +`selection.error`; a followed trial that fails records its reason the same +way. The **Parameters** band shows it as _Could not compute_, the **Metrics** +section shows the empty chart shell, and the next refine of that key — a later +navigation change — retries. When the study settles, following ends and the +point refines wherever the navigation stands — except after a cancellation, +which stops compute until the next move. + +The drawer's **Parameters** band edits `navigation` through +`setOptimizationNavigation`; the **Surface** section reads its positions from +the same field and commits picks the same way, while its contour fill still +walks the grid through `sampleDetachedObjective` +([the sweep surface](doc:experiments/sweep-surface) covers that walk). + +## What each side owns + +- **Python (optimizer-core)** owns the study: sampler, suggestion, ask/tell, + trial state and the best-so-far. It sees flat descriptors and scalar + objectives, nothing of the model. +- **The worker** owns Pyodide: loading, package installation, marshalling + proxies to plain objects, and pairing `evaluate` requests with replies by + request id. Cancelling resolves any pending evaluate as pruned so the loop + observes it. +- **The capability** owns run identity and the event log — dense sequence + numbers, replay from a cursor, terminal `complete`/`error` — and maps + channel failures to events: a refusal or abort prunes the trial; anything + else fails the run. +- **The channel** owns the mapping between a trial and a detached run: the + request, the outcome, and the study callbacks that let the record follow. +- **The experiments provider** owns simulation: compilation (cached per + study), backend choice, seeding, streaming frames and progress, and the + per-study queue. +- **The connected study** owns the drawer's local state: navigation, + following, and the refinement ladder at the navigated point. + +## Parity with the service + +Both drivers derive per-trial seeds with the same helper, and the sampler +runs the same Optuna version with the same seed. On the CPU pool the request +pins every run's seed (`runs: [{ seed }]`), so a trial's replicates run on the +seeds the CLI runs and the trial reports the service's objective value. The +parity in proposals depends on that: after its startup trials the TPE sampler +conditions each proposal on the objectives it was told, so the study proposes +the same parameter values step for step only while the objective values match. +Every connected study meets the condition, because the form cannot request a +GPU study: the objective is an expression metric, which the GPU backend cannot +compute, so `useGpuAvailability` reports it unavailable and the **Backend** +switch stays disabled. Were the GPU backend to take an objective, it would +derive its own run seeds from the base seed, and the objective values — and +the proposals after the startup trials — would diverge from the service's. The +runtime lock the worker installs from pins the Optuna version the core +package's own lockfile installs, and a test in the core package fails when they +differ; the service resolves the same version range in its own lockfile. What +remains different is where the study lives: in the tab. Closing the page ends +it, and because a connected run is never written to session storage, the next +load shows no record for it. diff --git a/libs/@local/petrinaut-arch-docs/content/optimizer/running-the-loop-locally.mdx b/libs/@local/petrinaut-arch-docs/content/optimizer/running-the-loop-locally.mdx index 471c55990a5..0886a80356b 100644 --- a/libs/@local/petrinaut-arch-docs/content/optimizer/running-the-loop-locally.mdx +++ b/libs/@local/petrinaut-arch-docs/content/optimizer/running-the-loop-locally.mdx @@ -69,14 +69,19 @@ same `/api/petrinaut-opt/*` proxy as the website's, and the story explains how to launch when the environment is missing. Remaining arguments go to Storybook (`turbo run dev --filter @hashintel/petrinaut -- --with-optimizer-service --port 6007`). -## Why plain `yarn dev` shows no Optimizations view - -The editor renders the Optimizations tab only when a -`PetrinautOptimizationContext` is mounted, and the website mounts one only on -the `/optimization` route with `VITE_PETRINAUT_OPT_PROVIDER=service` set. A -plain `turbo run dev` in `@apps/petrinaut-website` leaves the context null: -the tab is hidden and nothing optimization-related is reachable. Storybook -provides a fake optimizer for isolated UI work on the drawers. +## Which optimizer plain `yarn dev` offers + +The editor renders the Optimizations tab only when `useOptimizationSource()` +yields a source. The main demo (`/`) mounts the +[browser runtime](layer:core.optimization.browser) as a connected source, which +the hook returns only while the experimental **In-browser optimization** +setting is on: a plain `turbo run dev` in `@apps/petrinaut-website` shows the +tab once that setting is on, and the studies created there run in the tab with +no service involved ([the browser runtime](doc:optimizer/browser-runtime) +covers that path). The `/optimization` route mounts the service capability, and +only with `VITE_PETRINAUT_OPT_PROVIDER=service` set; without it the route is +not found. Storybook provides a fake optimizer for isolated UI work on the +drawers. ## The request path diff --git a/libs/@local/petrinaut-arch-docs/content/optimizer/subprocess-boundary.mdx b/libs/@local/petrinaut-arch-docs/content/optimizer/subprocess-boundary.mdx index e6945cdb4d9..3ef84968557 100644 --- a/libs/@local/petrinaut-arch-docs/content/optimizer/subprocess-boundary.mdx +++ b/libs/@local/petrinaut-arch-docs/content/optimizer/subprocess-boundary.mdx @@ -45,3 +45,19 @@ break; that is the seam working as intended. Nothing else crosses: the optimizer's HTTP surface, Optuna, and telemetry never reach the CLI, and the CLI never learns it is being driven by a service rather than a script. + +## One study, two drivers + +The study itself — parsing the description, seeding the sampler, suggesting +values, the ask/tell loop and the best-so-far — lives in +[optimizer-core](layer:optimizer-core), a pure-Python package with no threads, +I/O or process control. The service imports it and drives it from a worker +thread across the boundary above. The +[browser runtime](layer:core.optimization.browser) loads the same modules into +Pyodide inside a Web Worker and drives them through a host channel that +evaluates each trial as a batch of seeded runs on the experiments backend; +[the browser runtime](doc:optimizer/browser-runtime) walks through that flow. +Optuna never learns which driver it is under. The runtime lock the browser +installs from is asserted against the Optuna version the core package's own +lockfile installs; the service resolves the same version range in its own +lockfile. From 93c1751f21366611abea0620dcb3f5d7cab73123 Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Fri, 4 Sep 2026 04:08:11 +0200 Subject: [PATCH 2/5] Document the connected study's Surface as the optimizer's steps --- .../@hashintel/petrinaut/docs/optimization.md | 56 ++++++++++++------- .../content/experiments/sweep-surface.mdx | 11 ++-- .../content/optimizer/browser-runtime.mdx | 14 ++++- 3 files changed, 53 insertions(+), 28 deletions(-) diff --git a/libs/@hashintel/petrinaut/docs/optimization.md b/libs/@hashintel/petrinaut/docs/optimization.md index 8b7bf28c2b3..3de14e1fb2b 100644 --- a/libs/@hashintel/petrinaut/docs/optimization.md +++ b/libs/@hashintel/petrinaut/docs/optimization.md @@ -111,14 +111,24 @@ it is yours: step 4 of [Creating an optimization](#creating-an-optimization)). - A **Parameters** band with one slider per optimized numeric parameter and a switch per optimized boolean parameter. While the study runs, **Follow - steps** is on: the controls move to each step's values as it is evaluated and - the status line reads **Following step N**. Move any control and following - stops; the point you picked computes in escalating batches (8, 25, then 100 - runs) while the line reads **N of M runs — refining**. Turn **Follow steps** - back on to rejoin the step in flight. + steps** is on: the controls move to each step's values as it is evaluated, + disabled while they follow, and the status line reads **Following step N**. + Turn **Follow steps** off to take over early, or wait for the study to + finish; then move any control and the point you picked computes in + escalating batches (8, 25, then 100 runs) while the line reads **N of M + runs — refining**. Turn **Follow steps** back on to rejoin the step in + flight. - The **Surface** section whenever two or more numeric parameters are - optimized, without the Optimization surface setting. The ringed dot is the - Parameters band's position, and clicking or dragging the plot moves it. + optimized, without the Optimization surface setting. It draws only the + study's steps: each step is a dot at its parameters, the best emphasized, + pruned steps hollow, and the field is interpolated between them, so it fills + in as steps report — nothing is sampled behind the study. The ringed dot is + the step being evaluated, its running value streaming into the field as the + runs complete; the caption counts the steps placed and the best so far. + While the study runs with **Follow steps** on, the plot only displays. Once + the study is over, or **Follow steps** is off, the ringed dot is the point + the Parameters band holds: click or drag the plot to move it, and the + point's value enters the field as it refines. - A **Metrics** section with the objective metric's distribution over simulation time at that position: the step being evaluated while following, otherwise the point you picked. It streams again whenever the position @@ -130,12 +140,13 @@ The surface is experimental and off by default. Turn on **Optimization surface** under Simulation in the [settings dialog](visual-settings.md#optimization-surface-experimental) to see it. -A study with two or more optimized numeric parameters grows a **Surface** -section between the best parameters and the step list: an Optuna-style contour -of the objective over two parameters you pick. The study's own trials appear as rings (the best -trial highlighted), and the filled contour comes from points **computed -locally on your machine** — the study's model snapshot runs on a background -worker, a few runs per point, and the plot fills in coarse shape first. +A study run on the optimization service with two or more optimized numeric +parameters grows a **Surface** section between the best parameters and the +step list: an Optuna-style contour of the objective over two parameters you +pick. The study's own trials appear as rings (the best trial highlighted), and +the filled contour comes from points **computed locally on your machine** — +the study's model snapshot runs on a background worker, a few runs per point, +and the plot fills in coarse shape first. One slider per optimized parameter navigates the space; parameters not shown on the plot hold at their slider position, which starts at the best trial's @@ -148,8 +159,9 @@ Log-scale domains slide in log space, and integer domains snap to their step. Local points always reflect the model as it was when the study launched, even if you have edited the net since. -A study that runs in the browser shows the Surface without this setting -whenever it has two or more optimized numeric parameters, and its sliders are +A study that runs in the browser shows a different Surface without this +setting whenever it has two or more optimized numeric parameters: it samples +no grid of its own — the study's steps are its samples — and its controls are the drawer's **Parameters** band (see [Running in the browser](#running-in-the-browser)). @@ -175,11 +187,15 @@ experiments. optimization it stays greyed out and the steps run on the CPU: the objective is an expression metric, which the GPU backend cannot compute. - While the study runs, the drawer follows it: the **Parameters** band moves - to each step's values and the **Metrics** section streams the objective - metric over that step's runs as they complete. Move a slider, flip a switch, - or click the surface to look at any other point at any time — its objective - computes in escalating batches on the same backend — and turn **Follow - steps** back on to rejoin the study. + to each step's values, the **Surface** gains a dot per step with the field + filling in around it and the ringed dot streaming the step in flight, and + the **Metrics** section streams the objective metric over that step's runs + as they complete. The sliders and the plot only display while **Follow + steps** is on; turn it off to take over early. Then, or once the study is + over, move a slider, flip a switch, or click or drag the surface to look at + any other point — its objective computes in escalating batches on the same + backend and enters the field — and turn **Follow steps** back on to rejoin + the study. - When a point cannot be computed — the objective metric does not compile, the backend declines the model, or some of its runs fail — the **Parameters** band says why under **Could not compute** and the **Metrics** diff --git a/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx b/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx index 08d25957167..79e7c3ba615 100644 --- a/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx +++ b/libs/@local/petrinaut-arch-docs/content/experiments/sweep-surface.mdx @@ -111,9 +111,10 @@ single-worker lane that compiles the snapshot once per study (cached), runs its objective as an expression metric, and returns the finished frames. The component keeps its own per-position cache and climbs the same run ladder for the selected point, streaming the objective's mean and median. The study's -completed trials draw as projected markers over the locally computed fill — -the first form of the optimizer streaming its evaluations back. A study run in -the browser streams its trials into this view as Optuna tells them, and its -ring follows the record's `navigation` rather than component state, so the -drawer's Parameters band and the plot move together +completed trials draw as projected rings over the locally computed fill — the +optimizer's evaluations laid over a picture computed here. A study run in the +browser skips the walk: its trials are the field's samples, drawn as dots and +interpolated between, the step in flight streams its running objective at the +record's `navigation`, and the plot only displays until the study is over or +following is off, so the drawer's Parameters band and the plot move together ([the browser runtime](doc:optimizer/browser-runtime)). diff --git a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx index 22e51298eed..b9464a2a0b3 100644 --- a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx +++ b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx @@ -92,9 +92,17 @@ point refines wherever the navigation stands — except after a cancellation, which stops compute until the next move. The drawer's **Parameters** band edits `navigation` through -`setOptimizationNavigation`; the **Surface** section reads its positions from -the same field and commits picks the same way, while its contour fill still -walks the grid through `sampleDetachedObjective` +`setOptimizationNavigation`, its controls disabled while they follow a +running study. The **Surface** section samples nothing of its own: the +study's trials are the field (`trialSurfaceField` projects each trial with an +objective onto the shown axes as a sample and a dot, pruned trials as hollow +marks), and the selection stream's running objective enters the field at the +navigation (`navigatedSurfaceSample`) — the step in flight until its event +lands, the refined point afterwards — so the plot fills in as the study +streams. `surfaceInteraction` keeps the plot display-only while a running +study is followed and lets a click or drag commit `setOptimizationNavigation` +once the study is terminal or following is off. Only a remote study's surface +walks a grid through `sampleDetachedObjective` ([the sweep surface](doc:experiments/sweep-surface) covers that walk). ## What each side owns From d6b3fad199ee39d2e03ef8063f8d3f2182619135 Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Fri, 4 Sep 2026 05:26:27 +0200 Subject: [PATCH 3/5] Document stopping, continuing and parallel steps in the browser optimizer --- .../@hashintel/petrinaut/docs/optimization.md | 38 +++++++++++---- .../diagrams/browser-optimization-loop.d2 | 13 ++++-- .../content/optimizer/browser-runtime.mdx | 46 +++++++++++++++++-- 3 files changed, 81 insertions(+), 16 deletions(-) diff --git a/libs/@hashintel/petrinaut/docs/optimization.md b/libs/@hashintel/petrinaut/docs/optimization.md index 3de14e1fb2b..7b985154291 100644 --- a/libs/@hashintel/petrinaut/docs/optimization.md +++ b/libs/@hashintel/petrinaut/docs/optimization.md @@ -48,7 +48,8 @@ parameter behind the scenes. out, with the reason on hover: the objective is an expression metric, which the GPU backend cannot compute (see [Compute backend](experiments.md#compute-backend-experimental)), so the - steps run on the CPU. + steps run on the CPU. The in-browser optimizer also offers **Parallel + steps** (1 to 4, default `1`): how many steps it evaluates at once. 5. In **Parameters**, leave a parameter at its current **Value** or enable **Optimize** and enter its search range. At least one parameter must be optimized. @@ -94,8 +95,10 @@ steps arrive and shows: retaining aggregate progress and the current best result. Closing the drawer does not stop the optimization. Use **Cancel** to abort an -active run. Completed, cancelled, and failed records can be removed from their -result drawer. +active run on the optimization service, or **Stop** to end a study running in +the browser, which can then be continued (see [Running in the +browser](#running-in-the-browser)). Completed, cancelled, stopped, and failed +records can be removed from their result drawer. If a run fails, the drawer explains what happened — for example, a lost connection reports how many of the requested trials had completed and includes @@ -201,11 +204,30 @@ experiments. **Parameters** band says why under **Could not compute** and the **Metrics** section stays empty; a step that fails this way is pruned. Moving to another point, or back to this one, tries again. -- Once the study is finished the controls stay. The point they hold — the last - step, or wherever you moved them — refines up to 100 runs, and every point - you visit is kept for the record's lifetime, so returning to one is instant. - **Cancel** stops the current step and the study; the drawer keeps the - received steps and computes again only when you move. +- **Parallel steps** (1 to 4, default 1) sets how many steps the optimizer + evaluates at once. Above 1, the optimizer accounts for the steps still + running when it picks the next values, so the proposals differ from a + one-at-a-time study. The Parameters band follows the most recently started + step, and the Surface rings every step in flight with its running value. +- Under the Summary's steps bar, a thinner bar tracks the followed step's runs + over the runs per step, and a **"N computing"** chip appears while anything + computes — the steps in flight and the picked point's refinement — expanding + into one row per batch with its own progress. +- Once the study is finished, or you press **Stop**, the controls move to the + best step's point (if a step completed) and that point refines up to 100 + runs; a point you had already moved to stays where it is. Every point you + visit is kept for the record's lifetime, so returning to one is instant. A + point that cannot beat the best — its mean sits more than 2.5 standard + errors on the wrong side of the best value after its first 8 runs — stops + refining there, and the Parameters band says so: **8 runs · cannot beat the + best**. The best step's own point always refines to 100 runs. +- **Stop** discards the steps in flight — they count as pruned — and keeps + the study's sampler: the status reads **Stopped**, and the footer offers + **Continue** with a number of steps (the study's own step count by default). + Continue runs that many more steps on the same study, with everything it + learned so far, and the Summary's Steps counts them into the total. A + completed study can be continued the same way, as often as the 1,000-step + cap allows. Removing the study drops its sampler. - Keep the tab open. Closing or reloading the page ends the study, and the record is gone on the next load. - Given the same settings, each step of an in-browser optimization runs on the diff --git a/libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 b/libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 index 0908eb182b0..35aa2350d17 100644 --- a/libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 +++ b/libs/@local/petrinaut-arch-docs/content/diagrams/browser-optimization-loop.d2 @@ -9,9 +9,10 @@ channel: Channel experiments: Experiments provider provider -> capability: createOptimizationRun(manifest) -capability -> worker: "start { runId, description }" -worker -> optuna: run_browser_study -optuna -> worker: ask, suggest values +capability -> worker: "start { runId, description, parallelism }" +worker -> optuna: create_browser_study (kept handle) +worker -> optuna: run_browser_study (one segment of trials) +optuna -> worker: ask, suggest values (up to parallelism in flight) worker -> capability: "evaluate { requestId, trial, suggestedValues }" capability -> channel: evaluateTrial(request) channel -> experiments: "runDetachedObjective (runCount = seeds, seeds pinned on the CPU)" @@ -23,3 +24,9 @@ worker -> optuna: tell optuna -> worker: trial event with best worker -> capability: "trial { event }" capability -> provider: trial event through attachOptimizationRun +provider -> capability: "extendOptimizationRun(runId, trials) after Stop or Complete" +capability -> worker: "extend { runId, trials }" +worker -> optuna: run_browser_study (next segment, same study) +provider -> capability: releaseOptimizationRun(runId) when the record is removed +capability -> worker: "release { runId }" +worker -> optuna: release_browser_study diff --git a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx index b9464a2a0b3..dd8a1184cab 100644 --- a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx +++ b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx @@ -37,8 +37,12 @@ study's runs. A remote capability passes through the hook unchanged. 3. **Worker → Optuna.** On first use the worker loads Pyodide from the CDN pinned in the runtime lock, installs Optuna with micropip, writes the [optimizer-core](layer:optimizer-core) sources into the virtual filesystem - and imports them. `run_browser_study` asks the sampler for values and - awaits a JavaScript `evaluate` callback for each trial. + and imports them. `create_browser_study` builds the Optuna study once per + run (with the constant-liar TPE when the run asks for parallel steps) and + keeps it in a handle; `run_browser_study` runs one segment of trials on + that handle, asking the sampler for values and awaiting a JavaScript + `evaluate` callback for each trial, up to `parallelism` at a time; + `release_browser_study` drops the handle when the record is removed. 4. **Worker → channel.** Each `evaluate` crosses back to the main thread as a message; the capability resolves the trial's scenario parameter values and seeds (`deriveOptimizationTrialSeeds`, the CLI's helper) and calls @@ -87,9 +91,41 @@ fails stops the ladder at that key and records the batch's reason on `selection.error`; a followed trial that fails records its reason the same way. The **Parameters** band shows it as _Could not compute_, the **Metrics** section shows the empty chart shell, and the next refine of that key — a later -navigation change — retries. When the study settles, following ends and the -point refines wherever the navigation stands — except after a cancellation, -which stops compute until the next move. +navigation change — retries. When the study settles — complete or stopped — +while following is on, the navigation moves to the best trial's point and +refines there; a navigation the user moved earlier stays put. Between rungs +the refinement estimates the point's mean and standard error from the +objective's last distribution frame and stops with `selection.note` ("8 runs · +cannot beat the best") when the mean sits more than 2.5 standard errors on +the wrong side of the study's best (`shouldStopRefining`); the best trial's +own point is exempt and always climbs to the top rung. + +## Stop, Continue and parallel steps + +A connected run's log is a sequence of segments, each begun by `started` +(carrying the cumulative `requestedTrials`) and ended by a terminal event. +**Stop** calls `cancelOptimizationRun`: the worker prunes the trials in +flight, the segment ends with the cancelled terminal, and the study stays in +the worker. The provider keeps its attachment through the stop so the +record's cursor covers the whole segment: the stop marks the record cancelled +at once, the pruned trials leave it so, and the terminal event — `complete`, +or the cancellation — sets `OptimizationRecord.resumable` while the study's +local machinery is still held. **Continue** calls +`extendOptimizationRun(runId, trials)` on the +`PetrinautConnectedOptimizationCapability`, then re-runs the attach loop from +`record.lastSeq`: the new `started` sets the total, `followTrials` turns back +on, and the trials continue Optuna's numbering. Removing the record calls +`releaseOptimizationRun`, which drops the study. + +`createOptimizationRun(input, { parallelism })` keeps up to four trials in +flight. Each trial's detached run takes a `queueKey` of its own +(`${runId}:trial:${n}`) while sharing the study's compiled snapshot through +`cacheKey`, so trials overlap on the backend; the connected study follows the +most recently started trial and publishes every in-flight trial's running +objective as `record.inFlight`, which the Surface draws as rings. An +activity registry (`provider/activity-registry.ts`) lists each in-flight +trial and the refinement rung with its progress as `record.activity`, on a +100 ms throttle, for the Summary's "N computing" chip. The drawer's **Parameters** band edits `navigation` through `setOptimizationNavigation`, its controls disabled while they follow a From cfe9ab5c454066d70024b0042c56440f2cbe702e Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Fri, 4 Sep 2026 06:32:01 +0200 Subject: [PATCH 4/5] Describe the connected study drawer's layout --- .../@hashintel/petrinaut/docs/optimization.md | 82 +++++++++++-------- .../content/optimizer/browser-runtime.mdx | 10 +-- 2 files changed, 53 insertions(+), 39 deletions(-) diff --git a/libs/@hashintel/petrinaut/docs/optimization.md b/libs/@hashintel/petrinaut/docs/optimization.md index 7b985154291..c519c39bce6 100644 --- a/libs/@hashintel/petrinaut/docs/optimization.md +++ b/libs/@hashintel/petrinaut/docs/optimization.md @@ -83,16 +83,16 @@ not to the saved scenario. ## Watching results Open an optimization row to follow it while it runs. The drawer updates as -steps arrive and shows: +steps arrive. A study run on the optimization service shows: - Completed, pruned, and failed step counts. - The current best metric value. - The best flat scenario-parameter assignment. - The latest received steps, including their parameters, objective values, and colored state indicators. The step list sits at the bottom of the drawer and - scrolls on its own — the summary, the best parameters, and the surface above - it hold still. For long runs, the drawer displays the newest 200 steps while - retaining aggregate progress and the current best result. + scrolls on its own — the sections above it hold still. For long runs, the + drawer displays the newest 200 steps while retaining aggregate progress and + the current best result. Closing the drawer does not stop the optimization. Use **Cancel** to abort an active run on the optimization service, or **Stop** to end a study running in @@ -107,35 +107,49 @@ kept, and a **Retry** action starts a fresh run with the same settings. A study that runs in the browser (see [Running in the browser](#running-in-the-browser)) shows more, because the machine computing -it is yours: - -- A badge beside the **Summary** heading says where the steps run. It reads - **CPU**, because the GPU backend cannot compute an expression objective (see - step 4 of [Creating an optimization](#creating-an-optimization)). +it is yours, and lays it out so everything is in view at once — on a laptop +screen nothing needs scrolling while the study streams: + +- A summary strip across the top: the status, the steps finished over the + steps requested (with the runs per step and the parallel steps when they + are above one), the best value so far — hover it for the best step's + parameters — and a badge saying where the steps run. It reads **CPU**, + because the GPU backend cannot compute an expression objective (see step 4 + of [Creating an optimization](#creating-an-optimization)). Under the strip, + a progress bar for the steps, a thinner one for the runs of the step in + flight, and an **N computing** chip that expands into one row per batch + computing right now, each with its own progress. - A **Parameters** band with one slider per optimized numeric parameter and a - switch per optimized boolean parameter. While the study runs, **Follow - steps** is on: the controls move to each step's values as it is evaluated, - disabled while they follow, and the status line reads **Following step N**. - Turn **Follow steps** off to take over early, or wait for the study to - finish; then move any control and the point you picked computes in - escalating batches (8, 25, then 100 runs) while the line reads **N of M - runs — refining**. Turn **Follow steps** back on to rejoin the step in - flight. -- The **Surface** section whenever two or more numeric parameters are - optimized, without the Optimization surface setting. It draws only the - study's steps: each step is a dot at its parameters, the best emphasized, - pruned steps hollow, and the field is interpolated between them, so it fills - in as steps report — nothing is sampled behind the study. The ringed dot is - the step being evaluated, its running value streaming into the field as the - runs complete; the caption counts the steps placed and the best so far. + switch per optimized boolean parameter, two to a row when they fit. Its + heading carries the state line and the **Follow steps** switch. While the + study runs, **Follow steps** is on: the controls move to each step's values + as it is evaluated, disabled while they follow, and the line reads + **Following step N**. Turn **Follow steps** off to take over early, or wait + for the study to finish; then move any control and the point you picked + computes in escalating batches (8, 25, then 100 runs) while the line reads + **N of M runs — refining**. Turn **Follow steps** back on to rejoin the step + in flight. +- The **Surface**, on the left below the band, whenever two or more numeric + parameters are optimized, without the Optimization surface setting. It + draws only the study's steps: each step is a dot at its parameters, the + best emphasized, pruned steps hollow, and the field is interpolated between + them, so it fills in as steps report — nothing is sampled behind the study. + The ringed dot is the step being evaluated, its running value streaming into + the field as the runs complete; the caption counts the steps placed and the + best so far, and the info icon beside the axis pickers explains the marks. While the study runs with **Follow steps** on, the plot only displays. Once the study is over, or **Follow steps** is off, the ringed dot is the point the Parameters band holds: click or drag the plot to move it, and the point's value enters the field as it refines. -- A **Metrics** section with the objective metric's distribution over - simulation time at that position: the step being evaluated while following, - otherwise the point you picked. It streams again whenever the position - changes. +- The objective metric's chart, beside the surface: its distribution over + simulation time at that position — the step being evaluated while + following, otherwise the point you picked. It streams again whenever the + position changes, so the surface and the chart always describe the same + point. +- The steps table, filling whatever height is left: the newest steps first, + the best one starred and tinted, scrolling on its own. It shows a row or + two on a laptop screen and a page of them on a taller one; the strip's step + count and best value stay in view either way. ## The surface view @@ -192,8 +206,8 @@ experiments. - While the study runs, the drawer follows it: the **Parameters** band moves to each step's values, the **Surface** gains a dot per step with the field filling in around it and the ringed dot streaming the step in flight, and - the **Metrics** section streams the objective metric over that step's runs - as they complete. The sliders and the plot only display while **Follow + the objective's chart beside the surface streams the metric over that + step's runs as they complete. The sliders and the plot only display while **Follow steps** is on; turn it off to take over early. Then, or once the study is over, move a slider, flip a switch, or click or drag the surface to look at any other point — its objective computes in escalating batches on the same @@ -201,15 +215,15 @@ experiments. the study. - When a point cannot be computed — the objective metric does not compile, the backend declines the model, or some of its runs fail — the - **Parameters** band says why under **Could not compute** and the **Metrics** - section stays empty; a step that fails this way is pruned. Moving to another + **Parameters** band says why under **Could not compute** and the + objective's chart stays empty; a step that fails this way is pruned. Moving to another point, or back to this one, tries again. - **Parallel steps** (1 to 4, default 1) sets how many steps the optimizer evaluates at once. Above 1, the optimizer accounts for the steps still running when it picks the next values, so the proposals differ from a one-at-a-time study. The Parameters band follows the most recently started step, and the Surface rings every step in flight with its running value. -- Under the Summary's steps bar, a thinner bar tracks the followed step's runs +- Under the summary strip's steps bar, a thinner bar tracks the followed step's runs over the runs per step, and a **"N computing"** chip appears while anything computes — the steps in flight and the picked point's refinement — expanding into one row per batch with its own progress. @@ -225,7 +239,7 @@ experiments. the study's sampler: the status reads **Stopped**, and the footer offers **Continue** with a number of steps (the study's own step count by default). Continue runs that many more steps on the same study, with everything it - learned so far, and the Summary's Steps counts them into the total. A + learned so far, and the strip's **Steps** counts them into the total. A completed study can be continued the same way, as often as the 1,000-step cap allows. Removing the study drops its sampler. - Keep the tab open. Closing or reloading the page ends the study, and the diff --git a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx index dd8a1184cab..54b2bf0a255 100644 --- a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx +++ b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx @@ -80,8 +80,8 @@ The provider keeps one connected-study object per record `navigation.followTrials` is on — it starts on — `trialStarted` moves the navigation to the trial's values (`optimizationAxisPositionFor` per axis) and mirrors the batch's frames and progress stores into `selection` under the key -`trial:`, so the drawer's **Metrics** section streams the step being -evaluated. A user move clears `followTrials` and hands the navigation to point +`trial:`, so the objective's chart beside the drawer's Surface streams the +step being evaluated. A user move clears `followTrials` and hands the navigation to point refinement (`provider/point-refinement.ts`): the experiments' run ladder capped at 100, one batch per rung with `seed = sweepBatchSeed(studySeed, from)` and `runCount = target − from`, frames merged with `mergeMetricFramesAcrossCells`, @@ -89,9 +89,9 @@ results cached per navigation key for the record's lifetime. A key change cancels the batch in flight; returning to a cached key is instant. A rung that fails stops the ladder at that key and records the batch's reason on `selection.error`; a followed trial that fails records its reason the same -way. The **Parameters** band shows it as _Could not compute_, the **Metrics** -section shows the empty chart shell, and the next refine of that key — a later -navigation change — retries. When the study settles — complete or stopped — +way. The **Parameters** band shows it as _Could not compute_, the objective's +chart beside the Surface shows the empty shell, and the next refine of that +key — a later navigation change — retries. When the study settles — complete or stopped — while following is on, the navigation moves to the best trial's point and refines there; a navigation the user moved earlier stays put. Between rungs the refinement estimates the point's mean and standard error from the From 016ed28bc2b380d90f4767f6229fd324d109e51d Mon Sep 17 00:00:00 2001 From: Chris Feijoo Date: Fri, 4 Sep 2026 14:01:11 +0200 Subject: [PATCH 5/5] Document the study seed and the scaled start-up trials --- libs/@hashintel/petrinaut/docs/optimization.md | 12 +++++++++--- .../content/optimizer/browser-runtime.mdx | 8 +++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/libs/@hashintel/petrinaut/docs/optimization.md b/libs/@hashintel/petrinaut/docs/optimization.md index c519c39bce6..174f81d78ad 100644 --- a/libs/@hashintel/petrinaut/docs/optimization.md +++ b/libs/@hashintel/petrinaut/docs/optimization.md @@ -49,7 +49,11 @@ parameter behind the scenes. the GPU backend cannot compute (see [Compute backend](experiments.md#compute-backend-experimental)), so the steps run on the CPU. The in-browser optimizer also offers **Parallel - steps** (1 to 4, default `1`): how many steps it evaluates at once. + steps** (1 to 4, default `1`): how many steps it evaluates at once. The + **Seed** field starts at a fresh random value each time the form opens; it + seeds both the optimizer's proposals and the simulations' random draws, so + keep a seed to reproduce a study and change it to explore a different set + of steps. 5. In **Parameters**, leave a parameter at its current **Value** or enable **Optimize** and enter its search range. At least one parameter must be optimized. @@ -61,8 +65,10 @@ parameter behind the scenes. optimization objectives. 7. Click **Run**. -The metric is evaluated on the final frame of each run, and a step's objective -is the mean over its runs. The current model is reduced to an immutable snapshot +The optimizer draws its first steps at random, about a third of the requested +steps and never fewer than 2 or more than 10, then proposes each further step +from the results so far. The metric is evaluated on the final frame of each +run, and a step's objective is the mean over its runs. The current model is reduced to an immutable snapshot containing the selected scenario and objective metric when the optimization starts. Later edits do not change an in-flight run. diff --git a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx index 54b2bf0a255..6cad42c9784 100644 --- a/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx +++ b/libs/@local/petrinaut-arch-docs/content/optimizer/browser-runtime.mdx @@ -165,7 +165,13 @@ walks a grid through `sampleDetachedObjective` ## Parity with the service Both drivers derive per-trial seeds with the same helper, and the sampler -runs the same Optuna version with the same seed. On the CPU pool the request +runs the same Optuna version with the same seed and the same number of +startup trials: `create_study` in [optimizer-core](layer:optimizer-core) sets +the TPE sampler's `n_startup_trials` to a third of the requested trials, +between 2 and Optuna's default of 10, so a short study models its objective +instead of spending every step on random draws. The create form gives each +study a fresh random seed, so two studies over different models no longer +share their random draws. On the CPU pool the request pins every run's seed (`runs: [{ seed }]`), so a trial's replicates run on the seeds the CLI runs and the trial reports the service's objective value. The parity in proposals depends on that: after its startup trials the TPE sampler