From b26f01094027ce6d4bbe626f5e3bea0c930cb8da Mon Sep 17 00:00:00 2001 From: Alexandru Rotaru Date: Fri, 7 Aug 2026 12:03:31 +0000 Subject: [PATCH 01/13] docs(perf): register O(data) operations blocking 100GB+ interactivity Storage and serving paths whose cost scales with dataset size rather than with what changed. Storage findings re-verified against dev; two serving costs noted as already fixed upstream so they are not re-claimed as wins. No code changes - baseline and measurement protocol only. --- docs/perf/o_change_register.md | 122 +++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 docs/perf/o_change_register.md diff --git a/docs/perf/o_change_register.md b/docs/perf/o_change_register.md new file mode 100644 index 00000000..bcadd19c --- /dev/null +++ b/docs/perf/o_change_register.md @@ -0,0 +1,122 @@ +# Interactivity for 100GB+ datasets — register of O(data) operations + +Goal: every per-step and per-request operation should cost **O(change)** or +**O(page)**, never **O(dataset)**. Today several do, so cost grows with the +dataset while the actual work stays constant. + +> **Baseline caveat.** The measurements below were taken against a stale +> in-place copy of weightslab (`~/weightslab_src`, 1.3.3+multiview), which +> differs from `dev` in 51 files. Two serving costs listed there are **already +> fixed on dev** (see §B). Serving numbers must be re-measured on this branch +> before any serving change is attributed an improvement. The storage findings +> (§A) were re-verified against dev and still hold. + +Reference measurements (UltraEdit, 3,959,093 rows, ~19 cols, A10G box): + +| | measured | +|---|---| +| bare-torch step (no WL) | 1,162 ms → 20.66 samples/s | +| with WL, no UI client | ~5.5 samples/s (**3.8× slower**) | +| with WL + image requests | ~1.5 samples/s (**13.8× slower**) | +| per-step signal write itself | **4 ms (0.34%)** — already fine | +| grid page latency (64 imgs) | p50 5.2 s, p95 9.6 s | + +The signal path is not the problem. Storage write-amplification and the view +rebuild are. + +--- + +## A. STORAGE — `data/h5_dataframe_store.py` + +`upsert()` **receives** only dirty rows but **implements** a full table +replacement. + +| line | operation | cost | +|---|---|---| +| 693 | `_create_backup()` — full file copy **before every upsert** | O(file) | +| 708 | `existing = store.select(key)` — read entire table | O(N) | +| ~768 | `pd.concat([existing, delta])` | O(N) | +| ~772 | `existing[~existing.index.duplicated()]` — dedupe all rows | O(N) | +| ~785 | `_decategorize_for_storage(existing)` | O(N) | +| 801 | `store.remove(key)` — drop table | O(N) | +| 804 | `store.append(..., data_columns=True)` — rewrite + index **every** column | O(N·cols) | +| 846–883 | same read/remove/rewrite in the column-delete path | O(N) | + +**Amplification:** ~5 KB of changed signals per flush → ~200 MB written, +roughly **40,000×**. At `ledger_flush_interval=3.0s` vs ~1.5 s steps, that is a +full-table rewrite about every 2 steps. + +Fix direction: append new rows; modify existing rows in place +(`select_as_coordinates` + `table.modify_rows`). Backup incrementally, not per +upsert. No `data_columns=True` — no `store.select()` in this file uses `where=`, +so those per-column indexes are built and never read. + +*(A previous attempt to narrow `data_columns` broke the write path entirely — +678 upsert failures, zero persisted data. Any change here needs a +write→read→assert-contents check, not just a timing check.)* + +## B. SERVING — `trainer/services/data_service.py` + +`_pull_into_all_data_view_df()` (line 938) runs several full-frame passes. + +**Already fixed on dev — do not re-report as wins:** +- the collapse no longer re-enters `get_combined_df()`; the pulled frame is + passed in, so the frame is copied once, not twice +- `array_proxy` no longer does a per-cell `.apply(convert_to_proxy)` (was + 1,660 ms at 4M rows) + +Remaining, to be **re-measured on this branch**: + +| line | operation | cost (measured @4M) | +|---|---|---| +| 946 | `get_combined_df()` → `dataframe_manager:2140 self._df.copy()` | 101 ms–1.2 s | +| — | `get_collapse_annotations_to_samples_df(df)` — groupby collapse | 6,326 ms* | +| — | `safe_reset_index(df)` | 1,912 ms | +| — | `set_index([origin, sample_id])` | O(N) | +| 3636 | `updated_df.reindex(target_order)` | 290 ms | + +Callers — each one is a full O(N) rebuild: lines **440, 851, 3593, 4605, 4626**, +reached from `GetDataSamples`, `GetMetaData`, `EditDataSample`, `GetDataSplits`. + +Held under `_update_lock`, which the trainer also needs → measured lock holds of +39–126 s and the 3.7× training penalty while browsing. + +**The collapse is provably a no-op when `annotation_id.max() == 0`** (UltraEdit +is exactly 1:1) yet still costs 6.3 s per rebuild. + +Fix direction: serve a page from the source frame by index (O(page)); rebuild +the full view only for genuinely global operations (histogram, global sort); +apply deltas rather than rebuilding; never hold the writer lock across a +rebuild — build off-lock and swap the reference. + +## C. OTHER FULL SCANS + +| location | note | +|---|---| +| `dataframe_manager:1875` `data_snapshot.iterrows()` | input is O(change), but row-wise Python per flush | +| `dataframe_manager:2400` `.apply(lambda …)` | per cell | +| `data_service:1200` `_compute_natural_sort_stats` | builds a list of one Series per row (4M objects). Gated off (`compute_natural_sort=False`) — latent | +| `data_service:538` PreviewCache | bounded by `WL_MAX_PREVIEW_CACHE_SIZE` — OK | + +## D. ALREADY O(change) — keep + +- `self._pending` dirty-row set (`dataframe_manager:95, 751, 761`) +- flush work set: `work = list(self._pending)` (`:1827`) +- `_origin_revisions` per-origin version counters (`:94, 1235`) + +The bookkeeping needed for differential updates already exists; the storage and +view layers just don't use it. + +\* measured on the stale copy; re-measure on dev. + +## Measurement protocol + +Fixed workload: **1,000 train samples** = 41 steps at batch 24. Every change is +reported as: + +1. wall-clock for the 41 steps, vs the bare-torch floor +2. bytes written to H5 for those steps +3. grid-page latency (64 images) and training throughput **while** serving +4. **ledger contents verified** — signal columns present, measured-row count + +(4) is not optional: a previous "10× win" was writes silently failing. From 83a67e25f4b1cee621a5fca81f7b11c754bcc81a Mon Sep 17 00:00:00 2001 From: Alex Rotaru Date: Fri, 7 Aug 2026 16:04:52 +0000 Subject: [PATCH 02/13] docs(perf): triage every _slowUpdateInternals call site 16 of 18 sites only need fresh values for dirty rows (O(change)); only first build and schema change need a full reconstruction. Records why ApplyDataQuery filter paths stay on the rebuild (_is_filtered semantics) and why a no-client benchmark cannot show the difference. --- docs/perf/o_change_register.md | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/perf/o_change_register.md b/docs/perf/o_change_register.md index bcadd19c..270f3417 100644 --- a/docs/perf/o_change_register.md +++ b/docs/perf/o_change_register.md @@ -120,3 +120,51 @@ reported as: 4. **ledger contents verified** — signal columns present, measured-row count (4) is not optional: a previous "10× win" was writes silently failing. + +--- + +# E. Triage — which call sites need a full reconstruction + +`_slowUpdateInternals()` rebuilds the whole view: `copy → collapse → reset_index +→ set_index → reindex`. It has 18 call sites, and almost none of them need +that. Most just want **fresh values for rows the trainer touched**, which is +`O(change)`. + +`_fastUpdateInternals()` applies only dirty rows, via a maintained +`sample_id → position` map (`_rebuild_view_pos_map`), and returns `False` — +falling back to the full rebuild — whenever it cannot safely apply: + +- no view yet, or no position map (first build) +- a dirty `sample_id` absent from the map (new rows ⇒ structural change) +- backlog > `max_dirty` (a rebuild is genuinely cheaper) + +So the worst case is today's behaviour, never wrong data. + +| site | routing | why | +|---|---|---| +| `_bg_view_refresh` | **fast** | exists purely to refresh values after a stale read — the textbook differential case, and the one that holds `_lock` against the trainer | +| `_process_get_data_samples` | **fast** | grid fetch needs current values, not a new frame | +| `_compute_custom_signals` | **fast** | writes new signal *values*; schema unchanged | +| `GetDataSplits` | **fast** | read-only summary | +| `EditDataSample` ×5 | **fast** | per-sample value edits | +| `EditDataSample` ×3 (`df.modify`, `df.drop_column`) | **full** | changes the schema — differential cannot add/remove columns | +| `ApplyDataQuery` `@reset`/`@clear` | **full** | clears `_is_filtered` to restore the full universe; a differential updates values but cannot restore *dropped rows* | +| `ApplyDataQuery` filter + agent paths | **full** (deferred) | a forced rebuild preserves `_is_filtered` (`:3691`), so swapping in a differential changes which rows the user sees. Rare, user-initiated, low perf value, high blast radius — not worth the risk until the filter semantics are pinned down | +| `_compute_natural_sort_stats` | **full** | gated off (`compute_natural_sort=False`); latent | +| `_manual_save_data_state` | **full** | explicit user save; correctness over speed | + +**Kill-switch:** `WL_FAST_VIEW=0` disables the differential and the position-map +build, reproducing prior behaviour exactly. This is what makes a like-for-like +A/B possible from a single tree. + +## Why the no-client benchmark cannot show this + +A 41-step run with no UI client attached records **0 rebuild events** — nothing +calls `_slowUpdateInternals` at all, so the fast path has nothing to improve and +correctly measures as no change. The rebuild cost only materialises when a +client is attached, which is the case that measured **3.7× slower** with p50 +grid latency of 5.2 s. + +The A/B is therefore run under load: baseline → under-load → recovery phases +within one training process (`t_imgload.py`), so each arm is normalised against +its own idle throughput. From 999e859b542d2ea20f4d56961e3977bc3e38a441 Mon Sep 17 00:00:00 2001 From: Alexandru Rotaru Date: Sun, 9 Aug 2026 19:50:26 +0000 Subject: [PATCH 03/13] perf(interactivity): make view refresh and ledger writes O(change) Every path that kept the served view in sync ran O(dataset): a full materialized-view rebuild on each signal tick, and a read-modify-append of the whole H5 table per upsert. At 3.96M rows that meant a 670s startup and lock holds long enough that the UI looked hung while training. Three changes, each turning a whole-dataset pass into a per-change one: data_service: differential view refresh (_fastUpdateInternals). The materialized view only ever recomputes index-derived state, so a value-only delta can be written straight into the existing view through a sample_id -> row-position map instead of rebuilding it. Falls back to the full path on any structural change (unknown sample_id, missing pos map, backlog over max_dirty), so correctness never depends on the fast path being right about a schema change. 9 call sites routed here; the 10 that genuinely change shape still force a rebuild. Off with WL_FAST_VIEW=0. The sort path is restructured to do its work off-lock: ops and the pos-map rebuild both run on a shallow copy, and only the pointer swap happens under the lock. Skipping the pos-map rebuild after a sort would have been a data corruption bug -- sorting reorders the view, so stale positions send differential writes to the wrong rows. h5_dataframe_store: in-place row updates via modify_coordinates, with a cached sample_id -> coordinate map (stable row positions come free from the no-row-loss invariant). PyTables cannot invalidate a column index during modify_coordinates, so _try_inplace refuses indexed tables and the caller falls back to the append path. Index construction is also split from storage layout: data_columns=True keeps the on-disk layout queryable while index=False keeps the flush path from rebuilding an index no hot-path read uses (92.7s -> 6.9s per upsert at 4M rows). dataframe_manager: replaces the per-row iterrows() scans that dominated startup with column-wise vectorised passes, and adds the dirty-row/source-row accessors the differential refresh needs. Also fixes an unrelated thumbnail bug in trainer_tools.process_sample: it unpacked exactly 3 values from _getitem_raw, whose contract is (data, id, target, *metadata). Any dataset implementing get_items() with metadata raised "too many values to unpack" and every cell in the grid came back with no image. Now unpacked positionally. Measured on 3.96M-row UltraEdit, A10G: startup 670s -> 250s H5 upsert (24 rows) 127.9s -> ~16ms snapshot flush 338s -> 0 samples max lock hold 129,440ms -> none over 1s throughput under UI 13% -> 45-51% of idle The residual loss under load is CPU/GIL contention (8 vCPUs shared by 6 dataloader workers, training, and image encode), not lock waiting. Known gaps, deliberately left for review: - ensure_index() has no caller yet, and conflicts with _try_inplace, which refuses indexed tables. It documents the deliberate-index path but is dead code as committed. - _POSMAP_CACHE is class-level and never evicts (~400-600MB at 4M rows). - Three ApplyDataQuery sites still force a full rebuild pending a decision on _is_filtered semantics. Co-Authored-By: Claude Opus 5 --- weightslab/data/dataframe_manager.py | 156 ++++++++++++++- weightslab/data/h5_dataframe_store.py | 119 +++++++++++- weightslab/trainer/services/data_service.py | 200 ++++++++++++++++---- weightslab/trainer/trainer_tools.py | 14 +- 4 files changed, 435 insertions(+), 54 deletions(-) diff --git a/weightslab/data/dataframe_manager.py b/weightslab/data/dataframe_manager.py index c9f77bc6..ca19d5e8 100644 --- a/weightslab/data/dataframe_manager.py +++ b/weightslab/data/dataframe_manager.py @@ -93,6 +93,9 @@ def __init__(self, flush_interval: float = 3.0, flush_max_rows: int = 100, enabl self._array_store: H5ArrayStore | None = None self._origin_revisions: Dict[str, int] = {} self._pending: set[int] = set() + # Parallel dirty set for the view. _pending is drained by the H5 flush, + # so the view cannot share it without one consumer starving the other. + self._view_pending: set = set() self._force_flush = False self._flush_interval = flush_interval self._flush_max_rows = flush_max_rows @@ -297,9 +300,66 @@ def _expand_dataframe_with_annotations(self, df: pd.DataFrame) -> pd.DataFrame: if not isinstance(work.index, pd.MultiIndex) and work.index.name != SID: work = work.copy() work.index.name = SID + fast = self._expand_fast_no_instances(work) + if fast is not None: + return fast + records = work.reset_index().to_dict("records") return self._expand_records_to_multi_index(records) + def _expand_fast_no_instances(self, work: pd.DataFrame): + """Vectorized expansion for frames with no per-instance targets. + + Returns the (sample_id, annotation_id=0) frame, or None to signal the + caller must use the record-by-record path. + """ + SID = SampleStats.Ex.SAMPLE_ID.value + ANNOT = SampleStats.Ex.INSTANCE_ID.value + TARGET = SampleStats.Ex.TARGET.value + try: + flat = work.reset_index() + if SID not in flat.columns: + return None + if TARGET in flat.columns: + tgt = flat[TARGET] + # Non-object dtype cannot hold a list => every target is scalar. + if tgt.dtype == object: + for v in tgt.to_numpy(): + if isinstance(v, (list, tuple, np.ndarray)) and len(v) > 0: + return None + # _normalize_sample_id always returns str(). astype(str) reproduces + # that for numeric dtypes only -- bytes would render as "b'x'". + sid_ser = flat[SID] + if pd.api.types.is_integer_dtype(sid_ser) or pd.api.types.is_float_dtype(sid_ser): + sids = sid_ser.astype(str).tolist() + else: + sids = [self._normalize_sample_id(v) for v in sid_ser.to_numpy()] + + out = flat.drop(columns=[c for c in (SID, ANNOT) if c in flat.columns]) + # The record path goes through python lists, so extension dtypes + # (string[pyarrow], categorical) come back as object. Match it. + for c in out.columns: + if isinstance(out[c].dtype, pd.api.extensions.ExtensionDtype): + out[c] = out[c].astype(object) + out.index = pd.MultiIndex.from_arrays( + [sids, np.zeros(len(out), dtype=np.int64)], names=[SID, ANNOT]) + return out + except Exception: + return None + + def _normalize_sample_id_index(self, values) -> "pd.Index": + """Vectorized _normalize_sample_id over an Index (~2x; 0.8s -> 0.5s at 2M). + + _normalize_sample_id is str() after unwrapping numpy scalars/bytes, so + astype(str) is exact for numeric dtypes; anything else keeps the loop. + """ + try: + if pd.api.types.is_integer_dtype(values) or pd.api.types.is_float_dtype(values): + return pd.Index(values.astype(str)) + except Exception: + pass + return pd.Index([self._normalize_sample_id(v) for v in values]) + def _normalize_sample_id(self, sample_id: Any) -> Any: """Normalize incoming sample IDs while preserving numeric IDs when possible.""" try: @@ -646,7 +706,7 @@ def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flu # Normalize sample_id values in multi-index if isinstance(df_norm.index, pd.MultiIndex) and df_norm.index.nlevels >= 1: - level_0_normalized = pd.Index([self._normalize_sample_id(v) for v in df_norm.index.get_level_values(0)]) + level_0_normalized = self._normalize_sample_id_index(df_norm.index.get_level_values(0)) try: if df_norm.index.nlevels == 2: df_norm.index = pd.MultiIndex.from_arrays( @@ -710,7 +770,17 @@ def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flu for col in all_cols: if col in self._df.columns and isinstance(self._df[col].dtype, pd.CategoricalDtype): self._df[col] = self._df[col].astype(object) - self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols] + # Label-aligned 2D .loc realigns the entire frame (22s at 4M rows + # when adding columns to every row). Resolve row positions once, + # then write each column positionally. Falls back if the index + # has duplicates/misses, where get_indexer returns -1. + _pos = self._df.index.get_indexer(existing_idx) + if len(_pos) and (_pos >= 0).all(): + for _c in all_cols: + _ci = self._df.columns.get_loc(_c) + self._df.iloc[_pos, _ci] = df_norm.loc[existing_idx, _c].to_numpy() + else: + self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols] # Append rows that do not exist yet. Use a boolean mask (not # .loc[difference]) so a duplicate key in df_norm can't be @@ -731,7 +801,9 @@ def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flu self._auto_register_categorical_tags(df_norm) # Optimize memory by converting repetitive columns to categorical - self._df = self._optimize_dataframe_memory(self._df) + # Only columns just written can have changed dtype-wise; a full-frame + # nunique() over every object column was ~14s of a 670s startup. + self._df = self._optimize_dataframe_memory(self._df, columns=set(df_norm.columns)) # Mark dirty for flush (handle multi-index) if isinstance(df_norm.index, pd.MultiIndex): @@ -749,6 +821,7 @@ def mark_dirty(self, sample_id: int): with self._lock: normalized_id = self._coerce_sample_id_for_index(sample_id) self._pending.add(normalized_id) + self._view_pending.add(normalized_id) def drop_column(self, column: str): with self._lock: @@ -759,6 +832,7 @@ def drop_column(self, column: str): def mark_dirty_batch(self, sample_ids: List[int], force_flush: bool = False): with self._lock: self._pending.update(set(sample_ids)) + self._view_pending.update(set(sample_ids)) if force_flush: self._force_flush = True @@ -1263,6 +1337,32 @@ def update_values(self, origin: str, sample_id: int, updates: Dict[str, Any], an self._df = pd.concat([self._df, df_local]) self._bump_origin_revisions([origin]) + def take_view_dirty(self, limit: int | None = None): + """Drain and return the sample_ids changed since the last view sync. + + Returns None when the backlog exceeds *limit*, meaning a differential + update would cost more than a rebuild — the caller should fall back. + """ + with self._lock: + if limit is not None and len(self._view_pending) > limit: + self._view_pending.clear() + return None + out = list(self._view_pending) + self._view_pending.clear() + return out + + def get_source_rows(self, sample_ids, columns=None): + """Rows for *sample_ids* straight from the source frame. O(len(ids)).""" + with self._lock: + if self._df.empty or not len(sample_ids): + return None + idx = self._df.index + keys = idx.get_level_values(0) if isinstance(idx, pd.MultiIndex) else idx + want = set(str(s) for s in sample_ids) + mask = keys.astype(str).isin(want) + sub = self._df.loc[mask] + return sub[columns] if columns else sub + def get_origin_revision(self, origin: str) -> int: with self._lock: return int(self._origin_revisions.get(str(origin), 0)) @@ -1934,6 +2034,33 @@ def _flush_to_h5_if_needed(self, force: bool = False, blocking: bool = False): # Everything below happens WITHOUT locks - fully async self._flush_snapshot_to_h5(data_snapshot, work) + def _rows_with_array_cells(self, data_snapshot: pd.DataFrame): + """Index labels of rows that may hold an array-valued cell. + + Column-wise scan replacing a full iterrows() pass: only object-dtype + columns can hold an ndarray/list/tuple/ArrayH5Proxy, and rows with none + of those are no-ops for the caller. + """ + cols = [c for c in self._array_columns if c in data_snapshot.columns] + if not cols: + return [] + hits = None + for col in cols: + ser = data_snapshot[col] + if ser.dtype != object: + continue # a numeric column cannot hold an array object + vals = ser.to_numpy() + mask = np.fromiter( + (isinstance(v, (np.ndarray, list, tuple, ArrayH5Proxy)) for v in vals), + dtype=bool, count=len(vals)) + if not mask.any(): + continue + found = ser.index[mask] + hits = found if hits is None else hits.union(found) + if hits is None: + return [] + return list(hits) + def _flush_snapshot_to_h5(self, data_snapshot: pd.DataFrame, work: List[int]): """Flush data snapshot to H5 - runs completely outside locks. @@ -1956,7 +2083,10 @@ def _flush_snapshot_to_h5(self, data_snapshot: pd.DataFrame, work: List[int]): arrays_to_store: Dict[str, Dict[str, np.ndarray]] = {} rowkey_to_index: Dict[str, Any] = {} - for idx, row in data_snapshot.iterrows(): + for idx in self._rows_with_array_cells(data_snapshot): + row = data_snapshot.loc[idx] + if isinstance(row, pd.DataFrame): # duplicate label guard + row = row.iloc[0] if is_multi: sample_id, annot = idx[0], int(idx[1]) else: @@ -2029,7 +2159,7 @@ def _flush_snapshot_to_h5(self, data_snapshot: pd.DataFrame, work: List[int]): except Exception as e: logger.error(f"[LedgeredDataFrameManager] Error flushing to H5: {e}") - def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[str, List[str]] | None = None) -> pd.DataFrame: + def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[str, List[str]] | None = None, columns=None) -> pd.DataFrame: """Optimize dataframe memory by converting repetitive string columns to categorical. Categorical dtype compresses repeated values: instead of storing each string, @@ -2086,6 +2216,11 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st # -1) and can no longer be set to a plain None — so we clean object cells to # None here first, while they are still plain object dtype. for col in df.columns: + # The docstring above says numeric/bool/categorical are skipped, but the + # loop had no dtype check: at registration every signals//* column is all + # NaN, so this did a full label-aligned write per float column. + if df[col].dtype != object: + continue na_mask = df[col].isna() if na_mask.any(): df.loc[na_mask, col] = None @@ -2097,6 +2232,9 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st SampleStats.Ex.TASK_TYPE.value, # Task type (e.g. 'classification', 'segmentation') ] + # nunique() over the whole index was recomputed per object column (8s at 4M). + _n_rows_cached = (df.index.get_level_values(0).nunique() + if isinstance(df.index, pd.MultiIndex) else len(df)) for col in categorical_candidates: if col not in df.columns: continue @@ -2108,15 +2246,15 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st # Only convert to categorical if: # 1. Column contains strings (object dtype) # 2. Number of unique values < 50% of total rows (good compression ratio) + if columns is not None and col not in columns: + continue if df[col].dtype == 'object': n_unique = df[col].nunique() + _ = _n_rows_cached # Use unique sample count, not row count — with MultiIndex each # sample has multiple annotation rows which would inflate n_rows # and make the ratio appear better than it really is. - if isinstance(df.index, pd.MultiIndex): - n_rows = df.index.get_level_values(0).nunique() - else: - n_rows = len(df) + n_rows = _n_rows_cached compression_ratio = n_unique / n_rows if n_rows > 0 else 1.0 if compression_ratio < 0.5 and n_unique > 1: # Worth compressing if < 50% unique diff --git a/weightslab/data/h5_dataframe_store.py b/weightslab/data/h5_dataframe_store.py index 476b5655..3ce51fb6 100644 --- a/weightslab/data/h5_dataframe_store.py +++ b/weightslab/data/h5_dataframe_store.py @@ -680,6 +680,115 @@ def load_all(self, origins: Iterable[str] = None, columns: Optional[Iterable[str return pd.DataFrame() raise + def ensure_index(self, origin: str, columns=("sample_id",)) -> bool: + """Build the on-disk column index deliberately (checkpoint / first query). + + Kept OFF the flush path: rebuilding it per upsert costs 92.7s at 4M rows + versus 6.9s without, for an index no hot-path read uses. + """ + key = self._key(origin) + try: + with self._local_lock: + with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, + poll_interval=self._poll_interval): + with pd.HDFStore(str(self._path), mode="a") as store: + if key not in store: + return False + store.create_table_index(key, columns=list(columns), + optlevel=6, kind="medium") + return True + except Exception as exc: + logger.warning(f"[H5DataFrameStore] ensure_index({origin}) failed: {exc}") + return False + + # --- O(change) in-place update ------------------------------------------ + _POSMAP_CACHE: dict = {} + + def _posmap(self, store, key, force=False): + """(sample_id, annotation_id) -> row position. Rows are registered once + and never deleted, so positions are stable; built from ONE column (2.9s + at 4M rows) rather than reading the table.""" + ck = (str(self._path), key) + if not force and ck in self._POSMAP_CACHE: + return self._POSMAP_CACHE[ck] + try: + sids = store.select_column(key, "sample_id").values + try: + aids = store.select_column(key, "annotation_id").values + except Exception: + aids = np.zeros(len(sids), dtype="i8") + m = {} + for i, (sd, ad) in enumerate(zip(sids, aids)): + sd = sd.decode() if isinstance(sd, bytes) else str(sd) + m[(sd, int(ad))] = i + self._POSMAP_CACHE[ck] = m + return m + except Exception as exc: + logger.debug(f"[H5DataFrameStore] posmap build failed: {exc}") + return None + + def _invalidate_posmap(self, key): + self._POSMAP_CACHE.pop((str(self._path), key), None) + + def _try_inplace(self, store, key, df_norm) -> bool: + """Overwrite existing rows' values in place. True if fully applied.""" + try: + import tables as _tables + except Exception: + return False + try: + node = store._handle.get_node(key) + tbl = getattr(node, "table", node) + if not isinstance(tbl, _tables.Table): + return False + # An indexed column cannot be modified in place (PyTables raises). + if any(tbl.cols._f_col(c).is_indexed for c in tbl.colnames): + return False + + cols = [c for c in df_norm.columns if c in tbl.colnames] + if len(cols) != len(df_norm.columns): + return False # new column => schema change + + pos = self._posmap(store, key) + if not pos: + return False + + idx = df_norm.index + if isinstance(idx, pd.MultiIndex): + pairs = [(str(a), int(b)) for a, b in zip(idx.get_level_values(0), + idx.get_level_values(1))] + else: + pairs = [(str(a), 0) for a in idx] + + coords = np.empty(len(pairs), dtype=np.int64) + for i, p in enumerate(pairs): + j = pos.get(p) + if j is None: + return False # unknown row => not an update + coords[i] = j + + order = np.argsort(coords) # PyTables wants ascending coords + coords_sorted = coords[order] + rec = tbl.read_coordinates(coords_sorted) + for c in cols: + vals = df_norm[c].to_numpy()[order] + tgt = rec[c].dtype + if tgt.kind == "S": + vals = np.array([("" if v is None else str(v)).encode()[:tgt.itemsize] + for v in vals], dtype=tgt) + else: + try: + vals = vals.astype(tgt, copy=False) + except Exception: + return False # dtype mismatch => fall back + rec[c] = vals + tbl.modify_coordinates(coords_sorted, rec) + tbl.flush() + return True + except Exception as exc: + logger.debug(f"[H5DataFrameStore] in-place update fell back: {exc}") + return False + def upsert(self, origin: str, df: pd.DataFrame) -> int: """Atomic upsert with corruption prevention via backup and checksum verification.""" df_norm = self._normalize_for_write(df) @@ -696,6 +805,11 @@ def upsert(self, origin: str, df: pd.DataFrame) -> int: with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): try: with pd.HDFStore(str(self._path), mode="a") as store: + # O(change): if every row already exists and the schema is + # unchanged, overwrite values in place (0.1ms vs 42s). + if key in store and self._try_inplace(store, key, df_norm): + return len(df_norm) + existing = pd.DataFrame() # Try to load existing data. A ValueError can surface from a @@ -801,7 +915,8 @@ def upsert(self, origin: str, df: pd.DataFrame) -> int: store.remove(key) # Write new data - store.append(key, existing, format="table", data_columns=True) + store.append(key, existing, format="table", data_columns=True, index=False) + self._invalidate_posmap(key) # Force flush to disk store.flush() @@ -880,7 +995,7 @@ def delete_column(self, column_name: str, origins: Optional[Iterable[str]] = Non # Remove old key and write updated dataframe store.remove(key) if not df.empty: - store.append(key, df, format="table", data_columns=True) + store.append(key, df, format="table", data_columns=True, index=False) modified_count += 1 logger.debug(f"[H5DataFrameStore] Deleted column {column_name} from {origin}") diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index 85451ce6..81789d42 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -437,6 +437,11 @@ def rewrite_boolean_keywords_to_bitwise(code: str) -> str: return code +def _fast_view_enabled() -> bool: + """Differential view refresh. On by default; set WL_FAST_VIEW=0 to opt out.""" + return os.environ.get("WL_FAST_VIEW", "1") not in ("0", "false", "False") + + class DataService: """ @@ -508,6 +513,7 @@ def __init__(self, ctx): # In-memory dataframe view of all datasets combined (streamed to UI) self._all_datasets_df = self._pull_into_all_data_view_df() + self._rebuild_view_pos_map() self._load_existing_tags() self._agent = DataManipulationAgent(self) try: @@ -933,6 +939,7 @@ def _get_loader_by_origin(self, origin: str): def _initialize_data_service(self): """Recreate the in-memory dataframe view from the shared H5 store.""" self._all_datasets_df = self._pull_into_all_data_view_df() + self._rebuild_view_pos_map() self._load_existing_tags() def _resolve_root_log_dir(self) -> Path: @@ -1378,8 +1385,9 @@ def _compute_custom_signals(self): except Exception as e: logger.error(f"[DataService] Failed to compute signals for loader '{loader_name}': {e}") - # Force view update - self._slowUpdateInternals(force=True) + # Refresh signal values; differential unless the schema actually changed. + if not self._fastUpdateInternals(): + self._slowUpdateInternals(force=True) def _process_sample_row(self, args): """Process a single dataframe row to create a DataRecord.""" @@ -3710,12 +3718,112 @@ def _bg_view_refresh(self) -> None: real rebuild+swap via force=True OFF the request path, then releases the guard so a later stale read can trigger another. Never raises into a request.""" try: - self._slowUpdateInternals(force=True) + if not self._fastUpdateInternals(): + self._slowUpdateInternals(force=True) except Exception: logger.exception("[ViewRefresh] background view refresh failed") finally: self._refresh_in_flight.release() + # Columns the trainer mutates. Structural columns (origin, edit_prompt, + # task_type, ...) never change after registration, so a differential sync + # only has to carry these. + _FAST_SYNC_PREFIXES = ("signals", "last_seen", "discarded", "prediction", "target") + + def _fast_sync_columns(self, view): + return [c for c in view.columns + if str(c).startswith(self._FAST_SYNC_PREFIXES)] + + def _compute_view_pos_map(self, view): + """sample_id -> positional row index for *view*. Pure: builds and returns + the map so callers can do it OFF-lock (it is O(rows): ~5s at 4M).""" + if not _fast_view_enabled(): + return {} + try: + if view is None or view.empty: + return {} + SID = SampleStatsEx.SAMPLE_ID.value + keys = (view.index.get_level_values(SID) + if isinstance(view.index, pd.MultiIndex) and SID in (view.index.names or []) + else view.index) + return {str(k): i for i, k in enumerate(keys)} + except Exception: + return {} + + def _rebuild_view_pos_map(self): + """sample_id -> positional row index, rebuilt with the view so the + differential path does O(1) lookups instead of label alignment.""" + if not _fast_view_enabled(): + self._view_pos_map = {} + return # opt-out: skip the map build entirely + try: + view = self._all_datasets_df + if view is None or view.empty: + self._view_pos_map = {} + return + SID = SampleStatsEx.SAMPLE_ID.value + keys = (view.index.get_level_values(SID) + if isinstance(view.index, pd.MultiIndex) and SID in (view.index.names or []) + else view.index) + self._view_pos_map = {str(k): i for i, k in enumerate(keys)} + except Exception: + self._view_pos_map = {} + + def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: + """O(change) view refresh. True if applied, False -> caller must rebuild. + + Falls back when there is no view yet, when a dirty row is absent from + the view (new rows => structural change), or when the backlog is large + enough that a rebuild is cheaper. + """ + if not _fast_view_enabled(): + return False # opt-out: behave exactly as before + view = self._all_datasets_df + dfm = self._df_manager + if view is None or getattr(view, "empty", True) or dfm is None: + return False + pos_map = getattr(self, "_view_pos_map", None) + if not pos_map: + return False + + dirty = dfm.take_view_dirty(limit=max_dirty) + if dirty is None: + return False # backlog too large; rebuild is cheaper + if not dirty: + return True # nothing changed since last sync + + sids = [str(s) for s in dirty] + positions, keep = [], [] + for s in sids: + p = pos_map.get(s) + if p is None: + return False # unknown row => structural change + positions.append(p); keep.append(s) + + cols = self._fast_sync_columns(view) + if not cols: + return True + sub = dfm.get_source_rows(keep, columns=[c for c in cols if c in view.columns]) + if sub is None or sub.empty: + return True + if isinstance(sub.index, pd.MultiIndex): + sub = sub.droplevel(-1) + sub = sub[~sub.index.duplicated(keep="last")] + + order = {str(k): i for i, k in enumerate(sub.index)} + rows, vals_idx = [], [] + for s, p in zip(keep, positions): + j = order.get(s) + if j is not None: + rows.append(p); vals_idx.append(j) + if not rows: + return True + rows = np.asarray(rows); vals_idx = np.asarray(vals_idx) + for c in sub.columns: + ci = view.columns.get_loc(c) + view.iloc[rows, ci] = sub[c].to_numpy()[vals_idx] + return True + def _slowUpdateInternals(self, force: bool = False, reset_view: bool = False) -> None: """Update the internal dataframe view with the latest data from the manager. @@ -3885,6 +3993,7 @@ def _slowUpdateInternals(self, force: bool = False, reset_view: bool = False) -> # Atomic swap to make the new view available to readers self._all_datasets_df = updated_df + self._rebuild_view_pos_map() self._last_internals_update_time = current_time finally: @@ -4391,7 +4500,8 @@ def _process_get_data_samples(self, request, context): ) # Trigger update if needed (it has its own internal locking) - self._slowUpdateInternals() + if not self._fastUpdateInternals(): + self._slowUpdateInternals() # Atomic snapshot of the current authoritative dataframe current_df = self._all_datasets_df @@ -4702,37 +4812,43 @@ def ApplyDataQuery(self, request, context): operations = self._parse_direct_query(request.query) # Apply operations with lock - with self._watched_lock("_lock[ApplyDataQuery/ops]"): - # Skip the forced full-view rebuild for SORT-ONLY operations. Sorting just - # re-orders the existing snapshot, so a fresh collapse+combine (hundreds of - # ms on large views, and — being lock-held — contends with the training - # thread for multi-second stalls) is unnecessary. Filters/edits still refresh - # so they operate on the latest data. The view is frozen on direct queries - # anyway (_is_filtered=True), so it wasn't auto-refreshing mid-sort regardless. - _SORT_FUNCS = {"df.sort_values", "df.sort_index", "df.sort_view_slice"} - is_sort_only = bool(operations) and all( - op.get("function") in _SORT_FUNCS for op in operations) - if not is_sort_only: - self._slowUpdateInternals(force=True) # Refresh internals before applying non-sort operations - - # Work on a copy to allow concurrent readers to see a consistent state - df = self._all_datasets_df # Remove copy because memory waste and slowdown - messages = [] + _SORT_FUNCS = {"df.sort_values", "df.sort_index", "df.sort_view_slice"} + is_sort_only = bool(operations) and all( + op.get("function") in _SORT_FUNCS for op in operations) + def _run_ops(target): + out = [] for op in operations: - func = op.get("function") - params = op.get("params", {}) or {} - msg = self._apply_agent_operation(df, func, params) - messages.append(msg) - - final_message = " | ".join(messages) if messages else "No operation performed" - - # Atomic swap - self._all_datasets_df = df - - # Direct queries are manipulations -> Freeze the view - if operations: - self._is_filtered = True + out.append(self._apply_agent_operation( + target, op.get("function"), op.get("params", {}) or {})) + return " | ".join(out) if out else "No operation performed" + + if is_sort_only: + # Sorting 4M rows costs ~10s; under _lock that stalls the trainer + # for the whole duration (measured 11716ms). Sort a shallow copy + # off-lock, then hold the lock only for the reference swap. + base = self._all_datasets_df + df = base.copy(deep=False) if base is not None else base + final_message = _run_ops(df) + # Row ORDER changed, so sample_id -> position is stale; without a + # rebuild the differential refresh writes signals to wrong rows. + # Build it OFF-lock -- doing it inside the swap held the lock for + # 5320ms at 4M rows. + new_pos_map = self._compute_view_pos_map(df) + with self._watched_lock("_lock[ApplyDataQuery/swap]"): + self._all_datasets_df = df + self._view_pos_map = new_pos_map + if operations: + self._is_filtered = True + else: + with self._watched_lock("_lock[ApplyDataQuery/ops]"): + self._slowUpdateInternals(force=True) + df = self._all_datasets_df + final_message = _run_ops(df) + self._all_datasets_df = df + self._rebuild_view_pos_map() + if operations: + self._is_filtered = True return self._build_success_response( df=df, @@ -5456,7 +5572,8 @@ def EditDataSample(self, request, context): with self._watched_lock("_lock[EditDataSample/__copy_metadata__]"): try: - self._slowUpdateInternals() + if not self._fastUpdateInternals(): + self._slowUpdateInternals() if self._all_datasets_df is None or self._all_datasets_df.empty: return pb2.DataEditsResponse( success=False, @@ -5527,7 +5644,8 @@ def EditDataSample(self, request, context): with self._watched_lock("_lock[EditDataSample/delete-col]"): try: - self._slowUpdateInternals() + if not self._fastUpdateInternals(): + self._slowUpdateInternals() if self._all_datasets_df is None or self._all_datasets_df.empty: return pb2.DataEditsResponse( success=False, @@ -5565,7 +5683,8 @@ def EditDataSample(self, request, context): # Kick a background view-refresh (non-blocking) — the in-memory view # is already consistent after the drop above, so blocking inline rebuild # is unnecessary and causes the gRPC response to stall for 5-10 s. - self._slowUpdateInternals() + if not self._fastUpdateInternals(): + self._slowUpdateInternals() return pb2.DataEditsResponse( success=True, @@ -5588,7 +5707,8 @@ def EditDataSample(self, request, context): with self._watched_lock("_lock[EditDataSample/__discard_by_tag__]"): try: - self._slowUpdateInternals() + if not self._fastUpdateInternals(): + self._slowUpdateInternals() if self._all_datasets_df is None or self._all_datasets_df.empty: return pb2.DataEditsResponse( success=False, @@ -5597,7 +5717,8 @@ def EditDataSample(self, request, context): tag_col = f"{SampleStatsEx.TAG.value}:{tag_name}" if tag_col not in self._all_datasets_df.columns: - self._slowUpdateInternals() + if not self._fastUpdateInternals(): + self._slowUpdateInternals() df = safe_reset_index(self._all_datasets_df) if tag_col not in df.columns: return pb2.DataEditsResponse( @@ -5805,7 +5926,8 @@ def GetDataSplits(self, request, context): # IMPORTANT: keep lock ordering consistent (_update_lock -> _lock). # Calling _slowUpdateInternals() while holding _lock can deadlock # with concurrent readers/writers under high UI refresh pressure. - self._slowUpdateInternals() + if not self._fastUpdateInternals(): + self._slowUpdateInternals() if context is not None and not context.is_active(): return pb2.DataSplitsResponse(success=False, split_names=[]) diff --git a/weightslab/trainer/trainer_tools.py b/weightslab/trainer/trainer_tools.py index 136dd34c..30c5528d 100644 --- a/weightslab/trainer/trainer_tools.py +++ b/weightslab/trainer/trainer_tools.py @@ -434,10 +434,16 @@ def _get_input_tensor_for_sample(dataset, sample_id, device): def process_sample(sid, dataset, do_resize, resize_dims, experiment): try: - if hasattr(dataset, "_getitem_raw"): - tensor, idx, label = dataset._getitem_raw(id=sid) - else: - tensor, idx, label = dataset[sid] + # _getitem_raw returns (data, id, target, *metadata) by contract -- datasets + # implementing get_items() with metadata yield 4+ elements, so a fixed + # 3-way unpack raises "too many values to unpack" and kills every + # thumbnail. Unpack positionally instead. + _res = dataset._getitem_raw(id=sid) if hasattr(dataset, "_getitem_raw") else dataset[sid] + if not isinstance(_res, (tuple, list)): + _res = (_res, sid, None) + tensor = _res[0] + idx = _res[1] if len(_res) > 1 else sid + label = _res[2] if len(_res) > 2 else None if isinstance(tensor, torch.Tensor): img = tensor.detach().cpu() From cf0397df9052a183a6ee0b9ec31a7bdfb50f19de Mon Sep 17 00:00:00 2001 From: Alexandru Rotaru Date: Tue, 18 Aug 2026 13:42:58 +0000 Subject: [PATCH 04/13] fix(view): stop the served view silently diverging from the ledger The ledger was always correct; the view readers see was not, and every failure mode reported itself as success. On a 3.96M-sample run the UI showed ~2k samples with loss data at 34k steps, and toggling image modalities showed the source image twice. View correctness: * Address differential-sync rows by the SAMPLE_ID index level. The view is indexed (origin, sample_id), so get_level_values(0) returned origin and its intersection with the dirty sample_ids was always empty -- the sync wrote nothing and returned True, which suppressed the rebuild that would have repaired it. Positions now come from Index.get_indexer (cached hash engine), so it stays vectorised. * Rebuild when the ledger gains columns the view lacks. Per-sample signal columns are created on their first write, so on a fresh ledger the view predates them and could never gain them: sorting and histogramming failed with "column not in view" and last_seen served -1 forever. Checked before the dirty-set drain, since a schema gain is independent of dirty rows. * Keep the view-dirty backlog until a rebuild actually lands. It was discarded on overflow assuming the caller would rebuild, but the force path returns early on a contended lock -- those ids were then lost with nothing left to re-mark them. Cleared at the atomic view swap instead. * Log view-build failures as errors. They were swallowed at debug level and returned the previous view, so a broken build was indistinguishable from "no new data". Named image views: * Probe extra_images() on the unwrapped dataset -- WL's tracking wrapper does not forward it, so every named view was silently dropped. * Honour stats_to_retrieve for image views, but still advertise filtered-out views with an empty thumbnail so their toggles do not vanish from the panel. Cost: * Loss-shape autotagging runs on its own interval (WL_LOSS_SHAPE_INTERVAL_SECONDS, default 60s) instead of the 2s flush tick, where each pass cost ~990ms of GIL-held pandas work. * Signal-DAG history reads a bounded in-memory tail (WL_HISTORY_TAIL, default 16) instead of scanning per_sample -- 140ms per step at 20M rows, growing without bound. Neither an index nor a rewritten IN clause helped (1.1x/1.4x). * Skip array normalisation for columns the H5 write excludes: with predictions off it rasterised via get_mask, which reads the source image, for data that is never persisted. * Close inherited HDF5 fds in forked dataloader workers; they made HDF5 refuse the parent's read-write open and killed ledger persistence for 12 hours. Measured on the UltraEdit harness (859M params, batch 24, A10G) against an identical run with weightslab stubbed out: 1574ms -> ~1290ms/step versus a 1171ms baseline, i.e. +34% -> ~+10%. optrace.py is included: the @traced/hit markers the other files import are what located the sample_id level bug. Co-Authored-By: Claude Opus 5 --- weightslab/backend/dataloader_interface.py | 48 ++ weightslab/backend/logger.py | 87 +++- weightslab/backend/optrace.py | 348 +++++++++++++++ weightslab/data/dataframe_manager.py | 146 +++++- weightslab/data/h5_array_store.py | 64 +++ weightslab/data/h5_dataframe_store.py | 37 +- weightslab/src.py | 69 ++- weightslab/trainer/services/data_service.py | 420 ++++++++++++++---- .../trainer/services/experiment_service.py | 4 + weightslab/utils/logs.py | 2 +- 10 files changed, 1103 insertions(+), 122 deletions(-) create mode 100644 weightslab/backend/optrace.py diff --git a/weightslab/backend/dataloader_interface.py b/weightslab/backend/dataloader_interface.py index 9f76f660..21b5f5fc 100644 --- a/weightslab/backend/dataloader_interface.py +++ b/weightslab/backend/dataloader_interface.py @@ -43,6 +43,44 @@ _DENY_LIST_REFRESH_INTERVAL = 32 +def _close_inherited_h5_fds(worker_id: int = 0) -> None: + """DataLoader worker_init_fn: drop HDF5 handles inherited from the parent. + + torch forks workers, so every fd the parent had open at fork time is + duplicated into the child -- including the ledger store. The child never + uses them, but their mere existence makes HDF5 refuse the parent's + read-write open, which silently kills ledger persistence. + + Closes rather than just dropping the Python object: the fd is what holds + the file, and the child has no Python-level reference to it at all. + """ + import os + try: + fd_dir = "/proc/self/fd" + for entry in os.listdir(fd_dir): + try: + target = os.readlink(os.path.join(fd_dir, entry)) + except OSError: + continue + if target.endswith(".h5") or target.endswith(".h5.lock"): + try: + os.close(int(entry)) + except OSError: + pass + except Exception: + # Never let cleanup break a worker: a leaked handle degrades + # persistence, a raising worker_init_fn kills the run. + pass + + +def _with_worker_init(kwargs: dict, num_workers: int) -> dict: + """Attach the fd cleanup unless the caller supplied its own init.""" + if num_workers and not kwargs.get("worker_init_fn"): + kwargs = dict(kwargs) + kwargs["worker_init_fn"] = _close_inherited_h5_fds + return kwargs + + def _resolve_safe_num_workers(dataset: Any, num_workers: int, loader_name: Optional[str] = None) -> int: """Clamp worker count for datasets that cannot be pickled by Windows spawn.""" try: @@ -177,6 +215,12 @@ def _get_deny_list_revision(self) -> Optional[tuple[str, int]]: try: origin = self._get_current_origin() df_manager = get_dataframe() + if origin and df_manager is not None and hasattr(df_manager, "get_discard_revision"): + # Deliberately NOT get_origin_revision: that moves on every + # per-sample signal write, i.e. every training step, so the + # cache below could never hit and __len__ rescanned 3.96M rows + # per batch. Discard state is what the deny-list depends on. + return ("discard", int(df_manager.get_discard_revision(origin))) if origin and df_manager is not None and hasattr(df_manager, "get_origin_revision"): return ("origin", int(df_manager.get_origin_revision(origin))) except Exception: @@ -514,6 +558,7 @@ def __init__( self.tracked_dataset, batch_sampler=batch_sampler, num_workers=num_workers, + worker_init_fn=_close_inherited_h5_fds, pin_memory=pin_memory, collate_fn=collate_fn, persistent_workers=self._should_persist_workers(num_workers), @@ -1137,6 +1182,7 @@ def restore_iteration_state(self, state: dict) -> None: self.tracked_dataset, batch_sampler=sampler, num_workers=num_workers, + worker_init_fn=_close_inherited_h5_fds, pin_memory=pin_memory, collate_fn=collate_fn, # Ensure no conflicting args are passed alongside batch_sampler @@ -1223,6 +1269,7 @@ def set_batch_size(self, new_batch_size: int) -> None: self.tracked_dataset, batch_sampler=sampler, num_workers=num_workers, + worker_init_fn=_close_inherited_h5_fds, pin_memory=pin_memory, collate_fn=collate_fn, persistent_workers=self._should_persist_workers(num_workers), @@ -1235,6 +1282,7 @@ def set_batch_size(self, new_batch_size: int) -> None: batch_size=batch_size, shuffle=shuffle, num_workers=num_workers, + worker_init_fn=_close_inherited_h5_fds, drop_last=drop_last, pin_memory=pin_memory, collate_fn=collate_fn, diff --git a/weightslab/backend/logger.py b/weightslab/backend/logger.py index c86a21e9..0ad32966 100644 --- a/weightslab/backend/logger.py +++ b/weightslab/backend/logger.py @@ -35,13 +35,14 @@ import os import threading import time -from collections import defaultdict +from collections import defaultdict, deque import duckdb import pandas as pd import torch as th from weightslab.backend.ledgers import get_logger, register_logger, get_checkpoint_manager +from weightslab.backend.optrace import maybe_wrap_duckdb_conn logger = logging.getLogger(__name__) @@ -63,6 +64,31 @@ _STAGE_FLUSH_THRESHOLD = 50_000 # How often the background flush thread wakes up (see LoggerQueue._flush_loop). +def _default_loss_shape_interval_seconds() -> float: + """How often to re-derive loss-shape categoricals. + + Deliberately much slower than the flush tick: the label is display-only, + while each pass costs a classify + tag write + an upsert over the whole + ledger (~990ms at 4M rows) on a thread that holds the GIL against training. + """ + try: + return float(os.environ.get("WL_LOSS_SHAPE_INTERVAL_SECONDS", "60.0")) + except (TypeError, ValueError): + return 60.0 + + +def _default_history_tail() -> int: + """Recent points kept per sample for signal-DAG history reads. + + Bounded so history() costs O(batch) instead of scanning the whole + per_sample table (140ms at 20M rows, once per step, and growing). + """ + try: + return max(0, int(os.environ.get("WL_HISTORY_TAIL", "16"))) + except (TypeError, ValueError): + return 16 + + def _default_flush_interval_seconds() -> float: try: return float(os.environ.get("WL_LOGGER_FLUSH_INTERVAL_SECONDS", "2.0")) @@ -217,7 +243,7 @@ def __init__(self, register: bool = True, db_path: str = ":memory:") -> None: # DuckDB connection + write-staging buffers. self._lock = threading.RLock() self._db_path = db_path - self._conn = duckdb.connect(database=db_path) + self._conn = maybe_wrap_duckdb_conn(duckdb.connect(database=db_path)) self._stage_signals: list = [] self._stage_sample: list = [] self._stage_instance: list = [] @@ -237,7 +263,14 @@ def __init__(self, register: bool = True, db_path: str = ":memory:") -> None: # Cache size is env-configurable (WL_QUERY_CACHE_MAXSIZE, default 2048). _qps_maxsize = int(os.environ.get("WL_QUERY_CACHE_MAXSIZE", "2048")) self._qps_version: dict = defaultdict(int) + # {signal_name: {sample_id, ...}} staged since the last autotag pass. + # Lets the pass classify O(change) samples instead of the whole history. + self._qps_dirty: dict = defaultdict(set) self._qps_cache_step: int = -1 + # {signal: {sample_id: deque}} -- the recent tail of each sample's + # per-sample values, maintained on write so history() never scans. + self._tail_len = _default_history_tail() + self._recent_tail: dict = defaultdict(dict) self._qps_cache = functools.lru_cache(maxsize=_qps_maxsize)(self._query_per_sample_uncached) self._qps_step_cache = functools.lru_cache(maxsize=_qps_maxsize)(self._query_per_sample_at_step_uncached) @@ -365,7 +398,9 @@ def _autotag_loss_shapes(self) -> None: continue # no new per-sample data logged since the last pass tag_name, classifier = overrides.get(signal_name, (None, None)) try: - write_signal_shapes(signal_name, tag_name=tag_name, classifier=classifier) + write_signal_shapes(signal_name, tag_name=tag_name, + classifier=classifier, + only_sample_ids=self.drain_dirty_samples(signal_name)) self._loss_shape_last_version[signal_name] = current_version except Exception as exc: logger.debug( @@ -373,12 +408,20 @@ def _autotag_loss_shapes(self) -> None: def _flush_loop(self) -> None: interval = _default_flush_interval_seconds() + autotag_every = _default_loss_shape_interval_seconds() + next_autotag = 0.0 while not self._flush_stop.wait(interval): try: self.flush_to_disk() except Exception as exc: logger.debug(f"[LoggerQueue] background flush failed: {exc}") - self._autotag_loss_shapes() + # Autotagging on the flush tick re-derived the shape categorical + # every 2s; on its own slower clock it stops stealing the GIL from + # the training loop for a label nothing reads that often. + now = time.monotonic() + if now >= next_autotag: + next_autotag = now + autotag_every + self._autotag_loss_shapes() def stop_background_flush(self) -> None: """Stop the background flush/loss-shape thread (e.g. at shutdown or in tests).""" @@ -531,7 +574,7 @@ def set_db_path(self, db_path) -> None: # Adopt the on-disk file as the live connection. On resume this # is the source of truth; the fresh in-memory rows are ignored. self._conn.close() - self._conn = duckdb.connect(database=db_path) + self._conn = maybe_wrap_duckdb_conn(duckdb.connect(database=db_path)) self._db_path = db_path self._ensure_tables() self._invalidate_qps_cache() @@ -708,8 +751,42 @@ def _stage_sample_row(self, graph_name, exp_hash, sample_id, step, value): ) ) self._qps_version[graph_name] += 1 # invalidate this signal's cached reads + self._qps_dirty[graph_name].add(str(sample_id)) + if self._tail_len: + _tail = self._recent_tail[graph_name] + _sid = str(sample_id) + _q = _tail.get(_sid) + if _q is None: + _q = _tail[_sid] = deque(maxlen=self._tail_len) + _q.append(float(value)) self._maybe_autoflush() + def recent_per_sample(self, graph_name: str, sample_ids): + """Recent in-memory values per sample: ``{sample_id: [values]}``. + + O(batch). Samples not written in this process are absent rather than + empty-listed; callers treat both as "not enough history yet". + """ + tail = self._recent_tail.get(graph_name) or {} + out = {} + for s in sample_ids: + q = tail.get(str(s)) + if q: + out[s] = list(q) + return out + + def drain_dirty_samples(self, graph_name): + """Sample ids staged for *graph_name* since the last drain, then clear. + + The autotag pass uses this to re-classify only trajectories that gained + a point, rather than every sample ever logged.""" + with self._lock: + ids = self._qps_dirty.get(graph_name) + if not ids: + return set() + self._qps_dirty[graph_name] = set() + return ids + def _invalidate_qps_cache(self) -> None: """Drop both query caches + versions (step advance; bulk delete/clear).""" self._qps_cache.cache_clear() diff --git a/weightslab/backend/optrace.py b/weightslab/backend/optrace.py new file mode 100644 index 00000000..477b9ec4 --- /dev/null +++ b/weightslab/backend/optrace.py @@ -0,0 +1,348 @@ +"""Begin/end operation tracing for dataframe, array-store, duckdb and +experiment-service operations. + +Off by default (near-zero overhead: one bool check) — set ``WL_OPTRACE=1`` to +turn it on. Every traced call prints ONE line at start and ONE line at end to +stdout (unbuffered, same stream as main.py's ``[timing]`` prints), tagged +``[optrace]`` so a run's LOG file can be parsed the same way: + + grep -a "\\[optrace\\]" LOG | ... + +Line format (space-separated key=value tokens, so ``awk`` can pick fields by +name without caring about column position):: + + [optrace] BEGIN domain=dataframe op=dfm.upsert_df call=482 tid=140234 ts=1723300000.123456 site=dataframe_manager.py:680 n_in=24 bytes_in=196608 args=origin=train_loader,force_flush=False + [optrace] END domain=dataframe op=dfm.upsert_df call=482 tid=140234 ts=1723300000.234567 dur_ms=111.111 ok=True site=dataframe_manager.py:680 n_in=24 bytes_in=196608 args=origin=train_loader,force_flush=False mem_delta_kb=512 n_out=24 bytes_out=- + +``call=`` pairs a BEGIN with its END even when the same op runs concurrently +on multiple threads (same op+tid can otherwise appear twice before either +finishes). For call-count/timing/bytes/memory/object-count reports, the END +line alone carries every field -- see ``code/optrace_report.py``. + +When ``@traced``/``trace_op`` wraps a whole function (the normal case), the +extra fields beyond ``dur_ms``/``ok`` are filled in automatically: + + site file:line of the function's ``def`` (not the call site -- + stable across callers, and enough to jump to the code). + n_in/n_out best-effort element counts for arguments / return value + (numpy array .size, len() of dict/list/etc). + bytes_in/out best-effort byte counts (numpy .nbytes, len() of bytes), + summed recursively through dict/list/tuple containers. + args sanitized ``name=repr`` for each bound argument (arrays + collapse to ``ndarray(shape=...,dtype=...)`` rather than + dumping their contents) -- the "which sample_id did this" + detail needed to trace back a specific weird call. + mem_delta_kb RSS delta (psutil) across the call. Peak-agnostic and can + be noisy under concurrent threads sharing one process, but + cheap and good enough to spot a call that's allocating much + more than its neighbours. + +A bare ``with trace_op(domain, op, **extra):`` (not decorating a function, +e.g. ``TracingDuckDBConn``) has no function to introspect, so it only gets +whatever ``extra`` the caller passed plus ``mem_delta_kb`` -- no +site/n_in/n_out/bytes_in/bytes_out/args. +""" +import functools +import inspect +import itertools +import os +import sys +import threading +import time + +try: + import numpy as _np +except Exception: + _np = None + +try: + import pandas as _pd +except Exception: + _pd = None + +try: + import psutil as _psutil + _PROC = _psutil.Process() +except Exception: + _PROC = None + +_TRUTHY = {"1", "true", "yes", "on"} +_ENABLED = os.environ.get("WL_OPTRACE", "0").strip().lower() in _TRUTHY + +_counter = itertools.count() +_counter_lock = threading.Lock() + +# print(msg, flush=True) is two separate write()s under the hood (message, +# then the trailing newline) with no atomicity guarantee between them, so two +# threads tracing concurrently (training thread, flush thread, grpc workers) +# can interleave mid-line -- observed in practice as garbled/merged [optrace] +# lines. Serialize the full write+flush per line instead. +_print_lock = threading.Lock() + + +def _emit(line: str) -> None: + with _print_lock: + print(line, flush=True) + + +def trace_enabled() -> bool: + return _ENABLED + + +def _next_call_id() -> int: + with _counter_lock: + return next(_counter) + + +def _fmt_extra(extra: dict) -> str: + if not extra: + return "" + return " " + " ".join(f"{k}={v}" for k, v in extra.items()) + + +def sanitize(value, maxlen: int = 48) -> str: + """Collapse whitespace and truncate so a value is safe as a bare token + in the space-separated log line (e.g. a SQL statement).""" + s = " ".join(str(value).split()) + if len(s) > maxlen: + s = s[:maxlen] + "..." + return s.replace(" ", "_") + + +def _rss_kb(): + if _PROC is None: + return None + try: + return _PROC.memory_info().rss / 1024.0 + except Exception: + return None + + +def _obj_metrics(obj): + """Best-effort (count, bytes) size hints for an object; either may be None.""" + if obj is None: + return None, None + if _np is not None and isinstance(obj, _np.ndarray): + return obj.size, obj.nbytes + # deep=False ONLY. deep=True walks every element of every object/string + # column: measured at ~1000ms on a 3.96M-row frame vs ~1ms shallow (854x), + # and this runs on every traced call -- it turns tracing itself into the + # O(dataset) hot-path work this module exists to hunt down. Shallow + # undercounts object columns (it counts the 8-byte pointers, not the + # referenced strings), so bytes_in/out for string-heavy frames is a lower + # bound; that is the right trade for a diagnostic that must not distort + # what it measures. + if _pd is not None and isinstance(obj, _pd.DataFrame): + try: + return len(obj), int(obj.memory_usage(deep=False).sum()) + except Exception: + return len(obj), None + if _pd is not None and isinstance(obj, _pd.Series): + try: + return len(obj), int(obj.memory_usage(deep=False)) + except Exception: + return len(obj), None + if isinstance(obj, (bytes, bytearray, memoryview)): + return len(obj), len(obj) + if isinstance(obj, dict): + nbytes = 0 + for v in obj.values(): + _, vb = _obj_metrics(v) + if vb: + nbytes += vb + return len(obj), (nbytes or None) + if isinstance(obj, (list, tuple, set)): + nbytes = 0 + for v in obj: + _, vb = _obj_metrics(v) + if vb: + nbytes += vb + return len(obj), (nbytes or None) + if isinstance(obj, (str, int, float, bool)): + return None, None + if hasattr(obj, "__len__"): + try: + return len(obj), None + except Exception: + return None, None + return None, None + + +def _fmt_arg_value(value, maxlen: int = 40) -> str: + if _np is not None and isinstance(value, _np.ndarray): + return f"ndarray(shape={value.shape},dtype={value.dtype})" + if _pd is not None and isinstance(value, _pd.DataFrame): + return f"DataFrame(rows={len(value)},cols={value.shape[1]})" + if _pd is not None and isinstance(value, _pd.Series): + return f"Series(len={len(value)},dtype={value.dtype})" + s = repr(value) + return s if len(s) <= maxlen else s[: maxlen - 3] + "..." + + +def _in_metrics(sig, args, kwargs) -> dict: + """n_in/bytes_in/args extras for a decorated function's bound arguments.""" + if sig is None: + return {} + try: + bound = sig.bind_partial(*args, **kwargs) + bound.apply_defaults() + except Exception: + return {} + arg_items = [(n, v) for n, v in bound.arguments.items() if n != "self"] + n_in = b_in = 0 + has_n, has_b = False, False + for _, v in arg_items: + n, b = _obj_metrics(v) + if n is not None: + n_in += n + has_n = True + if b is not None: + b_in += b + has_b = True + out = {} + if has_n: + out["n_in"] = n_in + if has_b: + out["bytes_in"] = b_in + if arg_items: + args_str = ",".join(f"{n}={_fmt_arg_value(v)}" for n, v in arg_items) + out["args"] = sanitize(args_str, maxlen=160) + return out + + +def _out_metrics(result) -> dict: + n_out, b_out = _obj_metrics(result) + out = {} + if n_out is not None: + out["n_out"] = n_out + if b_out is not None: + out["bytes_out"] = b_out + return out + + +class trace_op: + """Context manager: logs BEGIN on enter, END (with duration) on exit. + + Also usable as a decorator: ``@trace_op("dfm.upsert_df")``. + """ + + __slots__ = ("domain", "op", "extra", "_call_id", "_t0", "_mem0") + + def __init__(self, domain: str, op: str, **extra): + self.domain = domain + self.op = op + self.extra = extra + + def set(self, **kv) -> None: + """Attach fields (e.g. n_out/bytes_out) to the END line, from inside + the ``with`` block, once they're known (e.g. after computing a + result).""" + self.extra.update(kv) + + def __call__(self, fn): + site = f"{os.path.basename(fn.__code__.co_filename)}:{fn.__code__.co_firstlineno}" + try: + sig = inspect.signature(fn) + except (TypeError, ValueError): + sig = None + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + if not _ENABLED: + return fn(*args, **kwargs) + call_extra = {"site": site} + call_extra.update(_in_metrics(sig, args, kwargs)) + call_extra.update(self.extra) + op_ctx = trace_op(self.domain, self.op, **call_extra) + with op_ctx: + result = fn(*args, **kwargs) + try: + op_ctx.set(**_out_metrics(result)) + except Exception: + pass + return result + return wrapper + + def __enter__(self): + if not _ENABLED: + return self + self._call_id = _next_call_id() + tid = threading.get_ident() + self._mem0 = _rss_kb() + self._t0 = time.perf_counter() + _emit(f"[optrace] BEGIN domain={self.domain} op={self.op} " + f"call={self._call_id} tid={tid} ts={time.time():.6f}" + f"{_fmt_extra(self.extra)}") + return self + + def __exit__(self, exc_type, exc, tb): + if not _ENABLED: + return False + dur_ms = (time.perf_counter() - self._t0) * 1000.0 + tid = threading.get_ident() + mem1 = _rss_kb() + if mem1 is not None and self._mem0 is not None: + self.extra["mem_delta_kb"] = f"{mem1 - self._mem0:.0f}" + _emit(f"[optrace] END domain={self.domain} op={self.op} " + f"call={self._call_id} tid={tid} ts={time.time():.6f} " + f"dur_ms={dur_ms:.3f} ok={exc_type is None}" + f"{_fmt_extra(self.extra)}") + return False + + +def hit(domain: str, op: str, **extra) -> None: + """Log a single one-line marker, iff tracing is enabled. + + Unlike ``trace_op``, this isn't a timed BEGIN/END pair -- it's for + confirming which branch of an if/else a call actually took (e.g. + fast-path vs fallback, in-place vs backup-and-rewrite) so a run's LOG can + answer "did the new code path get hit, and how often" via:: + + grep -a "\\[optrace\\] HIT" LOG | awk '...' + """ + if not _ENABLED: + return + _emit(f"[optrace] HIT domain={domain} op={op} tid={threading.get_ident()} " + f"ts={time.time():.6f}{_fmt_extra(extra)}") + + +def traced(domain: str, op: str = None): + """Method decorator: ``@traced("dataframe", "dfm.upsert_df")``. + + ``op`` defaults to the wrapped function's qualified name. + """ + def deco(fn): + name = op or fn.__qualname__ + return trace_op(domain, name)(fn) + return deco + + +class TracingDuckDBConn: + """Transparent proxy around a duckdb connection: traces ``execute``/ + ``sql``, delegates everything else (register/unregister/close/...) + untouched. Only construct this when tracing is enabled — with it off, + keep using the raw connection so there is zero added indirection. + """ + + __slots__ = ("_conn",) + + def __init__(self, conn): + self._conn = conn + + def execute(self, *args, **kwargs): + sql = sanitize(args[0]) if args else "" + with trace_op("duckdb", "duckdb.execute", sql=sql): + return self._conn.execute(*args, **kwargs) + + def sql(self, *args, **kwargs): + sql = sanitize(args[0]) if args else "" + with trace_op("duckdb", "duckdb.sql", sql=sql): + return self._conn.sql(*args, **kwargs) + + def __getattr__(self, name): + return getattr(self._conn, name) + + +def maybe_wrap_duckdb_conn(conn): + """Wrap ``conn`` for tracing iff WL_OPTRACE is on, else return it as-is.""" + return TracingDuckDBConn(conn) if _ENABLED else conn diff --git a/weightslab/data/dataframe_manager.py b/weightslab/data/dataframe_manager.py index ca19d5e8..d28b0e83 100644 --- a/weightslab/data/dataframe_manager.py +++ b/weightslab/data/dataframe_manager.py @@ -25,6 +25,7 @@ SAMPLES_STATS_TO_SAVE_TO_H5, ) from weightslab.backend.ledgers import get_hyperparams +from weightslab.backend.optrace import traced, hit pd.set_option('future.no_silent_downcasting', True) @@ -92,6 +93,11 @@ def __init__(self, flush_interval: float = 3.0, flush_max_rows: int = 100, enabl self._store: H5DataFrameStore | None = None self._array_store: H5ArrayStore | None = None self._origin_revisions: Dict[str, int] = {} + # Bumped ONLY when the `discarded` column changes. The deny-list cache + # keys on this instead of _origin_revisions, which training bumps every + # step (per-sample signal writes) and which therefore never lets a cache + # hit -- turning a per-batch len() into a 3.96M-row scan. + self._discard_revisions: Dict[str, int] = {} self._pending: set[int] = set() # Parallel dirty set for the view. _pending is drained by the H5 flush, # so the view cannot share it without one consumer starving the other. @@ -411,6 +417,11 @@ def set_array_store(self, array_store: H5ArrayStore): if self._enable_h5_persistence: self._array_store = array_store + def _bump_discard_revisions(self, origins: Sequence[Any]) -> None: + for origin in origins or []: + key = str(origin) + self._discard_revisions[key] = self._discard_revisions.get(key, 0) + 1 + def _bump_origin_revisions(self, origins: Sequence[Any]) -> None: for origin in origins: if origin is None or pd.isna(origin): @@ -476,6 +487,7 @@ def _merge_categories(self, name: str, categories, replace: bool = False) -> Lis self._categorical_tags[name] = list(dict.fromkeys([*existing, *cats])) return list(self._categorical_tags[name]) + @traced("dataframe", "dfm.register_categorical_tag") def register_categorical_tag(self, name: str, categories=None, replace: bool = False) -> List[str]: """Declare (or extend) a categorical tag and its allowed category values. @@ -565,6 +577,7 @@ def _load_tag_registry(self) -> None: except Exception as e: logger.debug(f"[LedgeredDataFrameManager] Failed to load tag registry: {e}") + @traced("dataframe", "dfm.register_split") def register_split(self, origin: str, df: List | pd.DataFrame, store: H5DataFrameStore | None = None, autoload_arrays: bool | list | set = False, return_proxies: bool = True, use_cache: bool = True): # Build the annotation-expanded (sample_id, annotation_id) frame. # Fast path: when given a list of record dicts, construct the EXPANDED frame @@ -598,6 +611,7 @@ def register_split(self, origin: str, df: List | pd.DataFrame, store: H5DataFram # Start flush thread if not already running self._ensure_flush_thread() + @traced("dataframe", "dfm._load_existing_data") def _load_existing_data(self, origin: str = None, autoload_arrays: bool | list | set = False, return_proxies: bool = True, use_cache: bool = True): # Restore the categorical tag registry so loaded string-valued tag columns # get their full allowed category set (not just the values present on disk). @@ -673,6 +687,7 @@ def _load_existing_data(self, origin: str = None, autoload_arrays: bool | list | else: logger.warning(f"[LedgeredDataFrameManager] Loaded data missing 'sample_id' column for origin={origin}. Skipping load.") + @traced("dataframe", "dfm.upsert_df") def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flush: bool = False): if df_local is None or (isinstance(df_local, pd.DataFrame) and df_local.empty) or len(df_local) == 0: return @@ -778,7 +793,19 @@ def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flu if len(_pos) and (_pos >= 0).all(): for _c in all_cols: _ci = self._df.columns.get_loc(_c) - self._df.iloc[_pos, _ci] = df_norm.loc[existing_idx, _c].to_numpy() + _vals = df_norm.loc[existing_idx, _c].to_numpy() + # An object array (None for "no value yet") written into + # a float column upcasts the WHOLE column to object and + # it never returns -- an 8x penalty on every later sort. + # Coerce to the target dtype so None becomes NaN instead. + try: + _tgt = self._df[_c].dtype + if (_vals.dtype == object + and getattr(_tgt, "kind", "") in "fiu"): + _vals = pd.to_numeric(_vals, errors="coerce") + except Exception: + pass + self._df.iloc[_pos, _ci] = _vals else: self._df.loc[existing_idx, all_cols] = df_norm.loc[existing_idx, all_cols] @@ -812,7 +839,10 @@ def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flu sample_ids = df_norm.index.tolist() self.mark_dirty_batch(sample_ids, force_flush=force_flush) self._bump_origin_revisions(affected_origins) + if SampleStats.Ex.DISCARDED.value in set(df_norm.columns): + self._bump_discard_revisions(affected_origins) + @traced("dataframe", "dfm.mark_dirty") def mark_dirty(self, sample_id: int): """Mark sample as dirty for H5 flush. @@ -823,12 +853,14 @@ def mark_dirty(self, sample_id: int): self._pending.add(normalized_id) self._view_pending.add(normalized_id) + @traced("dataframe", "dfm.drop_column") def drop_column(self, column: str): with self._lock: if column in self._df.columns: return self._df.pop(column) return None + @traced("dataframe", "dfm.mark_dirty_batch") def mark_dirty_batch(self, sample_ids: List[int], force_flush: bool = False): with self._lock: self._pending.update(set(sample_ids)) @@ -1003,6 +1035,7 @@ def _normalize_preds_raw_uint16(self, preds_raw: np.ndarray) -> np.ndarray: except Exception: return preds_raw + @traced("dataframe", "dfm.enqueue_batch") def enqueue_batch( self, sample_ids: Sequence[int], @@ -1121,6 +1154,7 @@ def index_batch(obj, batch_index, rec=False): self.first_init = False self.flush_async() + @traced("dataframe", "dfm.enqueue_instance_batch") def enqueue_instance_batch( self, sample_ids: Sequence[Any], @@ -1267,6 +1301,7 @@ def _index_target(obj, i): self.first_init = False self.flush_async() + @traced("dataframe", "dfm.update_values") def update_values(self, origin: str, sample_id: int, updates: Dict[str, Any], annotation_id: int = 0): """Update values for a sample (or specific annotation if multi-index). @@ -1337,6 +1372,7 @@ def update_values(self, origin: str, sample_id: int, updates: Dict[str, Any], an self._df = pd.concat([self._df, df_local]) self._bump_origin_revisions([origin]) + @traced("dataframe", "dfm.take_view_dirty") def take_view_dirty(self, limit: int | None = None): """Drain and return the sample_ids changed since the last view sync. @@ -1345,12 +1381,21 @@ def take_view_dirty(self, limit: int | None = None): """ with self._lock: if limit is not None and len(self._view_pending) > limit: - self._view_pending.clear() + # Do NOT clear here. The caller is expected to rebuild, but that + # rebuild can bail (contended update lock) -- and these ids would + # then be lost with nothing to re-mark them. clear_view_dirty() + # is called once the rebuilt view is actually swapped in. return None out = list(self._view_pending) self._view_pending.clear() return out + def clear_view_dirty(self): + """Drop the view-dirty backlog: a full rebuild has made the view current.""" + with self._lock: + self._view_pending.clear() + + @traced("dataframe", "dfm.get_source_rows") def get_source_rows(self, sample_ids, columns=None): """Rows for *sample_ids* straight from the source frame. O(len(ids)).""" with self._lock: @@ -1364,9 +1409,22 @@ def get_source_rows(self, sample_ids, columns=None): return sub[columns] if columns else sub def get_origin_revision(self, origin: str) -> int: - with self._lock: - return int(self._origin_revisions.get(str(origin), 0)) + # No lock: a dict read is atomic under the GIL, and this is polled from + # __len__ on every batch. Taking self._lock here put the training thread + # in contention with the flush thread on the hot path. + return int(self._origin_revisions.get(str(origin), 0)) + + def get_discard_revision(self, origin: str) -> int: + """Revision of the `discarded` column for *origin*. + + Changes only on discard/restore, so a consumer that depends purely on + deny-list state can cache against it across training steps. Read without + the lock: called per batch, and a stale-by-one read is harmless (the next + batch picks the change up), whereas lock contention here is not. + """ + return int(self._discard_revisions.get(str(origin), 0)) + @traced("dataframe", "dfm.update_by_groups_bulk") def update_by_groups_bulk(self, origin: str, group_ids: List[Any], updates_list: List[Dict[str, Any]]): """Broadcast updates to multiple groups in one pass.""" if not group_ids or not updates_list: @@ -1420,6 +1478,7 @@ def update_by_groups_bulk(self, origin: str, group_ids: List[Any], updates_list: if affected_ids: self.mark_dirty_batch(affected_ids) + @traced("dataframe", "dfm.get_tainted_group_ids") def get_tainted_group_ids(self, group_ids: List[Any], origin: str) -> set: """Return the subset of group_ids where at least one member is discarded. @@ -1489,6 +1548,7 @@ def get_group_column_values(self, group_ids: List[Any], origin: str, column: str return values + @traced("dataframe", "dfm.get_discarded_sample_ids") def get_discarded_sample_ids(self, sample_ids: List[Any], origin: str) -> set: """Return the subset of sample_ids that are marked as discarded. @@ -1580,6 +1640,7 @@ def get_sample_column_values(self, sample_ids: List[Any], column: str) -> Dict[A return values + @traced("dataframe", "dfm.get_row") def get_row(self, origin: str, sample_id: int, annotation_id: int = None) -> pd.Series | pd.DataFrame | None: """Get row(s) by sample_id and optional annotation_id. @@ -1619,12 +1680,14 @@ def get_row(self, origin: str, sample_id: int, annotation_id: int = None) -> pd. except (KeyError, TypeError): return None + @traced("dataframe", "dfm.get_value") def get_value(self, origin: str, sample_id: int, column: str): row = self.get_row(origin, sample_id) if row is None or column not in row: return None return row[column] + @traced("dataframe", "dfm.get_df_view") def get_df_view(self, column: str = None, limit: int = -1, copy: bool = False, value: str = None) -> pd.DataFrame: with self._lock: if self._df.empty: @@ -1640,10 +1703,12 @@ def get_df_view(self, column: str = None, limit: int = -1, copy: bool = False, v subset = subset.head(limit) return subset.copy() if copy else subset + @traced("dataframe", "dfm.set_dense") def set_dense(self, key: str, sample_id: int, value: np.ndarray): with self._lock: self._dense_store.setdefault(key, {})[str(sample_id)] = value + @traced("dataframe", "dfm.get_dense_map") def get_dense_map(self, origin: str) -> Dict[str, Dict[int, np.ndarray]]: with self._lock: origin_store = self._dense_store.get(origin, {}) @@ -1933,7 +1998,9 @@ def _apply_buffer_records(self, records: List[Dict[str, Any]]): self._apply_updates_frame_locked(instance_df, broadcast=False) self._apply_updates_frame_locked(sample_df, broadcast=True) # Keep newly-added signal columns float32 and empty object cells as None. - self._df = self._optimize_dataframe_memory(self._df) + self._df = self._optimize_dataframe_memory( + self._df, + columns=set(sample_df.columns) | set(instance_df.columns)) # Mark all as pending for h5 flush (outside lock) self.mark_dirty_batch(sample_ids) @@ -1971,13 +2038,22 @@ def _apply_buffer_records_nonblocking(self, records: List[Dict[str, Any]]): applied_index = written_s.append(written_i) if len(written_i) else written_s update_cols = sample_df.columns.union(instance_df.columns) # Keep newly-added signal columns float32 and empty object cells as None. - _df = self._optimize_dataframe_memory(self._df) + _df = self._optimize_dataframe_memory(self._df, columns=set(update_cols)) self._df = _df finally: self._lock.release() # Det→seg conversion / array normalization over all written rows. - if applied_index is not None and len(applied_index) > 0: + # This prepares cells for the H5 write, so it is only worth doing for + # columns that write will actually include. When predictions/targets are + # excluded from the save list (WEIGHTSLAB_SAVE_PREDICTIONS_IN_H5=0) the + # pass would otherwise call get_mask per row -- which reads the source + # image to size the mask -- for arrays that are never persisted. + _savable = set(_filter_columns_by_patterns( + list(update_cols), SAMPLES_STATS_TO_SAVE_TO_H5)) + _norm_cols = [c for c in update_cols + if c in self._array_columns and c in _savable] + if applied_index is not None and len(applied_index) > 0 and _norm_cols: if applied_index.has_duplicates: applied_index = applied_index[~applied_index.duplicated()] normalized_rows = self._df.loc[applied_index].apply( @@ -2061,6 +2137,7 @@ def _rows_with_array_cells(self, data_snapshot: pd.DataFrame): return [] return list(hits) + @traced("dataframe", "dfm._flush_snapshot_to_h5") def _flush_snapshot_to_h5(self, data_snapshot: pd.DataFrame, work: List[int]): """Flush data snapshot to H5 - runs completely outside locks. @@ -2186,12 +2263,41 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st if categorical_tags is None: categorical_tags = self._categorical_tags + # Honour `columns`: only the columns this flush actually wrote can have + # gained a float64 signal value or a fresh NaN, so scanning the rest is + # O(rows) of pure waste on every flush. + _scan_cols = (list(df.columns) if columns is None + else [c for c in df.columns if c in columns]) + hit("dataframe", "dfm._optimize_dataframe_memory", + scoped=columns is not None, n_scan=len(_scan_cols), n_total=len(df.columns)) + + # === 0) Repair signal columns that were upcast to object === + # A single None written into a float column converts it permanently, and + # object columns sort ~8x slower. signals//* are numeric by definition, + # so any object one is damage rather than intent. Runs BEFORE the + # NaN->None pass below, which only touches object columns and would + # otherwise keep them that way. + for col in _scan_cols: + if not str(col).startswith("signals") or df[col].dtype != object: + continue + try: + coerced = pd.to_numeric(df[col], errors="coerce") + # Only if nothing was lost: a genuine non-numeric value means the + # column is not what we think it is, so leave it alone. + if coerced.notna().sum() == df[col].notna().sum(): + df[col] = coerced.astype(np.float32) + logger.debug( + "[LedgeredDataFrameManager] restored '%s' object -> float32", col) + except Exception as exc: + logger.debug("[LedgeredDataFrameManager] dtype repair skipped for '%s': %s", + col, exc) + # === 1) Downcast float64 signal columns to float32 === # Per-sample / per-instance signal scalars (loss & metric values) don't need # float64 precision, so this halves the cost of every ``signals//*`` column # with no practical loss for monitoring. Numeric dtype is preserved (NaNs # stay NaN). Done before categorical conversion below. - for col in df.columns: + for col in _scan_cols: if str(col).startswith("signals") and df[col].dtype == np.float64: try: df[col] = df[col].astype(np.float32) @@ -2215,7 +2321,7 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st # into a Categorical, its missing cells are stuck as NaN (categorical code # -1) and can no longer be set to a plain None — so we clean object cells to # None here first, while they are still plain object dtype. - for col in df.columns: + for col in _scan_cols: # The docstring above says numeric/bool/categorical are skipped, but the # loop had no dtype check: at registration every signals//* column is all # NaN, so this did a full label-aligned write per float column. @@ -2232,9 +2338,17 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st SampleStats.Ex.TASK_TYPE.value, # Task type (e.g. 'classification', 'segmentation') ] - # nunique() over the whole index was recomputed per object column (8s at 4M). - _n_rows_cached = (df.index.get_level_values(0).nunique() - if isinstance(df.index, pd.MultiIndex) else len(df)) + # nunique() over a 4M-row MultiIndex costs ~8s AND holds the GIL, stalling + # the training thread inside forward/backward. Only an object-dtype + # candidate reads it, so compute it on first use rather than every flush. + _n_rows_cache = [] + + def _n_rows(): + if not _n_rows_cache: + _n_rows_cache.append( + df.index.get_level_values(0).nunique() + if isinstance(df.index, pd.MultiIndex) else len(df)) + return _n_rows_cache[0] for col in categorical_candidates: if col not in df.columns: continue @@ -2250,11 +2364,10 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st continue if df[col].dtype == 'object': n_unique = df[col].nunique() - _ = _n_rows_cached # Use unique sample count, not row count — with MultiIndex each # sample has multiple annotation rows which would inflate n_rows # and make the ratio appear better than it really is. - n_rows = _n_rows_cached + n_rows = _n_rows() compression_ratio = n_unique / n_rows if n_rows > 0 else 1.0 if compression_ratio < 0.5 and n_unique > 1: # Worth compressing if < 50% unique @@ -2348,6 +2461,7 @@ def stop(self): if self._flush_thread: self._flush_thread.join(timeout=2.0) + @traced("dataframe", "dfm.get_combined_df") def get_combined_df( self, autoload_arrays: bool | list | set = False, @@ -2383,6 +2497,7 @@ def get_combined_df( return df + @traced("dataframe", "dfm.get_collapse_annotations_to_samples_df") def get_collapse_annotations_to_samples_df(self, df: pd.DataFrame | None = None) -> pd.DataFrame: """Collapse a (sample_id, annotation_id) multi-index df to one row per sample. @@ -2631,6 +2746,7 @@ def _should_flush(self) -> bool: with self._lock: return len(self._pending) >= self._flush_max_rows or self._force_flush + @traced("dataframe", "dfm.flush_async") def flush_async(self): """Signal flush thread. Returns once buffer has been drained (not after H5 write). @@ -2655,6 +2771,7 @@ def flush_async(self): time.sleep(0.1) logger.warning("[LedgeredDataFrameManager] flush_async timed out waiting for buffer drain after 60s") + @traced("dataframe", "dfm.flush_if_needed_nonblocking") def flush_if_needed_nonblocking(self, force: bool = False): """Non-blocking flush - if can't acquire lock immediately, defer to next cycle.""" # Drain buffer quickly, then release lock before any DF/H5 work. @@ -2672,6 +2789,7 @@ def flush_if_needed_nonblocking(self, force: bool = False): self._flush_to_h5_if_needed(force=force) logger.debug(f"Completed non-blocking flush check. Pending count after flush: {len(self._pending)}.") + @traced("dataframe", "dfm.flush") def flush(self): """Blocking flush: buffer → DF → H5. diff --git a/weightslab/data/h5_array_store.py b/weightslab/data/h5_array_store.py index 02c3e7e3..20810d05 100644 --- a/weightslab/data/h5_array_store.py +++ b/weightslab/data/h5_array_store.py @@ -20,6 +20,7 @@ import h5py import numpy as np +from weightslab.backend.optrace import traced # Config global logger logger = logging.getLogger(__name__) @@ -382,6 +383,7 @@ def _compute_array_checksum(self, array: np.ndarray) -> str: logger.warning(f"[H5ArrayStore] Failed to compute checksum: {e}") return "" + @traced("arraystore", "arraystore._create_backup") def _create_backup(self) -> Optional[Path]: """Create backup of array file before write.""" if not self._path.exists(): @@ -395,6 +397,7 @@ def _create_backup(self) -> Optional[Path]: logger.warning(f"[H5ArrayStore] Failed to create backup: {e}") return None + @traced("arraystore", "arraystore._restore_backup") def _restore_backup(self, backup_path: Path) -> bool: """Restore array file from backup on write failure.""" try: @@ -425,6 +428,7 @@ def _parse_path_reference(self, path_ref: str) -> Tuple[int, str]: key_name = parts[1] return sample_id, key_name + @traced("arraystore", "arraystore.save_array") def save_array( self, sample_id: str, @@ -514,6 +518,53 @@ def save_array( finally: self._rw_lock.release_write() + def _try_inplace_batch(self, prepared): + """Overwrite existing datasets in place; None means "cannot, fall back". + + Two passes under the write lock: check every destination exists with a + matching shape and dtype, and only then write. A partial in-place write + followed by a fallback would corrupt silently, so nothing is written + until the whole batch is known to fit. + """ + if not self._path.exists(): + return None + with self._local_lock: + self._rw_lock.acquire_write() + try: + with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, + poll_interval=self._poll_interval): + with h5py.File(str(self._path), 'a') as f: + for group_name, key_data in prepared.items(): + grp = f.get(group_name) + if grp is None: + return None + for key_name, (array, _meta) in key_data.items(): + kg = grp.get(key_name) + if kg is None or 'data' not in kg: + return None + dset = kg['data'] + if dset.shape != array.shape or dset.dtype != array.dtype: + return None + for group_name, key_data in prepared.items(): + for key_name, (array, metadata) in key_data.items(): + kg = f[group_name][key_name] + kg['data'][...] = array + for mk, mv in metadata.items(): + kg.attrs[mk] = mv + return { + group_name: { + key_name: self._build_path_reference(group_name, key_name) + for key_name in key_data + } + for group_name, key_data in prepared.items() + } + except Exception as exc: + logger.debug(f"[H5ArrayStore] in-place batch fell back: {exc}") + return None + finally: + self._rw_lock.release_write() + + @traced("arraystore", "arraystore.save_arrays_batch") def save_arrays_batch( self, arrays_dict: Dict[int, Dict[str, np.ndarray]], @@ -562,6 +613,15 @@ def save_arrays_batch( if not prepared: return {} + # O(change): if every array already exists with the same shape and dtype, + # overwrite the values in place. That is not a structural change, so it + # needs neither the temp file nor the full-file backup (9.5GB per flush + # at current ledger size). Returns None if anything would need creating + # or resizing, and the original two-phase path below runs unchanged. + inplace_refs = self._try_inplace_batch(prepared) + if inplace_refs is not None: + return inplace_refs + tmp_path = self._path.with_suffix(f".h5.writing_{uuid.uuid4().hex[:8]}") try: with h5py.File(str(tmp_path), 'w') as f_tmp: @@ -650,6 +710,7 @@ def save_arrays_batch( finally: self._rw_lock.release_write() + @traced("arraystore", "arraystore.recover") def recover(self) -> None: """ Recover from a crash during save_arrays_batch. @@ -673,6 +734,7 @@ def recover(self) -> None: if self._restore_backup(backup_path): backup_path.unlink(missing_ok=True) + @traced("arraystore", "arraystore.load_array") def load_array(self, path_ref: str) -> Optional[np.ndarray]: """ Load array from path reference with LRU cache. @@ -741,6 +803,7 @@ def load_array(self, path_ref: str) -> Optional[np.ndarray]: finally: self._rw_lock.release_read() + @traced("arraystore", "arraystore.load_arrays_batch") def load_arrays_batch(self, path_refs: Dict[int, Dict[str, str]]) -> Dict[int, Dict[str, np.ndarray]]: """ Load multiple arrays in batch. @@ -805,6 +868,7 @@ def load_arrays_batch(self, path_refs: Dict[int, Dict[str, str]]) -> Dict[int, D finally: self._rw_lock.release_read() + @traced("arraystore", "arraystore.delete_sample") def delete_sample(self, sample_id: int) -> bool: """ Delete all arrays for a given sample_id. diff --git a/weightslab/data/h5_dataframe_store.py b/weightslab/data/h5_dataframe_store.py index 3ce51fb6..12d04dbd 100644 --- a/weightslab/data/h5_dataframe_store.py +++ b/weightslab/data/h5_dataframe_store.py @@ -14,6 +14,7 @@ from typing import Iterable, Optional, Union from weightslab.data.sample_stats import SampleStats +from weightslab.backend.optrace import traced, hit logger = logging.getLogger(__name__) # Initialize logger @@ -196,6 +197,7 @@ def _extract_tag_columns(self, df: pd.DataFrame) -> dict: # ------------------------------------------------------------------ # Categorical tag registry persistence # ------------------------------------------------------------------ + @traced("dataframe", "h5store.save_tag_registry") def save_tag_registry(self, registry: dict) -> None: """Persist the categorical tag registry ({tag_name: [categories]}) to H5. @@ -229,6 +231,7 @@ def save_tag_registry(self, registry: dict) -> None: else: time.sleep(self._poll_interval * attempt) + @traced("dataframe", "h5store.load_tag_registry") def load_tag_registry(self) -> dict: """Load the categorical tag registry from H5 into memory and return it.""" if not self._path.exists(): @@ -542,6 +545,7 @@ def _verify_checksum(self, store: pd.HDFStore, key: str, expected_checksum: str) logger.warning(f"[H5DataFrameStore] Failed to verify checksum: {e}") return False + @traced("dataframe", "h5store._create_backup") def _create_backup(self) -> Optional[Path]: """Create backup of H5 file before write. Returns backup path on success.""" if not self._path.exists(): @@ -556,6 +560,7 @@ def _create_backup(self) -> Optional[Path]: logger.warning(f"[H5DataFrameStore] Failed to create backup: {e}") return None + @traced("dataframe", "h5store._restore_backup") def _restore_backup(self, backup_path: Path): """Restore H5 file from backup on write failure.""" try: @@ -570,6 +575,7 @@ def _restore_backup(self, backup_path: Path): # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ + @traced("dataframe", "h5store.load") def load(self, origin: str, columns: Optional[Iterable[str]] = None, start: Optional[int] = None, stop: Optional[int] = None, non_blocking: bool = False) -> pd.DataFrame: """Load data from H5 store. @@ -602,6 +608,7 @@ def load(self, origin: str, columns: Optional[Iterable[str]] = None, start: Opti return self._normalize_for_read(df, origin) + @traced("dataframe", "h5store.load_all") def load_all(self, origins: Iterable[str] = None, columns: Optional[Iterable[str]] = None, non_blocking: bool = False) -> pd.DataFrame: """Load all origins in a single H5 transaction. @@ -680,6 +687,7 @@ def load_all(self, origins: Iterable[str] = None, columns: Optional[Iterable[str return pd.DataFrame() raise + @traced("dataframe", "h5store.ensure_index") def ensure_index(self, origin: str, columns=("sample_id",)) -> bool: """Build the on-disk column index deliberately (checkpoint / first query). @@ -717,10 +725,15 @@ def _posmap(self, store, key, force=False): aids = store.select_column(key, "annotation_id").values except Exception: aids = np.zeros(len(sids), dtype="i8") - m = {} - for i, (sd, ad) in enumerate(zip(sids, aids)): - sd = sd.decode() if isinstance(sd, bytes) else str(sd) - m[(sd, int(ad))] = i + # Hoist the normalisation out of the insert loop: the per-row + # decode/str/int calls interleaved with dict inserts cost 3.6s at 4M + # rows, against ~1.7s for dict(zip(...)) over pre-normalised lists. + # One pass that is also the type check: str hits the identity branch, + # so a mixed-dtype column stays correct without a second full scan. + sid_list = [s if type(s) is str + else (s.decode() if isinstance(s, bytes) else str(s)) + for s in sids.tolist()] + m = dict(zip(zip(sid_list, aids.tolist()), range(len(sid_list)))) self._POSMAP_CACHE[ck] = m return m except Exception as exc: @@ -789,6 +802,7 @@ def _try_inplace(self, store, key, df_norm) -> bool: logger.debug(f"[H5DataFrameStore] in-place update fell back: {exc}") return False + @traced("dataframe", "h5store.upsert") def upsert(self, origin: str, df: pd.DataFrame) -> int: """Atomic upsert with corruption prevention via backup and checksum verification.""" df_norm = self._normalize_for_write(df) @@ -798,8 +812,11 @@ def upsert(self, origin: str, df: pd.DataFrame) -> int: key = self._key(origin) self._ensure_parent() - # Create backup BEFORE any writes - backup_path = self._create_backup() + # The backup is a full copy of the file (696MB at 4M rows). Only the + # read-merge-rewrite path below can destroy the table -- the in-place + # path just overwrites values in already-allocated rows -- so the copy + # is deferred until we know we are taking the destructive route. + backup_path = None with self._local_lock: with _InterProcessFileLock(self._lock_path, timeout=self._lock_timeout, poll_interval=self._poll_interval): @@ -808,8 +825,15 @@ def upsert(self, origin: str, df: pd.DataFrame) -> int: # O(change): if every row already exists and the schema is # unchanged, overwrite values in place (0.1ms vs 42s). if key in store and self._try_inplace(store, key, df_norm): + hit("dataframe", "h5store.upsert", path="inplace", rows=len(df_norm)) return len(df_norm) + # Nothing has been written yet in this call; flush so the + # copy below captures a consistent on-disk file. + hit("dataframe", "h5store.upsert", path="backup_and_rewrite", rows=len(df_norm)) + store.flush() + backup_path = self._create_backup() + existing = pd.DataFrame() # Try to load existing data. A ValueError can surface from a @@ -944,6 +968,7 @@ def get_path(self) -> Path: def exists(self) -> bool: return self._path.exists() + @traced("dataframe", "h5store.delete_column") def delete_column(self, column_name: str, origins: Optional[Iterable[str]] = None) -> bool: """Delete a column from all specified origins (or all origins if None). diff --git a/weightslab/src.py b/weightslab/src.py index 21ae3b5f..adf72840 100644 --- a/weightslab/src.py +++ b/weightslab/src.py @@ -494,6 +494,16 @@ def history(self, signal_name): out = {s: [] for s in self.sample_ids} if self.logger is None: return out + # Read the bounded in-memory tail: O(batch). The full-history query this + # replaced scanned the entire per_sample table on every call (140ms at + # 20M rows, once per step, growing without bound). + if not os.environ.get("WL_HISTORY_FROM_DB"): + recent = self.logger.recent_per_sample(signal_name, self.sample_ids) + for s in self.sample_ids: + vals = recent.get(s) + if vals: + out[s] = list(vals) + return out # query_per_sample accepts a list of ids -> one scan for the whole batch. for sid, step, val, _ in self.logger.query_per_sample(signal_name, sample_ids=self.sample_ids): out.setdefault(int(sid), []).append(val) # rows already ordered by seq (= step order) @@ -1013,13 +1023,27 @@ def wrappered_fwd(original_forward, kwargs, reg_name, *a, **kw): # batch and call the signal once. It returns a length-B # array. Avoids B Python calls + B SignalContext allocs, # and lets the signal do batched ledger reads. + # inputs= must be populated here too: on the + # subscribe_to path the context was built without it, + # so b.inputs was {} and any signal declaring + # inputs=[...] raised KeyError on every call. The + # subscribed signal IS the declared input here, and + # its per-sample values are already in val_vec. + _decl = meta.get('inputs') or [] + _sub = meta.get('subscribe_to') + _vals = [float(v) for v in val_vec] + _bin = {} + for _d in _decl: + if _sub is None or _d == _sub or _d == reg_name: + _bin[_d] = _vals bctx = BatchSignalContext( sample_ids=[int(u) for u in ids_np], - subscribed_values=[float(v) for v in val_vec], + subscribed_values=_vals, logger=_lg, dataframe=df_proxy, origin=kwargs.get('origin', 'train'), step=step, + inputs=_bin, logits=preds_raw.detach() if hasattr(preds_raw, 'detach') else preds_raw, preds=preds.detach() if hasattr(preds, 'detach') else preds, targets=targets.detach() if hasattr(targets, 'detach') else targets, @@ -4880,7 +4904,11 @@ def resolve_signal_classifier(signal_name): return _GLOBAL_CLASSIFIER or classify_loss_shape -def write_signal_shapes(signal_name, tag_name=None, classifier=None): +_SHAPE_LABELS: dict = {} + + +def write_signal_shapes(signal_name, tag_name=None, classifier=None, + only_sample_ids=None): """Reusable engine: classify every sample's trajectory of *signal_name* into a categorical tag and return the ``{label: count}`` distribution. Works for ANY per-sample signal — loss, accuracy, a second loss, any metric. Reads the @@ -4892,17 +4920,46 @@ def write_signal_shapes(signal_name, tag_name=None, classifier=None): clf = classifier or resolve_signal_classifier(signal_name) if tag_name is None: tag_name = signal_name + "_shape" if signal_name.endswith('_loss') else signal_name + "_loss_shape" + + # O(change): with only_sample_ids we read and classify just the trajectories + # that gained a point since the last pass. Labels for everything else are + # carried in _SHAPE_LABELS, so the returned distribution still describes the + # whole dataset. Passing None keeps the original whole-history behaviour + # (what a one-shot end-of-run report wants). + cache = _SHAPE_LABELS.setdefault(signal_name, {}) + ids = None + if only_sample_ids is not None: + ids = [str(s) for s in only_sample_ids] + if not ids: + return {k: v for k, v in _label_counts(cache).items()} + + _lg = get_logger() + rows = (_lg.query_per_sample(signal_name, sample_ids=ids) + if _lg is not None else []) series = {} - for sid, step, val, _ in query_signal_history(signal_name): + for sid, step, val, _ in rows: series.setdefault(sid, []).append((step, val)) + by_label = {} for sid, pts in series.items(): label = clf([v for _, v in sorted(pts)]) - if label is not None: - by_label.setdefault(label, []).append(sid) + if label is None: + continue + if cache.get(sid) == label: + continue # unchanged -> no ledger write needed + cache[sid] = label + by_label.setdefault(label, []).append(sid) + for label, sids in by_label.items(): set_categorical_tag(sids, tag_name, label) - return {k: len(v) for k, v in by_label.items()} + return _label_counts(cache) + + +def _label_counts(cache): + out = {} + for lab in cache.values(): + out[lab] = out.get(lab, 0) + 1 + return out def write_loss_shapes(loss_signal="loss_sample", classifier=None): diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index 81789d42..ccf4e1a1 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -44,8 +44,9 @@ from weightslab.data import media_store from weightslab.trainer.trainer_tools import execute_df_operation, generate_overview, encode_image_to_raw_bytes from weightslab.data.data_utils import load_raw_image_array - +from weightslab.backend.optrace import traced, hit # Image encoding / mask compression / proto helpers (extracted) + from weightslab.trainer.services.data_image_utils import ( rle_encode_mask, create_data_stat, @@ -437,6 +438,18 @@ def rewrite_boolean_keywords_to_bitwise(code: str) -> str: return code +def _histogram_category_cap() -> int: + """Max distinct bars a categorical histogram returns (WL_HIST_CATEGORY_CAP). + + Beyond this the response is neither renderable nor informative -- the + remainder is folded into a single "(other)" bar. + """ + try: + return max(1, int(os.environ.get("WL_HIST_CATEGORY_CAP", "200"))) + except Exception: + return 200 + + def _fast_view_enabled() -> bool: """Differential view refresh. On by default; set WL_FAST_VIEW=0 to opt out.""" return os.environ.get("WL_FAST_VIEW", "1") not in ("0", "false", "False") @@ -513,7 +526,6 @@ def __init__(self, ctx): # In-memory dataframe view of all datasets combined (streamed to UI) self._all_datasets_df = self._pull_into_all_data_view_df() - self._rebuild_view_pos_map() self._load_existing_tags() self._agent = DataManipulationAgent(self) try: @@ -939,7 +951,6 @@ def _get_loader_by_origin(self, origin: str): def _initialize_data_service(self): """Recreate the in-memory dataframe view from the shared H5 store.""" self._all_datasets_df = self._pull_into_all_data_view_df() - self._rebuild_view_pos_map() self._load_existing_tags() def _resolve_root_log_dir(self) -> Path: @@ -1046,6 +1057,23 @@ def _pull_into_all_data_view_df(self): # merge + proxy conversion) a second time over the whole dataset every refresh. df = self._df_manager.get_collapse_annotations_to_samples_df(df) + # The collapse yields object dtype for signals//* columns. They are + # numeric by definition, and an object column sorts ~8x slower + # (6.17s vs 0.76s at 3.96M) while also slowing histogram binning, + # groupby and every differential write. Coerce them back here, at the + # single point the view is materialised. + for _c in df.columns: + if not str(_c).startswith("signals") or df[_c].dtype != object: + continue + try: + _num = pd.to_numeric(df[_c], errors="coerce") + # Only when nothing is lost: a genuinely non-numeric value + # means the column is not what we assume, so leave it be. + if _num.notna().sum() == df[_c].notna().sum(): + df[_c] = _num.astype("float32") + except Exception as _exc: + logger.debug("[DataService] dtype restore skipped for %r: %s", _c, _exc) + # Ensure sample_id is a column if it was the index df = safe_reset_index(df) @@ -1068,7 +1096,9 @@ def _pull_into_all_data_view_df(self): return df except Exception as e: - logger.debug(f"[DataService] Error pulling data view: {e}") + # Was debug: a swallowed failure here silently freezes the view at + # the previous snapshot, which looks exactly like "no new data". + logger.error("[DataService] Error pulling data view: %s", e, exc_info=True) # Use getattr to safely check for attribute during __init__ current_df = getattr(self, "_all_datasets_df", None) return current_df if current_df is not None else pd.DataFrame() @@ -1415,6 +1445,8 @@ def _process_sample_row(self, args): skip_prediction_for_request = metadata_only_request # ====== Step 2: Load dataset lazily (avoid unnecessary IO for metadata-only) ====== + # Views the client explicitly asked for; empty => send everything. + _wanted_stats = set(getattr(request, "stats_to_retrieve", None) or []) needs_dataset = bool(request.include_raw_data) or (not skip_label_for_request) dataset = self._get_dataset(origin) if needs_dataset else None @@ -2088,14 +2120,91 @@ def _json_default(o): target_height=target_height, ) - data_stats.append( - create_data_stat( - name='raw_data', - stat_type='bytes', - thumbnail=raw_data_bytes, - shape=raw_shape, + # An explicit stats_to_retrieve means the client knows which + # views it will draw; raw_data duplicates view rank 0, so + # only send it when actually asked for. + if not _wanted_stats or 'raw_data' in _wanted_stats: + data_stats.append( + create_data_stat( + name='raw_data', + stat_type='bytes', + thumbnail=raw_data_bytes, + shape=raw_shape, + ) ) - ) + + # Paired/multi-view datasets (e.g. a source+edited image + # pair) can optionally expose additional named views via + # extra_images() -- send each as its own 'image_' + # stat so the frontend renders it as its own grid column + # (isImageFieldName() already recognizes 'image_*'). This + # duck-typed hook is a no-op for datasets that don't + # define it. raw_data above already covers view rank 0 + # (e.g. 'source'); extra_images() may repeat that view + # under its own name too -- one small duplicated + # thumbnail, traded for not having to assume which named + # view is redundant across arbitrary datasets. + # Probe the UNWRAPPED dataset: `dataset` is WL's tracking + # wrapper and does not forward extra_images, so testing it + # silently disables every named view. + if hasattr(ds, "extra_images"): + try: + extra_views = ds.extra_images(ds_idx) or {} + except Exception as e: + extra_views = {} + logger.debug(f"extra_images failed for sample_id={sample_id}: {e}") + for view_name, view_pil in extra_views.items(): + if view_pil is None: + continue + # Filter BEFORE resize/encode -- that is the cost. + # Still ADVERTISE the view with an empty thumbnail: + # the panel builds its modality list from the stats + # present, so omitting it entirely would delete the + # toggle and make the view unrecoverable. + if (_wanted_stats + and ("image_%s" % view_name) not in _wanted_stats): + data_stats.append( + create_data_stat( + name="image_%s" % view_name, + stat_type='bytes', + thumbnail=b"", + shape=[], + ) + ) + continue + try: + resized_view = view_pil + if resized_view.size != (target_width, target_height): + _view_resample = ( + Image.Resampling.LANCZOS if is_full_resolution + else Image.Resampling.BILINEAR + ) + resized_view = resized_view.resize( + (target_width, target_height), _view_resample + ) + view_bytes, view_shape, _ = encode_image_to_raw_bytes( + np_img=None, + middle_pil=resized_view, + original_shape=[], + is_volumetric=False, + is_full_resolution=is_full_resolution, + target_width=target_width, + target_height=target_height, + ) + data_stats.append( + create_data_stat( + name=f"image_{view_name}", + stat_type='bytes', + thumbnail=view_bytes, + shape=view_shape, + ) + ) + del view_bytes, resized_view + except Exception as e: + logger.debug( + f"extra_images encode failed for sample_id={sample_id} " + f"view={view_name}: {e}" + ) # For video samples the bytes above are only the poster # frame, so advertise the clip's shape here. This lets the @@ -2212,6 +2321,7 @@ def _get_categorical_tag_defs(self) -> List["pb2.CategoricalTagDef"]: logger.debug(f"Error building categorical tag defs: {e}") return defs + @traced("dataservice", "sort.build_response") def _build_success_response( self, df, @@ -2260,6 +2370,7 @@ def _build_success_response( analysis_result=analysis_result ) + @traced("dataservice", "sort.parse_query") def _parse_direct_query(self, query: str) -> list: """ Parse a simple direct query string into operations list. @@ -2388,6 +2499,7 @@ def _sort_includes_sample_id(self, by) -> bool: by_list = [by] if isinstance(by, str) else list(by or []) return SampleStatsEx.SAMPLE_ID.value in by_list + @traced("dataservice", "sort.numeric_coerce") def _sample_id_sortable_series(self, values): """Return numeric values for sorting when all sample_ids are integer-like, else string values.""" numeric = pd.to_numeric(values, errors="coerce") @@ -2400,16 +2512,51 @@ def _sample_id_sortable_series(self, values): return numeric return values.astype(str) + @traced("dataservice", "sort.detect_numeric_cols") + def _numeric_like_sort_cols(self, df: pd.DataFrame, by) -> set: + """Sort columns whose values are strings but mean numbers. + + '1916469' < '191647' lexicographically but not numerically, so sorting a + numeric-valued string column by raw string order is simply wrong. Only + object/string columns are candidates; genuinely numeric dtypes already + sort correctly and must not be touched. + """ + out = set() + by_list = [by] if isinstance(by, str) else list(by or []) + for col in by_list: + if col == SampleStatsEx.SAMPLE_ID.value: + out.add(col) + continue + try: + s = df[col] if col in df.columns else None + if s is None or pd.api.types.is_numeric_dtype(s) or hasattr(s, "cat"): + continue + probe = s.dropna() + if probe.empty: + continue + # Cheap decision on a sample: a full 4M-row coercion here would + # cost more than the sort it is meant to correct. + head = probe.head(2048) + coerced = pd.to_numeric(head, errors="coerce") + if coerced.notna().all(): + out.add(col) + except Exception: + continue + return out + + @traced("dataservice", "sort.sort_values") def _sort_values_numeric_aware(self, df: pd.DataFrame, sort_params: dict) -> None: - """Sort dataframe while treating sample_id as numeric when possible.""" + """Sort dataframe, ordering numeric-valued string columns numerically.""" params = dict(sort_params) - if params.get("key") is None and self._sort_includes_sample_id(params.get("by")): - def _key(series: pd.Series): - if str(getattr(series, "name", "")) == SampleStatsEx.SAMPLE_ID.value: - return self._sample_id_sortable_series(series) - return series + if params.get("key") is None: + numeric_like = self._numeric_like_sort_cols(df, params.get("by")) + if numeric_like: + def _key(series: pd.Series): + if str(getattr(series, "name", "")) in numeric_like: + return self._sample_id_sortable_series(series) + return series - params["key"] = _key + params["key"] = _key df.sort_values(inplace=True, **params) @@ -3101,6 +3248,7 @@ def _mask_from_coerced_query(df, expr: str): return None return np.asarray(mask, dtype=bool) + @traced("dataservice", "sort.apply_operation") def _apply_agent_operation(self, df, func: str, params: dict) -> str: """ Apply an agent-described operation to df in-place. @@ -3456,6 +3604,38 @@ def _apply_agent_operation(self, df, func: str, params: dict) -> str: # silently stops refreshing). orig_index_names = [n for n in df.index.names if n is not None] + # Fast path: nothing in `by` is an index level, so the frame can be + # sorted where it stands. Avoids reset_index + astype(int)/astype(str) + # over every sample_id + a set_index that re-factorizes 3.96M string + # keys -- measured as the bulk of a ~20s sort. + _fp_by = params.get("by") + _fp_list = [_fp_by] if isinstance(_fp_by, str) else list(_fp_by or []) + _fp_res = [ + (c if c in df.columns + else ("signals//" + c if ("signals//" + c) in df.columns else c)) + for c in _fp_list + ] + if (_fp_res + and all(c in df.columns for c in _fp_res) + and not any(c in orig_index_names for c in _fp_list) + and not any(c in orig_index_names for c in _fp_res)): + _fp_params = dict(params) + _fp_params["by"] = (_fp_res if isinstance(_fp_by, (list, tuple)) + else _fp_res[0]) + try: + # Through the helper, NOT df.sort_values directly: a + # numeric-valued string column (group_id, target, ...) + # otherwise sorts lexicographically -- '1916469' before + # '191647'. + self._sort_values_numeric_aware(df, _fp_params) + return "Applied operation: sort_values" + except (TypeError, ValueError, KeyError) as _fp_exc: + # Mixed dtypes or an unexpected key: fall through to the + # original reset/restore path rather than failing the query. + logger.debug( + "[sort] fast path declined (%s); using index round-trip", + type(_fp_exc).__name__) + def _restore_index(): cols = [n for n in orig_index_names if n in df.columns] if cols and not isinstance(df.index, pd.MultiIndex): @@ -3734,41 +3914,9 @@ def _fast_sync_columns(self, view): return [c for c in view.columns if str(c).startswith(self._FAST_SYNC_PREFIXES)] - def _compute_view_pos_map(self, view): - """sample_id -> positional row index for *view*. Pure: builds and returns - the map so callers can do it OFF-lock (it is O(rows): ~5s at 4M).""" - if not _fast_view_enabled(): - return {} - try: - if view is None or view.empty: - return {} - SID = SampleStatsEx.SAMPLE_ID.value - keys = (view.index.get_level_values(SID) - if isinstance(view.index, pd.MultiIndex) and SID in (view.index.names or []) - else view.index) - return {str(k): i for i, k in enumerate(keys)} - except Exception: - return {} - def _rebuild_view_pos_map(self): - """sample_id -> positional row index, rebuilt with the view so the - differential path does O(1) lookups instead of label alignment.""" - if not _fast_view_enabled(): - self._view_pos_map = {} - return # opt-out: skip the map build entirely - try: - view = self._all_datasets_df - if view is None or view.empty: - self._view_pos_map = {} - return - SID = SampleStatsEx.SAMPLE_ID.value - keys = (view.index.get_level_values(SID) - if isinstance(view.index, pd.MultiIndex) and SID in (view.index.names or []) - else view.index) - self._view_pos_map = {str(k): i for i, k in enumerate(keys)} - except Exception: - self._view_pos_map = {} + @traced("dataservice", "dsvc._fastUpdateInternals") def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: """O(change) view refresh. True if applied, False -> caller must rebuild. @@ -3777,53 +3925,87 @@ def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: enough that a rebuild is cheaper. """ if not _fast_view_enabled(): + hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", reason="opt_out") return False # opt-out: behave exactly as before view = self._all_datasets_df dfm = self._df_manager if view is None or getattr(view, "empty", True) or dfm is None: + hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", reason="no_view") return False - pos_map = getattr(self, "_view_pos_map", None) - if not pos_map: - return False + # A column the ledger has but the view lacks can only arrive via a full + # rebuild -- the differential write below addresses existing columns + # only. Per-sample signal columns are created on their first write, so + # on a fresh ledger the view predates them and would never gain them. + try: + _src = getattr(dfm, "_df", None) + if _src is not None: + _have = set(view.columns) + _missing = [c for c in _src.columns + if str(c).startswith(self._FAST_SYNC_PREFIXES) + and c not in _have] + if _missing: + hit("dataservice", "dsvc._fastUpdateInternals", + outcome="fallback", reason="schema_gain", n_missing=len(_missing)) + return False + except Exception: + pass dirty = dfm.take_view_dirty(limit=max_dirty) if dirty is None: + hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", reason="backlog_too_large") return False # backlog too large; rebuild is cheaper if not dirty: + hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="no_change") return True # nothing changed since last sync sids = [str(s) for s in dirty] - positions, keep = [], [] - for s in sids: - p = pos_map.get(s) - if p is None: - return False # unknown row => structural change - positions.append(p); keep.append(s) + # Address rows by LABEL. pandas keeps a hash engine on the index, built + # in C and cached, so this needs no precomputed position map -- and a + # label that is absent surfaces below as a no-match rather than as a + # silently wrong row. + keep = sids cols = self._fast_sync_columns(view) if not cols: + hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="no_sync_cols") return True sub = dfm.get_source_rows(keep, columns=[c for c in cols if c in view.columns]) if sub is None or sub.empty: + hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="empty_source_rows", + n_dirty=len(sids)) return True if isinstance(sub.index, pd.MultiIndex): sub = sub.droplevel(-1) sub = sub[~sub.index.duplicated(keep="last")] - order = {str(k): i for i, k in enumerate(sub.index)} - rows, vals_idx = [], [] - for s, p in zip(keep, positions): - j = order.get(s) - if j is not None: - rows.append(p); vals_idx.append(j) - if not rows: + # Only rows the view actually holds; a structural change (new sample) + # must still fall back to the full rebuild rather than be invented here. + SID = SampleStatsEx.SAMPLE_ID.value + _names = list(getattr(view.index, "names", []) or []) + view_keys = (view.index.get_level_values(SID) + if isinstance(view.index, pd.MultiIndex) and SID in _names + else view.index) + # Positions via the Index hash engine: vectorised and cached, so this + # costs nothing like the O(rows) dict the position map used to rebuild. + _pos = pd.Index(view_keys.astype(str)).get_indexer(sub.index.astype(str)) + _ok = _pos >= 0 + if not _ok.any(): + hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="no_row_match", + n_dirty=len(sids), n_sub=len(sub.index)) return True - rows = np.asarray(rows); vals_idx = np.asarray(vals_idx) + if not _ok.all(): + hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", + reason="unknown_row", n_dirty=len(sids)) + return False for c in sub.columns: - ci = view.columns.get_loc(c) - view.iloc[rows, ci] = sub[c].to_numpy()[vals_idx] + _ci = view.columns.get_loc(c) + view.iloc[_pos, _ci] = sub[c].to_numpy() + hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="patched", + n_dirty=len(sids), n_sub=len(sub.index), n_pos=int(_ok.sum()), + n_rows=int(_ok.sum()), n_cols=len(sub.columns)) return True + @traced("dataservice", "dsvc._slowUpdateInternals") def _slowUpdateInternals(self, force: bool = False, reset_view: bool = False) -> None: """Update the internal dataframe view with the latest data from the manager. @@ -3993,8 +4175,14 @@ def _slowUpdateInternals(self, force: bool = False, reset_view: bool = False) -> # Atomic swap to make the new view available to readers self._all_datasets_df = updated_df - self._rebuild_view_pos_map() self._last_internals_update_time = current_time + # The rebuilt view reflects every row, so the differential backlog is + # satisfied. This is the only point at which that is true. + try: + if self._df_manager is not None: + self._df_manager.clear_view_dirty() + except Exception: + pass finally: held_ms = (time.time() - t_held_start) * 1000 @@ -4082,6 +4270,7 @@ def _signal_trajectory_curves(self, signal_name, sample_ids, max_points=None): resolved, len(sample_ids), len(curves)) return resolved, curves + @traced("dataservice", "dsvc._build_metadata_only_response") def _build_metadata_only_response(self, df_slice: pd.DataFrame, requested_cols=None): """Build a DataSamplesResponse of metadata DataRecords from dataframe columns only. @@ -4274,6 +4463,7 @@ def _get_all_metadata_column_names(self) -> list: logger.warning("Error enumerating metadata column names: %s", e) return [] + @traced("dataservice", "dsvc.GetMetaData") def GetMetaData(self, request, context): """Metadata-only retrieval, separated from GetDataSamples. @@ -4353,6 +4543,7 @@ def GetMetaData(self, request, context): grid_records=[], ) + @traced("dataservice", "dsvc.GetSignalTrajectory") def GetSignalTrajectory(self, request, context): """On-demand per-sample trajectory of one signal, for the samples shown. @@ -4461,6 +4652,7 @@ def _merge_multi_instance_signals(self, df_slice): merged_df = pd.DataFrame(merged_rows).reset_index(drop=True) return merged_df, signal_dict_mapping + @traced("dataservice", "dsvc._process_get_data_samples") def _process_get_data_samples(self, request, context): """ Actual implementation of GetDataSamples. @@ -4773,6 +4965,7 @@ def _parse_tags(self, tag_value: str) -> set: # RPC Implementations # =================== + @traced("dataservice", "dsvc.ApplyDataQuery") def ApplyDataQuery(self, request, context): """ Apply a query on the in-memory dataframe. @@ -4830,14 +5023,11 @@ def _run_ops(target): base = self._all_datasets_df df = base.copy(deep=False) if base is not None else base final_message = _run_ops(df) - # Row ORDER changed, so sample_id -> position is stale; without a - # rebuild the differential refresh writes signals to wrong rows. - # Build it OFF-lock -- doing it inside the swap held the lock for - # 5320ms at 4M rows. - new_pos_map = self._compute_view_pos_map(df) + # No position map to rebuild: the differential refresh addresses + # rows by label through pandas' own index engine, so reordering + # the view invalidates nothing. with self._watched_lock("_lock[ApplyDataQuery/swap]"): self._all_datasets_df = df - self._view_pos_map = new_pos_map if operations: self._is_filtered = True else: @@ -4846,7 +5036,6 @@ def _run_ops(target): df = self._all_datasets_df final_message = _run_ops(df) self._all_datasets_df = df - self._rebuild_view_pos_map() if operations: self._is_filtered = True @@ -5011,6 +5200,7 @@ def status_cb(msg: str): message=f"Failed to apply query: {str(e)}", ) + @traced("dataservice", "dsvc.GetDataSamples") def GetDataSamples(self, request, context): """ Retrieve samples from the dataframe with their data statistics. @@ -5029,6 +5219,7 @@ def GetDataSamples(self, request, context): data_records=[] ) + @traced("dataservice", "dsvc.GetHistogram") def GetHistogram(self, request, context): """Server-side histogram binning of one column (typed RPC). @@ -5048,17 +5239,37 @@ def GetHistogram(self, request, context): if df is None or df.empty: return pb2.HistogramResponse( success=False, message="empty dataframe view", total_rows=0, bins=[]) - df = safe_reset_index(df) + # safe_reset_index copies AND consolidates the entire frame (~70% of + # this RPC at 3.96M x 19 by py-spy). It is only needed to reach fields + # that live in the index; when they are already columns, use the frame + # as it stands. + def _field(frame, name): + """Series for *name* whether it is a column or an index level.""" + if name in frame.columns: + return frame[name] + names = list(getattr(frame.index, "names", []) or []) + if name in names: + return pd.Series(frame.index.get_level_values(name), + index=frame.index) + if getattr(frame.index, "name", None) == name: + return pd.Series(frame.index, index=frame.index) + return None + + # Only reset when the histogrammed column itself cannot be reached. + # safe_reset_index copies AND block-consolidates the whole frame + # (~82% of this RPC by py-spy); get_level_values is ~0.02s. + if _field(df, column) is None: + df = safe_reset_index(df) n = len(df) if column not in df.columns: return pb2.HistogramResponse( success=False, message=f"column '{column}' not in view", total_rows=n, bins=[]) - origin = (df["origin"].astype(str).to_numpy() if "origin" in df.columns - else np.full(n, "")) - disc = (df["discarded"].astype(bool).to_numpy() if "discarded" in df.columns - else np.zeros(n, bool)) + _o = _field(df, "origin") + origin = _o.astype(str).to_numpy() if _o is not None else np.full(n, "") + _d = _field(df, "discarded") + disc = _d.astype(bool).to_numpy() if _d is not None else np.zeros(n, bool) # Detect whether column is categorical (string/object) or numeric. # A column is numeric if ANY value coerces to a finite number — even @@ -5069,7 +5280,9 @@ def GetHistogram(self, request, context): # as a spurious "unset" bar). We therefore treat as categorical only # a genuine pandas ``category`` dtype, or a column whose values do # not coerce to any numeric value at all (pure strings). - col_series = df[column] + col_series = _field(df, column) + if col_series is None: + col_series = df[column] numeric_vals = pd.to_numeric(col_series, errors="coerce") is_category_dtype = ( str(col_series.dtype) == "category" or hasattr(col_series, "cat") @@ -5088,20 +5301,44 @@ def GetHistogram(self, request, context): if is_categorical: # --- Categorical path --- labels = col_series.astype(str).where(col_series.notna(), "") - gf = pd.DataFrame({"l": labels, "o": origin, "d": disc}) - total_count = gf.groupby("l")["l"].count().rename("count") + # Count first with a single-key value_counts, then restrict the + # three-key breakdown to the rows that survive the cap. Grouping + # all 3.96M rows by (label, discarded, origin) when the column is + # free text builds 722,870 groups to then discard all but 200. + total_count = labels.value_counts().rename("count") + _cap_pre = _histogram_category_cap() + _keep = set(total_count.iloc[:_cap_pre].index) + _m = labels.isin(_keep).to_numpy() sub_map: dict = {} - for (lbl, d, o), c in gf.groupby(["l", "d", "o"]).size().items(): - sub_map.setdefault(str(lbl), []).append( - pb2.HistogramSubBar(origin=str(o), discarded=bool(d), count=int(c))) + if _m.any(): + gf = pd.DataFrame({"l": labels.to_numpy()[_m], + "o": origin[_m], "d": disc[_m]}) + for (lbl, d, o), c in gf.groupby(["l", "d", "o"]).size().items(): + sub_map.setdefault(str(lbl), []).append( + pb2.HistogramSubBar(origin=str(o), discarded=bool(d), count=int(c))) + # Cap the output: a free-text column (e.g. edit_prompt) has one + # category per sample -- 722,870 bars / 54 MB / 30.5s measured, + # which no viewer can draw. Keep the top-N by count and fold the + # remainder into one "(other)" bar so the response stays bounded. + _ordered = total_count.sort_values(ascending=False) + _cap = _histogram_category_cap() + _head, _tail = _ordered.iloc[:_cap], _ordered.iloc[_cap:] cat_bars = [ pb2.CategoricalHistogramBar( label=str(lbl), count=int(cnt), sub_bars=sub_map.get(str(lbl), []), ) - for lbl, cnt in total_count.sort_values(ascending=False).items() + for lbl, cnt in _head.items() ] + if len(_tail): + cat_bars.append(pb2.CategoricalHistogramBar( + label="(other: %d categories)" % len(_tail), + count=int(_tail.sum()), + sub_bars=[], + )) + logger.info("[HistCat] column=%s capped %d categories -> %d bars", + column, len(_ordered), len(cat_bars)) logger.info("[HistCat] column=%s rows=%d categories=%d", column, n, len(cat_bars)) return pb2.HistogramResponse( @@ -5355,6 +5592,7 @@ def _media_cache_put(self, key, value) -> None: while len(self._media_cache) > self._MEDIA_CACHE_ENTRIES: self._media_cache.pop(next(iter(self._media_cache))) + @traced("dataservice", "dsvc.GetPointCloud") def GetPointCloud(self, request, context): """Stream one sample's raw point cloud as binary float32 chunks. @@ -5530,6 +5768,7 @@ def _manual_save_data_state(self, force_enable_h5: bool = False): message="Data state saved to H5 (JSON snapshot not available).", ) + @traced("dataservice", "dsvc.EditDataSample") def EditDataSample(self, request, context): """ Edit sample metadata (tags and discarded). @@ -5913,6 +6152,7 @@ def EditDataSample(self, request, context): message=f"Failed to edit samples: {str(e)}", ) + @traced("dataservice", "dsvc.GetDataSplits") def GetDataSplits(self, request, context): """ Return the list of available dataset splits (train, test, val, etc.) diff --git a/weightslab/trainer/services/experiment_service.py b/weightslab/trainer/services/experiment_service.py index b3113b2f..ea6d2e87 100644 --- a/weightslab/trainer/services/experiment_service.py +++ b/weightslab/trainer/services/experiment_service.py @@ -20,6 +20,7 @@ from weightslab.trainer.services.notebook_service import NotebookService from weightslab.data.sample_stats import SampleStatsEx from weightslab.components.evaluation_controller import eval_controller +from weightslab.backend.optrace import traced # Logger @@ -225,6 +226,7 @@ def _kick_eval_worker(self) -> None: # ------------------------------------------------------------------------- # Logger queue sync for WeightsStudio # ------------------------------------------------------------------------- + @traced("experiment", "expsvc.GetLatestLoggerData") def GetLatestLoggerData(self, request, context): """ Returns logger data for WeightsStudio polling. @@ -480,6 +482,7 @@ def _get_latest_logger_data_impl(self, request, context): return pb2.GetLatestLoggerDataResponse(points=points) + @traced("experiment", "expsvc.RestoreCheckpoint") def RestoreCheckpoint(self, request, context): """ Restore a checkpoint from a given experiment hash. @@ -865,6 +868,7 @@ def _delayed_exit(): # Training & hyperparameter commands # ------------------------------------------------------------------------- + @traced("experiment", "expsvc.ExperimentCommand") def ExperimentCommand(self, request, context): if request.HasField("restart_operation"): return self._handle_restart_instance() diff --git a/weightslab/utils/logs.py b/weightslab/utils/logs.py index dd75872f..717165b4 100644 --- a/weightslab/utils/logs.py +++ b/weightslab/utils/logs.py @@ -9,7 +9,7 @@ # Define the log format to include timestamp, level, module name, and function name -FORMAT = '%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(funcName)s: %(message)s' +FORMAT = '%(asctime)s.%(msecs)03d %(levelname)s:%(name)s:%(filename)s:%(lineno)d:%(funcName)s: %(message)s' DATE_FORMAT = '%d/%m/%Y-%H:%M:%S' # Global variables to track the log file path and handler From 9466dc13f42545dd568bbc5b727305c1e6ecbd74 Mon Sep 17 00:00:00 2001 From: Alexandru Rotaru Date: Tue, 18 Aug 2026 14:04:38 +0000 Subject: [PATCH 05/13] fix(histogram): bin over rows that carry a value, not every row in the view The numeric path cut bin boundaries by row position across the WHOLE view and dropped non-finite values only afterwards, so every bucket spanned len(view)/max_bins rows regardless of the data. On a 3,963,189-row view with 512 bins that is 7,740 rows per bucket -- so a column where only ~33k samples carry a value (any signal early in a run) collapsed into the first four buckets, and the remaining 500 sliced empty space into 1-6 sample slivers. Visible as four fat bars followed by a long tail of random-looking spikes, with the same 7740/7741 counts appearing on unrelated columns because the number came from the row count, not the data. These bars are a search surface over the loss landscape: each should be a click-target holding a comparable number of samples. Mask first, then cut equal-population boundaries over the finite subset. Row ORDER is untouched, so this is still "bin the current view by row order" -- it just stops counting rows that have nothing to show. Also fixes the per-(origin, discarded) sub-bars, which were grouped by the same positional bins. Co-Authored-By: Claude Opus 5 --- weightslab/trainer/services/data_service.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index ccf4e1a1..fcad5c0f 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -5351,12 +5351,25 @@ def _field(frame, name): ) # --- Numeric path (unchanged) --- - bars = max(1, min(n, max_bins)) vals = numeric_vals.to_numpy() - edges = (np.arange(bars + 1) * n) // bars - bin_of_row = np.searchsorted(edges, np.arange(n), side="right") - 1 + # Mask BEFORE choosing boundaries: bins are a search surface, so each + # one should hold a comparable number of SAMPLES THAT HAVE A VALUE. + # Cutting by position across the whole view instead made every bucket + # span len(view)/max_bins rows, so a sparsely-populated column landed + # entirely in the first few buckets. fin = np.isfinite(vals) - gf = pd.DataFrame({"b": bin_of_row[fin], "v": vals[fin], + n_fin = int(fin.sum()) + if n_fin == 0: + return pb2.HistogramResponse( + success=True, + message=f"histogram {column}: no rows carry a value", + total_rows=n, bins=[], is_categorical=False, categorical_bars=[]) + bars = max(1, min(n_fin, max_bins)) + # Positions WITHIN the finite subset; vals[fin] keeps the view's row + # order, so equal-population still means equal-population by order. + edges = (np.arange(bars + 1) * n_fin) // bars + bin_of_row = np.searchsorted(edges, np.arange(n_fin), side="right") - 1 + gf = pd.DataFrame({"b": bin_of_row, "v": vals[fin], "o": origin[fin], "d": disc[fin]}) stats = gf.groupby("b")["v"].agg(["min", "max", "mean", "count"]) sub_by_bin = {} From 2c37d71431781ef8467dc50f5a1ab2ab6362f2c3 Mon Sep 17 00:00:00 2001 From: Alexandru Rotaru Date: Tue, 18 Aug 2026 21:02:58 +0000 Subject: [PATCH 06/13] fix(ledger): make the NB_SEEN lookup O(batch), and regenerate the protos Three things, all needed to make the branch runnable on top of dev at 4M rows. 1. Regenerated experiment_service_pb2{,_grpc}.py dev's .proto declares AnnotationExportFormat / EXPORT_FORMAT_CVAT but the committed gencode predates it, so a clean checkout of dev does not import at all: AttributeError: module 'weightslab.proto.experiment_service_pb2' has no attribute 'EXPORT_FORMAT_CVAT' Regenerated with grpcio-tools 1.68.1 (protoc 5.28.1), matching the runtime version already pinned in the file, so the gencode major does not move. 2. Cache the level-0 index for sample-id coercion _coerce_sample_id_for_index() called index.get_level_values(0) on every invocation. That materialises a fresh Index over all rows, and a fresh Index carries a fresh hash engine, so each `sid in level_0_values` paid a full engine build -- twice per sample when the int probe missed. enqueue_batch does that per sample, 24x a step: training sat at 0 iterations with the main thread pinned at 100% CPU inside pandas __contains__ (py-spy: active+gil). Cached on the index object's identity. pandas Index is immutable, so any reindex or rebuild yields a new object and invalidates it; membership semantics are unchanged, the engine is simply reused. 3. Positional NB_SEEN lookup get_sample_column_values() then still materialised both index levels, ran isin over every row and copied a boolean-masked frame -- a full pass plus a copy over 3.96M rows to read 24 integers, ~1.2s/step (signals 6ms -> 1230ms, total 1290ms -> 2500ms). The wanted rows are exactly (sample_id, 0), so resolve their positions with Index.get_indexer instead. Falls back to the original scan when the index is not unique. Measured on the UltraEdit harness (859M params, batch 24, A10G, 3.96M samples): signals 1230ms -> 28-41ms total 2500ms -> 1310-1338ms (1171ms with weightslab stubbed out) NB_SEEN now actually increments (it was stuck at 0 before dev's fix), verified against the ledger: rows with nb_seen>0 equals rows with last_seen>=0. The UI contract suite passes 21/21 on this build. Co-Authored-By: Claude Opus 5 --- weightslab/data/dataframe_manager.py | 68 ++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 15 deletions(-) diff --git a/weightslab/data/dataframe_manager.py b/weightslab/data/dataframe_manager.py index d28b0e83..a7db2ade 100644 --- a/weightslab/data/dataframe_manager.py +++ b/weightslab/data/dataframe_manager.py @@ -382,6 +382,22 @@ def _normalize_sample_id(self, sample_id: Any) -> Any: return str(sample_id) + def _level0_index(self): + """Level-0 (sample_id) values of the ledger index, cached. + + Keyed on the index object's identity: pandas Index is immutable, so a + reindex or rebuild yields a new object and invalidates this. Reusing the + object also reuses its hash engine, which is what makes a membership + probe O(1) instead of O(rows). + """ + idx = self._df.index + key = id(idx) + if getattr(self, "_lvl0_key", None) != key: + self._lvl0_key = key + self._lvl0 = (idx.get_level_values(0) + if isinstance(idx, pd.MultiIndex) else idx) + return self._lvl0 + def _coerce_sample_id_for_index(self, sample_id: Any) -> Any: """Coerce sample_id to match current dataframe index representation. @@ -394,8 +410,10 @@ def _coerce_sample_id_for_index(self, sample_id: Any) -> Any: # Check if multi-index if isinstance(self._df.index, pd.MultiIndex): - # Get level 0 (sample_id level) values - level_0_values = self._df.index.get_level_values(0) + # Cached: get_level_values(0) built a new Index over every row on each + # call, and a new object means a new hash engine, so this probe was + # O(rows) per sample. + level_0_values = self._level0_index() if sid in level_0_values: return sid sid_str = str(sid) @@ -1621,22 +1639,42 @@ def get_sample_column_values(self, sample_ids: List[Any], column: str) -> Dict[A coerced_ids = [self._coerce_sample_id_for_index(sid) for sid in sample_ids] + idx = self._df.index try: - if isinstance(self._df.index, pd.MultiIndex) and self._df.index.nlevels >= 2: - sample_level = self._df.index.get_level_values(0) - anno_level = self._df.index.get_level_values(1) - mask = sample_level.isin(coerced_ids) & (anno_level == 0) - slice_df = self._df[mask] - sids = slice_df.index.get_level_values(0) + # _positional NB_SEEN lookup: the wanted rows are exactly + # (sample_id, 0), so resolve their POSITIONS instead of scanning. + # The masked form below materialised both index levels and copied + # a boolean-masked frame over every row -- ~1.2s/step at 3.96M + # just to read a batch of integers. + if idx.has_duplicates: + raise ValueError("non-unique index; use the scan path") + if isinstance(idx, pd.MultiIndex) and idx.nlevels >= 2: + pos = idx.get_indexer([(cid, 0) for cid in coerced_ids]) else: - mask = self._df.index.isin(coerced_ids) - slice_df = self._df[mask] - sids = slice_df.index - - for sid, val in zip(sids, slice_df[column]): - values[self._normalize_sample_id(sid)] = val + pos = idx.get_indexer(list(coerced_ids)) + col = self._df[column].to_numpy() + for cid, p in zip(coerced_ids, pos): + if p >= 0: + values[self._normalize_sample_id(cid)] = col[p] except Exception: - pass + # Fallback: original scan, for a non-unique index or any dtype + # mismatch get_indexer will not tolerate. + try: + if isinstance(idx, pd.MultiIndex) and idx.nlevels >= 2: + sample_level = idx.get_level_values(0) + anno_level = idx.get_level_values(1) + mask = sample_level.isin(coerced_ids) & (anno_level == 0) + slice_df = self._df[mask] + sids = slice_df.index.get_level_values(0) + else: + mask = idx.isin(coerced_ids) + slice_df = self._df[mask] + sids = slice_df.index + + for sid, val in zip(sids, slice_df[column]): + values[self._normalize_sample_id(sid)] = val + except Exception: + pass return values From 0ef0a1ce31f8fe2122b0fd91c1f241de6349e446 Mon Sep 17 00:00:00 2001 From: Alexandru Rotaru Date: Wed, 19 Aug 2026 13:58:21 +0000 Subject: [PATCH 07/13] fix(data_service): bin the numeric histogram over the whole view again Each bar must cover total_rows / max_bins samples so the chart carries density: a column that is only 0.2% populated should show a few filled bars and the rest empty. Binning over just the rows that carry a value made the chart look equally full at any coverage, which reads as "every sample already has a loss". Co-Authored-By: Claude Opus 5 --- weightslab/trainer/services/data_service.py | 26 +++++++-------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index fcad5c0f..c08e9006 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -5351,25 +5351,17 @@ def _field(frame, name): ) # --- Numeric path (unchanged) --- + # Each bar covers a fixed slice of the VIEW: total_rows / max_bins + # samples. That is what makes the chart show density -- a column only + # 0.2% populated shows a few filled bars and the rest empty. Binning + # over just the rows that carry a value makes the chart look equally + # full at any coverage, which reads as "everything has a value". + bars = max(1, min(n, max_bins)) vals = numeric_vals.to_numpy() - # Mask BEFORE choosing boundaries: bins are a search surface, so each - # one should hold a comparable number of SAMPLES THAT HAVE A VALUE. - # Cutting by position across the whole view instead made every bucket - # span len(view)/max_bins rows, so a sparsely-populated column landed - # entirely in the first few buckets. + edges = (np.arange(bars + 1) * n) // bars + bin_of_row = np.searchsorted(edges, np.arange(n), side="right") - 1 fin = np.isfinite(vals) - n_fin = int(fin.sum()) - if n_fin == 0: - return pb2.HistogramResponse( - success=True, - message=f"histogram {column}: no rows carry a value", - total_rows=n, bins=[], is_categorical=False, categorical_bars=[]) - bars = max(1, min(n_fin, max_bins)) - # Positions WITHIN the finite subset; vals[fin] keeps the view's row - # order, so equal-population still means equal-population by order. - edges = (np.arange(bars + 1) * n_fin) // bars - bin_of_row = np.searchsorted(edges, np.arange(n_fin), side="right") - 1 - gf = pd.DataFrame({"b": bin_of_row, "v": vals[fin], + gf = pd.DataFrame({"b": bin_of_row[fin], "v": vals[fin], "o": origin[fin], "d": disc[fin]}) stats = gf.groupby("b")["v"].agg(["min", "max", "mean", "count"]) sub_by_bin = {} From 53ac28a24360914a272144db4c9a90ebc74ab473 Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 19 Aug 2026 16:41:03 +0000 Subject: [PATCH 08/13] proto: regenerate with package-relative imports after the dev merge dev checks in generated code that does a flat 'import experiment_service_pb2', which only resolves if weightslab/proto is itself on sys.path. Imported as a package -- which is how the trainer loads it -- startup dies with ModuleNotFoundError. Regenerated from the merged .proto at the repo root so dev's new RPCs are kept and the import is package-relative again. Co-Authored-By: Claude Opus 5 --- weightslab/proto/experiment_service_pb2.py | 452 +++++++++--------- .../proto/experiment_service_pb2_grpc.py | 448 ++++++++--------- 2 files changed, 450 insertions(+), 450 deletions(-) diff --git a/weightslab/proto/experiment_service_pb2.py b/weightslab/proto/experiment_service_pb2.py index 09791483..9d92712b 100644 --- a/weightslab/proto/experiment_service_pb2.py +++ b/weightslab/proto/experiment_service_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE -# source: experiment_service.proto +# source: weightslab/proto/experiment_service.proto # Protobuf Python Version: 5.28.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor @@ -15,7 +15,7 @@ 28, 1, '', - 'experiment_service.proto' + 'weightslab/proto/experiment_service.proto' ) # @@protoc_insertion_point(imports) @@ -24,11 +24,11 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x65xperiment_service.proto\"\x81\x02\n\x1aGetLatestLoggerDataRequest\x12\x1c\n\x14request_full_history\x18\x01 \x01(\x08\x12\x12\n\nmax_points\x18\x02 \x01(\x05\x12\x17\n\x0f\x62reak_by_slices\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\x12\x12\n\ngraph_name\x18\x05 \x01(\t\x12\r\n\x05x_min\x18\x06 \x01(\x03\x12\r\n\x05x_max\x18\x07 \x01(\x03\x12\x13\n\x0bhas_x_range\x18\x08 \x01(\x08\x12\x14\n\x0cmetric_names\x18\t \x03(\t\x12\x19\n\x11\x65xperiment_hashes\x18\n \x03(\t\x12\x12\n\nindex_only\x18\x0b \x01(\x08\"|\n\x10SignalCurveIndex\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x12\n\nfirst_step\x18\x03 \x01(\x03\x12\x11\n\tlast_step\x18\x04 \x01(\x03\x12\x13\n\x0bpoint_count\x18\x05 \x01(\x03\"1\n\rSignalOutlier\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"\xd2\x03\n\x0fLoggerDataPoint\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x11\n\tmodel_age\x18\x02 \x01(\x05\x12\x14\n\x0cmetric_value\x18\x03 \x01(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tsample_id\x18\x06 \x01(\t\x12\x1c\n\x14is_evaluation_marker\x18\x07 \x01(\x08\x12\x12\n\nsplit_name\x18\x08 \x01(\t\x12\x17\n\x0f\x65valuation_tags\x18\t \x03(\t\x12\x12\n\npoint_note\x18\n \x01(\t\x12\x12\n\naudit_mode\x18\x0b \x01(\x08\x12 \n\x08outliers\x18\x0c \x03(\x0b\x32\x0e.SignalOutlier\x12\x15\n\routlier_count\x18\r \x01(\x05\x12\x14\n\x0csample_count\x18\x0e \x01(\x05\x12\x13\n\x0btrend_value\x18\x0f \x01(\x02\x12\x14\n\x0ctrend_margin\x18\x10 \x01(\x02\x12\x16\n\x0ehas_trend_band\x18\x11 \x01(\x08\x12\x11\n\tvalue_min\x18\x12 \x01(\x02\x12\x11\n\tvalue_max\x18\x13 \x01(\x02\x12\x17\n\x0fhas_value_range\x18\x14 \x01(\x08\"\x9a\x01\n\x1bGetLatestLoggerDataResponse\x12 \n\x06points\x18\x01 \x03(\x0b\x32\x10.LoggerDataPoint\x12\x1a\n\x12weightslab_version\x18\x02 \x01(\t\x12!\n\x06\x63urves\x18\x03 \x03(\x0b\x32\x11.SignalCurveIndex\x12\x1a\n\x12\x61pplied_max_points\x18\x04 \x01(\x05\"\x07\n\x05\x45mpty\"/\n\x08NeuronId\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tneuron_id\x18\x02 \x01(\x05\"\x91\x02\n\x0fWeightOperation\x12*\n\x07op_type\x18\x01 \x01(\x0e\x32\x14.WeightOperationTypeH\x00\x88\x01\x01\x12\x15\n\x08layer_id\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\nneuron_ids\x18\x03 \x03(\x0b\x32\t.NeuronId\x12\x16\n\x0eneurons_to_add\x18\t \x01(\x05\x12 \n\x18zerofy_from_incoming_ids\x18\x0b \x03(\x05\x12\x1c\n\x14zerofy_to_neuron_ids\x18\x0c \x03(\x05\x12+\n\x11zerofy_predicates\x18\r \x03(\x0e\x32\x10.ZerofyPredicateB\n\n\x08_op_typeB\x0b\n\t_layer_id\"_\n\x17WeightsOperationRequest\x12/\n\x10weight_operation\x18\x01 \x01(\x0b\x32\x10.WeightOperationH\x00\x88\x01\x01\x42\x13\n\x11_weight_operation\"<\n\x18WeightsOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xc1\x05\n\x0fHyperParameters\x12\x1c\n\x0f\x65xperiment_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12!\n\x14training_steps_to_do\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x17\n\nbatch_size\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12 \n\x13\x66ull_eval_frequency\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12 \n\x13\x63heckpont_frequency\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bis_training\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x15\n\x08nb_steps\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x19\n\x0c\x61uditor_mode\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1d\n\x10train_batch_size\x18\n \x01(\x05H\t\x88\x01\x01\x12\x1b\n\x0eval_batch_size\x18\x0b \x01(\x05H\n\x88\x01\x01\x12\x1c\n\x0ftest_batch_size\x18\x0c \x01(\x05H\x0b\x88\x01\x01\x12\x1c\n\x0f\x65valuation_mode\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11\x65valuation_config\x18\x0e \x01(\tH\r\x88\x01\x01\x42\x12\n\x10_experiment_nameB\x17\n\x15_training_steps_to_doB\x10\n\x0e_learning_rateB\r\n\x0b_batch_sizeB\x16\n\x14_full_eval_frequencyB\x16\n\x14_checkpont_frequencyB\x0e\n\x0c_is_trainingB\x0b\n\t_nb_stepsB\x0f\n\r_auditor_modeB\x13\n\x11_train_batch_sizeB\x11\n\x0f_val_batch_sizeB\x12\n\x10_test_batch_sizeB\x12\n\x10_evaluation_modeB\x14\n\x12_evaluation_config\",\n\rMetricsStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"~\n\rAnnotatStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x08metadata\x18\x02 \x03(\x0b\x32\x1c.AnnotatStatus.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x90\x02\n\x10TrainingStatusEx\x12\x16\n\ttimestamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x0f\x65xperiment_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmodel_age\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12+\n\x0emetrics_status\x18\x04 \x01(\x0b\x32\x0e.MetricsStatusH\x03\x88\x01\x01\x12+\n\x0e\x61nnotat_status\x18\x05 \x01(\x0b\x32\x0e.AnnotatStatusH\x04\x88\x01\x01\x42\x0c\n\n_timestampB\x12\n\x10_experiment_nameB\x0c\n\n_model_ageB\x11\n\x0f_metrics_statusB\x11\n\x0f_annotat_status\"]\n\x15HyperParameterCommand\x12/\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32\x10.HyperParametersH\x00\x88\x01\x01\x42\x13\n\x11_hyper_parameters\">\n\x14\x44\x65nySamplesOperation\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\"0\n\x17LoadCheckpointOperation\x12\x15\n\rcheckpoint_id\x18\x01 \x01(\x05\"b\n\x11PlotNoteOperation\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x0c\n\x04note\x18\x04 \x01(\t\"L\n\x17SaveCheckpointOperation\x12\x19\n\x11save_architecture\x18\x01 \x01(\x08\x12\x16\n\x0esave_optimizer\x18\x02 \x01(\x08\"\x1a\n\x18RestartInstanceOperation\"\x8d\x08\n\x0eTrainerCommand\x12\x1c\n\x14get_hyper_parameters\x18\x04 \x01(\x08\x12\x1e\n\x16get_interactive_layers\x18\x05 \x01(\x08\x12\x1d\n\x10get_data_records\x18\x06 \x01(\tH\x00\x88\x01\x01\x12%\n\x18get_single_layer_info_id\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12;\n\x16hyper_parameter_change\x18\x01 \x01(\x0b\x32\x16.HyperParameterCommandH\x02\x88\x01\x01\x12:\n\x16\x64\x65ny_samples_operation\x18\x07 \x01(\x0b\x32\x15.DenySamplesOperationH\x03\x88\x01\x01\x12?\n\x1b\x64\x65ny_eval_samples_operation\x18\n \x01(\x0b\x32\x15.DenySamplesOperationH\x04\x88\x01\x01\x12@\n\x19load_checkpoint_operation\x18\t \x01(\x0b\x32\x18.LoadCheckpointOperationH\x05\x88\x01\x01\x12\x42\n\x1eremove_from_denylist_operation\x18\x0b \x01(\x0b\x32\x15.DenySamplesOperationH\x06\x88\x01\x01\x12G\n#remove_eval_from_denylist_operation\x18\x0c \x01(\x0b\x32\x15.DenySamplesOperationH\x07\x88\x01\x01\x12\x34\n\x13plot_note_operation\x18\r \x01(\x0b\x32\x12.PlotNoteOperationH\x08\x88\x01\x01\x12@\n\x19save_checkpoint_operation\x18\x0e \x01(\x0b\x32\x18.SaveCheckpointOperationH\t\x88\x01\x01\x12\x39\n\x11restart_operation\x18\x0f \x01(\x0b\x32\x19.RestartInstanceOperationH\n\x88\x01\x01\x42\x13\n\x11_get_data_recordsB\x1b\n\x19_get_single_layer_info_idB\x19\n\x17_hyper_parameter_changeB\x19\n\x17_deny_samples_operationB\x1e\n\x1c_deny_eval_samples_operationB\x1c\n\x1a_load_checkpoint_operationB!\n\x1f_remove_from_denylist_operationB&\n$_remove_eval_from_denylist_operationB\x16\n\x14_plot_note_operationB\x1c\n\x1a_save_checkpoint_operationB\x14\n\x12_restart_operation\"\x9d\x01\n\x12HyperParameterDesc\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1c\n\x0fnumerical_value\x18\x04 \x01(\x02H\x00\x88\x01\x01\x12\x19\n\x0cstring_value\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\x12\n\x10_numerical_valueB\x0f\n\r_string_value\"\xf2\x02\n\x10NeuronStatistics\x12!\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronIdH\x00\x88\x01\x01\x12\x17\n\nneuron_age\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1f\n\x12train_trigger_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x65val_trigger_rate\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x07 \x01(\x02H\x04\x88\x01\x01\x12\x36\n\x0bincoming_lr\x18\x08 \x03(\x0b\x32!.NeuronStatistics.IncomingLrEntry\x1a\x31\n\x0fIncomingLrEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\n_neuron_idB\r\n\x0b_neuron_ageB\x15\n\x13_train_trigger_rateB\x14\n\x12_eval_trigger_rateB\x10\n\x0e_learning_rate\"\xf0\x02\n\x13LayerRepresentation\x12\x15\n\x08layer_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rneurons_count\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16incoming_neurons_count\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x13\n\x06stride\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12-\n\x12neurons_statistics\x18\n \x03(\x0b\x32\x11.NeuronStatisticsB\x0b\n\t_layer_idB\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x10\n\x0e_neurons_countB\x19\n\x17_incoming_neurons_countB\x0e\n\x0c_kernel_sizeB\t\n\x07_stride\"H\n\x11\x41\x63tivationRequest\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tsample_id\x18\x02 \x01(\t\x12\x0e\n\x06origin\x18\x03 \x01(\t\"H\n\rActivationMap\x12\x11\n\tneuron_id\x18\x01 \x01(\x05\x12\x0e\n\x06values\x18\x02 \x03(\x02\x12\t\n\x01H\x18\x03 \x01(\x05\x12\t\n\x01W\x18\x04 \x01(\x05\"d\n\x12\x41\x63tivationResponse\x12\x12\n\nlayer_type\x18\x01 \x01(\t\x12\x15\n\rneurons_count\x18\x02 \x01(\x05\x12#\n\x0b\x61\x63tivations\x18\x03 \x03(\x0b\x32\x0e.ActivationMap\"\x93\x01\n\tTaskField\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x02H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x05H\x00\x12\x16\n\x0cstring_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x05 \x01(\x0cH\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x42\x07\n\x05value\"\x87\x03\n\x0eRecordMetadata\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x14\n\x0csample_label\x18\x02 \x03(\x05\x12\x19\n\x11sample_prediction\x18\x03 \x03(\x05\x12=\n\x10sample_last_loss\x18\x04 \x03(\x0b\x32#.RecordMetadata.SampleLastLossEntry\x12\x19\n\x11sample_encounters\x18\x05 \x01(\x05\x12\x18\n\x10sample_discarded\x18\x06 \x01(\x08\x12 \n\x0c\x65xtra_fields\x18\x07 \x03(\x0b\x32\n.TaskField\x12\x16\n\x0eprediction_raw\x18\t \x01(\x0c\x12\x11\n\ttask_type\x18\n \x01(\t\x12\x19\n\x11sample_label_text\x18\x0b \x03(\t\x12\x1e\n\x16sample_prediction_text\x18\x0c \x03(\t\x1a\x35\n\x13SampleLastLossEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x93\x01\n\x10SampleStatistics\x12\x13\n\x06origin\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0csample_count\x18\x07 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\ttask_type\x18\t \x01(\t\x12 \n\x07records\x18\x08 \x03(\x0b\x32\x0f.RecordMetadataB\t\n\x07_originB\x0f\n\r_sample_count\"\xe6\x01\n\x0f\x43ommandResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x33\n\x16hyper_parameters_descs\x18\x03 \x03(\x0b\x32\x13.HyperParameterDesc\x12\x33\n\x15layer_representations\x18\x04 \x03(\x0b\x32\x14.LayerRepresentation\x12\x31\n\x11sample_statistics\x18\x05 \x01(\x0b\x32\x11.SampleStatisticsH\x00\x88\x01\x01\x42\x14\n\x12_sample_statistics\"U\n\rSampleRequest\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_origin\"\xad\x02\n\x15SampleRequestResponse\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x12\n\x05label\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04\x64\x61ta\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x1a\n\rerror_message\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08raw_data\x18\x06 \x01(\x0cH\x05\x88\x01\x01\x12\x11\n\x04mask\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x12\x17\n\nprediction\x18\x08 \x01(\x0cH\x07\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_originB\x08\n\x06_labelB\x07\n\x05_dataB\x10\n\x0e_error_messageB\x0b\n\t_raw_dataB\x07\n\x05_maskB\r\n\x0b_prediction\"\x92\x01\n\x12\x42\x61tchSampleRequest\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x19\n\x0cresize_width\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x1a\n\rresize_height\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x0f\n\r_resize_widthB\x10\n\x0e_resize_height\">\n\x13\x42\x61tchSampleResponse\x12\'\n\x07samples\x18\x01 \x03(\x0b\x32\x16.SampleRequestResponse\".\n\x0eWeightsRequest\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\"\x9d\x02\n\x0fWeightsResponse\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08incoming\x18\x04 \x01(\x05\x12\x10\n\x08outgoing\x18\x05 \x01(\x05\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x02\x88\x01\x01\x12\x0f\n\x07weights\x18\x07 \x03(\x02\x12\x0f\n\x07success\x18\x0b \x01(\x08\x12\x1a\n\rerror_message\x18\x0c \x01(\tH\x03\x88\x01\x01\x42\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x0e\n\x0c_kernel_sizeB\x10\n\x0e_error_message\"R\n\x10\x44\x61taQueryRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\x12\x1b\n\x13is_natural_language\x18\x03 \x01(\x08\"5\n\x11\x43\x61tegoricalTagDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ncategories\x18\x02 \x03(\t\"\xa9\x02\n\x11\x44\x61taQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1d\n\x15number_of_all_samples\x18\x03 \x01(\x05\x12%\n\x1dnumber_of_samples_in_the_loop\x18\x04 \x01(\x05\x12#\n\x1bnumber_of_discarded_samples\x18\x05 \x01(\x05\x12\x13\n\x0bunique_tags\x18\x06 \x03(\t\x12+\n\x11\x61gent_intent_type\x18\x07 \x01(\x0e\x32\x10.AgentIntentType\x12\x17\n\x0f\x61nalysis_result\x18\x08 \x01(\t\x12,\n\x10\x63\x61tegorical_tags\x18\t \x03(\x0b\x32\x12.CategoricalTagDef\"\xc2\x01\n\x12\x44\x61taSamplesRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12 \n\x18include_transformed_data\x18\x03 \x01(\x08\x12\x18\n\x10include_raw_data\x18\x04 \x01(\x08\x12\x19\n\x11stats_to_retrieve\x18\x05 \x03(\t\x12\x14\n\x0cresize_width\x18\x06 \x01(\x05\x12\x15\n\rresize_height\x18\x07 \x01(\x05\"m\n\x08\x44\x61taStat\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x05\x12\r\n\x05value\x18\x04 \x03(\x02\x12\x14\n\x0cvalue_string\x18\x05 \x01(\t\x12\x11\n\tthumbnail\x18\x06 \x01(\x0c\">\n\nDataRecord\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x1d\n\ndata_stats\x18\x02 \x03(\x0b\x32\t.DataStat\"Z\n\x13\x44\x61taSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12!\n\x0c\x64\x61ta_records\x18\x03 \x03(\x0b\x32\x0b.DataRecord\"C\n\x0fHistogramSubBar\x12\x0e\n\x06origin\x18\x01 \x01(\t\x12\x11\n\tdiscarded\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\"h\n\x0cHistogramBin\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\x12\x0b\n\x03\x61vg\x18\x03 \x01(\x01\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\x12\"\n\x08sub_bars\x18\x05 \x03(\x0b\x32\x10.HistogramSubBar\"[\n\x17\x43\x61tegoricalHistogramBar\x12\r\n\x05label\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x03\x12\"\n\x08sub_bars\x18\x03 \x03(\x0b\x32\x10.HistogramSubBar\"4\n\x10HistogramRequest\x12\x0e\n\x06\x63olumn\x18\x01 \x01(\t\x12\x10\n\x08max_bins\x18\x02 \x01(\x05\"\xb2\x01\n\x11HistogramResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\ntotal_rows\x18\x03 \x01(\x03\x12\x1b\n\x04\x62ins\x18\x04 \x03(\x0b\x32\r.HistogramBin\x12\x16\n\x0eis_categorical\x18\x05 \x01(\x08\x12\x32\n\x10\x63\x61tegorical_bars\x18\x06 \x03(\x0b\x32\x18.CategoricalHistogramBar\"W\n\x12GetMetaDataRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12\x17\n\x0fmodal_sample_id\x18\x03 \x01(\t\"\x99\x01\n\x13GetMetaDataResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1a\n\x12\x61ll_metadata_names\x18\x03 \x03(\t\x12!\n\x0cgrid_records\x18\x04 \x03(\x0b\x32\x0b.DataRecord\x12!\n\x0cmodal_record\x18\x05 \x01(\x0b\x32\x0b.DataRecord\"j\n\x12StepSamplesRequest\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x13\n\x0bmax_samples\x18\x04 \x01(\x05\"{\n\x13StepSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nsample_ids\x18\x03 \x03(\t\x12\x17\n\x0ftotal_available\x18\x04 \x01(\x05\x12\x15\n\rsample_values\x18\x05 \x03(\x02\"Y\n\x1aGetSignalTrajectoryRequest\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12\x12\n\nsample_ids\x18\x02 \x03(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"4\n\x10SignalTrajectory\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x03(\x02\"}\n\x1bGetSignalTrajectoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12\'\n\x0ctrajectories\x18\x04 \x03(\x0b\x32\x11.SignalTrajectory\"Y\n\x11PointCloudRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\x12\r\n\x05\x66ield\x18\x04 \x01(\t\"\xbf\x01\n\x0fPointCloudChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nnum_points\x18\x03 \x01(\x05\x12\x14\n\x0cnum_features\x18\x04 \x01(\x05\x12\x10\n\x08pc_range\x18\x05 \x03(\x02\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\x07 \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x08 \x01(\x05\x12\x15\n\rfeature_names\x18\t \x03(\t\"b\n\x0cMediaRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x0c\n\x04kind\x18\x03 \x01(\t\x12\x12\n\nmax_frames\x18\x04 \x01(\x05\x12\r\n\x05\x66ield\x18\x05 \x01(\t\"\x92\x02\n\nMediaChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0b\x66rame_count\x18\x04 \x01(\x05\x12\x0b\n\x03\x66ps\x18\x05 \x01(\x02\x12\x11\n\thas_audio\x18\x06 \x01(\x08\x12\r\n\x05width\x18\x07 \x01(\x05\x12\x0e\n\x06height\x18\x08 \x01(\x05\x12\x13\n\x0btotal_bytes\x18\t \x01(\x05\x12\x18\n\x10\x64uration_seconds\x18\n \x01(\x02\x12\x13\n\x0bsample_rate\x18\x0b \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x0c \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\r \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x0e \x01(\x05\"\x8c\x02\n\x10\x44\x61taEditsRequest\x12\x11\n\tstat_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0cstring_value\x18\x03 \x01(\t\x12\x12\n\nbool_value\x18\x04 \x01(\x08\x12\x1d\n\x04type\x18\x05 \x01(\x0e\x32\x0f.SampleEditType\x12\x13\n\x0bsamples_ids\x18\x06 \x03(\t\x12\x16\n\x0esample_origins\x18\x07 \x03(\t\x12\x16\n\x0eis_categorical\x18\x08 \x01(\x08\x12\x12\n\ncategories\x18\t \x03(\t\x12\x15\n\rsample_values\x18\n \x03(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x0b \x01(\t\"5\n\x11\x44\x61taEditsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x12\x44\x61taSplitsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0bsplit_names\x18\x02 \x03(\t\"9\n\x13\x41gentHealthResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"^\n\x16InitializeAgentRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12$\n\x08provider\x18\x02 \x01(\x0e\x32\x12.AgentProviderType\x12\r\n\x05model\x18\x03 \x01(\t\";\n\x17InitializeAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"(\n\x17\x43hangeAgentModelRequest\x12\r\n\x05model\x18\x01 \x01(\t\"<\n\x18\x43hangeAgentModelResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x17\n\x15GetAgentModelsRequest\"J\n\x16GetAgentModelsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06models\x18\x02 \x03(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"6\n\x12ResetAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"=\n\x19\x43learAgentHistoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"?\n\x1b\x43ompactAgentHistoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xe5\x01\n\x1cGetAgentContextUsageResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\r\n\x05model\x18\x03 \x01(\t\x12\x16\n\x0e\x63ontext_window\x18\x04 \x01(\x03\x12\x14\n\x0cinput_tokens\x18\x05 \x01(\x03\x12\x15\n\routput_tokens\x18\x06 \x01(\x03\x12\x18\n\x10reasoning_tokens\x18\x07 \x01(\x03\x12\x19\n\x11\x63\x61\x63he_read_tokens\x18\x08 \x01(\x03\x12\x1a\n\x12\x63\x61\x63he_write_tokens\x18\t \x01(\x03\"3\n\x18RestoreCheckpointRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\"=\n\x19RestoreCheckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xa8\x01\n\x11\x45xperimentRunInfo\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_name\x18\x02 \x01(\t\x12\r\n\x05notes\x18\x03 \x01(\t\x12\x0f\n\x07\x63reated\x18\x04 \x01(\t\x12\x11\n\tlast_used\x18\x05 \x01(\t\x12\x1a\n\x12latest_weight_step\x18\x06 \x01(\x05\x12\x12\n\nis_current\x18\x07 \x01(\x08\"\x1b\n\x19ListExperimentRunsRequest\">\n\x1aListExperimentRunsResponse\x12 \n\x04runs\x18\x01 \x03(\x0b\x32\x12.ExperimentRunInfo\"G\n\x1aRenameExperimentRunRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\x12\x10\n\x08new_name\x18\x02 \x01(\t\"?\n\x1bRenameExperimentRunResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"F\n\x1cSetExperimentRunNotesRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\x12\r\n\x05notes\x18\x02 \x01(\t\"A\n\x1dSetExperimentRunNotesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"R\n\x18TriggerEvaluationRequest\x12\x12\n\nsplit_name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x14\n\x0cuse_full_set\x18\x03 \x01(\x08\"=\n\x19TriggerEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1c\n\x1aGetEvaluationStatusRequest\"\x81\x01\n\x1bGetEvaluationStatusResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x63urrent\x18\x02 \x01(\x05\x12\r\n\x05total\x18\x03 \x01(\x05\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x12\n\nsplit_name\x18\x06 \x01(\t\")\n\x17\x43\x61ncelEvaluationRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\"<\n\x18\x43\x61ncelEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"7\n\x16RunNotebookCellRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x65ll_id\x18\x02 \x01(\t\"2\n\x10NotebookCellDone\x12\x12\n\nexec_count\x18\x01 \x01(\x05\x12\n\n\x02ok\x18\x02 \x01(\x08\"\x1e\n\x1cInterruptNotebookCellRequest\":\n\x1dInterruptNotebookCellResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\"\xbd\x01\n\x11NotebookCellChunk\x12\x0f\n\x07\x63\x65ll_id\x18\x01 \x01(\t\x12\x10\n\x06stdout\x18\x02 \x01(\tH\x00\x12\x10\n\x06stderr\x18\x03 \x01(\tH\x00\x12\x15\n\x0bresult_text\x18\x04 \x01(\tH\x00\x12\x13\n\timage_png\x18\x05 \x01(\x0cH\x00\x12\x19\n\x0f\x65rror_traceback\x18\x06 \x01(\tH\x00\x12!\n\x04\x64one\x18\x07 \x01(\x0b\x32\x11.NotebookCellDoneH\x00\x42\t\n\x07payload\"S\n\x10NotebookResponse\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0f\n\x07\x65xisted\x18\x02 \x01(\x08\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"7\n\x13SaveNotebookRequest\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"M\n\x14SaveNotebookResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"C\n\x1bGenerateNotebookCodeRequest\x12\x0e\n\x06prompt\x18\x01 \x01(\t\x12\x14\n\x0c\x63ontext_code\x18\x02 \x01(\t\"\\\n\x1cGenerateNotebookCodeResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x13\n\x0b\x65xplanation\x18\x02 \x01(\t\x12\n\n\x02ok\x18\x03 \x01(\x08\x12\r\n\x05\x65rror\x18\x04 \x01(\t\"~\n\x18\x45xportAnnotationsRequest\x12\'\n\x06\x66ormat\x18\x01 \x01(\x0e\x32\x17.AnnotationExportFormat\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x1b\n\x13include_predictions\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\"\x88\x01\n\x19\x45xportAnnotationsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x10\n\x08\x66ilename\x18\x04 \x01(\t\x12\x11\n\tmime_type\x18\x05 \x01(\t\x12\x13\n\x0bimage_count\x18\x06 \x01(\x05*d\n\x13WeightOperationType\x12\n\n\x06ZEROFY\x10\x00\x12\x10\n\x0cREINITIALIZE\x10\x01\x12\n\n\x06\x46REEZE\x10\x02\x12\x12\n\x0eREMOVE_NEURONS\x10\t\x12\x0f\n\x0b\x41\x44\x44_NEURONS\x10\n*o\n\x0fZerofyPredicate\x12\x19\n\x15ZEROFY_PREDICATE_NONE\x10\x00\x12 \n\x1cZEROFY_PREDICATE_WITH_FROZEN\x10\x01\x12\x1f\n\x1bZEROFY_PREDICATE_WITH_OLDER\x10\x02*M\n\x0f\x41gentIntentType\x12\x12\n\x0eINTENT_UNKNOWN\x10\x00\x12\x11\n\rINTENT_FILTER\x10\x01\x12\x13\n\x0fINTENT_ANALYSIS\x10\x02*I\n\x0eSampleEditType\x12\x11\n\rEDIT_OVERRIDE\x10\x00\x12\x13\n\x0f\x45\x44IT_ACCUMULATE\x10\x01\x12\x0f\n\x0b\x45\x44IT_REMOVE\x10\x02*C\n\x11\x41gentProviderType\x12\x17\n\x13PROVIDER_OPENROUTER\x10\x00\x12\x15\n\x11PROVIDER_OPENCODE\x10\x01*m\n\x16\x41nnotationExportFormat\x12\x16\n\x12\x45XPORT_FORMAT_CVAT\x10\x00\x12\x1e\n\x1a\x45XPORT_FORMAT_LABEL_STUDIO\x10\x01\x12\x1b\n\x17\x45XPORT_FORMAT_V7_DARWIN\x10\x02\x32\xfe\x12\n\x11\x45xperimentService\x12P\n\x13GetLatestLoggerData\x12\x1b.GetLatestLoggerDataRequest\x1a\x1c.GetLatestLoggerDataResponse\x12\x36\n\x11\x45xperimentCommand\x12\x0f.TrainerCommand\x1a\x10.CommandResponse\x12H\n\x11ManipulateWeights\x12\x18.WeightsOperationRequest\x1a\x19.WeightsOperationResponse\x12/\n\nGetWeights\x12\x0f.WeightsRequest\x1a\x10.WeightsResponse\x12\x39\n\x0eGetActivations\x12\x12.ActivationRequest\x1a\x13.ActivationResponse\x12\x37\n\nGetSamples\x12\x13.BatchSampleRequest\x1a\x14.BatchSampleResponse\x12\x37\n\x0e\x41pplyDataQuery\x12\x11.DataQueryRequest\x1a\x12.DataQueryResponse\x12;\n\x0eGetDataSamples\x12\x13.DataSamplesRequest\x1a\x14.DataSamplesResponse\x12\x35\n\x0cGetHistogram\x12\x11.HistogramRequest\x1a\x12.HistogramResponse\x12\x38\n\x0bGetMetaData\x12\x13.GetMetaDataRequest\x1a\x14.GetMetaDataResponse\x12P\n\x13GetSignalTrajectory\x12\x1b.GetSignalTrajectoryRequest\x1a\x1c.GetSignalTrajectoryResponse\x12;\n\x0eGetStepSamples\x12\x13.StepSamplesRequest\x1a\x14.StepSamplesResponse\x12\x37\n\rGetPointCloud\x12\x12.PointCloudRequest\x1a\x10.PointCloudChunk0\x01\x12(\n\x08GetMedia\x12\r.MediaRequest\x1a\x0b.MediaChunk0\x01\x12\x37\n\x0e\x45\x64itDataSample\x12\x11.DataEditsRequest\x1a\x12.DataEditsResponse\x12,\n\rGetDataSplits\x12\x06.Empty\x1a\x13.DataSplitsResponse\x12\x30\n\x10\x43heckAgentHealth\x12\x06.Empty\x1a\x14.AgentHealthResponse\x12\x44\n\x0fInitializeAgent\x12\x17.InitializeAgentRequest\x1a\x18.InitializeAgentResponse\x12G\n\x10\x43hangeAgentModel\x12\x18.ChangeAgentModelRequest\x1a\x19.ChangeAgentModelResponse\x12\x41\n\x0eGetAgentModels\x12\x16.GetAgentModelsRequest\x1a\x17.GetAgentModelsResponse\x12)\n\nResetAgent\x12\x06.Empty\x1a\x13.ResetAgentResponse\x12\x37\n\x11\x43learAgentHistory\x12\x06.Empty\x1a\x1a.ClearAgentHistoryResponse\x12;\n\x13\x43ompactAgentHistory\x12\x06.Empty\x1a\x1c.CompactAgentHistoryResponse\x12=\n\x14GetAgentContextUsage\x12\x06.Empty\x1a\x1d.GetAgentContextUsageResponse\x12@\n\x0fRunNotebookCell\x12\x17.RunNotebookCellRequest\x1a\x12.NotebookCellChunk0\x01\x12V\n\x15InterruptNotebookCell\x12\x1d.InterruptNotebookCellRequest\x1a\x1e.InterruptNotebookCellResponse\x12(\n\x0bGetNotebook\x12\x06.Empty\x1a\x11.NotebookResponse\x12;\n\x0cSaveNotebook\x12\x14.SaveNotebookRequest\x1a\x15.SaveNotebookResponse\x12S\n\x14GenerateNotebookCode\x12\x1c.GenerateNotebookCodeRequest\x1a\x1d.GenerateNotebookCodeResponse\x12J\n\x11RestoreCheckpoint\x12\x19.RestoreCheckpointRequest\x1a\x1a.RestoreCheckpointResponse\x12M\n\x12ListExperimentRuns\x12\x1a.ListExperimentRunsRequest\x1a\x1b.ListExperimentRunsResponse\x12P\n\x13RenameExperimentRun\x12\x1b.RenameExperimentRunRequest\x1a\x1c.RenameExperimentRunResponse\x12V\n\x15SetExperimentRunNotes\x12\x1d.SetExperimentRunNotesRequest\x1a\x1e.SetExperimentRunNotesResponse\x12J\n\x11TriggerEvaluation\x12\x19.TriggerEvaluationRequest\x1a\x1a.TriggerEvaluationResponse\x12P\n\x13GetEvaluationStatus\x12\x1b.GetEvaluationStatusRequest\x1a\x1c.GetEvaluationStatusResponse\x12G\n\x10\x43\x61ncelEvaluation\x12\x18.CancelEvaluationRequest\x1a\x19.CancelEvaluationResponse\x12J\n\x11\x45xportAnnotations\x12\x19.ExportAnnotationsRequest\x1a\x1a.ExportAnnotationsResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)weightslab/proto/experiment_service.proto\"\x81\x02\n\x1aGetLatestLoggerDataRequest\x12\x1c\n\x14request_full_history\x18\x01 \x01(\x08\x12\x12\n\nmax_points\x18\x02 \x01(\x05\x12\x17\n\x0f\x62reak_by_slices\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\x12\x12\n\ngraph_name\x18\x05 \x01(\t\x12\r\n\x05x_min\x18\x06 \x01(\x03\x12\r\n\x05x_max\x18\x07 \x01(\x03\x12\x13\n\x0bhas_x_range\x18\x08 \x01(\x08\x12\x14\n\x0cmetric_names\x18\t \x03(\t\x12\x19\n\x11\x65xperiment_hashes\x18\n \x03(\t\x12\x12\n\nindex_only\x18\x0b \x01(\x08\"|\n\x10SignalCurveIndex\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x12\n\nfirst_step\x18\x03 \x01(\x03\x12\x11\n\tlast_step\x18\x04 \x01(\x03\x12\x13\n\x0bpoint_count\x18\x05 \x01(\x03\"1\n\rSignalOutlier\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"\xd2\x03\n\x0fLoggerDataPoint\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x11\n\tmodel_age\x18\x02 \x01(\x05\x12\x14\n\x0cmetric_value\x18\x03 \x01(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tsample_id\x18\x06 \x01(\t\x12\x1c\n\x14is_evaluation_marker\x18\x07 \x01(\x08\x12\x12\n\nsplit_name\x18\x08 \x01(\t\x12\x17\n\x0f\x65valuation_tags\x18\t \x03(\t\x12\x12\n\npoint_note\x18\n \x01(\t\x12\x12\n\naudit_mode\x18\x0b \x01(\x08\x12 \n\x08outliers\x18\x0c \x03(\x0b\x32\x0e.SignalOutlier\x12\x15\n\routlier_count\x18\r \x01(\x05\x12\x14\n\x0csample_count\x18\x0e \x01(\x05\x12\x13\n\x0btrend_value\x18\x0f \x01(\x02\x12\x14\n\x0ctrend_margin\x18\x10 \x01(\x02\x12\x16\n\x0ehas_trend_band\x18\x11 \x01(\x08\x12\x11\n\tvalue_min\x18\x12 \x01(\x02\x12\x11\n\tvalue_max\x18\x13 \x01(\x02\x12\x17\n\x0fhas_value_range\x18\x14 \x01(\x08\"\x9a\x01\n\x1bGetLatestLoggerDataResponse\x12 \n\x06points\x18\x01 \x03(\x0b\x32\x10.LoggerDataPoint\x12\x1a\n\x12weightslab_version\x18\x02 \x01(\t\x12!\n\x06\x63urves\x18\x03 \x03(\x0b\x32\x11.SignalCurveIndex\x12\x1a\n\x12\x61pplied_max_points\x18\x04 \x01(\x05\"\x07\n\x05\x45mpty\"/\n\x08NeuronId\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tneuron_id\x18\x02 \x01(\x05\"\x91\x02\n\x0fWeightOperation\x12*\n\x07op_type\x18\x01 \x01(\x0e\x32\x14.WeightOperationTypeH\x00\x88\x01\x01\x12\x15\n\x08layer_id\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\nneuron_ids\x18\x03 \x03(\x0b\x32\t.NeuronId\x12\x16\n\x0eneurons_to_add\x18\t \x01(\x05\x12 \n\x18zerofy_from_incoming_ids\x18\x0b \x03(\x05\x12\x1c\n\x14zerofy_to_neuron_ids\x18\x0c \x03(\x05\x12+\n\x11zerofy_predicates\x18\r \x03(\x0e\x32\x10.ZerofyPredicateB\n\n\x08_op_typeB\x0b\n\t_layer_id\"_\n\x17WeightsOperationRequest\x12/\n\x10weight_operation\x18\x01 \x01(\x0b\x32\x10.WeightOperationH\x00\x88\x01\x01\x42\x13\n\x11_weight_operation\"<\n\x18WeightsOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xc1\x05\n\x0fHyperParameters\x12\x1c\n\x0f\x65xperiment_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12!\n\x14training_steps_to_do\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x17\n\nbatch_size\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12 \n\x13\x66ull_eval_frequency\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12 \n\x13\x63heckpont_frequency\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bis_training\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x15\n\x08nb_steps\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x19\n\x0c\x61uditor_mode\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1d\n\x10train_batch_size\x18\n \x01(\x05H\t\x88\x01\x01\x12\x1b\n\x0eval_batch_size\x18\x0b \x01(\x05H\n\x88\x01\x01\x12\x1c\n\x0ftest_batch_size\x18\x0c \x01(\x05H\x0b\x88\x01\x01\x12\x1c\n\x0f\x65valuation_mode\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11\x65valuation_config\x18\x0e \x01(\tH\r\x88\x01\x01\x42\x12\n\x10_experiment_nameB\x17\n\x15_training_steps_to_doB\x10\n\x0e_learning_rateB\r\n\x0b_batch_sizeB\x16\n\x14_full_eval_frequencyB\x16\n\x14_checkpont_frequencyB\x0e\n\x0c_is_trainingB\x0b\n\t_nb_stepsB\x0f\n\r_auditor_modeB\x13\n\x11_train_batch_sizeB\x11\n\x0f_val_batch_sizeB\x12\n\x10_test_batch_sizeB\x12\n\x10_evaluation_modeB\x14\n\x12_evaluation_config\",\n\rMetricsStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"~\n\rAnnotatStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x08metadata\x18\x02 \x03(\x0b\x32\x1c.AnnotatStatus.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x90\x02\n\x10TrainingStatusEx\x12\x16\n\ttimestamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x0f\x65xperiment_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmodel_age\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12+\n\x0emetrics_status\x18\x04 \x01(\x0b\x32\x0e.MetricsStatusH\x03\x88\x01\x01\x12+\n\x0e\x61nnotat_status\x18\x05 \x01(\x0b\x32\x0e.AnnotatStatusH\x04\x88\x01\x01\x42\x0c\n\n_timestampB\x12\n\x10_experiment_nameB\x0c\n\n_model_ageB\x11\n\x0f_metrics_statusB\x11\n\x0f_annotat_status\"]\n\x15HyperParameterCommand\x12/\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32\x10.HyperParametersH\x00\x88\x01\x01\x42\x13\n\x11_hyper_parameters\">\n\x14\x44\x65nySamplesOperation\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\"0\n\x17LoadCheckpointOperation\x12\x15\n\rcheckpoint_id\x18\x01 \x01(\x05\"b\n\x11PlotNoteOperation\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x0c\n\x04note\x18\x04 \x01(\t\"L\n\x17SaveCheckpointOperation\x12\x19\n\x11save_architecture\x18\x01 \x01(\x08\x12\x16\n\x0esave_optimizer\x18\x02 \x01(\x08\"\x1a\n\x18RestartInstanceOperation\"\x8d\x08\n\x0eTrainerCommand\x12\x1c\n\x14get_hyper_parameters\x18\x04 \x01(\x08\x12\x1e\n\x16get_interactive_layers\x18\x05 \x01(\x08\x12\x1d\n\x10get_data_records\x18\x06 \x01(\tH\x00\x88\x01\x01\x12%\n\x18get_single_layer_info_id\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12;\n\x16hyper_parameter_change\x18\x01 \x01(\x0b\x32\x16.HyperParameterCommandH\x02\x88\x01\x01\x12:\n\x16\x64\x65ny_samples_operation\x18\x07 \x01(\x0b\x32\x15.DenySamplesOperationH\x03\x88\x01\x01\x12?\n\x1b\x64\x65ny_eval_samples_operation\x18\n \x01(\x0b\x32\x15.DenySamplesOperationH\x04\x88\x01\x01\x12@\n\x19load_checkpoint_operation\x18\t \x01(\x0b\x32\x18.LoadCheckpointOperationH\x05\x88\x01\x01\x12\x42\n\x1eremove_from_denylist_operation\x18\x0b \x01(\x0b\x32\x15.DenySamplesOperationH\x06\x88\x01\x01\x12G\n#remove_eval_from_denylist_operation\x18\x0c \x01(\x0b\x32\x15.DenySamplesOperationH\x07\x88\x01\x01\x12\x34\n\x13plot_note_operation\x18\r \x01(\x0b\x32\x12.PlotNoteOperationH\x08\x88\x01\x01\x12@\n\x19save_checkpoint_operation\x18\x0e \x01(\x0b\x32\x18.SaveCheckpointOperationH\t\x88\x01\x01\x12\x39\n\x11restart_operation\x18\x0f \x01(\x0b\x32\x19.RestartInstanceOperationH\n\x88\x01\x01\x42\x13\n\x11_get_data_recordsB\x1b\n\x19_get_single_layer_info_idB\x19\n\x17_hyper_parameter_changeB\x19\n\x17_deny_samples_operationB\x1e\n\x1c_deny_eval_samples_operationB\x1c\n\x1a_load_checkpoint_operationB!\n\x1f_remove_from_denylist_operationB&\n$_remove_eval_from_denylist_operationB\x16\n\x14_plot_note_operationB\x1c\n\x1a_save_checkpoint_operationB\x14\n\x12_restart_operation\"\x9d\x01\n\x12HyperParameterDesc\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1c\n\x0fnumerical_value\x18\x04 \x01(\x02H\x00\x88\x01\x01\x12\x19\n\x0cstring_value\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\x12\n\x10_numerical_valueB\x0f\n\r_string_value\"\xf2\x02\n\x10NeuronStatistics\x12!\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronIdH\x00\x88\x01\x01\x12\x17\n\nneuron_age\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1f\n\x12train_trigger_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x65val_trigger_rate\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x07 \x01(\x02H\x04\x88\x01\x01\x12\x36\n\x0bincoming_lr\x18\x08 \x03(\x0b\x32!.NeuronStatistics.IncomingLrEntry\x1a\x31\n\x0fIncomingLrEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\n_neuron_idB\r\n\x0b_neuron_ageB\x15\n\x13_train_trigger_rateB\x14\n\x12_eval_trigger_rateB\x10\n\x0e_learning_rate\"\xf0\x02\n\x13LayerRepresentation\x12\x15\n\x08layer_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rneurons_count\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16incoming_neurons_count\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x13\n\x06stride\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12-\n\x12neurons_statistics\x18\n \x03(\x0b\x32\x11.NeuronStatisticsB\x0b\n\t_layer_idB\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x10\n\x0e_neurons_countB\x19\n\x17_incoming_neurons_countB\x0e\n\x0c_kernel_sizeB\t\n\x07_stride\"H\n\x11\x41\x63tivationRequest\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tsample_id\x18\x02 \x01(\t\x12\x0e\n\x06origin\x18\x03 \x01(\t\"H\n\rActivationMap\x12\x11\n\tneuron_id\x18\x01 \x01(\x05\x12\x0e\n\x06values\x18\x02 \x03(\x02\x12\t\n\x01H\x18\x03 \x01(\x05\x12\t\n\x01W\x18\x04 \x01(\x05\"d\n\x12\x41\x63tivationResponse\x12\x12\n\nlayer_type\x18\x01 \x01(\t\x12\x15\n\rneurons_count\x18\x02 \x01(\x05\x12#\n\x0b\x61\x63tivations\x18\x03 \x03(\x0b\x32\x0e.ActivationMap\"\x93\x01\n\tTaskField\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x02H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x05H\x00\x12\x16\n\x0cstring_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x05 \x01(\x0cH\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x42\x07\n\x05value\"\x87\x03\n\x0eRecordMetadata\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x14\n\x0csample_label\x18\x02 \x03(\x05\x12\x19\n\x11sample_prediction\x18\x03 \x03(\x05\x12=\n\x10sample_last_loss\x18\x04 \x03(\x0b\x32#.RecordMetadata.SampleLastLossEntry\x12\x19\n\x11sample_encounters\x18\x05 \x01(\x05\x12\x18\n\x10sample_discarded\x18\x06 \x01(\x08\x12 \n\x0c\x65xtra_fields\x18\x07 \x03(\x0b\x32\n.TaskField\x12\x16\n\x0eprediction_raw\x18\t \x01(\x0c\x12\x11\n\ttask_type\x18\n \x01(\t\x12\x19\n\x11sample_label_text\x18\x0b \x03(\t\x12\x1e\n\x16sample_prediction_text\x18\x0c \x03(\t\x1a\x35\n\x13SampleLastLossEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x93\x01\n\x10SampleStatistics\x12\x13\n\x06origin\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0csample_count\x18\x07 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\ttask_type\x18\t \x01(\t\x12 \n\x07records\x18\x08 \x03(\x0b\x32\x0f.RecordMetadataB\t\n\x07_originB\x0f\n\r_sample_count\"\xe6\x01\n\x0f\x43ommandResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x33\n\x16hyper_parameters_descs\x18\x03 \x03(\x0b\x32\x13.HyperParameterDesc\x12\x33\n\x15layer_representations\x18\x04 \x03(\x0b\x32\x14.LayerRepresentation\x12\x31\n\x11sample_statistics\x18\x05 \x01(\x0b\x32\x11.SampleStatisticsH\x00\x88\x01\x01\x42\x14\n\x12_sample_statistics\"U\n\rSampleRequest\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_origin\"\xad\x02\n\x15SampleRequestResponse\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x12\n\x05label\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04\x64\x61ta\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x1a\n\rerror_message\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08raw_data\x18\x06 \x01(\x0cH\x05\x88\x01\x01\x12\x11\n\x04mask\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x12\x17\n\nprediction\x18\x08 \x01(\x0cH\x07\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_originB\x08\n\x06_labelB\x07\n\x05_dataB\x10\n\x0e_error_messageB\x0b\n\t_raw_dataB\x07\n\x05_maskB\r\n\x0b_prediction\"\x92\x01\n\x12\x42\x61tchSampleRequest\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x19\n\x0cresize_width\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x1a\n\rresize_height\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x0f\n\r_resize_widthB\x10\n\x0e_resize_height\">\n\x13\x42\x61tchSampleResponse\x12\'\n\x07samples\x18\x01 \x03(\x0b\x32\x16.SampleRequestResponse\".\n\x0eWeightsRequest\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\"\x9d\x02\n\x0fWeightsResponse\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08incoming\x18\x04 \x01(\x05\x12\x10\n\x08outgoing\x18\x05 \x01(\x05\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x02\x88\x01\x01\x12\x0f\n\x07weights\x18\x07 \x03(\x02\x12\x0f\n\x07success\x18\x0b \x01(\x08\x12\x1a\n\rerror_message\x18\x0c \x01(\tH\x03\x88\x01\x01\x42\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x0e\n\x0c_kernel_sizeB\x10\n\x0e_error_message\"R\n\x10\x44\x61taQueryRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\x12\x1b\n\x13is_natural_language\x18\x03 \x01(\x08\"5\n\x11\x43\x61tegoricalTagDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ncategories\x18\x02 \x03(\t\"\xa9\x02\n\x11\x44\x61taQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1d\n\x15number_of_all_samples\x18\x03 \x01(\x05\x12%\n\x1dnumber_of_samples_in_the_loop\x18\x04 \x01(\x05\x12#\n\x1bnumber_of_discarded_samples\x18\x05 \x01(\x05\x12\x13\n\x0bunique_tags\x18\x06 \x03(\t\x12+\n\x11\x61gent_intent_type\x18\x07 \x01(\x0e\x32\x10.AgentIntentType\x12\x17\n\x0f\x61nalysis_result\x18\x08 \x01(\t\x12,\n\x10\x63\x61tegorical_tags\x18\t \x03(\x0b\x32\x12.CategoricalTagDef\"\xc2\x01\n\x12\x44\x61taSamplesRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12 \n\x18include_transformed_data\x18\x03 \x01(\x08\x12\x18\n\x10include_raw_data\x18\x04 \x01(\x08\x12\x19\n\x11stats_to_retrieve\x18\x05 \x03(\t\x12\x14\n\x0cresize_width\x18\x06 \x01(\x05\x12\x15\n\rresize_height\x18\x07 \x01(\x05\"m\n\x08\x44\x61taStat\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x05\x12\r\n\x05value\x18\x04 \x03(\x02\x12\x14\n\x0cvalue_string\x18\x05 \x01(\t\x12\x11\n\tthumbnail\x18\x06 \x01(\x0c\">\n\nDataRecord\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x1d\n\ndata_stats\x18\x02 \x03(\x0b\x32\t.DataStat\"Z\n\x13\x44\x61taSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12!\n\x0c\x64\x61ta_records\x18\x03 \x03(\x0b\x32\x0b.DataRecord\"C\n\x0fHistogramSubBar\x12\x0e\n\x06origin\x18\x01 \x01(\t\x12\x11\n\tdiscarded\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\"h\n\x0cHistogramBin\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\x12\x0b\n\x03\x61vg\x18\x03 \x01(\x01\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\x12\"\n\x08sub_bars\x18\x05 \x03(\x0b\x32\x10.HistogramSubBar\"[\n\x17\x43\x61tegoricalHistogramBar\x12\r\n\x05label\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x03\x12\"\n\x08sub_bars\x18\x03 \x03(\x0b\x32\x10.HistogramSubBar\"4\n\x10HistogramRequest\x12\x0e\n\x06\x63olumn\x18\x01 \x01(\t\x12\x10\n\x08max_bins\x18\x02 \x01(\x05\"\xb2\x01\n\x11HistogramResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\ntotal_rows\x18\x03 \x01(\x03\x12\x1b\n\x04\x62ins\x18\x04 \x03(\x0b\x32\r.HistogramBin\x12\x16\n\x0eis_categorical\x18\x05 \x01(\x08\x12\x32\n\x10\x63\x61tegorical_bars\x18\x06 \x03(\x0b\x32\x18.CategoricalHistogramBar\"W\n\x12GetMetaDataRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12\x17\n\x0fmodal_sample_id\x18\x03 \x01(\t\"\x99\x01\n\x13GetMetaDataResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1a\n\x12\x61ll_metadata_names\x18\x03 \x03(\t\x12!\n\x0cgrid_records\x18\x04 \x03(\x0b\x32\x0b.DataRecord\x12!\n\x0cmodal_record\x18\x05 \x01(\x0b\x32\x0b.DataRecord\"j\n\x12StepSamplesRequest\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x13\n\x0bmax_samples\x18\x04 \x01(\x05\"{\n\x13StepSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nsample_ids\x18\x03 \x03(\t\x12\x17\n\x0ftotal_available\x18\x04 \x01(\x05\x12\x15\n\rsample_values\x18\x05 \x03(\x02\"Y\n\x1aGetSignalTrajectoryRequest\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12\x12\n\nsample_ids\x18\x02 \x03(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"4\n\x10SignalTrajectory\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x03(\x02\"}\n\x1bGetSignalTrajectoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12\'\n\x0ctrajectories\x18\x04 \x03(\x0b\x32\x11.SignalTrajectory\"Y\n\x11PointCloudRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\x12\r\n\x05\x66ield\x18\x04 \x01(\t\"\xbf\x01\n\x0fPointCloudChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nnum_points\x18\x03 \x01(\x05\x12\x14\n\x0cnum_features\x18\x04 \x01(\x05\x12\x10\n\x08pc_range\x18\x05 \x03(\x02\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\x07 \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x08 \x01(\x05\x12\x15\n\rfeature_names\x18\t \x03(\t\"b\n\x0cMediaRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x0c\n\x04kind\x18\x03 \x01(\t\x12\x12\n\nmax_frames\x18\x04 \x01(\x05\x12\r\n\x05\x66ield\x18\x05 \x01(\t\"\x92\x02\n\nMediaChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x11\n\tmime_type\x18\x03 \x01(\t\x12\x13\n\x0b\x66rame_count\x18\x04 \x01(\x05\x12\x0b\n\x03\x66ps\x18\x05 \x01(\x02\x12\x11\n\thas_audio\x18\x06 \x01(\x08\x12\r\n\x05width\x18\x07 \x01(\x05\x12\x0e\n\x06height\x18\x08 \x01(\x05\x12\x13\n\x0btotal_bytes\x18\t \x01(\x05\x12\x18\n\x10\x64uration_seconds\x18\n \x01(\x02\x12\x13\n\x0bsample_rate\x18\x0b \x01(\x05\x12\x0c\n\x04\x64\x61ta\x18\x0c \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\r \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x0e \x01(\x05\"\x8c\x02\n\x10\x44\x61taEditsRequest\x12\x11\n\tstat_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0cstring_value\x18\x03 \x01(\t\x12\x12\n\nbool_value\x18\x04 \x01(\x08\x12\x1d\n\x04type\x18\x05 \x01(\x0e\x32\x0f.SampleEditType\x12\x13\n\x0bsamples_ids\x18\x06 \x03(\t\x12\x16\n\x0esample_origins\x18\x07 \x03(\t\x12\x16\n\x0eis_categorical\x18\x08 \x01(\x08\x12\x12\n\ncategories\x18\t \x03(\t\x12\x15\n\rsample_values\x18\n \x03(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x0b \x01(\t\"5\n\x11\x44\x61taEditsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x12\x44\x61taSplitsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0bsplit_names\x18\x02 \x03(\t\"9\n\x13\x41gentHealthResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"^\n\x16InitializeAgentRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12$\n\x08provider\x18\x02 \x01(\x0e\x32\x12.AgentProviderType\x12\r\n\x05model\x18\x03 \x01(\t\";\n\x17InitializeAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"(\n\x17\x43hangeAgentModelRequest\x12\r\n\x05model\x18\x01 \x01(\t\"<\n\x18\x43hangeAgentModelResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x17\n\x15GetAgentModelsRequest\"J\n\x16GetAgentModelsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06models\x18\x02 \x03(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"6\n\x12ResetAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"=\n\x19\x43learAgentHistoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"?\n\x1b\x43ompactAgentHistoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xe5\x01\n\x1cGetAgentContextUsageResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\r\n\x05model\x18\x03 \x01(\t\x12\x16\n\x0e\x63ontext_window\x18\x04 \x01(\x03\x12\x14\n\x0cinput_tokens\x18\x05 \x01(\x03\x12\x15\n\routput_tokens\x18\x06 \x01(\x03\x12\x18\n\x10reasoning_tokens\x18\x07 \x01(\x03\x12\x19\n\x11\x63\x61\x63he_read_tokens\x18\x08 \x01(\x03\x12\x1a\n\x12\x63\x61\x63he_write_tokens\x18\t \x01(\x03\"3\n\x18RestoreCheckpointRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\"=\n\x19RestoreCheckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xa8\x01\n\x11\x45xperimentRunInfo\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_name\x18\x02 \x01(\t\x12\r\n\x05notes\x18\x03 \x01(\t\x12\x0f\n\x07\x63reated\x18\x04 \x01(\t\x12\x11\n\tlast_used\x18\x05 \x01(\t\x12\x1a\n\x12latest_weight_step\x18\x06 \x01(\x05\x12\x12\n\nis_current\x18\x07 \x01(\x08\"\x1b\n\x19ListExperimentRunsRequest\">\n\x1aListExperimentRunsResponse\x12 \n\x04runs\x18\x01 \x03(\x0b\x32\x12.ExperimentRunInfo\"G\n\x1aRenameExperimentRunRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\x12\x10\n\x08new_name\x18\x02 \x01(\t\"?\n\x1bRenameExperimentRunResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"F\n\x1cSetExperimentRunNotesRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\x12\r\n\x05notes\x18\x02 \x01(\t\"A\n\x1dSetExperimentRunNotesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"R\n\x18TriggerEvaluationRequest\x12\x12\n\nsplit_name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x14\n\x0cuse_full_set\x18\x03 \x01(\x08\"=\n\x19TriggerEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1c\n\x1aGetEvaluationStatusRequest\"\x81\x01\n\x1bGetEvaluationStatusResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x63urrent\x18\x02 \x01(\x05\x12\r\n\x05total\x18\x03 \x01(\x05\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x12\n\nsplit_name\x18\x06 \x01(\t\")\n\x17\x43\x61ncelEvaluationRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\"<\n\x18\x43\x61ncelEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"7\n\x16RunNotebookCellRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x65ll_id\x18\x02 \x01(\t\"2\n\x10NotebookCellDone\x12\x12\n\nexec_count\x18\x01 \x01(\x05\x12\n\n\x02ok\x18\x02 \x01(\x08\"\x1e\n\x1cInterruptNotebookCellRequest\":\n\x1dInterruptNotebookCellResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\"\xbd\x01\n\x11NotebookCellChunk\x12\x0f\n\x07\x63\x65ll_id\x18\x01 \x01(\t\x12\x10\n\x06stdout\x18\x02 \x01(\tH\x00\x12\x10\n\x06stderr\x18\x03 \x01(\tH\x00\x12\x15\n\x0bresult_text\x18\x04 \x01(\tH\x00\x12\x13\n\timage_png\x18\x05 \x01(\x0cH\x00\x12\x19\n\x0f\x65rror_traceback\x18\x06 \x01(\tH\x00\x12!\n\x04\x64one\x18\x07 \x01(\x0b\x32\x11.NotebookCellDoneH\x00\x42\t\n\x07payload\"S\n\x10NotebookResponse\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0f\n\x07\x65xisted\x18\x02 \x01(\x08\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"7\n\x13SaveNotebookRequest\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"M\n\x14SaveNotebookResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"C\n\x1bGenerateNotebookCodeRequest\x12\x0e\n\x06prompt\x18\x01 \x01(\t\x12\x14\n\x0c\x63ontext_code\x18\x02 \x01(\t\"\\\n\x1cGenerateNotebookCodeResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x13\n\x0b\x65xplanation\x18\x02 \x01(\t\x12\n\n\x02ok\x18\x03 \x01(\x08\x12\r\n\x05\x65rror\x18\x04 \x01(\t\"~\n\x18\x45xportAnnotationsRequest\x12\'\n\x06\x66ormat\x18\x01 \x01(\x0e\x32\x17.AnnotationExportFormat\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x1b\n\x13include_predictions\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\"\x88\x01\n\x19\x45xportAnnotationsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x10\n\x08\x66ilename\x18\x04 \x01(\t\x12\x11\n\tmime_type\x18\x05 \x01(\t\x12\x13\n\x0bimage_count\x18\x06 \x01(\x05*d\n\x13WeightOperationType\x12\n\n\x06ZEROFY\x10\x00\x12\x10\n\x0cREINITIALIZE\x10\x01\x12\n\n\x06\x46REEZE\x10\x02\x12\x12\n\x0eREMOVE_NEURONS\x10\t\x12\x0f\n\x0b\x41\x44\x44_NEURONS\x10\n*o\n\x0fZerofyPredicate\x12\x19\n\x15ZEROFY_PREDICATE_NONE\x10\x00\x12 \n\x1cZEROFY_PREDICATE_WITH_FROZEN\x10\x01\x12\x1f\n\x1bZEROFY_PREDICATE_WITH_OLDER\x10\x02*M\n\x0f\x41gentIntentType\x12\x12\n\x0eINTENT_UNKNOWN\x10\x00\x12\x11\n\rINTENT_FILTER\x10\x01\x12\x13\n\x0fINTENT_ANALYSIS\x10\x02*I\n\x0eSampleEditType\x12\x11\n\rEDIT_OVERRIDE\x10\x00\x12\x13\n\x0f\x45\x44IT_ACCUMULATE\x10\x01\x12\x0f\n\x0b\x45\x44IT_REMOVE\x10\x02*C\n\x11\x41gentProviderType\x12\x17\n\x13PROVIDER_OPENROUTER\x10\x00\x12\x15\n\x11PROVIDER_OPENCODE\x10\x01*m\n\x16\x41nnotationExportFormat\x12\x16\n\x12\x45XPORT_FORMAT_CVAT\x10\x00\x12\x1e\n\x1a\x45XPORT_FORMAT_LABEL_STUDIO\x10\x01\x12\x1b\n\x17\x45XPORT_FORMAT_V7_DARWIN\x10\x02\x32\xfe\x12\n\x11\x45xperimentService\x12P\n\x13GetLatestLoggerData\x12\x1b.GetLatestLoggerDataRequest\x1a\x1c.GetLatestLoggerDataResponse\x12\x36\n\x11\x45xperimentCommand\x12\x0f.TrainerCommand\x1a\x10.CommandResponse\x12H\n\x11ManipulateWeights\x12\x18.WeightsOperationRequest\x1a\x19.WeightsOperationResponse\x12/\n\nGetWeights\x12\x0f.WeightsRequest\x1a\x10.WeightsResponse\x12\x39\n\x0eGetActivations\x12\x12.ActivationRequest\x1a\x13.ActivationResponse\x12\x37\n\nGetSamples\x12\x13.BatchSampleRequest\x1a\x14.BatchSampleResponse\x12\x37\n\x0e\x41pplyDataQuery\x12\x11.DataQueryRequest\x1a\x12.DataQueryResponse\x12;\n\x0eGetDataSamples\x12\x13.DataSamplesRequest\x1a\x14.DataSamplesResponse\x12\x35\n\x0cGetHistogram\x12\x11.HistogramRequest\x1a\x12.HistogramResponse\x12\x38\n\x0bGetMetaData\x12\x13.GetMetaDataRequest\x1a\x14.GetMetaDataResponse\x12P\n\x13GetSignalTrajectory\x12\x1b.GetSignalTrajectoryRequest\x1a\x1c.GetSignalTrajectoryResponse\x12;\n\x0eGetStepSamples\x12\x13.StepSamplesRequest\x1a\x14.StepSamplesResponse\x12\x37\n\rGetPointCloud\x12\x12.PointCloudRequest\x1a\x10.PointCloudChunk0\x01\x12(\n\x08GetMedia\x12\r.MediaRequest\x1a\x0b.MediaChunk0\x01\x12\x37\n\x0e\x45\x64itDataSample\x12\x11.DataEditsRequest\x1a\x12.DataEditsResponse\x12,\n\rGetDataSplits\x12\x06.Empty\x1a\x13.DataSplitsResponse\x12\x30\n\x10\x43heckAgentHealth\x12\x06.Empty\x1a\x14.AgentHealthResponse\x12\x44\n\x0fInitializeAgent\x12\x17.InitializeAgentRequest\x1a\x18.InitializeAgentResponse\x12G\n\x10\x43hangeAgentModel\x12\x18.ChangeAgentModelRequest\x1a\x19.ChangeAgentModelResponse\x12\x41\n\x0eGetAgentModels\x12\x16.GetAgentModelsRequest\x1a\x17.GetAgentModelsResponse\x12)\n\nResetAgent\x12\x06.Empty\x1a\x13.ResetAgentResponse\x12\x37\n\x11\x43learAgentHistory\x12\x06.Empty\x1a\x1a.ClearAgentHistoryResponse\x12;\n\x13\x43ompactAgentHistory\x12\x06.Empty\x1a\x1c.CompactAgentHistoryResponse\x12=\n\x14GetAgentContextUsage\x12\x06.Empty\x1a\x1d.GetAgentContextUsageResponse\x12@\n\x0fRunNotebookCell\x12\x17.RunNotebookCellRequest\x1a\x12.NotebookCellChunk0\x01\x12V\n\x15InterruptNotebookCell\x12\x1d.InterruptNotebookCellRequest\x1a\x1e.InterruptNotebookCellResponse\x12(\n\x0bGetNotebook\x12\x06.Empty\x1a\x11.NotebookResponse\x12;\n\x0cSaveNotebook\x12\x14.SaveNotebookRequest\x1a\x15.SaveNotebookResponse\x12S\n\x14GenerateNotebookCode\x12\x1c.GenerateNotebookCodeRequest\x1a\x1d.GenerateNotebookCodeResponse\x12J\n\x11RestoreCheckpoint\x12\x19.RestoreCheckpointRequest\x1a\x1a.RestoreCheckpointResponse\x12M\n\x12ListExperimentRuns\x12\x1a.ListExperimentRunsRequest\x1a\x1b.ListExperimentRunsResponse\x12P\n\x13RenameExperimentRun\x12\x1b.RenameExperimentRunRequest\x1a\x1c.RenameExperimentRunResponse\x12V\n\x15SetExperimentRunNotes\x12\x1d.SetExperimentRunNotesRequest\x1a\x1e.SetExperimentRunNotesResponse\x12J\n\x11TriggerEvaluation\x12\x19.TriggerEvaluationRequest\x1a\x1a.TriggerEvaluationResponse\x12P\n\x13GetEvaluationStatus\x12\x1b.GetEvaluationStatusRequest\x1a\x1c.GetEvaluationStatusResponse\x12G\n\x10\x43\x61ncelEvaluation\x12\x18.CancelEvaluationRequest\x1a\x19.CancelEvaluationResponse\x12J\n\x11\x45xportAnnotations\x12\x19.ExportAnnotationsRequest\x1a\x1a.ExportAnnotationsResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'experiment_service_pb2', _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'weightslab.proto.experiment_service_pb2', _globals) if not _descriptor._USE_C_DESCRIPTORS: DESCRIPTOR._loaded_options = None _globals['_ANNOTATSTATUS_METADATAENTRY']._loaded_options = None @@ -37,226 +37,226 @@ _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_options = b'8\001' _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._loaded_options = None _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_options = b'8\001' - _globals['_WEIGHTOPERATIONTYPE']._serialized_start=13373 - _globals['_WEIGHTOPERATIONTYPE']._serialized_end=13473 - _globals['_ZEROFYPREDICATE']._serialized_start=13475 - _globals['_ZEROFYPREDICATE']._serialized_end=13586 - _globals['_AGENTINTENTTYPE']._serialized_start=13588 - _globals['_AGENTINTENTTYPE']._serialized_end=13665 - _globals['_SAMPLEEDITTYPE']._serialized_start=13667 - _globals['_SAMPLEEDITTYPE']._serialized_end=13740 - _globals['_AGENTPROVIDERTYPE']._serialized_start=13742 - _globals['_AGENTPROVIDERTYPE']._serialized_end=13809 - _globals['_ANNOTATIONEXPORTFORMAT']._serialized_start=13811 - _globals['_ANNOTATIONEXPORTFORMAT']._serialized_end=13920 - _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_start=29 - _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_end=286 - _globals['_SIGNALCURVEINDEX']._serialized_start=288 - _globals['_SIGNALCURVEINDEX']._serialized_end=412 - _globals['_SIGNALOUTLIER']._serialized_start=414 - _globals['_SIGNALOUTLIER']._serialized_end=463 - _globals['_LOGGERDATAPOINT']._serialized_start=466 - _globals['_LOGGERDATAPOINT']._serialized_end=932 - _globals['_GETLATESTLOGGERDATARESPONSE']._serialized_start=935 - _globals['_GETLATESTLOGGERDATARESPONSE']._serialized_end=1089 - _globals['_EMPTY']._serialized_start=1091 - _globals['_EMPTY']._serialized_end=1098 - _globals['_NEURONID']._serialized_start=1100 - _globals['_NEURONID']._serialized_end=1147 - _globals['_WEIGHTOPERATION']._serialized_start=1150 - _globals['_WEIGHTOPERATION']._serialized_end=1423 - _globals['_WEIGHTSOPERATIONREQUEST']._serialized_start=1425 - _globals['_WEIGHTSOPERATIONREQUEST']._serialized_end=1520 - _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_start=1522 - _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_end=1582 - _globals['_HYPERPARAMETERS']._serialized_start=1585 - _globals['_HYPERPARAMETERS']._serialized_end=2290 - _globals['_METRICSSTATUS']._serialized_start=2292 - _globals['_METRICSSTATUS']._serialized_end=2336 - _globals['_ANNOTATSTATUS']._serialized_start=2338 - _globals['_ANNOTATSTATUS']._serialized_end=2464 - _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_start=2417 - _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_end=2464 - _globals['_TRAININGSTATUSEX']._serialized_start=2467 - _globals['_TRAININGSTATUSEX']._serialized_end=2739 - _globals['_HYPERPARAMETERCOMMAND']._serialized_start=2741 - _globals['_HYPERPARAMETERCOMMAND']._serialized_end=2834 - _globals['_DENYSAMPLESOPERATION']._serialized_start=2836 - _globals['_DENYSAMPLESOPERATION']._serialized_end=2898 - _globals['_LOADCHECKPOINTOPERATION']._serialized_start=2900 - _globals['_LOADCHECKPOINTOPERATION']._serialized_end=2948 - _globals['_PLOTNOTEOPERATION']._serialized_start=2950 - _globals['_PLOTNOTEOPERATION']._serialized_end=3048 - _globals['_SAVECHECKPOINTOPERATION']._serialized_start=3050 - _globals['_SAVECHECKPOINTOPERATION']._serialized_end=3126 - _globals['_RESTARTINSTANCEOPERATION']._serialized_start=3128 - _globals['_RESTARTINSTANCEOPERATION']._serialized_end=3154 - _globals['_TRAINERCOMMAND']._serialized_start=3157 - _globals['_TRAINERCOMMAND']._serialized_end=4194 - _globals['_HYPERPARAMETERDESC']._serialized_start=4197 - _globals['_HYPERPARAMETERDESC']._serialized_end=4354 - _globals['_NEURONSTATISTICS']._serialized_start=4357 - _globals['_NEURONSTATISTICS']._serialized_end=4727 - _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_start=4586 - _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_end=4635 - _globals['_LAYERREPRESENTATION']._serialized_start=4730 - _globals['_LAYERREPRESENTATION']._serialized_end=5098 - _globals['_ACTIVATIONREQUEST']._serialized_start=5100 - _globals['_ACTIVATIONREQUEST']._serialized_end=5172 - _globals['_ACTIVATIONMAP']._serialized_start=5174 - _globals['_ACTIVATIONMAP']._serialized_end=5246 - _globals['_ACTIVATIONRESPONSE']._serialized_start=5248 - _globals['_ACTIVATIONRESPONSE']._serialized_end=5348 - _globals['_TASKFIELD']._serialized_start=5351 - _globals['_TASKFIELD']._serialized_end=5498 - _globals['_RECORDMETADATA']._serialized_start=5501 - _globals['_RECORDMETADATA']._serialized_end=5892 - _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_start=5839 - _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_end=5892 - _globals['_SAMPLESTATISTICS']._serialized_start=5895 - _globals['_SAMPLESTATISTICS']._serialized_end=6042 - _globals['_COMMANDRESPONSE']._serialized_start=6045 - _globals['_COMMANDRESPONSE']._serialized_end=6275 - _globals['_SAMPLEREQUEST']._serialized_start=6277 - _globals['_SAMPLEREQUEST']._serialized_end=6362 - _globals['_SAMPLEREQUESTRESPONSE']._serialized_start=6365 - _globals['_SAMPLEREQUESTRESPONSE']._serialized_end=6666 - _globals['_BATCHSAMPLEREQUEST']._serialized_start=6669 - _globals['_BATCHSAMPLEREQUEST']._serialized_end=6815 - _globals['_BATCHSAMPLERESPONSE']._serialized_start=6817 - _globals['_BATCHSAMPLERESPONSE']._serialized_end=6879 - _globals['_WEIGHTSREQUEST']._serialized_start=6881 - _globals['_WEIGHTSREQUEST']._serialized_end=6927 - _globals['_WEIGHTSRESPONSE']._serialized_start=6930 - _globals['_WEIGHTSRESPONSE']._serialized_end=7215 - _globals['_DATAQUERYREQUEST']._serialized_start=7217 - _globals['_DATAQUERYREQUEST']._serialized_end=7299 - _globals['_CATEGORICALTAGDEF']._serialized_start=7301 - _globals['_CATEGORICALTAGDEF']._serialized_end=7354 - _globals['_DATAQUERYRESPONSE']._serialized_start=7357 - _globals['_DATAQUERYRESPONSE']._serialized_end=7654 - _globals['_DATASAMPLESREQUEST']._serialized_start=7657 - _globals['_DATASAMPLESREQUEST']._serialized_end=7851 - _globals['_DATASTAT']._serialized_start=7853 - _globals['_DATASTAT']._serialized_end=7962 - _globals['_DATARECORD']._serialized_start=7964 - _globals['_DATARECORD']._serialized_end=8026 - _globals['_DATASAMPLESRESPONSE']._serialized_start=8028 - _globals['_DATASAMPLESRESPONSE']._serialized_end=8118 - _globals['_HISTOGRAMSUBBAR']._serialized_start=8120 - _globals['_HISTOGRAMSUBBAR']._serialized_end=8187 - _globals['_HISTOGRAMBIN']._serialized_start=8189 - _globals['_HISTOGRAMBIN']._serialized_end=8293 - _globals['_CATEGORICALHISTOGRAMBAR']._serialized_start=8295 - _globals['_CATEGORICALHISTOGRAMBAR']._serialized_end=8386 - _globals['_HISTOGRAMREQUEST']._serialized_start=8388 - _globals['_HISTOGRAMREQUEST']._serialized_end=8440 - _globals['_HISTOGRAMRESPONSE']._serialized_start=8443 - _globals['_HISTOGRAMRESPONSE']._serialized_end=8621 - _globals['_GETMETADATAREQUEST']._serialized_start=8623 - _globals['_GETMETADATAREQUEST']._serialized_end=8710 - _globals['_GETMETADATARESPONSE']._serialized_start=8713 - _globals['_GETMETADATARESPONSE']._serialized_end=8866 - _globals['_STEPSAMPLESREQUEST']._serialized_start=8868 - _globals['_STEPSAMPLESREQUEST']._serialized_end=8974 - _globals['_STEPSAMPLESRESPONSE']._serialized_start=8976 - _globals['_STEPSAMPLESRESPONSE']._serialized_end=9099 - _globals['_GETSIGNALTRAJECTORYREQUEST']._serialized_start=9101 - _globals['_GETSIGNALTRAJECTORYREQUEST']._serialized_end=9190 - _globals['_SIGNALTRAJECTORY']._serialized_start=9192 - _globals['_SIGNALTRAJECTORY']._serialized_end=9244 - _globals['_GETSIGNALTRAJECTORYRESPONSE']._serialized_start=9246 - _globals['_GETSIGNALTRAJECTORYRESPONSE']._serialized_end=9371 - _globals['_POINTCLOUDREQUEST']._serialized_start=9373 - _globals['_POINTCLOUDREQUEST']._serialized_end=9462 - _globals['_POINTCLOUDCHUNK']._serialized_start=9465 - _globals['_POINTCLOUDCHUNK']._serialized_end=9656 - _globals['_MEDIAREQUEST']._serialized_start=9658 - _globals['_MEDIAREQUEST']._serialized_end=9756 - _globals['_MEDIACHUNK']._serialized_start=9759 - _globals['_MEDIACHUNK']._serialized_end=10033 - _globals['_DATAEDITSREQUEST']._serialized_start=10036 - _globals['_DATAEDITSREQUEST']._serialized_end=10304 - _globals['_DATAEDITSRESPONSE']._serialized_start=10306 - _globals['_DATAEDITSRESPONSE']._serialized_end=10359 - _globals['_DATASPLITSRESPONSE']._serialized_start=10361 - _globals['_DATASPLITSRESPONSE']._serialized_end=10419 - _globals['_AGENTHEALTHRESPONSE']._serialized_start=10421 - _globals['_AGENTHEALTHRESPONSE']._serialized_end=10478 - _globals['_INITIALIZEAGENTREQUEST']._serialized_start=10480 - _globals['_INITIALIZEAGENTREQUEST']._serialized_end=10574 - _globals['_INITIALIZEAGENTRESPONSE']._serialized_start=10576 - _globals['_INITIALIZEAGENTRESPONSE']._serialized_end=10635 - _globals['_CHANGEAGENTMODELREQUEST']._serialized_start=10637 - _globals['_CHANGEAGENTMODELREQUEST']._serialized_end=10677 - _globals['_CHANGEAGENTMODELRESPONSE']._serialized_start=10679 - _globals['_CHANGEAGENTMODELRESPONSE']._serialized_end=10739 - _globals['_GETAGENTMODELSREQUEST']._serialized_start=10741 - _globals['_GETAGENTMODELSREQUEST']._serialized_end=10764 - _globals['_GETAGENTMODELSRESPONSE']._serialized_start=10766 - _globals['_GETAGENTMODELSRESPONSE']._serialized_end=10840 - _globals['_RESETAGENTRESPONSE']._serialized_start=10842 - _globals['_RESETAGENTRESPONSE']._serialized_end=10896 - _globals['_CLEARAGENTHISTORYRESPONSE']._serialized_start=10898 - _globals['_CLEARAGENTHISTORYRESPONSE']._serialized_end=10959 - _globals['_COMPACTAGENTHISTORYRESPONSE']._serialized_start=10961 - _globals['_COMPACTAGENTHISTORYRESPONSE']._serialized_end=11024 - _globals['_GETAGENTCONTEXTUSAGERESPONSE']._serialized_start=11027 - _globals['_GETAGENTCONTEXTUSAGERESPONSE']._serialized_end=11256 - _globals['_RESTORECHECKPOINTREQUEST']._serialized_start=11258 - _globals['_RESTORECHECKPOINTREQUEST']._serialized_end=11309 - _globals['_RESTORECHECKPOINTRESPONSE']._serialized_start=11311 - _globals['_RESTORECHECKPOINTRESPONSE']._serialized_end=11372 - _globals['_EXPERIMENTRUNINFO']._serialized_start=11375 - _globals['_EXPERIMENTRUNINFO']._serialized_end=11543 - _globals['_LISTEXPERIMENTRUNSREQUEST']._serialized_start=11545 - _globals['_LISTEXPERIMENTRUNSREQUEST']._serialized_end=11572 - _globals['_LISTEXPERIMENTRUNSRESPONSE']._serialized_start=11574 - _globals['_LISTEXPERIMENTRUNSRESPONSE']._serialized_end=11636 - _globals['_RENAMEEXPERIMENTRUNREQUEST']._serialized_start=11638 - _globals['_RENAMEEXPERIMENTRUNREQUEST']._serialized_end=11709 - _globals['_RENAMEEXPERIMENTRUNRESPONSE']._serialized_start=11711 - _globals['_RENAMEEXPERIMENTRUNRESPONSE']._serialized_end=11774 - _globals['_SETEXPERIMENTRUNNOTESREQUEST']._serialized_start=11776 - _globals['_SETEXPERIMENTRUNNOTESREQUEST']._serialized_end=11846 - _globals['_SETEXPERIMENTRUNNOTESRESPONSE']._serialized_start=11848 - _globals['_SETEXPERIMENTRUNNOTESRESPONSE']._serialized_end=11913 - _globals['_TRIGGEREVALUATIONREQUEST']._serialized_start=11915 - _globals['_TRIGGEREVALUATIONREQUEST']._serialized_end=11997 - _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_start=11999 - _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_end=12060 - _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_start=12062 - _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_end=12090 - _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_start=12093 - _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_end=12222 - _globals['_CANCELEVALUATIONREQUEST']._serialized_start=12224 - _globals['_CANCELEVALUATIONREQUEST']._serialized_end=12265 - _globals['_CANCELEVALUATIONRESPONSE']._serialized_start=12267 - _globals['_CANCELEVALUATIONRESPONSE']._serialized_end=12327 - _globals['_RUNNOTEBOOKCELLREQUEST']._serialized_start=12329 - _globals['_RUNNOTEBOOKCELLREQUEST']._serialized_end=12384 - _globals['_NOTEBOOKCELLDONE']._serialized_start=12386 - _globals['_NOTEBOOKCELLDONE']._serialized_end=12436 - _globals['_INTERRUPTNOTEBOOKCELLREQUEST']._serialized_start=12438 - _globals['_INTERRUPTNOTEBOOKCELLREQUEST']._serialized_end=12468 - _globals['_INTERRUPTNOTEBOOKCELLRESPONSE']._serialized_start=12470 - _globals['_INTERRUPTNOTEBOOKCELLRESPONSE']._serialized_end=12528 - _globals['_NOTEBOOKCELLCHUNK']._serialized_start=12531 - _globals['_NOTEBOOKCELLCHUNK']._serialized_end=12720 - _globals['_NOTEBOOKRESPONSE']._serialized_start=12722 - _globals['_NOTEBOOKRESPONSE']._serialized_end=12805 - _globals['_SAVENOTEBOOKREQUEST']._serialized_start=12807 - _globals['_SAVENOTEBOOKREQUEST']._serialized_end=12862 - _globals['_SAVENOTEBOOKRESPONSE']._serialized_start=12864 - _globals['_SAVENOTEBOOKRESPONSE']._serialized_end=12941 - _globals['_GENERATENOTEBOOKCODEREQUEST']._serialized_start=12943 - _globals['_GENERATENOTEBOOKCODEREQUEST']._serialized_end=13010 - _globals['_GENERATENOTEBOOKCODERESPONSE']._serialized_start=13012 - _globals['_GENERATENOTEBOOKCODERESPONSE']._serialized_end=13104 - _globals['_EXPORTANNOTATIONSREQUEST']._serialized_start=13106 - _globals['_EXPORTANNOTATIONSREQUEST']._serialized_end=13232 - _globals['_EXPORTANNOTATIONSRESPONSE']._serialized_start=13235 - _globals['_EXPORTANNOTATIONSRESPONSE']._serialized_end=13371 - _globals['_EXPERIMENTSERVICE']._serialized_start=13923 - _globals['_EXPERIMENTSERVICE']._serialized_end=16353 + _globals['_WEIGHTOPERATIONTYPE']._serialized_start=13390 + _globals['_WEIGHTOPERATIONTYPE']._serialized_end=13490 + _globals['_ZEROFYPREDICATE']._serialized_start=13492 + _globals['_ZEROFYPREDICATE']._serialized_end=13603 + _globals['_AGENTINTENTTYPE']._serialized_start=13605 + _globals['_AGENTINTENTTYPE']._serialized_end=13682 + _globals['_SAMPLEEDITTYPE']._serialized_start=13684 + _globals['_SAMPLEEDITTYPE']._serialized_end=13757 + _globals['_AGENTPROVIDERTYPE']._serialized_start=13759 + _globals['_AGENTPROVIDERTYPE']._serialized_end=13826 + _globals['_ANNOTATIONEXPORTFORMAT']._serialized_start=13828 + _globals['_ANNOTATIONEXPORTFORMAT']._serialized_end=13937 + _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_start=46 + _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_end=303 + _globals['_SIGNALCURVEINDEX']._serialized_start=305 + _globals['_SIGNALCURVEINDEX']._serialized_end=429 + _globals['_SIGNALOUTLIER']._serialized_start=431 + _globals['_SIGNALOUTLIER']._serialized_end=480 + _globals['_LOGGERDATAPOINT']._serialized_start=483 + _globals['_LOGGERDATAPOINT']._serialized_end=949 + _globals['_GETLATESTLOGGERDATARESPONSE']._serialized_start=952 + _globals['_GETLATESTLOGGERDATARESPONSE']._serialized_end=1106 + _globals['_EMPTY']._serialized_start=1108 + _globals['_EMPTY']._serialized_end=1115 + _globals['_NEURONID']._serialized_start=1117 + _globals['_NEURONID']._serialized_end=1164 + _globals['_WEIGHTOPERATION']._serialized_start=1167 + _globals['_WEIGHTOPERATION']._serialized_end=1440 + _globals['_WEIGHTSOPERATIONREQUEST']._serialized_start=1442 + _globals['_WEIGHTSOPERATIONREQUEST']._serialized_end=1537 + _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_start=1539 + _globals['_WEIGHTSOPERATIONRESPONSE']._serialized_end=1599 + _globals['_HYPERPARAMETERS']._serialized_start=1602 + _globals['_HYPERPARAMETERS']._serialized_end=2307 + _globals['_METRICSSTATUS']._serialized_start=2309 + _globals['_METRICSSTATUS']._serialized_end=2353 + _globals['_ANNOTATSTATUS']._serialized_start=2355 + _globals['_ANNOTATSTATUS']._serialized_end=2481 + _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_start=2434 + _globals['_ANNOTATSTATUS_METADATAENTRY']._serialized_end=2481 + _globals['_TRAININGSTATUSEX']._serialized_start=2484 + _globals['_TRAININGSTATUSEX']._serialized_end=2756 + _globals['_HYPERPARAMETERCOMMAND']._serialized_start=2758 + _globals['_HYPERPARAMETERCOMMAND']._serialized_end=2851 + _globals['_DENYSAMPLESOPERATION']._serialized_start=2853 + _globals['_DENYSAMPLESOPERATION']._serialized_end=2915 + _globals['_LOADCHECKPOINTOPERATION']._serialized_start=2917 + _globals['_LOADCHECKPOINTOPERATION']._serialized_end=2965 + _globals['_PLOTNOTEOPERATION']._serialized_start=2967 + _globals['_PLOTNOTEOPERATION']._serialized_end=3065 + _globals['_SAVECHECKPOINTOPERATION']._serialized_start=3067 + _globals['_SAVECHECKPOINTOPERATION']._serialized_end=3143 + _globals['_RESTARTINSTANCEOPERATION']._serialized_start=3145 + _globals['_RESTARTINSTANCEOPERATION']._serialized_end=3171 + _globals['_TRAINERCOMMAND']._serialized_start=3174 + _globals['_TRAINERCOMMAND']._serialized_end=4211 + _globals['_HYPERPARAMETERDESC']._serialized_start=4214 + _globals['_HYPERPARAMETERDESC']._serialized_end=4371 + _globals['_NEURONSTATISTICS']._serialized_start=4374 + _globals['_NEURONSTATISTICS']._serialized_end=4744 + _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_start=4603 + _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_end=4652 + _globals['_LAYERREPRESENTATION']._serialized_start=4747 + _globals['_LAYERREPRESENTATION']._serialized_end=5115 + _globals['_ACTIVATIONREQUEST']._serialized_start=5117 + _globals['_ACTIVATIONREQUEST']._serialized_end=5189 + _globals['_ACTIVATIONMAP']._serialized_start=5191 + _globals['_ACTIVATIONMAP']._serialized_end=5263 + _globals['_ACTIVATIONRESPONSE']._serialized_start=5265 + _globals['_ACTIVATIONRESPONSE']._serialized_end=5365 + _globals['_TASKFIELD']._serialized_start=5368 + _globals['_TASKFIELD']._serialized_end=5515 + _globals['_RECORDMETADATA']._serialized_start=5518 + _globals['_RECORDMETADATA']._serialized_end=5909 + _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_start=5856 + _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_end=5909 + _globals['_SAMPLESTATISTICS']._serialized_start=5912 + _globals['_SAMPLESTATISTICS']._serialized_end=6059 + _globals['_COMMANDRESPONSE']._serialized_start=6062 + _globals['_COMMANDRESPONSE']._serialized_end=6292 + _globals['_SAMPLEREQUEST']._serialized_start=6294 + _globals['_SAMPLEREQUEST']._serialized_end=6379 + _globals['_SAMPLEREQUESTRESPONSE']._serialized_start=6382 + _globals['_SAMPLEREQUESTRESPONSE']._serialized_end=6683 + _globals['_BATCHSAMPLEREQUEST']._serialized_start=6686 + _globals['_BATCHSAMPLEREQUEST']._serialized_end=6832 + _globals['_BATCHSAMPLERESPONSE']._serialized_start=6834 + _globals['_BATCHSAMPLERESPONSE']._serialized_end=6896 + _globals['_WEIGHTSREQUEST']._serialized_start=6898 + _globals['_WEIGHTSREQUEST']._serialized_end=6944 + _globals['_WEIGHTSRESPONSE']._serialized_start=6947 + _globals['_WEIGHTSRESPONSE']._serialized_end=7232 + _globals['_DATAQUERYREQUEST']._serialized_start=7234 + _globals['_DATAQUERYREQUEST']._serialized_end=7316 + _globals['_CATEGORICALTAGDEF']._serialized_start=7318 + _globals['_CATEGORICALTAGDEF']._serialized_end=7371 + _globals['_DATAQUERYRESPONSE']._serialized_start=7374 + _globals['_DATAQUERYRESPONSE']._serialized_end=7671 + _globals['_DATASAMPLESREQUEST']._serialized_start=7674 + _globals['_DATASAMPLESREQUEST']._serialized_end=7868 + _globals['_DATASTAT']._serialized_start=7870 + _globals['_DATASTAT']._serialized_end=7979 + _globals['_DATARECORD']._serialized_start=7981 + _globals['_DATARECORD']._serialized_end=8043 + _globals['_DATASAMPLESRESPONSE']._serialized_start=8045 + _globals['_DATASAMPLESRESPONSE']._serialized_end=8135 + _globals['_HISTOGRAMSUBBAR']._serialized_start=8137 + _globals['_HISTOGRAMSUBBAR']._serialized_end=8204 + _globals['_HISTOGRAMBIN']._serialized_start=8206 + _globals['_HISTOGRAMBIN']._serialized_end=8310 + _globals['_CATEGORICALHISTOGRAMBAR']._serialized_start=8312 + _globals['_CATEGORICALHISTOGRAMBAR']._serialized_end=8403 + _globals['_HISTOGRAMREQUEST']._serialized_start=8405 + _globals['_HISTOGRAMREQUEST']._serialized_end=8457 + _globals['_HISTOGRAMRESPONSE']._serialized_start=8460 + _globals['_HISTOGRAMRESPONSE']._serialized_end=8638 + _globals['_GETMETADATAREQUEST']._serialized_start=8640 + _globals['_GETMETADATAREQUEST']._serialized_end=8727 + _globals['_GETMETADATARESPONSE']._serialized_start=8730 + _globals['_GETMETADATARESPONSE']._serialized_end=8883 + _globals['_STEPSAMPLESREQUEST']._serialized_start=8885 + _globals['_STEPSAMPLESREQUEST']._serialized_end=8991 + _globals['_STEPSAMPLESRESPONSE']._serialized_start=8993 + _globals['_STEPSAMPLESRESPONSE']._serialized_end=9116 + _globals['_GETSIGNALTRAJECTORYREQUEST']._serialized_start=9118 + _globals['_GETSIGNALTRAJECTORYREQUEST']._serialized_end=9207 + _globals['_SIGNALTRAJECTORY']._serialized_start=9209 + _globals['_SIGNALTRAJECTORY']._serialized_end=9261 + _globals['_GETSIGNALTRAJECTORYRESPONSE']._serialized_start=9263 + _globals['_GETSIGNALTRAJECTORYRESPONSE']._serialized_end=9388 + _globals['_POINTCLOUDREQUEST']._serialized_start=9390 + _globals['_POINTCLOUDREQUEST']._serialized_end=9479 + _globals['_POINTCLOUDCHUNK']._serialized_start=9482 + _globals['_POINTCLOUDCHUNK']._serialized_end=9673 + _globals['_MEDIAREQUEST']._serialized_start=9675 + _globals['_MEDIAREQUEST']._serialized_end=9773 + _globals['_MEDIACHUNK']._serialized_start=9776 + _globals['_MEDIACHUNK']._serialized_end=10050 + _globals['_DATAEDITSREQUEST']._serialized_start=10053 + _globals['_DATAEDITSREQUEST']._serialized_end=10321 + _globals['_DATAEDITSRESPONSE']._serialized_start=10323 + _globals['_DATAEDITSRESPONSE']._serialized_end=10376 + _globals['_DATASPLITSRESPONSE']._serialized_start=10378 + _globals['_DATASPLITSRESPONSE']._serialized_end=10436 + _globals['_AGENTHEALTHRESPONSE']._serialized_start=10438 + _globals['_AGENTHEALTHRESPONSE']._serialized_end=10495 + _globals['_INITIALIZEAGENTREQUEST']._serialized_start=10497 + _globals['_INITIALIZEAGENTREQUEST']._serialized_end=10591 + _globals['_INITIALIZEAGENTRESPONSE']._serialized_start=10593 + _globals['_INITIALIZEAGENTRESPONSE']._serialized_end=10652 + _globals['_CHANGEAGENTMODELREQUEST']._serialized_start=10654 + _globals['_CHANGEAGENTMODELREQUEST']._serialized_end=10694 + _globals['_CHANGEAGENTMODELRESPONSE']._serialized_start=10696 + _globals['_CHANGEAGENTMODELRESPONSE']._serialized_end=10756 + _globals['_GETAGENTMODELSREQUEST']._serialized_start=10758 + _globals['_GETAGENTMODELSREQUEST']._serialized_end=10781 + _globals['_GETAGENTMODELSRESPONSE']._serialized_start=10783 + _globals['_GETAGENTMODELSRESPONSE']._serialized_end=10857 + _globals['_RESETAGENTRESPONSE']._serialized_start=10859 + _globals['_RESETAGENTRESPONSE']._serialized_end=10913 + _globals['_CLEARAGENTHISTORYRESPONSE']._serialized_start=10915 + _globals['_CLEARAGENTHISTORYRESPONSE']._serialized_end=10976 + _globals['_COMPACTAGENTHISTORYRESPONSE']._serialized_start=10978 + _globals['_COMPACTAGENTHISTORYRESPONSE']._serialized_end=11041 + _globals['_GETAGENTCONTEXTUSAGERESPONSE']._serialized_start=11044 + _globals['_GETAGENTCONTEXTUSAGERESPONSE']._serialized_end=11273 + _globals['_RESTORECHECKPOINTREQUEST']._serialized_start=11275 + _globals['_RESTORECHECKPOINTREQUEST']._serialized_end=11326 + _globals['_RESTORECHECKPOINTRESPONSE']._serialized_start=11328 + _globals['_RESTORECHECKPOINTRESPONSE']._serialized_end=11389 + _globals['_EXPERIMENTRUNINFO']._serialized_start=11392 + _globals['_EXPERIMENTRUNINFO']._serialized_end=11560 + _globals['_LISTEXPERIMENTRUNSREQUEST']._serialized_start=11562 + _globals['_LISTEXPERIMENTRUNSREQUEST']._serialized_end=11589 + _globals['_LISTEXPERIMENTRUNSRESPONSE']._serialized_start=11591 + _globals['_LISTEXPERIMENTRUNSRESPONSE']._serialized_end=11653 + _globals['_RENAMEEXPERIMENTRUNREQUEST']._serialized_start=11655 + _globals['_RENAMEEXPERIMENTRUNREQUEST']._serialized_end=11726 + _globals['_RENAMEEXPERIMENTRUNRESPONSE']._serialized_start=11728 + _globals['_RENAMEEXPERIMENTRUNRESPONSE']._serialized_end=11791 + _globals['_SETEXPERIMENTRUNNOTESREQUEST']._serialized_start=11793 + _globals['_SETEXPERIMENTRUNNOTESREQUEST']._serialized_end=11863 + _globals['_SETEXPERIMENTRUNNOTESRESPONSE']._serialized_start=11865 + _globals['_SETEXPERIMENTRUNNOTESRESPONSE']._serialized_end=11930 + _globals['_TRIGGEREVALUATIONREQUEST']._serialized_start=11932 + _globals['_TRIGGEREVALUATIONREQUEST']._serialized_end=12014 + _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_start=12016 + _globals['_TRIGGEREVALUATIONRESPONSE']._serialized_end=12077 + _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_start=12079 + _globals['_GETEVALUATIONSTATUSREQUEST']._serialized_end=12107 + _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_start=12110 + _globals['_GETEVALUATIONSTATUSRESPONSE']._serialized_end=12239 + _globals['_CANCELEVALUATIONREQUEST']._serialized_start=12241 + _globals['_CANCELEVALUATIONREQUEST']._serialized_end=12282 + _globals['_CANCELEVALUATIONRESPONSE']._serialized_start=12284 + _globals['_CANCELEVALUATIONRESPONSE']._serialized_end=12344 + _globals['_RUNNOTEBOOKCELLREQUEST']._serialized_start=12346 + _globals['_RUNNOTEBOOKCELLREQUEST']._serialized_end=12401 + _globals['_NOTEBOOKCELLDONE']._serialized_start=12403 + _globals['_NOTEBOOKCELLDONE']._serialized_end=12453 + _globals['_INTERRUPTNOTEBOOKCELLREQUEST']._serialized_start=12455 + _globals['_INTERRUPTNOTEBOOKCELLREQUEST']._serialized_end=12485 + _globals['_INTERRUPTNOTEBOOKCELLRESPONSE']._serialized_start=12487 + _globals['_INTERRUPTNOTEBOOKCELLRESPONSE']._serialized_end=12545 + _globals['_NOTEBOOKCELLCHUNK']._serialized_start=12548 + _globals['_NOTEBOOKCELLCHUNK']._serialized_end=12737 + _globals['_NOTEBOOKRESPONSE']._serialized_start=12739 + _globals['_NOTEBOOKRESPONSE']._serialized_end=12822 + _globals['_SAVENOTEBOOKREQUEST']._serialized_start=12824 + _globals['_SAVENOTEBOOKREQUEST']._serialized_end=12879 + _globals['_SAVENOTEBOOKRESPONSE']._serialized_start=12881 + _globals['_SAVENOTEBOOKRESPONSE']._serialized_end=12958 + _globals['_GENERATENOTEBOOKCODEREQUEST']._serialized_start=12960 + _globals['_GENERATENOTEBOOKCODEREQUEST']._serialized_end=13027 + _globals['_GENERATENOTEBOOKCODERESPONSE']._serialized_start=13029 + _globals['_GENERATENOTEBOOKCODERESPONSE']._serialized_end=13121 + _globals['_EXPORTANNOTATIONSREQUEST']._serialized_start=13123 + _globals['_EXPORTANNOTATIONSREQUEST']._serialized_end=13249 + _globals['_EXPORTANNOTATIONSRESPONSE']._serialized_start=13252 + _globals['_EXPORTANNOTATIONSRESPONSE']._serialized_end=13388 + _globals['_EXPERIMENTSERVICE']._serialized_start=13940 + _globals['_EXPERIMENTSERVICE']._serialized_end=16370 # @@protoc_insertion_point(module_scope) diff --git a/weightslab/proto/experiment_service_pb2_grpc.py b/weightslab/proto/experiment_service_pb2_grpc.py index 7050452f..dd5d1209 100644 --- a/weightslab/proto/experiment_service_pb2_grpc.py +++ b/weightslab/proto/experiment_service_pb2_grpc.py @@ -3,7 +3,7 @@ import grpc import warnings -import experiment_service_pb2 as experiment__service__pb2 +from weightslab.proto import experiment_service_pb2 as weightslab_dot_proto_dot_experiment__service__pb2 GRPC_GENERATED_VERSION = '1.68.1' GRPC_VERSION = grpc.__version__ @@ -18,7 +18,7 @@ if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' - + f' but the generated code in experiment_service_pb2_grpc.py depends on' + + f' but the generated code in weightslab/proto/experiment_service_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' @@ -36,188 +36,188 @@ def __init__(self, channel): """ self.GetLatestLoggerData = channel.unary_unary( '/ExperimentService/GetLatestLoggerData', - request_serializer=experiment__service__pb2.GetLatestLoggerDataRequest.SerializeToString, - response_deserializer=experiment__service__pb2.GetLatestLoggerDataResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetLatestLoggerDataRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetLatestLoggerDataResponse.FromString, _registered_method=True) self.ExperimentCommand = channel.unary_unary( '/ExperimentService/ExperimentCommand', - request_serializer=experiment__service__pb2.TrainerCommand.SerializeToString, - response_deserializer=experiment__service__pb2.CommandResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.TrainerCommand.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.CommandResponse.FromString, _registered_method=True) self.ManipulateWeights = channel.unary_unary( '/ExperimentService/ManipulateWeights', - request_serializer=experiment__service__pb2.WeightsOperationRequest.SerializeToString, - response_deserializer=experiment__service__pb2.WeightsOperationResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsOperationRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsOperationResponse.FromString, _registered_method=True) self.GetWeights = channel.unary_unary( '/ExperimentService/GetWeights', - request_serializer=experiment__service__pb2.WeightsRequest.SerializeToString, - response_deserializer=experiment__service__pb2.WeightsResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsResponse.FromString, _registered_method=True) self.GetActivations = channel.unary_unary( '/ExperimentService/GetActivations', - request_serializer=experiment__service__pb2.ActivationRequest.SerializeToString, - response_deserializer=experiment__service__pb2.ActivationResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ActivationRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ActivationResponse.FromString, _registered_method=True) self.GetSamples = channel.unary_unary( '/ExperimentService/GetSamples', - request_serializer=experiment__service__pb2.BatchSampleRequest.SerializeToString, - response_deserializer=experiment__service__pb2.BatchSampleResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.BatchSampleRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.BatchSampleResponse.FromString, _registered_method=True) self.ApplyDataQuery = channel.unary_unary( '/ExperimentService/ApplyDataQuery', - request_serializer=experiment__service__pb2.DataQueryRequest.SerializeToString, - response_deserializer=experiment__service__pb2.DataQueryResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.DataQueryRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.DataQueryResponse.FromString, _registered_method=True) self.GetDataSamples = channel.unary_unary( '/ExperimentService/GetDataSamples', - request_serializer=experiment__service__pb2.DataSamplesRequest.SerializeToString, - response_deserializer=experiment__service__pb2.DataSamplesResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.DataSamplesRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.DataSamplesResponse.FromString, _registered_method=True) self.GetHistogram = channel.unary_unary( '/ExperimentService/GetHistogram', - request_serializer=experiment__service__pb2.HistogramRequest.SerializeToString, - response_deserializer=experiment__service__pb2.HistogramResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.HistogramRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.HistogramResponse.FromString, _registered_method=True) self.GetMetaData = channel.unary_unary( '/ExperimentService/GetMetaData', - request_serializer=experiment__service__pb2.GetMetaDataRequest.SerializeToString, - response_deserializer=experiment__service__pb2.GetMetaDataResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetMetaDataRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetMetaDataResponse.FromString, _registered_method=True) self.GetSignalTrajectory = channel.unary_unary( '/ExperimentService/GetSignalTrajectory', - request_serializer=experiment__service__pb2.GetSignalTrajectoryRequest.SerializeToString, - response_deserializer=experiment__service__pb2.GetSignalTrajectoryResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetSignalTrajectoryRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetSignalTrajectoryResponse.FromString, _registered_method=True) self.GetStepSamples = channel.unary_unary( '/ExperimentService/GetStepSamples', - request_serializer=experiment__service__pb2.StepSamplesRequest.SerializeToString, - response_deserializer=experiment__service__pb2.StepSamplesResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.StepSamplesRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.StepSamplesResponse.FromString, _registered_method=True) self.GetPointCloud = channel.unary_stream( '/ExperimentService/GetPointCloud', - request_serializer=experiment__service__pb2.PointCloudRequest.SerializeToString, - response_deserializer=experiment__service__pb2.PointCloudChunk.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.PointCloudRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.PointCloudChunk.FromString, _registered_method=True) self.GetMedia = channel.unary_stream( '/ExperimentService/GetMedia', - request_serializer=experiment__service__pb2.MediaRequest.SerializeToString, - response_deserializer=experiment__service__pb2.MediaChunk.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.MediaRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.MediaChunk.FromString, _registered_method=True) self.EditDataSample = channel.unary_unary( '/ExperimentService/EditDataSample', - request_serializer=experiment__service__pb2.DataEditsRequest.SerializeToString, - response_deserializer=experiment__service__pb2.DataEditsResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.DataEditsRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.DataEditsResponse.FromString, _registered_method=True) self.GetDataSplits = channel.unary_unary( '/ExperimentService/GetDataSplits', - request_serializer=experiment__service__pb2.Empty.SerializeToString, - response_deserializer=experiment__service__pb2.DataSplitsResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.DataSplitsResponse.FromString, _registered_method=True) self.CheckAgentHealth = channel.unary_unary( '/ExperimentService/CheckAgentHealth', - request_serializer=experiment__service__pb2.Empty.SerializeToString, - response_deserializer=experiment__service__pb2.AgentHealthResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.AgentHealthResponse.FromString, _registered_method=True) self.InitializeAgent = channel.unary_unary( '/ExperimentService/InitializeAgent', - request_serializer=experiment__service__pb2.InitializeAgentRequest.SerializeToString, - response_deserializer=experiment__service__pb2.InitializeAgentResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.InitializeAgentRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.InitializeAgentResponse.FromString, _registered_method=True) self.ChangeAgentModel = channel.unary_unary( '/ExperimentService/ChangeAgentModel', - request_serializer=experiment__service__pb2.ChangeAgentModelRequest.SerializeToString, - response_deserializer=experiment__service__pb2.ChangeAgentModelResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ChangeAgentModelRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ChangeAgentModelResponse.FromString, _registered_method=True) self.GetAgentModels = channel.unary_unary( '/ExperimentService/GetAgentModels', - request_serializer=experiment__service__pb2.GetAgentModelsRequest.SerializeToString, - response_deserializer=experiment__service__pb2.GetAgentModelsResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetAgentModelsRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetAgentModelsResponse.FromString, _registered_method=True) self.ResetAgent = channel.unary_unary( '/ExperimentService/ResetAgent', - request_serializer=experiment__service__pb2.Empty.SerializeToString, - response_deserializer=experiment__service__pb2.ResetAgentResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ResetAgentResponse.FromString, _registered_method=True) self.ClearAgentHistory = channel.unary_unary( '/ExperimentService/ClearAgentHistory', - request_serializer=experiment__service__pb2.Empty.SerializeToString, - response_deserializer=experiment__service__pb2.ClearAgentHistoryResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ClearAgentHistoryResponse.FromString, _registered_method=True) self.CompactAgentHistory = channel.unary_unary( '/ExperimentService/CompactAgentHistory', - request_serializer=experiment__service__pb2.Empty.SerializeToString, - response_deserializer=experiment__service__pb2.CompactAgentHistoryResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.CompactAgentHistoryResponse.FromString, _registered_method=True) self.GetAgentContextUsage = channel.unary_unary( '/ExperimentService/GetAgentContextUsage', - request_serializer=experiment__service__pb2.Empty.SerializeToString, - response_deserializer=experiment__service__pb2.GetAgentContextUsageResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetAgentContextUsageResponse.FromString, _registered_method=True) self.RunNotebookCell = channel.unary_stream( '/ExperimentService/RunNotebookCell', - request_serializer=experiment__service__pb2.RunNotebookCellRequest.SerializeToString, - response_deserializer=experiment__service__pb2.NotebookCellChunk.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.RunNotebookCellRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.NotebookCellChunk.FromString, _registered_method=True) self.InterruptNotebookCell = channel.unary_unary( '/ExperimentService/InterruptNotebookCell', - request_serializer=experiment__service__pb2.InterruptNotebookCellRequest.SerializeToString, - response_deserializer=experiment__service__pb2.InterruptNotebookCellResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.InterruptNotebookCellRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.InterruptNotebookCellResponse.FromString, _registered_method=True) self.GetNotebook = channel.unary_unary( '/ExperimentService/GetNotebook', - request_serializer=experiment__service__pb2.Empty.SerializeToString, - response_deserializer=experiment__service__pb2.NotebookResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.NotebookResponse.FromString, _registered_method=True) self.SaveNotebook = channel.unary_unary( '/ExperimentService/SaveNotebook', - request_serializer=experiment__service__pb2.SaveNotebookRequest.SerializeToString, - response_deserializer=experiment__service__pb2.SaveNotebookResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.SaveNotebookRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.SaveNotebookResponse.FromString, _registered_method=True) self.GenerateNotebookCode = channel.unary_unary( '/ExperimentService/GenerateNotebookCode', - request_serializer=experiment__service__pb2.GenerateNotebookCodeRequest.SerializeToString, - response_deserializer=experiment__service__pb2.GenerateNotebookCodeResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GenerateNotebookCodeRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GenerateNotebookCodeResponse.FromString, _registered_method=True) self.RestoreCheckpoint = channel.unary_unary( '/ExperimentService/RestoreCheckpoint', - request_serializer=experiment__service__pb2.RestoreCheckpointRequest.SerializeToString, - response_deserializer=experiment__service__pb2.RestoreCheckpointResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.RestoreCheckpointRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.RestoreCheckpointResponse.FromString, _registered_method=True) self.ListExperimentRuns = channel.unary_unary( '/ExperimentService/ListExperimentRuns', - request_serializer=experiment__service__pb2.ListExperimentRunsRequest.SerializeToString, - response_deserializer=experiment__service__pb2.ListExperimentRunsResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ListExperimentRunsRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ListExperimentRunsResponse.FromString, _registered_method=True) self.RenameExperimentRun = channel.unary_unary( '/ExperimentService/RenameExperimentRun', - request_serializer=experiment__service__pb2.RenameExperimentRunRequest.SerializeToString, - response_deserializer=experiment__service__pb2.RenameExperimentRunResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.RenameExperimentRunRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.RenameExperimentRunResponse.FromString, _registered_method=True) self.SetExperimentRunNotes = channel.unary_unary( '/ExperimentService/SetExperimentRunNotes', - request_serializer=experiment__service__pb2.SetExperimentRunNotesRequest.SerializeToString, - response_deserializer=experiment__service__pb2.SetExperimentRunNotesResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.SetExperimentRunNotesRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.SetExperimentRunNotesResponse.FromString, _registered_method=True) self.TriggerEvaluation = channel.unary_unary( '/ExperimentService/TriggerEvaluation', - request_serializer=experiment__service__pb2.TriggerEvaluationRequest.SerializeToString, - response_deserializer=experiment__service__pb2.TriggerEvaluationResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.TriggerEvaluationRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.TriggerEvaluationResponse.FromString, _registered_method=True) self.GetEvaluationStatus = channel.unary_unary( '/ExperimentService/GetEvaluationStatus', - request_serializer=experiment__service__pb2.GetEvaluationStatusRequest.SerializeToString, - response_deserializer=experiment__service__pb2.GetEvaluationStatusResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetEvaluationStatusRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetEvaluationStatusResponse.FromString, _registered_method=True) self.CancelEvaluation = channel.unary_unary( '/ExperimentService/CancelEvaluation', - request_serializer=experiment__service__pb2.CancelEvaluationRequest.SerializeToString, - response_deserializer=experiment__service__pb2.CancelEvaluationResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationResponse.FromString, _registered_method=True) self.ExportAnnotations = channel.unary_unary( '/ExperimentService/ExportAnnotations', - request_serializer=experiment__service__pb2.ExportAnnotationsRequest.SerializeToString, - response_deserializer=experiment__service__pb2.ExportAnnotationsResponse.FromString, + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsResponse.FromString, _registered_method=True) @@ -499,188 +499,188 @@ def add_ExperimentServiceServicer_to_server(servicer, server): rpc_method_handlers = { 'GetLatestLoggerData': grpc.unary_unary_rpc_method_handler( servicer.GetLatestLoggerData, - request_deserializer=experiment__service__pb2.GetLatestLoggerDataRequest.FromString, - response_serializer=experiment__service__pb2.GetLatestLoggerDataResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetLatestLoggerDataRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetLatestLoggerDataResponse.SerializeToString, ), 'ExperimentCommand': grpc.unary_unary_rpc_method_handler( servicer.ExperimentCommand, - request_deserializer=experiment__service__pb2.TrainerCommand.FromString, - response_serializer=experiment__service__pb2.CommandResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.TrainerCommand.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.CommandResponse.SerializeToString, ), 'ManipulateWeights': grpc.unary_unary_rpc_method_handler( servicer.ManipulateWeights, - request_deserializer=experiment__service__pb2.WeightsOperationRequest.FromString, - response_serializer=experiment__service__pb2.WeightsOperationResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsOperationRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsOperationResponse.SerializeToString, ), 'GetWeights': grpc.unary_unary_rpc_method_handler( servicer.GetWeights, - request_deserializer=experiment__service__pb2.WeightsRequest.FromString, - response_serializer=experiment__service__pb2.WeightsResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.WeightsResponse.SerializeToString, ), 'GetActivations': grpc.unary_unary_rpc_method_handler( servicer.GetActivations, - request_deserializer=experiment__service__pb2.ActivationRequest.FromString, - response_serializer=experiment__service__pb2.ActivationResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ActivationRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ActivationResponse.SerializeToString, ), 'GetSamples': grpc.unary_unary_rpc_method_handler( servicer.GetSamples, - request_deserializer=experiment__service__pb2.BatchSampleRequest.FromString, - response_serializer=experiment__service__pb2.BatchSampleResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.BatchSampleRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.BatchSampleResponse.SerializeToString, ), 'ApplyDataQuery': grpc.unary_unary_rpc_method_handler( servicer.ApplyDataQuery, - request_deserializer=experiment__service__pb2.DataQueryRequest.FromString, - response_serializer=experiment__service__pb2.DataQueryResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.DataQueryRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.DataQueryResponse.SerializeToString, ), 'GetDataSamples': grpc.unary_unary_rpc_method_handler( servicer.GetDataSamples, - request_deserializer=experiment__service__pb2.DataSamplesRequest.FromString, - response_serializer=experiment__service__pb2.DataSamplesResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.DataSamplesRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.DataSamplesResponse.SerializeToString, ), 'GetHistogram': grpc.unary_unary_rpc_method_handler( servicer.GetHistogram, - request_deserializer=experiment__service__pb2.HistogramRequest.FromString, - response_serializer=experiment__service__pb2.HistogramResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.HistogramRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.HistogramResponse.SerializeToString, ), 'GetMetaData': grpc.unary_unary_rpc_method_handler( servicer.GetMetaData, - request_deserializer=experiment__service__pb2.GetMetaDataRequest.FromString, - response_serializer=experiment__service__pb2.GetMetaDataResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetMetaDataRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetMetaDataResponse.SerializeToString, ), 'GetSignalTrajectory': grpc.unary_unary_rpc_method_handler( servicer.GetSignalTrajectory, - request_deserializer=experiment__service__pb2.GetSignalTrajectoryRequest.FromString, - response_serializer=experiment__service__pb2.GetSignalTrajectoryResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetSignalTrajectoryRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetSignalTrajectoryResponse.SerializeToString, ), 'GetStepSamples': grpc.unary_unary_rpc_method_handler( servicer.GetStepSamples, - request_deserializer=experiment__service__pb2.StepSamplesRequest.FromString, - response_serializer=experiment__service__pb2.StepSamplesResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.StepSamplesRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.StepSamplesResponse.SerializeToString, ), 'GetPointCloud': grpc.unary_stream_rpc_method_handler( servicer.GetPointCloud, - request_deserializer=experiment__service__pb2.PointCloudRequest.FromString, - response_serializer=experiment__service__pb2.PointCloudChunk.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.PointCloudRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.PointCloudChunk.SerializeToString, ), 'GetMedia': grpc.unary_stream_rpc_method_handler( servicer.GetMedia, - request_deserializer=experiment__service__pb2.MediaRequest.FromString, - response_serializer=experiment__service__pb2.MediaChunk.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.MediaRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.MediaChunk.SerializeToString, ), 'EditDataSample': grpc.unary_unary_rpc_method_handler( servicer.EditDataSample, - request_deserializer=experiment__service__pb2.DataEditsRequest.FromString, - response_serializer=experiment__service__pb2.DataEditsResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.DataEditsRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.DataEditsResponse.SerializeToString, ), 'GetDataSplits': grpc.unary_unary_rpc_method_handler( servicer.GetDataSplits, - request_deserializer=experiment__service__pb2.Empty.FromString, - response_serializer=experiment__service__pb2.DataSplitsResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.DataSplitsResponse.SerializeToString, ), 'CheckAgentHealth': grpc.unary_unary_rpc_method_handler( servicer.CheckAgentHealth, - request_deserializer=experiment__service__pb2.Empty.FromString, - response_serializer=experiment__service__pb2.AgentHealthResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.AgentHealthResponse.SerializeToString, ), 'InitializeAgent': grpc.unary_unary_rpc_method_handler( servicer.InitializeAgent, - request_deserializer=experiment__service__pb2.InitializeAgentRequest.FromString, - response_serializer=experiment__service__pb2.InitializeAgentResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.InitializeAgentRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.InitializeAgentResponse.SerializeToString, ), 'ChangeAgentModel': grpc.unary_unary_rpc_method_handler( servicer.ChangeAgentModel, - request_deserializer=experiment__service__pb2.ChangeAgentModelRequest.FromString, - response_serializer=experiment__service__pb2.ChangeAgentModelResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ChangeAgentModelRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ChangeAgentModelResponse.SerializeToString, ), 'GetAgentModels': grpc.unary_unary_rpc_method_handler( servicer.GetAgentModels, - request_deserializer=experiment__service__pb2.GetAgentModelsRequest.FromString, - response_serializer=experiment__service__pb2.GetAgentModelsResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetAgentModelsRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetAgentModelsResponse.SerializeToString, ), 'ResetAgent': grpc.unary_unary_rpc_method_handler( servicer.ResetAgent, - request_deserializer=experiment__service__pb2.Empty.FromString, - response_serializer=experiment__service__pb2.ResetAgentResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ResetAgentResponse.SerializeToString, ), 'ClearAgentHistory': grpc.unary_unary_rpc_method_handler( servicer.ClearAgentHistory, - request_deserializer=experiment__service__pb2.Empty.FromString, - response_serializer=experiment__service__pb2.ClearAgentHistoryResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ClearAgentHistoryResponse.SerializeToString, ), 'CompactAgentHistory': grpc.unary_unary_rpc_method_handler( servicer.CompactAgentHistory, - request_deserializer=experiment__service__pb2.Empty.FromString, - response_serializer=experiment__service__pb2.CompactAgentHistoryResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.CompactAgentHistoryResponse.SerializeToString, ), 'GetAgentContextUsage': grpc.unary_unary_rpc_method_handler( servicer.GetAgentContextUsage, - request_deserializer=experiment__service__pb2.Empty.FromString, - response_serializer=experiment__service__pb2.GetAgentContextUsageResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetAgentContextUsageResponse.SerializeToString, ), 'RunNotebookCell': grpc.unary_stream_rpc_method_handler( servicer.RunNotebookCell, - request_deserializer=experiment__service__pb2.RunNotebookCellRequest.FromString, - response_serializer=experiment__service__pb2.NotebookCellChunk.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.RunNotebookCellRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.NotebookCellChunk.SerializeToString, ), 'InterruptNotebookCell': grpc.unary_unary_rpc_method_handler( servicer.InterruptNotebookCell, - request_deserializer=experiment__service__pb2.InterruptNotebookCellRequest.FromString, - response_serializer=experiment__service__pb2.InterruptNotebookCellResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.InterruptNotebookCellRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.InterruptNotebookCellResponse.SerializeToString, ), 'GetNotebook': grpc.unary_unary_rpc_method_handler( servicer.GetNotebook, - request_deserializer=experiment__service__pb2.Empty.FromString, - response_serializer=experiment__service__pb2.NotebookResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.Empty.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.NotebookResponse.SerializeToString, ), 'SaveNotebook': grpc.unary_unary_rpc_method_handler( servicer.SaveNotebook, - request_deserializer=experiment__service__pb2.SaveNotebookRequest.FromString, - response_serializer=experiment__service__pb2.SaveNotebookResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.SaveNotebookRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.SaveNotebookResponse.SerializeToString, ), 'GenerateNotebookCode': grpc.unary_unary_rpc_method_handler( servicer.GenerateNotebookCode, - request_deserializer=experiment__service__pb2.GenerateNotebookCodeRequest.FromString, - response_serializer=experiment__service__pb2.GenerateNotebookCodeResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GenerateNotebookCodeRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GenerateNotebookCodeResponse.SerializeToString, ), 'RestoreCheckpoint': grpc.unary_unary_rpc_method_handler( servicer.RestoreCheckpoint, - request_deserializer=experiment__service__pb2.RestoreCheckpointRequest.FromString, - response_serializer=experiment__service__pb2.RestoreCheckpointResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.RestoreCheckpointRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.RestoreCheckpointResponse.SerializeToString, ), 'ListExperimentRuns': grpc.unary_unary_rpc_method_handler( servicer.ListExperimentRuns, - request_deserializer=experiment__service__pb2.ListExperimentRunsRequest.FromString, - response_serializer=experiment__service__pb2.ListExperimentRunsResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ListExperimentRunsRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ListExperimentRunsResponse.SerializeToString, ), 'RenameExperimentRun': grpc.unary_unary_rpc_method_handler( servicer.RenameExperimentRun, - request_deserializer=experiment__service__pb2.RenameExperimentRunRequest.FromString, - response_serializer=experiment__service__pb2.RenameExperimentRunResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.RenameExperimentRunRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.RenameExperimentRunResponse.SerializeToString, ), 'SetExperimentRunNotes': grpc.unary_unary_rpc_method_handler( servicer.SetExperimentRunNotes, - request_deserializer=experiment__service__pb2.SetExperimentRunNotesRequest.FromString, - response_serializer=experiment__service__pb2.SetExperimentRunNotesResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.SetExperimentRunNotesRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.SetExperimentRunNotesResponse.SerializeToString, ), 'TriggerEvaluation': grpc.unary_unary_rpc_method_handler( servicer.TriggerEvaluation, - request_deserializer=experiment__service__pb2.TriggerEvaluationRequest.FromString, - response_serializer=experiment__service__pb2.TriggerEvaluationResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.TriggerEvaluationRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.TriggerEvaluationResponse.SerializeToString, ), 'GetEvaluationStatus': grpc.unary_unary_rpc_method_handler( servicer.GetEvaluationStatus, - request_deserializer=experiment__service__pb2.GetEvaluationStatusRequest.FromString, - response_serializer=experiment__service__pb2.GetEvaluationStatusResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.GetEvaluationStatusRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.GetEvaluationStatusResponse.SerializeToString, ), 'CancelEvaluation': grpc.unary_unary_rpc_method_handler( servicer.CancelEvaluation, - request_deserializer=experiment__service__pb2.CancelEvaluationRequest.FromString, - response_serializer=experiment__service__pb2.CancelEvaluationResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationResponse.SerializeToString, ), 'ExportAnnotations': grpc.unary_unary_rpc_method_handler( servicer.ExportAnnotations, - request_deserializer=experiment__service__pb2.ExportAnnotationsRequest.FromString, - response_serializer=experiment__service__pb2.ExportAnnotationsResponse.SerializeToString, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( @@ -708,8 +708,8 @@ def GetLatestLoggerData(request, request, target, '/ExperimentService/GetLatestLoggerData', - experiment__service__pb2.GetLatestLoggerDataRequest.SerializeToString, - experiment__service__pb2.GetLatestLoggerDataResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.GetLatestLoggerDataRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.GetLatestLoggerDataResponse.FromString, options, channel_credentials, insecure, @@ -735,8 +735,8 @@ def ExperimentCommand(request, request, target, '/ExperimentService/ExperimentCommand', - experiment__service__pb2.TrainerCommand.SerializeToString, - experiment__service__pb2.CommandResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.TrainerCommand.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.CommandResponse.FromString, options, channel_credentials, insecure, @@ -762,8 +762,8 @@ def ManipulateWeights(request, request, target, '/ExperimentService/ManipulateWeights', - experiment__service__pb2.WeightsOperationRequest.SerializeToString, - experiment__service__pb2.WeightsOperationResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.WeightsOperationRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.WeightsOperationResponse.FromString, options, channel_credentials, insecure, @@ -789,8 +789,8 @@ def GetWeights(request, request, target, '/ExperimentService/GetWeights', - experiment__service__pb2.WeightsRequest.SerializeToString, - experiment__service__pb2.WeightsResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.WeightsRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.WeightsResponse.FromString, options, channel_credentials, insecure, @@ -816,8 +816,8 @@ def GetActivations(request, request, target, '/ExperimentService/GetActivations', - experiment__service__pb2.ActivationRequest.SerializeToString, - experiment__service__pb2.ActivationResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.ActivationRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.ActivationResponse.FromString, options, channel_credentials, insecure, @@ -843,8 +843,8 @@ def GetSamples(request, request, target, '/ExperimentService/GetSamples', - experiment__service__pb2.BatchSampleRequest.SerializeToString, - experiment__service__pb2.BatchSampleResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.BatchSampleRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.BatchSampleResponse.FromString, options, channel_credentials, insecure, @@ -870,8 +870,8 @@ def ApplyDataQuery(request, request, target, '/ExperimentService/ApplyDataQuery', - experiment__service__pb2.DataQueryRequest.SerializeToString, - experiment__service__pb2.DataQueryResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.DataQueryRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.DataQueryResponse.FromString, options, channel_credentials, insecure, @@ -897,8 +897,8 @@ def GetDataSamples(request, request, target, '/ExperimentService/GetDataSamples', - experiment__service__pb2.DataSamplesRequest.SerializeToString, - experiment__service__pb2.DataSamplesResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.DataSamplesRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.DataSamplesResponse.FromString, options, channel_credentials, insecure, @@ -924,8 +924,8 @@ def GetHistogram(request, request, target, '/ExperimentService/GetHistogram', - experiment__service__pb2.HistogramRequest.SerializeToString, - experiment__service__pb2.HistogramResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.HistogramRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.HistogramResponse.FromString, options, channel_credentials, insecure, @@ -951,8 +951,8 @@ def GetMetaData(request, request, target, '/ExperimentService/GetMetaData', - experiment__service__pb2.GetMetaDataRequest.SerializeToString, - experiment__service__pb2.GetMetaDataResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.GetMetaDataRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.GetMetaDataResponse.FromString, options, channel_credentials, insecure, @@ -978,8 +978,8 @@ def GetSignalTrajectory(request, request, target, '/ExperimentService/GetSignalTrajectory', - experiment__service__pb2.GetSignalTrajectoryRequest.SerializeToString, - experiment__service__pb2.GetSignalTrajectoryResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.GetSignalTrajectoryRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.GetSignalTrajectoryResponse.FromString, options, channel_credentials, insecure, @@ -1005,8 +1005,8 @@ def GetStepSamples(request, request, target, '/ExperimentService/GetStepSamples', - experiment__service__pb2.StepSamplesRequest.SerializeToString, - experiment__service__pb2.StepSamplesResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.StepSamplesRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.StepSamplesResponse.FromString, options, channel_credentials, insecure, @@ -1032,8 +1032,8 @@ def GetPointCloud(request, request, target, '/ExperimentService/GetPointCloud', - experiment__service__pb2.PointCloudRequest.SerializeToString, - experiment__service__pb2.PointCloudChunk.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.PointCloudRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.PointCloudChunk.FromString, options, channel_credentials, insecure, @@ -1059,8 +1059,8 @@ def GetMedia(request, request, target, '/ExperimentService/GetMedia', - experiment__service__pb2.MediaRequest.SerializeToString, - experiment__service__pb2.MediaChunk.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.MediaRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.MediaChunk.FromString, options, channel_credentials, insecure, @@ -1086,8 +1086,8 @@ def EditDataSample(request, request, target, '/ExperimentService/EditDataSample', - experiment__service__pb2.DataEditsRequest.SerializeToString, - experiment__service__pb2.DataEditsResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.DataEditsRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.DataEditsResponse.FromString, options, channel_credentials, insecure, @@ -1113,8 +1113,8 @@ def GetDataSplits(request, request, target, '/ExperimentService/GetDataSplits', - experiment__service__pb2.Empty.SerializeToString, - experiment__service__pb2.DataSplitsResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.DataSplitsResponse.FromString, options, channel_credentials, insecure, @@ -1140,8 +1140,8 @@ def CheckAgentHealth(request, request, target, '/ExperimentService/CheckAgentHealth', - experiment__service__pb2.Empty.SerializeToString, - experiment__service__pb2.AgentHealthResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.AgentHealthResponse.FromString, options, channel_credentials, insecure, @@ -1167,8 +1167,8 @@ def InitializeAgent(request, request, target, '/ExperimentService/InitializeAgent', - experiment__service__pb2.InitializeAgentRequest.SerializeToString, - experiment__service__pb2.InitializeAgentResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.InitializeAgentRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.InitializeAgentResponse.FromString, options, channel_credentials, insecure, @@ -1194,8 +1194,8 @@ def ChangeAgentModel(request, request, target, '/ExperimentService/ChangeAgentModel', - experiment__service__pb2.ChangeAgentModelRequest.SerializeToString, - experiment__service__pb2.ChangeAgentModelResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.ChangeAgentModelRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.ChangeAgentModelResponse.FromString, options, channel_credentials, insecure, @@ -1221,8 +1221,8 @@ def GetAgentModels(request, request, target, '/ExperimentService/GetAgentModels', - experiment__service__pb2.GetAgentModelsRequest.SerializeToString, - experiment__service__pb2.GetAgentModelsResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.GetAgentModelsRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.GetAgentModelsResponse.FromString, options, channel_credentials, insecure, @@ -1248,8 +1248,8 @@ def ResetAgent(request, request, target, '/ExperimentService/ResetAgent', - experiment__service__pb2.Empty.SerializeToString, - experiment__service__pb2.ResetAgentResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.ResetAgentResponse.FromString, options, channel_credentials, insecure, @@ -1275,8 +1275,8 @@ def ClearAgentHistory(request, request, target, '/ExperimentService/ClearAgentHistory', - experiment__service__pb2.Empty.SerializeToString, - experiment__service__pb2.ClearAgentHistoryResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.ClearAgentHistoryResponse.FromString, options, channel_credentials, insecure, @@ -1302,8 +1302,8 @@ def CompactAgentHistory(request, request, target, '/ExperimentService/CompactAgentHistory', - experiment__service__pb2.Empty.SerializeToString, - experiment__service__pb2.CompactAgentHistoryResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.CompactAgentHistoryResponse.FromString, options, channel_credentials, insecure, @@ -1329,8 +1329,8 @@ def GetAgentContextUsage(request, request, target, '/ExperimentService/GetAgentContextUsage', - experiment__service__pb2.Empty.SerializeToString, - experiment__service__pb2.GetAgentContextUsageResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.GetAgentContextUsageResponse.FromString, options, channel_credentials, insecure, @@ -1356,8 +1356,8 @@ def RunNotebookCell(request, request, target, '/ExperimentService/RunNotebookCell', - experiment__service__pb2.RunNotebookCellRequest.SerializeToString, - experiment__service__pb2.NotebookCellChunk.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.RunNotebookCellRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.NotebookCellChunk.FromString, options, channel_credentials, insecure, @@ -1383,8 +1383,8 @@ def InterruptNotebookCell(request, request, target, '/ExperimentService/InterruptNotebookCell', - experiment__service__pb2.InterruptNotebookCellRequest.SerializeToString, - experiment__service__pb2.InterruptNotebookCellResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.InterruptNotebookCellRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.InterruptNotebookCellResponse.FromString, options, channel_credentials, insecure, @@ -1410,8 +1410,8 @@ def GetNotebook(request, request, target, '/ExperimentService/GetNotebook', - experiment__service__pb2.Empty.SerializeToString, - experiment__service__pb2.NotebookResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.Empty.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.NotebookResponse.FromString, options, channel_credentials, insecure, @@ -1437,8 +1437,8 @@ def SaveNotebook(request, request, target, '/ExperimentService/SaveNotebook', - experiment__service__pb2.SaveNotebookRequest.SerializeToString, - experiment__service__pb2.SaveNotebookResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.SaveNotebookRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.SaveNotebookResponse.FromString, options, channel_credentials, insecure, @@ -1464,8 +1464,8 @@ def GenerateNotebookCode(request, request, target, '/ExperimentService/GenerateNotebookCode', - experiment__service__pb2.GenerateNotebookCodeRequest.SerializeToString, - experiment__service__pb2.GenerateNotebookCodeResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.GenerateNotebookCodeRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.GenerateNotebookCodeResponse.FromString, options, channel_credentials, insecure, @@ -1491,8 +1491,8 @@ def RestoreCheckpoint(request, request, target, '/ExperimentService/RestoreCheckpoint', - experiment__service__pb2.RestoreCheckpointRequest.SerializeToString, - experiment__service__pb2.RestoreCheckpointResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.RestoreCheckpointRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.RestoreCheckpointResponse.FromString, options, channel_credentials, insecure, @@ -1518,8 +1518,8 @@ def ListExperimentRuns(request, request, target, '/ExperimentService/ListExperimentRuns', - experiment__service__pb2.ListExperimentRunsRequest.SerializeToString, - experiment__service__pb2.ListExperimentRunsResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.ListExperimentRunsRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.ListExperimentRunsResponse.FromString, options, channel_credentials, insecure, @@ -1545,8 +1545,8 @@ def RenameExperimentRun(request, request, target, '/ExperimentService/RenameExperimentRun', - experiment__service__pb2.RenameExperimentRunRequest.SerializeToString, - experiment__service__pb2.RenameExperimentRunResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.RenameExperimentRunRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.RenameExperimentRunResponse.FromString, options, channel_credentials, insecure, @@ -1572,8 +1572,8 @@ def SetExperimentRunNotes(request, request, target, '/ExperimentService/SetExperimentRunNotes', - experiment__service__pb2.SetExperimentRunNotesRequest.SerializeToString, - experiment__service__pb2.SetExperimentRunNotesResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.SetExperimentRunNotesRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.SetExperimentRunNotesResponse.FromString, options, channel_credentials, insecure, @@ -1599,8 +1599,8 @@ def TriggerEvaluation(request, request, target, '/ExperimentService/TriggerEvaluation', - experiment__service__pb2.TriggerEvaluationRequest.SerializeToString, - experiment__service__pb2.TriggerEvaluationResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.TriggerEvaluationRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.TriggerEvaluationResponse.FromString, options, channel_credentials, insecure, @@ -1626,8 +1626,8 @@ def GetEvaluationStatus(request, request, target, '/ExperimentService/GetEvaluationStatus', - experiment__service__pb2.GetEvaluationStatusRequest.SerializeToString, - experiment__service__pb2.GetEvaluationStatusResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.GetEvaluationStatusRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.GetEvaluationStatusResponse.FromString, options, channel_credentials, insecure, @@ -1653,8 +1653,8 @@ def CancelEvaluation(request, request, target, '/ExperimentService/CancelEvaluation', - experiment__service__pb2.CancelEvaluationRequest.SerializeToString, - experiment__service__pb2.CancelEvaluationResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationResponse.FromString, options, channel_credentials, insecure, @@ -1680,8 +1680,8 @@ def ExportAnnotations(request, request, target, '/ExperimentService/ExportAnnotations', - experiment__service__pb2.ExportAnnotationsRequest.SerializeToString, - experiment__service__pb2.ExportAnnotationsResponse.FromString, + weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsResponse.FromString, options, channel_credentials, insecure, From ac608f6974f155218a28f0198dce63ba9325fa05 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 20 Aug 2026 22:03:24 +0000 Subject: [PATCH 09/13] fix(logger): import deque alongside defaultdict The merge re-applied our in-memory history tail onto dev logger.py, which imports only defaultdict. Every per-sample write then raised NameError inside _stage_sample_row. The caller swallows per-signal exceptions, so nothing crashed: the tail just stayed empty, sig/loss_debiased failed every step, and loss_shape had no history to classify. Co-Authored-By: Claude Opus 5 --- weightslab/backend/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weightslab/backend/logger.py b/weightslab/backend/logger.py index fea0aaa6..9b48ba8c 100644 --- a/weightslab/backend/logger.py +++ b/weightslab/backend/logger.py @@ -36,7 +36,7 @@ import os import threading import time -from collections import defaultdict +from collections import defaultdict, deque import duckdb import pandas as pd From 70caddeef4e1faa53d80d69ad04e67101b0e5e74 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 20 Aug 2026 22:30:29 +0000 Subject: [PATCH 10/13] fix(shapes): keep the label cache on top of dev write_signal_shapes dev rewrite keeps the O(change) read and adds exp_hash scoping, both kept. What it dropped is the label cache, which two behaviours depended on: - an incremental pass still returns a distribution over the WHOLE dataset, not just the samples it happened to touch; - a sample whose label did not change is not re-written to the ledger. Both are asserted by e2e_autotag (distribution_covers_dataset, incremental_writes_bounded), which failed on the merge until this went back. The test also moves to dev parameter name, sample_ids. Co-Authored-By: Claude Opus 5 --- weightslab/src.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/weightslab/src.py b/weightslab/src.py index 8e7fc7bb..7c852a66 100644 --- a/weightslab/src.py +++ b/weightslab/src.py @@ -4888,6 +4888,16 @@ def resolve_signal_classifier(signal_name): return _GLOBAL_CLASSIFIER or classify_loss_shape +_SHAPE_LABELS: dict = {} + + +def _label_counts(cache): + out = {} + for lab in cache.values(): + out[lab] = out.get(lab, 0) + 1 + return out + + def write_signal_shapes(signal_name, tag_name=None, classifier=None, exp_hash=None, sample_ids=None): """Reusable engine: classify each sample's own trajectory of *signal_name* into a categorical tag and return the ``{label: count}`` distribution. @@ -4906,17 +4916,29 @@ def write_signal_shapes(signal_name, tag_name=None, classifier=None, exp_hash=No clf = classifier or resolve_signal_classifier(signal_name) if tag_name is None: tag_name = signal_name + "_shape" if signal_name.endswith('_loss') else signal_name + "_loss_shape" + + # Labels for samples this pass does not reclassify are carried here, so an + # incremental call still returns a distribution over the WHOLE dataset, and + # a sample whose label is unchanged is not re-written to the ledger. + cache = _SHAPE_LABELS.setdefault(signal_name, {}) + if sample_ids is not None and not list(sample_ids): + return _label_counts(cache) + series = {} for sid, step, val, _ in query_signal_history(signal_name, exp_hash=exp_hash, sample_ids=sample_ids): series.setdefault(sid, []).append((step, val)) by_label = {} for sid, pts in series.items(): label = clf([v for _, v in sorted(pts)]) - if label is not None: - by_label.setdefault(label, []).append(sid) + if label is None: + continue + if cache.get(sid) == label: + continue # unchanged -> no ledger write needed + cache[sid] = label + by_label.setdefault(label, []).append(sid) for label, sids in by_label.items(): set_categorical_tag(sids, tag_name, label) - return {k: len(v) for k, v in by_label.items()} + return _label_counts(cache) def write_loss_shapes(loss_signal="loss_sample", classifier=None): From da3536615ce3b2d50b00b909f4a52a91a51cb41b Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 21 Aug 2026 03:46:44 +0000 Subject: [PATCH 11/13] fix(signals): restore inputs= on the batched subscribe_to path On the subscribe_to path BatchSignalContext was built without inputs=, so b.inputs was {} and any signal declaring inputs=[...] raised KeyError on every call. sig/loss_debiased does exactly that: it failed 12,079 times in one five hour run -- once per step -- and because wrappered_fwd swallows per-signal exceptions nothing crashed, the column just silently never got values. We had already fixed this; taking dev src.py whole during the merge reverted it, since dev never carried the fix. Same class as the deque import and the label cache: dev has no equivalent, so a wholesale take drops it. Co-Authored-By: Claude Opus 5 --- weightslab/src.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/weightslab/src.py b/weightslab/src.py index 7c852a66..56b2a664 100644 --- a/weightslab/src.py +++ b/weightslab/src.py @@ -1023,9 +1023,23 @@ def wrappered_fwd(original_forward, kwargs, reg_name, *a, **kw): # batch and call the signal once. It returns a length-B # array. Avoids B Python calls + B SignalContext allocs, # and lets the signal do batched ledger reads. + # inputs= must be populated here too: on the + # subscribe_to path the context was built without it, + # so b.inputs was {} and any signal declaring + # inputs=[...] raised KeyError on every call. The + # subscribed signal IS the declared input here, and + # its per-sample values are already in val_vec. + _decl = meta.get('inputs') or [] + _sub = meta.get('subscribe_to') + _vals = [float(v) for v in val_vec] + _bin = {} + for _d in _decl: + if _sub is None or _d == _sub or _d == reg_name: + _bin[_d] = _vals bctx = BatchSignalContext( sample_ids=[int(u) for u in ids_np], - subscribed_values=[float(v) for v in val_vec], + subscribed_values=_vals, + inputs=_bin, logger=_lg, dataframe=df_proxy, origin=kwargs.get('origin', 'train'), From a360234d51d7f7722a9ab18d235c86614cb11ee1 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 21 Aug 2026 11:03:10 +0000 Subject: [PATCH 12/13] Remove optrace tracing from weightslab Drops the optrace module and every call site: 64 @traced decorators, 13 hit() markers and 5 imports across the data stores, the dataframe manager and the two services. Pure deletion -- 437 lines out, 0 in. Every hit() was verified to be a bare statement rather than an expression, so removing it cannot change a value, and removal was parenthesis-balanced because several spanned three lines. Each file is compiled after editing, which is what would catch a removal that left an empty block. The tracing was built to find where interactivity time went on a 100GB dataset. It has served that purpose: the O(change) view sync, the O(batch) NB_SEEN lookup and the flush accounting all came out of it. Co-Authored-By: Claude Opus 5 --- weightslab/backend/optrace.py | 348 ------------------ weightslab/data/dataframe_manager.py | 29 -- weightslab/data/h5_array_store.py | 9 - weightslab/data/h5_dataframe_store.py | 12 - weightslab/trainer/services/data_service.py | 35 -- .../trainer/services/experiment_service.py | 4 - 6 files changed, 437 deletions(-) delete mode 100644 weightslab/backend/optrace.py diff --git a/weightslab/backend/optrace.py b/weightslab/backend/optrace.py deleted file mode 100644 index 477b9ec4..00000000 --- a/weightslab/backend/optrace.py +++ /dev/null @@ -1,348 +0,0 @@ -"""Begin/end operation tracing for dataframe, array-store, duckdb and -experiment-service operations. - -Off by default (near-zero overhead: one bool check) — set ``WL_OPTRACE=1`` to -turn it on. Every traced call prints ONE line at start and ONE line at end to -stdout (unbuffered, same stream as main.py's ``[timing]`` prints), tagged -``[optrace]`` so a run's LOG file can be parsed the same way: - - grep -a "\\[optrace\\]" LOG | ... - -Line format (space-separated key=value tokens, so ``awk`` can pick fields by -name without caring about column position):: - - [optrace] BEGIN domain=dataframe op=dfm.upsert_df call=482 tid=140234 ts=1723300000.123456 site=dataframe_manager.py:680 n_in=24 bytes_in=196608 args=origin=train_loader,force_flush=False - [optrace] END domain=dataframe op=dfm.upsert_df call=482 tid=140234 ts=1723300000.234567 dur_ms=111.111 ok=True site=dataframe_manager.py:680 n_in=24 bytes_in=196608 args=origin=train_loader,force_flush=False mem_delta_kb=512 n_out=24 bytes_out=- - -``call=`` pairs a BEGIN with its END even when the same op runs concurrently -on multiple threads (same op+tid can otherwise appear twice before either -finishes). For call-count/timing/bytes/memory/object-count reports, the END -line alone carries every field -- see ``code/optrace_report.py``. - -When ``@traced``/``trace_op`` wraps a whole function (the normal case), the -extra fields beyond ``dur_ms``/``ok`` are filled in automatically: - - site file:line of the function's ``def`` (not the call site -- - stable across callers, and enough to jump to the code). - n_in/n_out best-effort element counts for arguments / return value - (numpy array .size, len() of dict/list/etc). - bytes_in/out best-effort byte counts (numpy .nbytes, len() of bytes), - summed recursively through dict/list/tuple containers. - args sanitized ``name=repr`` for each bound argument (arrays - collapse to ``ndarray(shape=...,dtype=...)`` rather than - dumping their contents) -- the "which sample_id did this" - detail needed to trace back a specific weird call. - mem_delta_kb RSS delta (psutil) across the call. Peak-agnostic and can - be noisy under concurrent threads sharing one process, but - cheap and good enough to spot a call that's allocating much - more than its neighbours. - -A bare ``with trace_op(domain, op, **extra):`` (not decorating a function, -e.g. ``TracingDuckDBConn``) has no function to introspect, so it only gets -whatever ``extra`` the caller passed plus ``mem_delta_kb`` -- no -site/n_in/n_out/bytes_in/bytes_out/args. -""" -import functools -import inspect -import itertools -import os -import sys -import threading -import time - -try: - import numpy as _np -except Exception: - _np = None - -try: - import pandas as _pd -except Exception: - _pd = None - -try: - import psutil as _psutil - _PROC = _psutil.Process() -except Exception: - _PROC = None - -_TRUTHY = {"1", "true", "yes", "on"} -_ENABLED = os.environ.get("WL_OPTRACE", "0").strip().lower() in _TRUTHY - -_counter = itertools.count() -_counter_lock = threading.Lock() - -# print(msg, flush=True) is two separate write()s under the hood (message, -# then the trailing newline) with no atomicity guarantee between them, so two -# threads tracing concurrently (training thread, flush thread, grpc workers) -# can interleave mid-line -- observed in practice as garbled/merged [optrace] -# lines. Serialize the full write+flush per line instead. -_print_lock = threading.Lock() - - -def _emit(line: str) -> None: - with _print_lock: - print(line, flush=True) - - -def trace_enabled() -> bool: - return _ENABLED - - -def _next_call_id() -> int: - with _counter_lock: - return next(_counter) - - -def _fmt_extra(extra: dict) -> str: - if not extra: - return "" - return " " + " ".join(f"{k}={v}" for k, v in extra.items()) - - -def sanitize(value, maxlen: int = 48) -> str: - """Collapse whitespace and truncate so a value is safe as a bare token - in the space-separated log line (e.g. a SQL statement).""" - s = " ".join(str(value).split()) - if len(s) > maxlen: - s = s[:maxlen] + "..." - return s.replace(" ", "_") - - -def _rss_kb(): - if _PROC is None: - return None - try: - return _PROC.memory_info().rss / 1024.0 - except Exception: - return None - - -def _obj_metrics(obj): - """Best-effort (count, bytes) size hints for an object; either may be None.""" - if obj is None: - return None, None - if _np is not None and isinstance(obj, _np.ndarray): - return obj.size, obj.nbytes - # deep=False ONLY. deep=True walks every element of every object/string - # column: measured at ~1000ms on a 3.96M-row frame vs ~1ms shallow (854x), - # and this runs on every traced call -- it turns tracing itself into the - # O(dataset) hot-path work this module exists to hunt down. Shallow - # undercounts object columns (it counts the 8-byte pointers, not the - # referenced strings), so bytes_in/out for string-heavy frames is a lower - # bound; that is the right trade for a diagnostic that must not distort - # what it measures. - if _pd is not None and isinstance(obj, _pd.DataFrame): - try: - return len(obj), int(obj.memory_usage(deep=False).sum()) - except Exception: - return len(obj), None - if _pd is not None and isinstance(obj, _pd.Series): - try: - return len(obj), int(obj.memory_usage(deep=False)) - except Exception: - return len(obj), None - if isinstance(obj, (bytes, bytearray, memoryview)): - return len(obj), len(obj) - if isinstance(obj, dict): - nbytes = 0 - for v in obj.values(): - _, vb = _obj_metrics(v) - if vb: - nbytes += vb - return len(obj), (nbytes or None) - if isinstance(obj, (list, tuple, set)): - nbytes = 0 - for v in obj: - _, vb = _obj_metrics(v) - if vb: - nbytes += vb - return len(obj), (nbytes or None) - if isinstance(obj, (str, int, float, bool)): - return None, None - if hasattr(obj, "__len__"): - try: - return len(obj), None - except Exception: - return None, None - return None, None - - -def _fmt_arg_value(value, maxlen: int = 40) -> str: - if _np is not None and isinstance(value, _np.ndarray): - return f"ndarray(shape={value.shape},dtype={value.dtype})" - if _pd is not None and isinstance(value, _pd.DataFrame): - return f"DataFrame(rows={len(value)},cols={value.shape[1]})" - if _pd is not None and isinstance(value, _pd.Series): - return f"Series(len={len(value)},dtype={value.dtype})" - s = repr(value) - return s if len(s) <= maxlen else s[: maxlen - 3] + "..." - - -def _in_metrics(sig, args, kwargs) -> dict: - """n_in/bytes_in/args extras for a decorated function's bound arguments.""" - if sig is None: - return {} - try: - bound = sig.bind_partial(*args, **kwargs) - bound.apply_defaults() - except Exception: - return {} - arg_items = [(n, v) for n, v in bound.arguments.items() if n != "self"] - n_in = b_in = 0 - has_n, has_b = False, False - for _, v in arg_items: - n, b = _obj_metrics(v) - if n is not None: - n_in += n - has_n = True - if b is not None: - b_in += b - has_b = True - out = {} - if has_n: - out["n_in"] = n_in - if has_b: - out["bytes_in"] = b_in - if arg_items: - args_str = ",".join(f"{n}={_fmt_arg_value(v)}" for n, v in arg_items) - out["args"] = sanitize(args_str, maxlen=160) - return out - - -def _out_metrics(result) -> dict: - n_out, b_out = _obj_metrics(result) - out = {} - if n_out is not None: - out["n_out"] = n_out - if b_out is not None: - out["bytes_out"] = b_out - return out - - -class trace_op: - """Context manager: logs BEGIN on enter, END (with duration) on exit. - - Also usable as a decorator: ``@trace_op("dfm.upsert_df")``. - """ - - __slots__ = ("domain", "op", "extra", "_call_id", "_t0", "_mem0") - - def __init__(self, domain: str, op: str, **extra): - self.domain = domain - self.op = op - self.extra = extra - - def set(self, **kv) -> None: - """Attach fields (e.g. n_out/bytes_out) to the END line, from inside - the ``with`` block, once they're known (e.g. after computing a - result).""" - self.extra.update(kv) - - def __call__(self, fn): - site = f"{os.path.basename(fn.__code__.co_filename)}:{fn.__code__.co_firstlineno}" - try: - sig = inspect.signature(fn) - except (TypeError, ValueError): - sig = None - - @functools.wraps(fn) - def wrapper(*args, **kwargs): - if not _ENABLED: - return fn(*args, **kwargs) - call_extra = {"site": site} - call_extra.update(_in_metrics(sig, args, kwargs)) - call_extra.update(self.extra) - op_ctx = trace_op(self.domain, self.op, **call_extra) - with op_ctx: - result = fn(*args, **kwargs) - try: - op_ctx.set(**_out_metrics(result)) - except Exception: - pass - return result - return wrapper - - def __enter__(self): - if not _ENABLED: - return self - self._call_id = _next_call_id() - tid = threading.get_ident() - self._mem0 = _rss_kb() - self._t0 = time.perf_counter() - _emit(f"[optrace] BEGIN domain={self.domain} op={self.op} " - f"call={self._call_id} tid={tid} ts={time.time():.6f}" - f"{_fmt_extra(self.extra)}") - return self - - def __exit__(self, exc_type, exc, tb): - if not _ENABLED: - return False - dur_ms = (time.perf_counter() - self._t0) * 1000.0 - tid = threading.get_ident() - mem1 = _rss_kb() - if mem1 is not None and self._mem0 is not None: - self.extra["mem_delta_kb"] = f"{mem1 - self._mem0:.0f}" - _emit(f"[optrace] END domain={self.domain} op={self.op} " - f"call={self._call_id} tid={tid} ts={time.time():.6f} " - f"dur_ms={dur_ms:.3f} ok={exc_type is None}" - f"{_fmt_extra(self.extra)}") - return False - - -def hit(domain: str, op: str, **extra) -> None: - """Log a single one-line marker, iff tracing is enabled. - - Unlike ``trace_op``, this isn't a timed BEGIN/END pair -- it's for - confirming which branch of an if/else a call actually took (e.g. - fast-path vs fallback, in-place vs backup-and-rewrite) so a run's LOG can - answer "did the new code path get hit, and how often" via:: - - grep -a "\\[optrace\\] HIT" LOG | awk '...' - """ - if not _ENABLED: - return - _emit(f"[optrace] HIT domain={domain} op={op} tid={threading.get_ident()} " - f"ts={time.time():.6f}{_fmt_extra(extra)}") - - -def traced(domain: str, op: str = None): - """Method decorator: ``@traced("dataframe", "dfm.upsert_df")``. - - ``op`` defaults to the wrapped function's qualified name. - """ - def deco(fn): - name = op or fn.__qualname__ - return trace_op(domain, name)(fn) - return deco - - -class TracingDuckDBConn: - """Transparent proxy around a duckdb connection: traces ``execute``/ - ``sql``, delegates everything else (register/unregister/close/...) - untouched. Only construct this when tracing is enabled — with it off, - keep using the raw connection so there is zero added indirection. - """ - - __slots__ = ("_conn",) - - def __init__(self, conn): - self._conn = conn - - def execute(self, *args, **kwargs): - sql = sanitize(args[0]) if args else "" - with trace_op("duckdb", "duckdb.execute", sql=sql): - return self._conn.execute(*args, **kwargs) - - def sql(self, *args, **kwargs): - sql = sanitize(args[0]) if args else "" - with trace_op("duckdb", "duckdb.sql", sql=sql): - return self._conn.sql(*args, **kwargs) - - def __getattr__(self, name): - return getattr(self._conn, name) - - -def maybe_wrap_duckdb_conn(conn): - """Wrap ``conn`` for tracing iff WL_OPTRACE is on, else return it as-is.""" - return TracingDuckDBConn(conn) if _ENABLED else conn diff --git a/weightslab/data/dataframe_manager.py b/weightslab/data/dataframe_manager.py index 25ceb5e0..3bfcc31f 100644 --- a/weightslab/data/dataframe_manager.py +++ b/weightslab/data/dataframe_manager.py @@ -25,7 +25,6 @@ SAMPLES_STATS_TO_SAVE_TO_H5, ) from weightslab.backend.ledgers import get_hyperparams -from weightslab.backend.optrace import traced, hit pd.set_option('future.no_silent_downcasting', True) @@ -579,7 +578,6 @@ def _merge_categories(self, name: str, categories, replace: bool = False) -> Lis self._categorical_tags[name] = list(dict.fromkeys([*existing, *cats])) return list(self._categorical_tags[name]) - @traced("dataframe", "dfm.register_categorical_tag") def register_categorical_tag(self, name: str, categories=None, replace: bool = False) -> List[str]: """Declare (or extend) a categorical tag and its allowed category values. @@ -669,7 +667,6 @@ def _load_tag_registry(self) -> None: except Exception as e: logger.debug(f"[LedgeredDataFrameManager] Failed to load tag registry: {e}") - @traced("dataframe", "dfm.register_split") def register_split(self, origin: str, df: List | pd.DataFrame, store: H5DataFrameStore | None = None, autoload_arrays: bool | list | set = False, return_proxies: bool = True, use_cache: bool = True): # Build the annotation-expanded (sample_id, annotation_id) frame. # Fast path: when given a list of record dicts, construct the EXPANDED frame @@ -703,7 +700,6 @@ def register_split(self, origin: str, df: List | pd.DataFrame, store: H5DataFram # Start flush thread if not already running self._ensure_flush_thread() - @traced("dataframe", "dfm._load_existing_data") def _load_existing_data(self, origin: str = None, autoload_arrays: bool | list | set = False, return_proxies: bool = True, use_cache: bool = True): # Restore the categorical tag registry so loaded string-valued tag columns # get their full allowed category set (not just the values present on disk). @@ -779,7 +775,6 @@ def _load_existing_data(self, origin: str = None, autoload_arrays: bool | list | else: logger.warning(f"[LedgeredDataFrameManager] Loaded data missing 'sample_id' column for origin={origin}. Skipping load.") - @traced("dataframe", "dfm.upsert_df") def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flush: bool = False): if df_local is None or (isinstance(df_local, pd.DataFrame) and df_local.empty) or len(df_local) == 0: return @@ -934,7 +929,6 @@ def upsert_df(self, df_local: List | pd.DataFrame, origin: str = None, force_flu if SampleStats.Ex.DISCARDED.value in set(df_norm.columns): self._bump_discard_revisions(affected_origins) - @traced("dataframe", "dfm.mark_dirty") def mark_dirty(self, sample_id: int): """Mark sample as dirty for H5 flush. @@ -945,14 +939,12 @@ def mark_dirty(self, sample_id: int): self._pending.add(normalized_id) self._view_pending.add(normalized_id) - @traced("dataframe", "dfm.drop_column") def drop_column(self, column: str): with self._lock: if column in self._df.columns: return self._df.pop(column) return None - @traced("dataframe", "dfm.mark_dirty_batch") def mark_dirty_batch(self, sample_ids: List[int], force_flush: bool = False): with self._lock: self._pending.update(set(sample_ids)) @@ -1127,7 +1119,6 @@ def _normalize_preds_raw_uint16(self, preds_raw: np.ndarray) -> np.ndarray: except Exception: return preds_raw - @traced("dataframe", "dfm.enqueue_batch") def enqueue_batch( self, sample_ids: Sequence[int], @@ -1246,7 +1237,6 @@ def index_batch(obj, batch_index, rec=False): self.first_init = False self.flush_async() - @traced("dataframe", "dfm.enqueue_instance_batch") def enqueue_instance_batch( self, sample_ids: Sequence[Any], @@ -1393,7 +1383,6 @@ def _index_target(obj, i): self.first_init = False self.flush_async() - @traced("dataframe", "dfm.update_values") def update_values(self, origin: str, sample_id: int, updates: Dict[str, Any], annotation_id: int = 0): """Update values for a sample (or specific annotation if multi-index). @@ -1464,7 +1453,6 @@ def update_values(self, origin: str, sample_id: int, updates: Dict[str, Any], an self._df = pd.concat([self._df, df_local]) self._bump_origin_revisions([origin]) - @traced("dataframe", "dfm.take_view_dirty") def take_view_dirty(self, limit: int | None = None): """Drain and return the sample_ids changed since the last view sync. @@ -1487,7 +1475,6 @@ def clear_view_dirty(self): with self._lock: self._view_pending.clear() - @traced("dataframe", "dfm.get_source_rows") def get_source_rows(self, sample_ids, columns=None): """Rows for *sample_ids* straight from the source frame. O(len(ids)).""" with self._lock: @@ -1516,7 +1503,6 @@ def get_discard_revision(self, origin: str) -> int: """ return int(self._discard_revisions.get(str(origin), 0)) - @traced("dataframe", "dfm.update_by_groups_bulk") def update_by_groups_bulk(self, origin: str, group_ids: List[Any], updates_list: List[Dict[str, Any]]): """Broadcast updates to multiple groups in one pass.""" if not group_ids or not updates_list: @@ -1570,7 +1556,6 @@ def update_by_groups_bulk(self, origin: str, group_ids: List[Any], updates_list: if affected_ids: self.mark_dirty_batch(affected_ids) - @traced("dataframe", "dfm.get_tainted_group_ids") def get_tainted_group_ids(self, group_ids: List[Any], origin: str) -> set: """Return the subset of group_ids where at least one member is discarded. @@ -1640,7 +1625,6 @@ def get_group_column_values(self, group_ids: List[Any], origin: str, column: str return values - @traced("dataframe", "dfm.get_discarded_sample_ids") def get_discarded_sample_ids(self, sample_ids: List[Any], origin: str) -> set: """Return the subset of sample_ids that are marked as discarded. @@ -1752,7 +1736,6 @@ def get_sample_column_values(self, sample_ids: List[Any], column: str) -> Dict[A return values - @traced("dataframe", "dfm.get_row") def get_row(self, origin: str, sample_id: int, annotation_id: int = None) -> pd.Series | pd.DataFrame | None: """Get row(s) by sample_id and optional annotation_id. @@ -1792,14 +1775,12 @@ def get_row(self, origin: str, sample_id: int, annotation_id: int = None) -> pd. except (KeyError, TypeError): return None - @traced("dataframe", "dfm.get_value") def get_value(self, origin: str, sample_id: int, column: str): row = self.get_row(origin, sample_id) if row is None or column not in row: return None return row[column] - @traced("dataframe", "dfm.get_df_view") def get_df_view(self, column: str = None, limit: int = -1, copy: bool = False, value: str = None) -> pd.DataFrame: with self._lock: if self._df.empty: @@ -1815,12 +1796,10 @@ def get_df_view(self, column: str = None, limit: int = -1, copy: bool = False, v subset = subset.head(limit) return subset.copy() if copy else subset - @traced("dataframe", "dfm.set_dense") def set_dense(self, key: str, sample_id: int, value: np.ndarray): with self._lock: self._dense_store.setdefault(key, {})[str(sample_id)] = value - @traced("dataframe", "dfm.get_dense_map") def get_dense_map(self, origin: str) -> Dict[str, Dict[int, np.ndarray]]: with self._lock: origin_store = self._dense_store.get(origin, {}) @@ -2249,7 +2228,6 @@ def _rows_with_array_cells(self, data_snapshot: pd.DataFrame): return [] return list(hits) - @traced("dataframe", "dfm._flush_snapshot_to_h5") def _flush_snapshot_to_h5(self, data_snapshot: pd.DataFrame, work: List[int]): """Flush data snapshot to H5 - runs completely outside locks. @@ -2380,8 +2358,6 @@ def _optimize_dataframe_memory(self, df: pd.DataFrame, categorical_tags: Dict[st # O(rows) of pure waste on every flush. _scan_cols = (list(df.columns) if columns is None else [c for c in df.columns if c in columns]) - hit("dataframe", "dfm._optimize_dataframe_memory", - scoped=columns is not None, n_scan=len(_scan_cols), n_total=len(df.columns)) # === 0) Repair signal columns that were upcast to object === # A single None written into a float column converts it permanently, and @@ -2573,7 +2549,6 @@ def stop(self): if self._flush_thread: self._flush_thread.join(timeout=2.0) - @traced("dataframe", "dfm.get_combined_df") def get_combined_df( self, autoload_arrays: bool | list | set = False, @@ -2609,7 +2584,6 @@ def get_combined_df( return df - @traced("dataframe", "dfm.get_collapse_annotations_to_samples_df") def get_collapse_annotations_to_samples_df(self, df: pd.DataFrame | None = None) -> pd.DataFrame: """Collapse a (sample_id, annotation_id) multi-index df to one row per sample. @@ -2858,7 +2832,6 @@ def _should_flush(self) -> bool: with self._lock: return len(self._pending) >= self._flush_max_rows or self._force_flush - @traced("dataframe", "dfm.flush_async") def flush_async(self): """Signal flush thread. Returns once buffer has been drained (not after H5 write). @@ -2883,7 +2856,6 @@ def flush_async(self): time.sleep(0.1) logger.warning("[LedgeredDataFrameManager] flush_async timed out waiting for buffer drain after 60s") - @traced("dataframe", "dfm.flush_if_needed_nonblocking") def flush_if_needed_nonblocking(self, force: bool = False): """Non-blocking flush - if can't acquire lock immediately, defer to next cycle.""" # Drain buffer quickly, then release lock before any DF/H5 work. @@ -2901,7 +2873,6 @@ def flush_if_needed_nonblocking(self, force: bool = False): self._flush_to_h5_if_needed(force=force) logger.debug(f"Completed non-blocking flush check. Pending count after flush: {len(self._pending)}.") - @traced("dataframe", "dfm.flush") def flush(self): """Blocking flush: buffer → DF → H5. diff --git a/weightslab/data/h5_array_store.py b/weightslab/data/h5_array_store.py index 20810d05..405cf02b 100644 --- a/weightslab/data/h5_array_store.py +++ b/weightslab/data/h5_array_store.py @@ -20,7 +20,6 @@ import h5py import numpy as np -from weightslab.backend.optrace import traced # Config global logger logger = logging.getLogger(__name__) @@ -383,7 +382,6 @@ def _compute_array_checksum(self, array: np.ndarray) -> str: logger.warning(f"[H5ArrayStore] Failed to compute checksum: {e}") return "" - @traced("arraystore", "arraystore._create_backup") def _create_backup(self) -> Optional[Path]: """Create backup of array file before write.""" if not self._path.exists(): @@ -397,7 +395,6 @@ def _create_backup(self) -> Optional[Path]: logger.warning(f"[H5ArrayStore] Failed to create backup: {e}") return None - @traced("arraystore", "arraystore._restore_backup") def _restore_backup(self, backup_path: Path) -> bool: """Restore array file from backup on write failure.""" try: @@ -428,7 +425,6 @@ def _parse_path_reference(self, path_ref: str) -> Tuple[int, str]: key_name = parts[1] return sample_id, key_name - @traced("arraystore", "arraystore.save_array") def save_array( self, sample_id: str, @@ -564,7 +560,6 @@ def _try_inplace_batch(self, prepared): finally: self._rw_lock.release_write() - @traced("arraystore", "arraystore.save_arrays_batch") def save_arrays_batch( self, arrays_dict: Dict[int, Dict[str, np.ndarray]], @@ -710,7 +705,6 @@ def save_arrays_batch( finally: self._rw_lock.release_write() - @traced("arraystore", "arraystore.recover") def recover(self) -> None: """ Recover from a crash during save_arrays_batch. @@ -734,7 +728,6 @@ def recover(self) -> None: if self._restore_backup(backup_path): backup_path.unlink(missing_ok=True) - @traced("arraystore", "arraystore.load_array") def load_array(self, path_ref: str) -> Optional[np.ndarray]: """ Load array from path reference with LRU cache. @@ -803,7 +796,6 @@ def load_array(self, path_ref: str) -> Optional[np.ndarray]: finally: self._rw_lock.release_read() - @traced("arraystore", "arraystore.load_arrays_batch") def load_arrays_batch(self, path_refs: Dict[int, Dict[str, str]]) -> Dict[int, Dict[str, np.ndarray]]: """ Load multiple arrays in batch. @@ -868,7 +860,6 @@ def load_arrays_batch(self, path_refs: Dict[int, Dict[str, str]]) -> Dict[int, D finally: self._rw_lock.release_read() - @traced("arraystore", "arraystore.delete_sample") def delete_sample(self, sample_id: int) -> bool: """ Delete all arrays for a given sample_id. diff --git a/weightslab/data/h5_dataframe_store.py b/weightslab/data/h5_dataframe_store.py index 12d04dbd..431425d1 100644 --- a/weightslab/data/h5_dataframe_store.py +++ b/weightslab/data/h5_dataframe_store.py @@ -14,7 +14,6 @@ from typing import Iterable, Optional, Union from weightslab.data.sample_stats import SampleStats -from weightslab.backend.optrace import traced, hit logger = logging.getLogger(__name__) # Initialize logger @@ -197,7 +196,6 @@ def _extract_tag_columns(self, df: pd.DataFrame) -> dict: # ------------------------------------------------------------------ # Categorical tag registry persistence # ------------------------------------------------------------------ - @traced("dataframe", "h5store.save_tag_registry") def save_tag_registry(self, registry: dict) -> None: """Persist the categorical tag registry ({tag_name: [categories]}) to H5. @@ -231,7 +229,6 @@ def save_tag_registry(self, registry: dict) -> None: else: time.sleep(self._poll_interval * attempt) - @traced("dataframe", "h5store.load_tag_registry") def load_tag_registry(self) -> dict: """Load the categorical tag registry from H5 into memory and return it.""" if not self._path.exists(): @@ -545,7 +542,6 @@ def _verify_checksum(self, store: pd.HDFStore, key: str, expected_checksum: str) logger.warning(f"[H5DataFrameStore] Failed to verify checksum: {e}") return False - @traced("dataframe", "h5store._create_backup") def _create_backup(self) -> Optional[Path]: """Create backup of H5 file before write. Returns backup path on success.""" if not self._path.exists(): @@ -560,7 +556,6 @@ def _create_backup(self) -> Optional[Path]: logger.warning(f"[H5DataFrameStore] Failed to create backup: {e}") return None - @traced("dataframe", "h5store._restore_backup") def _restore_backup(self, backup_path: Path): """Restore H5 file from backup on write failure.""" try: @@ -575,7 +570,6 @@ def _restore_backup(self, backup_path: Path): # ------------------------------------------------------------------ # Public API # ------------------------------------------------------------------ - @traced("dataframe", "h5store.load") def load(self, origin: str, columns: Optional[Iterable[str]] = None, start: Optional[int] = None, stop: Optional[int] = None, non_blocking: bool = False) -> pd.DataFrame: """Load data from H5 store. @@ -608,7 +602,6 @@ def load(self, origin: str, columns: Optional[Iterable[str]] = None, start: Opti return self._normalize_for_read(df, origin) - @traced("dataframe", "h5store.load_all") def load_all(self, origins: Iterable[str] = None, columns: Optional[Iterable[str]] = None, non_blocking: bool = False) -> pd.DataFrame: """Load all origins in a single H5 transaction. @@ -687,7 +680,6 @@ def load_all(self, origins: Iterable[str] = None, columns: Optional[Iterable[str return pd.DataFrame() raise - @traced("dataframe", "h5store.ensure_index") def ensure_index(self, origin: str, columns=("sample_id",)) -> bool: """Build the on-disk column index deliberately (checkpoint / first query). @@ -802,7 +794,6 @@ def _try_inplace(self, store, key, df_norm) -> bool: logger.debug(f"[H5DataFrameStore] in-place update fell back: {exc}") return False - @traced("dataframe", "h5store.upsert") def upsert(self, origin: str, df: pd.DataFrame) -> int: """Atomic upsert with corruption prevention via backup and checksum verification.""" df_norm = self._normalize_for_write(df) @@ -825,12 +816,10 @@ def upsert(self, origin: str, df: pd.DataFrame) -> int: # O(change): if every row already exists and the schema is # unchanged, overwrite values in place (0.1ms vs 42s). if key in store and self._try_inplace(store, key, df_norm): - hit("dataframe", "h5store.upsert", path="inplace", rows=len(df_norm)) return len(df_norm) # Nothing has been written yet in this call; flush so the # copy below captures a consistent on-disk file. - hit("dataframe", "h5store.upsert", path="backup_and_rewrite", rows=len(df_norm)) store.flush() backup_path = self._create_backup() @@ -968,7 +957,6 @@ def get_path(self) -> Path: def exists(self) -> bool: return self._path.exists() - @traced("dataframe", "h5store.delete_column") def delete_column(self, column_name: str, origins: Optional[Iterable[str]] = None) -> bool: """Delete a column from all specified origins (or all origins if None). diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index f59ce7f3..30c0dfe5 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -44,7 +44,6 @@ from weightslab.data import media_store from weightslab.trainer.trainer_tools import execute_df_operation, generate_overview, encode_image_to_raw_bytes from weightslab.data.data_utils import load_raw_image_array -from weightslab.backend.optrace import traced, hit # Image encoding / mask compression / proto helpers (extracted) from weightslab.trainer.services.data_image_utils import ( @@ -2372,7 +2371,6 @@ def _get_categorical_tag_defs(self) -> List["pb2.CategoricalTagDef"]: logger.debug(f"Error building categorical tag defs: {e}") return defs - @traced("dataservice", "sort.build_response") def _build_success_response( self, df, @@ -2421,7 +2419,6 @@ def _build_success_response( analysis_result=analysis_result ) - @traced("dataservice", "sort.parse_query") def _parse_direct_query(self, query: str) -> list: """ Parse a simple direct query string into operations list. @@ -2550,7 +2547,6 @@ def _sort_includes_sample_id(self, by) -> bool: by_list = [by] if isinstance(by, str) else list(by or []) return SampleStatsEx.SAMPLE_ID.value in by_list - @traced("dataservice", "sort.numeric_coerce") def _sample_id_sortable_series(self, values): """Return numeric values for sorting when all sample_ids are integer-like, else string values.""" numeric = pd.to_numeric(values, errors="coerce") @@ -2563,7 +2559,6 @@ def _sample_id_sortable_series(self, values): return numeric return values.astype(str) - @traced("dataservice", "sort.detect_numeric_cols") def _numeric_like_sort_cols(self, df: pd.DataFrame, by) -> set: """Sort columns whose values are strings but mean numbers. @@ -2595,7 +2590,6 @@ def _numeric_like_sort_cols(self, df: pd.DataFrame, by) -> set: continue return out - @traced("dataservice", "sort.sort_values") def _sort_values_numeric_aware(self, df: pd.DataFrame, sort_params: dict) -> None: """Sort dataframe, ordering numeric-valued string columns numerically.""" params = dict(sort_params) @@ -3300,7 +3294,6 @@ def _mask_from_coerced_query(df, expr: str): return None return np.asarray(mask, dtype=bool) - @traced("dataservice", "sort.apply_operation") def _apply_agent_operation(self, df, func: str, params: dict) -> str: """ Apply an agent-described operation to df in-place. @@ -3968,7 +3961,6 @@ def _fast_sync_columns(self, view): - @traced("dataservice", "dsvc._fastUpdateInternals") def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: """O(change) view refresh. True if applied, False -> caller must rebuild. @@ -3977,12 +3969,10 @@ def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: enough that a rebuild is cheaper. """ if not _fast_view_enabled(): - hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", reason="opt_out") return False # opt-out: behave exactly as before view = self._all_datasets_df dfm = self._df_manager if view is None or getattr(view, "empty", True) or dfm is None: - hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", reason="no_view") return False # A column the ledger has but the view lacks can only arrive via a full # rebuild -- the differential write below addresses existing columns @@ -3996,18 +3986,14 @@ def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: if str(c).startswith(self._FAST_SYNC_PREFIXES) and c not in _have] if _missing: - hit("dataservice", "dsvc._fastUpdateInternals", - outcome="fallback", reason="schema_gain", n_missing=len(_missing)) return False except Exception: pass dirty = dfm.take_view_dirty(limit=max_dirty) if dirty is None: - hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", reason="backlog_too_large") return False # backlog too large; rebuild is cheaper if not dirty: - hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="no_change") return True # nothing changed since last sync sids = [str(s) for s in dirty] @@ -4019,12 +4005,9 @@ def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: cols = self._fast_sync_columns(view) if not cols: - hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="no_sync_cols") return True sub = dfm.get_source_rows(keep, columns=[c for c in cols if c in view.columns]) if sub is None or sub.empty: - hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="empty_source_rows", - n_dirty=len(sids)) return True if isinstance(sub.index, pd.MultiIndex): sub = sub.droplevel(-1) @@ -4042,22 +4025,14 @@ def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: _pos = pd.Index(view_keys.astype(str)).get_indexer(sub.index.astype(str)) _ok = _pos >= 0 if not _ok.any(): - hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="no_row_match", - n_dirty=len(sids), n_sub=len(sub.index)) return True if not _ok.all(): - hit("dataservice", "dsvc._fastUpdateInternals", outcome="fallback", - reason="unknown_row", n_dirty=len(sids)) return False for c in sub.columns: _ci = view.columns.get_loc(c) view.iloc[_pos, _ci] = sub[c].to_numpy() - hit("dataservice", "dsvc._fastUpdateInternals", outcome="applied", reason="patched", - n_dirty=len(sids), n_sub=len(sub.index), n_pos=int(_ok.sum()), - n_rows=int(_ok.sum()), n_cols=len(sub.columns)) return True - @traced("dataservice", "dsvc._slowUpdateInternals") def _slowUpdateInternals(self, force: bool = False, reset_view: bool = False) -> None: """Update the internal dataframe view with the latest data from the manager. @@ -4334,7 +4309,6 @@ def _signal_trajectory_curves(self, signal_name, sample_ids, max_points=None): resolved, len(sample_ids), len(curves)) return resolved, curves - @traced("dataservice", "dsvc._build_metadata_only_response") def _build_metadata_only_response(self, df_slice: pd.DataFrame, requested_cols=None): """Build a DataSamplesResponse of metadata DataRecords from dataframe columns only. @@ -4527,7 +4501,6 @@ def _get_all_metadata_column_names(self) -> list: logger.warning("Error enumerating metadata column names: %s", e) return [] - @traced("dataservice", "dsvc.GetMetaData") def GetMetaData(self, request, context): """Metadata-only retrieval, separated from GetDataSamples. @@ -4607,7 +4580,6 @@ def GetMetaData(self, request, context): grid_records=[], ) - @traced("dataservice", "dsvc.GetSignalTrajectory") def GetSignalTrajectory(self, request, context): """On-demand per-sample trajectory of one signal, for the samples shown. @@ -4716,7 +4688,6 @@ def _merge_multi_instance_signals(self, df_slice): merged_df = pd.DataFrame(merged_rows).reset_index(drop=True) return merged_df, signal_dict_mapping - @traced("dataservice", "dsvc._process_get_data_samples") def _process_get_data_samples(self, request, context): """ Actual implementation of GetDataSamples. @@ -5029,7 +5000,6 @@ def _parse_tags(self, tag_value: str) -> set: # RPC Implementations # =================== - @traced("dataservice", "dsvc.ApplyDataQuery") def ApplyDataQuery(self, request, context): """ Apply a query on the in-memory dataframe. @@ -5264,7 +5234,6 @@ def status_cb(msg: str): message=f"Failed to apply query: {str(e)}", ) - @traced("dataservice", "dsvc.GetDataSamples") def GetDataSamples(self, request, context): """ Retrieve samples from the dataframe with their data statistics. @@ -5283,7 +5252,6 @@ def GetDataSamples(self, request, context): data_records=[] ) - @traced("dataservice", "dsvc.GetHistogram") def GetHistogram(self, request, context): """Server-side histogram binning of one column (typed RPC). @@ -5661,7 +5629,6 @@ def _media_cache_put(self, key, value) -> None: while len(self._media_cache) > self._MEDIA_CACHE_ENTRIES: self._media_cache.pop(next(iter(self._media_cache))) - @traced("dataservice", "dsvc.GetPointCloud") def GetPointCloud(self, request, context): """Stream one sample's raw point cloud as binary float32 chunks. @@ -5888,7 +5855,6 @@ def _register_tag(self, tag_name: str): message=f"Tag '{tag_name}' registered", ) - @traced("dataservice", "dsvc.EditDataSample") def EditDataSample(self, request, context): """ Edit sample metadata (tags and discarded). @@ -6380,7 +6346,6 @@ def EditDataSample(self, request, context): message=f"Failed to edit samples: {str(e)}", ) - @traced("dataservice", "dsvc.GetDataSplits") def GetDataSplits(self, request, context): """ Return the list of available dataset splits (train, test, val, etc.) diff --git a/weightslab/trainer/services/experiment_service.py b/weightslab/trainer/services/experiment_service.py index 878897ed..d0c57bdd 100644 --- a/weightslab/trainer/services/experiment_service.py +++ b/weightslab/trainer/services/experiment_service.py @@ -20,7 +20,6 @@ from weightslab.trainer.services.notebook_service import NotebookService from weightslab.data.sample_stats import SampleStatsEx from weightslab.components.evaluation_controller import eval_controller -from weightslab.backend.optrace import traced # Logger @@ -226,7 +225,6 @@ def _kick_eval_worker(self) -> None: # ------------------------------------------------------------------------- # Logger queue sync for WeightsStudio # ------------------------------------------------------------------------- - @traced("experiment", "expsvc.GetLatestLoggerData") def GetLatestLoggerData(self, request, context): """ Returns logger data for WeightsStudio polling. @@ -542,7 +540,6 @@ def _get_latest_logger_data_impl(self, request, context): return pb2.GetLatestLoggerDataResponse(points=points) - @traced("experiment", "expsvc.RestoreCheckpoint") def RestoreCheckpoint(self, request, context): """ Restore a checkpoint from a given experiment hash. @@ -1013,7 +1010,6 @@ def _delayed_exit(): # Training & hyperparameter commands # ------------------------------------------------------------------------- - @traced("experiment", "expsvc.ExperimentCommand") def ExperimentCommand(self, request, context): if request.HasField("restart_operation"): return self._handle_restart_instance() From 00443e21bb4efa03dee4a8a184ba7d294b3d37cb Mon Sep 17 00:00:00 2001 From: Alexandru Rotaru Date: Thu, 3 Sep 2026 13:38:49 +0200 Subject: [PATCH 13/13] ci: fix the code-quality and gRPC-test failures on this branch Three fixes, one per failing check. ruff F841, trainer_tools.process_sample: the positional unpack of _getitem_raw bound _res[1] to idx, which nothing reads -- the function returns sid. Dropped. ruff F401, examples/.../wl-video-generation/utils/data.py: unused `os` import. Pre-existing on dev and untouched by this branch; it only surfaces here because the lint step appends ./weightslab to the changed-file list, so ruff scans the whole package on any PR that touches it. Removing it is what unblocks the gate. AttributeError in tests/gRPC/test_grpc_user_actions.py: _fastUpdateInternals duck-types take_view_dirty and get_source_rows on the df manager. Both are new on this branch, so _FakeDFManager -- and any third-party manager -- raised AttributeError instead of taking the fallback. Guarded: a manager without dirty tracking cannot serve a delta, which is the same "structural change" case the method already falls back on, and the caller then runs _slowUpdateInternals exactly as before. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RHV8zUd5aCtHtQourmwUJC --- weightslab/examples/PyTorch/wl-video-generation/utils/data.py | 1 - weightslab/trainer/services/data_service.py | 3 +++ weightslab/trainer/trainer_tools.py | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/weightslab/examples/PyTorch/wl-video-generation/utils/data.py b/weightslab/examples/PyTorch/wl-video-generation/utils/data.py index ddb4d3ab..42112fba 100644 --- a/weightslab/examples/PyTorch/wl-video-generation/utils/data.py +++ b/weightslab/examples/PyTorch/wl-video-generation/utils/data.py @@ -31,7 +31,6 @@ """ import csv import logging -import os import subprocess import shutil import wave diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index 30c0dfe5..93a541ca 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -3974,6 +3974,9 @@ def _fastUpdateInternals(self, max_dirty: int = 250_000) -> bool: dfm = self._df_manager if view is None or getattr(view, "empty", True) or dfm is None: return False + # A manager without dirty tracking cannot serve a delta -- rebuild. + if not (hasattr(dfm, "take_view_dirty") and hasattr(dfm, "get_source_rows")): + return False # A column the ledger has but the view lacks can only arrive via a full # rebuild -- the differential write below addresses existing columns # only. Per-sample signal columns are created on their first write, so diff --git a/weightslab/trainer/trainer_tools.py b/weightslab/trainer/trainer_tools.py index 30c5528d..1b2d467c 100644 --- a/weightslab/trainer/trainer_tools.py +++ b/weightslab/trainer/trainer_tools.py @@ -442,7 +442,6 @@ def process_sample(sid, dataset, do_resize, resize_dims, experiment): if not isinstance(_res, (tuple, list)): _res = (_res, sid, None) tensor = _res[0] - idx = _res[1] if len(_res) > 1 else sid label = _res[2] if len(_res) > 2 else None if isinstance(tensor, torch.Tensor):