diff --git a/contrib/calibrate_idle_offset.py b/contrib/calibrate_idle_offset.py index 73c0bfd..2e4a999 100644 --- a/contrib/calibrate_idle_offset.py +++ b/contrib/calibrate_idle_offset.py @@ -50,9 +50,10 @@ from __future__ import annotations import argparse -import math +import json import sqlite3 -from datetime import datetime + +from wallmonitor import calibration, thermal def _rows(conn: sqlite3.Connection, sql: str, params: tuple) -> list[dict]: @@ -61,37 +62,9 @@ def _rows(conn: sqlite3.Connection, sql: str, params: tuple) -> list[dict]: return [dict(zip(cols, row)) for row in cur.fetchall()] -def _interp(series: list[tuple[float, float]], ts: float, max_gap: float = 300.0) -> float | None: - """Linear interpolation with a gap guard: None when the bracketing - samples are too far apart to trust the line between them.""" - if not series or ts < series[0][0] or ts > series[-1][0]: - return None - lo, hi = 0, len(series) - 1 - while hi - lo > 1: - mid = (lo + hi) // 2 - if series[mid][0] <= ts: - lo = mid - else: - hi = mid - (t0, v0), (t1, v1) = series[lo], series[hi] - if t1 - t0 > max_gap: - return None - if t1 == t0: - return v0 - return v0 + (ts - t0) / (t1 - t0) * (v1 - v0) - - -def _mean_sd(vals: list[float]) -> tuple[float, float]: - mu = sum(vals) / len(vals) - if len(vals) < 2: - return mu, float("nan") - return mu, math.sqrt(sum((v - mu) ** 2 for v in vals) / (len(vals) - 1)) - - def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0] if __doc__ else None) parser.add_argument("--db", required=True, help="path to a wallmonitor.db (read-only; use a copy)") - parser.add_argument("--source", default="ecowitt", help="ambient source tag (default %(default)s)") parser.add_argument("--lookback-days", type=float, default=30.0) parser.add_argument("--settle-hours", type=float, default=1.0, help="idle time required after charging before samples count") @@ -101,132 +74,67 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--min-seg-samples", type=int, default=10) args = parser.parse_args(argv) + # The estimator itself lives in the package (wallmonitor.calibration) and + # is what the running monitor refits from daily; this script is the same + # code over the same SQL, for a copy of a database on any machine. conn = sqlite3.connect(f"file:{args.db}?mode=ro", uri=True) now = _rows(conn, "SELECT MAX(ts) AS t FROM vitals_samples", ())[0]["t"] t_from = now - args.lookback_days * 86400.0 - - amb = [ - (r["ts"], r["temp_c"]) - for r in _rows( - conn, - "SELECT ts, temp_c FROM ambient_samples " - "WHERE ts >= ? AND source = ? AND temp_c IS NOT NULL ORDER BY ts", - (t_from, args.source), - ) - ] + amb = [(r["ts"], r["temp_c"]) for r in _rows(conn, calibration.AMBIENT_SQL, + (t_from, now + 1, calibration.CAR_SOURCE))] if len(amb) < 100: - print(f"only {len(amb)} '{args.source}' ambient samples in range — nothing to calibrate against") + print(f"only {len(amb)} stationary ambient samples in range — nothing to calibrate against") return 1 - - vit = _rows( - conn, - "SELECT ts, total_power_w, contactor_closed, vehicle_current_a, " - "CASE WHEN handle_temp_c >= 255 THEN NULL ELSE handle_temp_c END AS handle_temp_c " - "FROM vitals_samples WHERE ts >= ? ORDER BY ts", - (t_from,), - ) - - settle_s = args.settle_hours * 3600.0 - last_charge = None - samples: list[tuple[float, float, float]] = [] # ts, handle, ambient - for r in vit: - if (r["total_power_w"] or 0) > 50: - last_charge = r["ts"] - continue - if r["contactor_closed"] or (r["vehicle_current_a"] or 0) >= 1: - continue - if r["handle_temp_c"] is None: - continue - if last_charge is not None and r["ts"] - last_charge < settle_s: - continue - a_now = _interp(amb, r["ts"]) - a_prev = _interp(amb, r["ts"] - 1800.0) - if a_now is None or a_prev is None or abs(a_now - a_prev) >= args.max_drift_c: - continue - samples.append((r["ts"], r["handle_temp_c"], a_now)) - - if not samples: - print("no settled quasi-static idle samples — is the sensor overlapping idle periods?") - return 1 - - # Collapse to contiguous segments (split on >10 min gaps). - segs: list[list[tuple[float, float, float]]] = [[samples[0]]] - for s in samples[1:]: - if s[0] - segs[-1][-1][0] > 600.0: - segs.append([s]) - else: - segs[-1].append(s) - segs = [g for g in segs - if len(g) >= args.min_seg_samples and g[-1][0] - g[0][0] >= args.min_seg_span_s] - if len(segs) < 8: - print(f"only {len(segs)} usable idle segments — too few for inference; widen the lookback") + vit = _rows(conn, calibration.VITALS_SQL, (t_from, now + 1)) + cal = calibration.estimate(vit, amb, settle_s=args.settle_hours * 3600.0, max_drift_c=args.max_drift_c, + min_seg_span_s=args.min_seg_span_s, min_seg_samples=args.min_seg_samples) + if cal is None: + print("too few settled, quasi-static idle segments for inference — widen the lookback, " + "or check that the sensor overlaps idle periods") return 1 - seg_off = [sum(h - a for _, h, a in g) / len(g) for g in segs] - seg_amb = [sum(a for _, _, a in g) / len(g) for g in segs] - seg_day = [datetime.fromtimestamp((g[0][0] + g[-1][0]) / 2).strftime("%Y-%m-%d") for g in segs] - seg_hour = [datetime.fromtimestamp((g[0][0] + g[-1][0]) / 2).hour for g in segs] - - n = len(segs) - mean, sd = _mean_sd(seg_off) - se = sd / math.sqrt(n) - tcrit = 2.045 if n < 60 else 2.0 - print(f"{len(samples)} settled quasi-static idle samples in {n} segments over " - f"{len(set(seg_day))} days") - print(f"mean offset {mean:.2f} C (sd {sd:.2f}, 95% CI " - f"[{mean - tcrit * se:.2f}, {mean + tcrit * se:.2f}])") - - day = [o for o, h in zip(seg_off, seg_hour) if 8 <= h < 20] - night = [o for o, h in zip(seg_off, seg_hour) if not 8 <= h < 20] - for label, vals in (("day (08-20)", day), ("night", night)): - if len(vals) >= 2: - mu, s = _mean_sd(vals) - print(f" {label}: mean {mu:.2f} sd {s:.2f} (n={len(vals)})") - - # offset ~ ambient regression, day-jackknifed slope error - mx, _ = _mean_sd(seg_amb) - my = mean - sxx = sum((x - mx) ** 2 for x in seg_amb) - slope = sum((x - mx) * (y - my) for x, y in zip(seg_amb, seg_off)) / sxx - days = sorted(set(seg_day)) - jk = [] - for d in days: - keep = [(x, y) for x, y, sd_ in zip(seg_amb, seg_off, seg_day) if sd_ != d] - kx = [x for x, _ in keep] - ky = [y for _, y in keep] - kmx = sum(kx) / len(kx) - kmy = sum(ky) / len(ky) - ksxx = sum((x - kmx) ** 2 for x in kx) - jk.append(sum((x - kmx) * (y - kmy) for x, y in zip(kx, ky)) / ksxx) - mj = sum(jk) / len(jk) - se_slope = math.sqrt((len(jk) - 1) / len(jk) * sum((v - mj) ** 2 for v in jk)) - lo_a, hi_a = min(seg_amb), max(seg_amb) - print(f"offset vs ambient: slope {slope:.4f} C/C " - f"(day-jackknife se {se_slope:.4f}, t {slope / se_slope:.1f}), " - f"coverage {lo_a:.1f}..{hi_a:.1f} C") - - ref = 30.0 - off_ref = my + slope * (ref - mx) + print(f"{cal.n_samples} settled quasi-static idle samples in {cal.n_segments} segments over {cal.n_days} days") + print(f"mean offset {cal.mean_offset_c:.2f} C (sd {cal.sd_c:.2f}, 95% CI " + f"[{cal.ci95_c[0]:.2f}, {cal.ci95_c[1]:.2f}])") + if cal.day_mean_c is not None: + print(f" day (08-20): mean {cal.day_mean_c:.2f}") + if cal.night_mean_c is not None: + print(f" night: mean {cal.night_mean_c:.2f}") + t_slope = cal.slope / cal.slope_se if cal.slope_se and cal.slope_se == cal.slope_se and cal.slope_se > 0 else float("nan") + print(f"offset vs ambient: slope {cal.slope:.4f} C/C (day-jackknife se {cal.slope_se:.4f}, t {t_slope:.1f}), " + f"coverage {cal.ambient_lo_c:.1f}..{cal.ambient_hi_c:.1f} C; segment residual sd {cal.residual_sd_c:.2f} C") + + model = calibration.proposed_model(cal, now) + why = calibration.gate(cal) print() - print("drop-in constants for thermal.py (linear model):") - print(f" IDLE_OFFSET_REF_C = {off_ref:.2f}") - print(f" IDLE_OFFSET_SLOPE = {slope:.4f}") - print(f" IDLE_OFFSET_AMBIENT_REF_C = {ref:.1f}") - print(f" IDLE_OFFSET_AMBIENT_RANGE_C = ({math.floor(lo_a):.1f}, {math.ceil(hi_a * 2) / 2:.1f})") - - try: - from wallmonitor import thermal - - rms_cur = math.sqrt(sum((o - thermal.idle_offset_c(a)) ** 2 - for o, a in zip(seg_off, seg_amb)) / n) - rms_new = math.sqrt(sum((o - (off_ref + slope * (a - ref))) ** 2 - for o, a in zip(seg_off, seg_amb)) / n) - print(f"segment RMS error — current thermal.py model: {rms_cur:.2f} C, " - f"this fit: {rms_new:.2f} C") - except ImportError: - pass + print("model this implies (what the running monitor would adopt" + (f" — but gated: {why})" if why else "):")) + print(json.dumps(model, indent=2)) + print() + print("drop-in constants for thermal.py, if you prefer to change the seed:") + print(f" IDLE_OFFSET_REF_C = {model['ref_c']:.2f}") + print(f" IDLE_OFFSET_SLOPE = {model['slope']:.4f}") + print(f" IDLE_OFFSET_AMBIENT_REF_C = {model['ambient_ref_c']:.1f}") + print(f" IDLE_OFFSET_AMBIENT_RANGE_C = ({model['ambient_range_c'][0]:.1f}, {model['ambient_range_c'][1]:.1f})") + current = thermal.load_idle_offset(_Settings(conn)) if _has_settings(conn) else thermal.BUILTIN_IDLE_OFFSET + print(f"currently in effect on this database: {current.source} " + f"({current.ref_c:.2f} C at {current.ambient_ref_c:.0f} C, slope {current.slope:.4f})") return 0 +class _Settings: + """Just enough of Database for thermal.load_idle_offset over a raw connection.""" + + def __init__(self, conn: sqlite3.Connection): + self._conn = conn + + def get_setting(self, key: str) -> str | None: + row = self._conn.execute("SELECT value FROM settings WHERE key = ?", (key,)).fetchone() + return row[0] if row else None + + +def _has_settings(conn: sqlite3.Connection) -> bool: + return bool(conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='settings'").fetchone()) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/docs/ambient-sensors.md b/docs/ambient-sensors.md index d577e13..96b1f17 100644 --- a/docs/ambient-sensors.md +++ b/docs/ambient-sensors.md @@ -15,6 +15,31 @@ the live forecast, and the idle ambient tile. No configuration: the sensor can appear, disappear, or never exist, and every path falls back to the handle proxy. +A stationary sensor does one more thing: it **calibrates the handle proxy +to your install**. The proxy's idle-offset model (how far above garage air +the idle handle settles, as a function of ambient) ships as a seed fitted +on one install. Once your sensor has overlapped a few days of settled idle +time, the daily maintenance pass refits that model from your own history — +settled, quasi-static idle segments only, per-segment means, a +day-jackknifed slope — and adopts it when it passes sanity gates and moves +materially from what is stored. From then on every handle-derived ambient +read (pre-charge idle, cool-down tails, the idle tile when the sensor is +briefly silent) goes through *your* model; `/api/thermal` reports it under +`model.idle_offset` with `source: "calibrated"`, the segment count and +coverage, and the dashboard's model note says so. Adoption is recorded as +an `idle_offset_calibrated` event and marked on the rise-vs-date chart, +because it reinterprets every proxy-tier fit in history at once — a step +there after a recalibration is the correction working, not the connector +changing. Re-anchor the verified baseline afterwards if the step is large. + +Without a sensor the seed stands, labelled `source: "built-in"`, and every +proxy-derived ambient carries a stated uncertainty (`ambient_se_c`, +±1.5 °C) that the idle tile and model-basis forecasts show. That +uncertainty cannot be reduced without ground truth — it is the honest +price of running sensor-less, and the reason a $20 sensor is worth it. +`contrib/calibrate_idle_offset.py` runs the same estimator over a copy of +any database and prints what the monitor would adopt. + Two dialects are accepted: - **Ecowitt gateway (GW1100/GW1200)** — in the gateway's local web UI, set diff --git a/docs/thermal-model.md b/docs/thermal-model.md index 0cd39d3..e5eeaff 100644 --- a/docs/thermal-model.md +++ b/docs/thermal-model.md @@ -24,10 +24,13 @@ the trip happens. against the default τ, so only charges of ~22 min or more at steady current teach the model there. - **The charger is its own thermometer.** Idle, the handle sits ~1–2 °C above - ambient (a calibrated, ambient-dependent offset — see - `contrib/calibrate_idle_offset.py`), so ambient can be read without any - extra sensor. A LAN ambient sensor or the car's thermometer, when present, - take precedence — see [Ambient sensing](ambient-sensors.md). + ambient (an ambient-dependent offset), so ambient can be read without any + extra sensor. The offset model ships as a seed from one install and is + **recalibrated to yours automatically** once a stationary ambient sensor + has overlapped a few days of idle time; without one it stands, labelled, + with a stated ±1.5 °C uncertainty on every proxy read. A LAN ambient + sensor or the car's thermometer, when present, take precedence — see + [Ambient sensing](ambient-sensors.md). ## What gets fitted: segments, not sessions diff --git a/tests/test_calibration.py b/tests/test_calibration.py new file mode 100644 index 0000000..15a72b4 --- /dev/null +++ b/tests/test_calibration.py @@ -0,0 +1,177 @@ +"""Per-install idle-offset calibration: estimator, gates, hysteresis, and +the model's reach into the proxy-tier fits and the forecast.""" + +import json +import math +import time + +import pytest + +from wallmonitor import calibration, thermal +from wallmonitor.db import Database + + +@pytest.fixture +def db(tmp_path): + database = Database(str(tmp_path / "test.db")) + yield database + database.close() + + +TRUE = thermal.IdleOffset(ref_c=3.0, slope=-0.08, ambient_ref_c=30.0, ambient_range_c=(15.0, 45.0)) + + +def _seed_idle_days(db, now, days, true_model=TRUE, source="ecowitt", sensor=True, tau_s=660.0): + """`days` of a garage: ambient swings 24->34 C daily, the handle sits at + true_model.handle_c(ambient) with first-order lag, one 40-min charge + each afternoon so settle gating has something to gate. Sensor samples + every 60 s when `sensor`.""" + t = now - days * 86400.0 + handle = true_model.handle_c(24.0) + while t < now: + hour = (t % 86400.0) / 3600.0 + ambient = 29.0 + 5.0 * math.sin((hour - 9.0) / 24.0 * 2 * math.pi) + charging = 14.0 <= hour < 14.67 + target = ambient + 30.0 if charging else true_model.handle_c(ambient) + handle += 10.0 * (target - handle) / tau_s + db.insert_vitals(t, { + "vehicle_connected": 1 if charging else 0, "contactor_closed": 1 if charging else 0, + "vehicle_current_a": 48.0 if charging else 0.0, "handle_temp_c": round(handle, 3), + "pcba_temp_c": 40.0, "mcu_temp_c": 45.0, + }, None, 11000.0 if charging else 0.0) + if sensor and int(t) % 60 == 0: + db.insert_ambient(t, round(ambient, 2), source=source) + t += 10.0 + + +def test_estimator_recovers_seeded_offset_model(db): + now = time.time() + now -= now % 60 + _seed_idle_days(db, now, 6) + cal = calibration.calibrate(db, now, lookback_days=10) + assert cal is not None and cal.n_days >= 5 and cal.n_segments >= 8 + # Offset at the reference ambient and the slope both land on the truth. + assert abs(cal.offset_ref_c - TRUE.offset_c(30.0)) < 0.25 + assert abs(cal.slope - TRUE.slope) < 0.03 + assert cal.residual_sd_c < 0.3 + assert calibration.gate(cal) is None + + +def test_car_source_is_not_ground_truth(db): + now = time.time() + now -= now % 60 + _seed_idle_days(db, now, 6, source="car") + assert calibration.calibrate(db, now, lookback_days=10) is None + + +def test_no_sensor_keeps_builtin(db): + now = time.time() + _seed_idle_days(db, now, 3, sensor=False) + old, new, why = calibration.maybe_adopt(db, now, thermal.IDLE_OFFSET_SETTING) + assert new is None and why is None + assert thermal.load_idle_offset(db) is thermal.BUILTIN_IDLE_OFFSET + + +def test_adoption_gates_and_hysteresis(db): + now = time.time() + now -= now % 60 + _seed_idle_days(db, now, 6) + old, new, why = calibration.maybe_adopt(db, now, thermal.IDLE_OFFSET_SETTING) + assert why is None and old is None and new["source"] == "calibrated" + stored = thermal.load_idle_offset(db) + assert stored.source == "calibrated" and abs(stored.offset_c(30.0) - TRUE.offset_c(30.0)) < 0.25 + assert stored.ambient_se_c < 0.5 # the calibration's own scatter, not the 1.5 default + # Same data again: no material change, nothing rewritten. + old2, new2, why2 = calibration.maybe_adopt(db, now, thermal.IDLE_OFFSET_SETTING) + assert new2 is None and why2 == "no material change" and old2 == new + # Gates: an implausible fit never becomes the model. + bad = calibration.Calibration( + n_samples=1000, n_segments=20, n_days=5, mean_offset_c=8.0, sd_c=0.3, ci95_c=(7.8, 8.2), + slope=0.0, slope_se=0.01, ambient_lo_c=20.0, ambient_hi_c=35.0, offset_ref_c=8.0, + residual_sd_c=0.3, day_mean_c=None, night_mean_c=None, from_ts=0.0, to_ts=1.0) + assert "outside" in calibration.gate(bad) + steep = calibration.Calibration(**{**bad.as_dict(), "offset_ref_c": 1.0, "mean_offset_c": 1.0, "slope": -0.9}) + assert "implausible" in calibration.gate(steep) + thin = calibration.Calibration(**{**bad.as_dict(), "offset_ref_c": 1.0, "mean_offset_c": 1.0, "n_days": 2}) + assert "days" in calibration.gate(thin) + # Narrow coverage can't support a slope: the model degrades to a constant. + narrow = calibration.Calibration(**{**bad.as_dict(), "offset_ref_c": 1.0, "mean_offset_c": 1.2, + "slope": -0.3, "ambient_lo_c": 27.0, "ambient_hi_c": 29.0}) + model = calibration.proposed_model(narrow, now) + assert model["slope"] == 0.0 and abs(model["ref_c"] - 1.2) < 1e-6 + + +def test_load_idle_offset_rejects_garbage(db): + db.set_setting(thermal.IDLE_OFFSET_SETTING, "not json") + assert thermal.load_idle_offset(db) is thermal.BUILTIN_IDLE_OFFSET + db.set_setting(thermal.IDLE_OFFSET_SETTING, json.dumps({"ref_c": 1, "slope": -1.2, "ambient_ref_c": 30, + "ambient_range_c": [20, 30]})) + assert thermal.load_idle_offset(db) is thermal.BUILTIN_IDLE_OFFSET # slope would blow up the inversion + db.set_setting(thermal.IDLE_OFFSET_SETTING, json.dumps({"ref_c": 2.5, "slope": -0.1, "ambient_ref_c": 30, + "ambient_range_c": [20, 40], "source": "calibrated", + "residual_sd_c": 0.4})) + model = thermal.load_idle_offset(db) + assert model.source == "calibrated" and abs(model.ambient_from_handle(model.handle_c(27.0)) - 27.0) < 1e-9 + + +def test_proxy_fits_follow_the_calibrated_model(db): + # A garage whose real idle offset is 3 C, no sensor at charge time: under + # the built-in seed (1.4 C) every pre-idle fit reads ambient ~1.6 C high + # and the rise ~1.6 C low. With the install's own model adopted, the + # same rows fit the seeded rise. + now = time.time() + rise, ambient = 36.0, 27.0 + # Seed ramps whose idle lead-in sits at the *true* handle temperature. + for i in range(3): + start = now - (4 - i) * 7200 + _seed_idle_true(db, start - 1800, start, ambient) + _seed_ramp(db, start, ambient, rise, TRUE) + naive = thermal.fit_sessions(db, now) + assert len(naive) == 3 + bias = TRUE.offset_c(ambient) - thermal.BUILTIN_IDLE_OFFSET.offset_c(ambient) + assert bias > 1.0 + for fit in naive: + assert fit["ambient_source"] == "pre_idle" + assert abs(fit["rise_ref_c"] - (rise - bias)) < 1.0 # biased low by the offset error + db.set_setting(thermal.IDLE_OFFSET_SETTING, json.dumps({ + "ref_c": TRUE.ref_c, "slope": TRUE.slope, "ambient_ref_c": TRUE.ambient_ref_c, + "ambient_range_c": list(TRUE.ambient_range_c), "source": "calibrated", "residual_sd_c": 0.3})) + calibrated = thermal.fit_sessions(db, now) + for fit in calibrated: + assert abs(fit["rise_ref_c"] - rise) < 1.0 + # The forecast reports whose model it is, and the proxy's uncertainty. + out = thermal.predict(db, now, thermal.fit_history(db, now, fits=calibrated)) + assert out["model"]["idle_offset"]["source"] == "calibrated" + assert out["model"]["idle_offset"]["ambient_se_c"] == 0.3 + + +def test_idle_forecast_states_proxy_uncertainty(db): + now = time.time() + _seed_idle_true(db, now - 900, now, 26.0) + out = thermal.predict(db, now, thermal.ThermalParams()) + assert out["state"] == "idle" and out["ambient_source"] == "idle_handle" + assert out["ambient_se_c"] == thermal.IDLE_OFFSET_UNCALIBRATED_SE_C + assert out["model"]["idle_offset"]["source"] == "built-in" + + +def _seed_idle_true(db, t_from, t_to, ambient_c, model=TRUE, dt=10.0): + ts = t_from + while ts < t_to: + db.insert_vitals(ts, {"vehicle_connected": 1, "contactor_closed": 0, "vehicle_current_a": 0.0, + "handle_temp_c": round(model.handle_c(ambient_c), 2), + "pcba_temp_c": 38.0, "mcu_temp_c": 46.0}, None, 0.0) + ts += dt + + +def _seed_ramp(db, start_ts, ambient_c, rise_ref_c, model=TRUE, tau_s=720.0, amps=48.6, charge_s=1500.0, dt=10.0): + sid = db.start_session(start_ts) + t0 = model.handle_c(ambient_c) + t_inf = ambient_c + rise_ref_c * (amps / thermal.REF_CURRENT_A) ** 2 + ts = start_ts + while ts <= start_ts + charge_s: + temp = t_inf - (t_inf - t0) * math.exp(-(ts - start_ts) / tau_s) + db.insert_vitals(ts, {"vehicle_connected": 1, "contactor_closed": 1, "vehicle_current_a": amps, + "handle_temp_c": round(temp, 3), "pcba_temp_c": 55.0, "mcu_temp_c": 50.0}, + sid, amps * 233.0) + ts += dt + db.close_session(sid, start_ts + charge_s, "vehicle_disconnected") diff --git a/wallmonitor/__main__.py b/wallmonitor/__main__.py index 2b96b3c..7cfc433 100644 --- a/wallmonitor/__main__.py +++ b/wallmonitor/__main__.py @@ -30,13 +30,17 @@ log = logging.getLogger("wallmonitor") -async def _maintenance(db: Database, cfg) -> None: - """Background housekeeping: the one-time diagnostics backfill, then — - when retention is enabled — a daily raw-JSON trim of samples older - than the retention window. Chunked throughout, so live polling only - ever waits a moment.""" +async def _maintenance(db: Database, cfg, poller: Poller) -> None: + """Background housekeeping: the one-time diagnostics backfill, then a + daily pass that (a) refits the idle-offset model from this install's + own settled-idle history when a stationary ambient sensor has been + reporting, adopting it only on a material, sane change, and (b) when + retention is enabled, trims raw JSON older than the window. Chunked + throughout, so live polling only ever waits a moment.""" import time as _time + from . import calibration, thermal + def report(done: int, total: int) -> None: if done % 200_000 < 10_000 or done >= total: log.info("diagnostics backfill: %d / %d rows", done, total) @@ -44,15 +48,27 @@ def report(done: int, total: int) -> None: touched = await asyncio.to_thread(db.backfill_diag_columns, 10_000, report) if touched: log.info("diagnostics backfill complete: %d rows", touched) - if not cfg.retain_raw_days: - return while True: - cutoff = _time.time() - cfg.retain_raw_days * 86400.0 - counts = await asyncio.to_thread(db.trim_raw, cutoff) - total = sum(counts.values()) - if total: - log.info("retention: trimmed raw JSON from %d rows (%s)", total, - ", ".join(f"{k}={v}" for k, v in counts.items() if v)) + now = _time.time() + try: + old, new, why = await asyncio.to_thread( + calibration.maybe_adopt, db, now, thermal.IDLE_OFFSET_SETTING) + if new is not None: + log.info("idle-offset calibration adopted: %.2f C at %.0f C, slope %.4f, %d segments over %d days", + new["ref_c"], new["ambient_ref_c"], new["slope"], new["segments"], new["days"]) + await poller._event(now, "idle_offset_calibrated", {"old": old, "new": new}) + poller.invalidate_thermal() + elif why and why != "no material change": + log.info("idle-offset calibration not adopted: %s", why) + except Exception: + log.exception("idle-offset calibration failed") + if cfg.retain_raw_days: + cutoff = now - cfg.retain_raw_days * 86400.0 + counts = await asyncio.to_thread(db.trim_raw, cutoff) + total = sum(counts.values()) + if total: + log.info("retention: trimmed raw JSON from %d rows (%s)", total, + ", ".join(f"{k}={v}" for k, v in counts.items() if v)) await asyncio.sleep(24 * 3600.0) @@ -97,7 +113,7 @@ async def run(argv: list[str] | None = None) -> None: client = aiohttp.ClientSession() poller = Poller(cfg, db, bus, client) await poller.start() - backfill = asyncio.create_task(_maintenance(db, cfg), name="wallmonitor-maintenance") + backfill = asyncio.create_task(_maintenance(db, cfg, poller), name="wallmonitor-maintenance") app = make_app(db, bus, poller) runner = web.AppRunner(app) diff --git a/wallmonitor/calibration.py b/wallmonitor/calibration.py new file mode 100644 index 0000000..cd9b361 --- /dev/null +++ b/wallmonitor/calibration.py @@ -0,0 +1,311 @@ +"""Per-install calibration of the idle handle offset. + +The handle-proxy ambient (``ambient ≈ idle handle − offset``) rests on a +model of how far above garage air the idle handle settles. The built-in +model was fitted on one install; this module refits it from *this* +install's own history whenever a stationary ambient sensor gives ground +truth, so the shipped constants are the seed every install starts from +rather than the calibration every install is stuck with. + +Estimator (shared, verbatim, with ``contrib/calibrate_idle_offset.py``): + +- **Settled idle only.** Contactor open, no current, and at least + ``settle_s`` since the last charging sample — the handle needs a few time + constants to shed charge heat; including the decay biases the offset up. +- **Quasi-static ambient only.** The handle lags air by ~tau, so during fast + ambient swings the pairing (handle now, ambient now) is wrong even though + both sensors are right. Samples count only when ambient moved less than + ``max_drift_c`` over the prior 30 min. +- **Per-segment means, not per-sample stats.** Consecutive samples are + massively autocorrelated; treating them as independent would shrink the + error bars by ~sqrt(n) for free. Contiguous idle runs collapse to one + observation each and all inference runs on segment means. +- **Day-clustered slope error.** Segments within a day share weather, so + the offset-vs-ambient slope's significance is jackknifed leave-one-day-out. + +Adoption is separate from estimation and deliberately conservative: sanity +gates (segment count, coverage, a bounded slope — the proxy inverts through +``1 + slope``), and hysteresis so the model only moves on a material +change. Every proxy-tier fit in history is reinterpreted under the adopted +model on its next read, and a drift comparison spanning the change sees a +step — so adoption is recorded as an event, not done silently. +""" + +from __future__ import annotations + +import json +import math +from dataclasses import asdict, dataclass +from datetime import datetime + +# Ambient-sample sources that are stationary garage thermometers. A parked +# car's sensor (source "car") drives away and reads high after drives; it +# is never calibration ground truth. +CAR_SOURCE = "car" + +# The one SQL both loaders use, so the in-process estimator and the contrib +# script see identical rows (255 is the handle sensor's glitch sentinel). +VITALS_SQL = ( + "SELECT ts, total_power_w, contactor_closed, vehicle_current_a, " + "CASE WHEN handle_temp_c >= 255 THEN NULL ELSE handle_temp_c END AS handle_temp_c " + "FROM vitals_samples WHERE ts >= ? AND ts < ? ORDER BY ts" +) +AMBIENT_SQL = ( + "SELECT ts, temp_c FROM ambient_samples " + "WHERE ts >= ? AND ts < ? AND temp_c IS NOT NULL AND (source IS NULL OR source != ?) ORDER BY ts" +) + +AMBIENT_REF_C = 30.0 # the offset is reported at this ambient + + +@dataclass(frozen=True) +class Calibration: + """One estimate: what the data says, with its uncertainty attached.""" + + n_samples: int + n_segments: int + n_days: int + mean_offset_c: float + sd_c: float + ci95_c: tuple[float, float] + slope: float + slope_se: float + ambient_lo_c: float + ambient_hi_c: float + offset_ref_c: float # offset at AMBIENT_REF_C from the regression line + residual_sd_c: float # segment scatter around the fitted line + day_mean_c: float | None + night_mean_c: float | None + from_ts: float + to_ts: float + + def as_dict(self) -> dict: + return asdict(self) + + +def _interp(series: list[tuple[float, float]], ts: float, max_gap: float = 300.0) -> float | None: + """Linear interpolation with a gap guard: None when the bracketing + samples are too far apart to trust the line between them.""" + if not series or ts < series[0][0] or ts > series[-1][0]: + return None + lo, hi = 0, len(series) - 1 + while hi - lo > 1: + mid = (lo + hi) // 2 + if series[mid][0] <= ts: + lo = mid + else: + hi = mid + (t0, v0), (t1, v1) = series[lo], series[hi] + if t1 - t0 > max_gap: + return None + if t1 == t0: + return v0 + return v0 + (ts - t0) / (t1 - t0) * (v1 - v0) + + +def _mean_sd(vals: list[float]) -> tuple[float, float]: + mu = sum(vals) / len(vals) + if len(vals) < 2: + return mu, float("nan") + return mu, math.sqrt(sum((v - mu) ** 2 for v in vals) / (len(vals) - 1)) + + +def estimate(vitals: list[dict], ambient: list[tuple[float, float]], *, settle_s: float = 3600.0, + max_drift_c: float = 0.5, min_seg_span_s: float = 1200.0, min_seg_samples: int = 10, + min_segments: int = 8) -> Calibration | None: + """The estimator. ``vitals`` are raw rows (VITALS_SQL shape, oldest + first); ``ambient`` is the stationary sensor series. None when the + data can't support an estimate — never a number from too little.""" + if len(ambient) < 100: + return None + last_charge = None + samples: list[tuple[float, float, float]] = [] # ts, handle, ambient + for r in vitals: + if (r["total_power_w"] or 0) > 50: + last_charge = r["ts"] + continue + if r["contactor_closed"] or (r["vehicle_current_a"] or 0) >= 1: + continue + if r["handle_temp_c"] is None: + continue + if last_charge is not None and r["ts"] - last_charge < settle_s: + continue + a_now = _interp(ambient, r["ts"]) + a_prev = _interp(ambient, r["ts"] - 1800.0) + if a_now is None or a_prev is None or abs(a_now - a_prev) >= max_drift_c: + continue + samples.append((r["ts"], r["handle_temp_c"], a_now)) + if not samples: + return None + + segs: list[list[tuple[float, float, float]]] = [[samples[0]]] + for s in samples[1:]: + if s[0] - segs[-1][-1][0] > 600.0: + segs.append([s]) + else: + segs[-1].append(s) + segs = [g for g in segs if len(g) >= min_seg_samples and g[-1][0] - g[0][0] >= min_seg_span_s] + if len(segs) < min_segments: + return None + + seg_off = [sum(h - a for _, h, a in g) / len(g) for g in segs] + seg_amb = [sum(a for _, _, a in g) / len(g) for g in segs] + seg_day = [datetime.fromtimestamp((g[0][0] + g[-1][0]) / 2).strftime("%Y-%m-%d") for g in segs] + seg_hour = [datetime.fromtimestamp((g[0][0] + g[-1][0]) / 2).hour for g in segs] + + n = len(segs) + mean, sd = _mean_sd(seg_off) + se = sd / math.sqrt(n) + tcrit = 2.045 if n < 60 else 2.0 + + day = [o for o, h in zip(seg_off, seg_hour) if 8 <= h < 20] + night = [o for o, h in zip(seg_off, seg_hour) if not 8 <= h < 20] + + mx, _ = _mean_sd(seg_amb) + sxx = sum((x - mx) ** 2 for x in seg_amb) + slope = sum((x - mx) * (y - mean) for x, y in zip(seg_amb, seg_off)) / sxx if sxx > 0 else 0.0 + days = sorted(set(seg_day)) + jk = [] + for d in days: + keep = [(x, y) for x, y, sd_ in zip(seg_amb, seg_off, seg_day) if sd_ != d] + if len(keep) < 2: + continue + kx = [x for x, _ in keep] + ky = [y for _, y in keep] + kmx = sum(kx) / len(kx) + kmy = sum(ky) / len(ky) + ksxx = sum((x - kmx) ** 2 for x in kx) + if ksxx > 0: + jk.append(sum((x - kmx) * (y - kmy) for x, y in zip(kx, ky)) / ksxx) + if len(jk) >= 2: + mj = sum(jk) / len(jk) + slope_se = math.sqrt((len(jk) - 1) / len(jk) * sum((v - mj) ** 2 for v in jk)) + else: + slope_se = float("nan") + offset_ref = mean + slope * (AMBIENT_REF_C - mx) + resid = [o - (offset_ref + slope * (a - AMBIENT_REF_C)) for o, a in zip(seg_off, seg_amb)] + residual_sd = math.sqrt(sum(r * r for r in resid) / max(n - 2, 1)) + return Calibration( + n_samples=len(samples), + n_segments=n, + n_days=len(days), + mean_offset_c=mean, + sd_c=sd, + ci95_c=(mean - tcrit * se, mean + tcrit * se), + slope=slope, + slope_se=slope_se, + ambient_lo_c=min(seg_amb), + ambient_hi_c=max(seg_amb), + offset_ref_c=offset_ref, + residual_sd_c=residual_sd, + day_mean_c=_mean_sd(day)[0] if len(day) >= 2 else None, + night_mean_c=_mean_sd(night)[0] if len(night) >= 2 else None, + from_ts=samples[0][0], + to_ts=samples[-1][0], + ) + + +def calibrate(db, now: float, lookback_days: float = 30.0, **kwargs) -> Calibration | None: + """Estimate from a Database, day-chunked so a month of raw rows never + sits in memory at once.""" + t_from = now - lookback_days * 86400.0 + ambient = db.ambient_series(t_from, now, exclude_source=CAR_SOURCE) + if len(ambient) < 100: + return None + vitals: list[dict] = [] + t = t_from + while t < now: + vitals.extend(db.idle_calibration_rows(t, min(t + 86400.0, now))) + t += 86400.0 + return estimate(vitals, ambient, **kwargs) + + +# ---------------- adoption ---------------- + +# Sanity gates: a fit outside these is more likely a contaminated sensor or +# a pathological window than physics, and must not become the proxy model. +OFFSET_REF_RANGE_C = (-1.0, 5.0) +MAX_ABS_SLOPE = 0.5 # the proxy inverts through (1 + slope) +MIN_RANGE_FOR_SLOPE_C = 3.0 # narrower coverage can't support a slope: use a constant +MIN_DAYS = 3 +# Hysteresis: re-estimate daily, adopt only on material change, so the +# proxy-tier fit history doesn't creep a little every day. +ADOPT_DELTA_REF_C = 0.25 +ADOPT_DELTA_SLOPE = 0.03 +ADOPT_RANGE_EXTEND_C = 2.0 + + +def gate(cal: Calibration) -> str | None: + """Why this calibration must not be adopted, or None if it may be.""" + if cal.n_days < MIN_DAYS: + return f"only {cal.n_days} days of settled idle" + lo, hi = OFFSET_REF_RANGE_C + if not (lo <= cal.offset_ref_c <= hi): + return f"offset {cal.offset_ref_c:.2f} C at {AMBIENT_REF_C:.0f} C outside {lo}..{hi}" + if abs(cal.slope) > MAX_ABS_SLOPE and cal.ambient_hi_c - cal.ambient_lo_c >= MIN_RANGE_FOR_SLOPE_C: + return f"slope {cal.slope:.3f} C/C implausible" + return None + + +def proposed_model(cal: Calibration, now: float) -> dict: + """The idle-offset model this calibration implies, as the settings + JSON shape thermal.IdleOffset reads. A coverage too narrow to support + a slope yields a constant offset at the covered ambient.""" + width = cal.ambient_hi_c - cal.ambient_lo_c + slope = cal.slope if width >= MIN_RANGE_FOR_SLOPE_C else 0.0 + if width >= MIN_RANGE_FOR_SLOPE_C: + ref_c, ambient_ref_c = cal.offset_ref_c, AMBIENT_REF_C + else: + ref_c, ambient_ref_c = cal.mean_offset_c, (cal.ambient_lo_c + cal.ambient_hi_c) / 2.0 + return { + "ref_c": round(ref_c, 3), + "slope": round(slope, 4), + "ambient_ref_c": round(ambient_ref_c, 2), + "ambient_range_c": [round(math.floor(cal.ambient_lo_c * 2) / 2, 1), + round(math.ceil(cal.ambient_hi_c * 2) / 2, 1)], + "source": "calibrated", + "segments": cal.n_segments, + "days": cal.n_days, + "residual_sd_c": round(cal.residual_sd_c, 3), + "calibrated_ts": now, + } + + +def material_change(old: dict | None, new: dict) -> bool: + """Hysteresis: is `new` different enough from the stored model to be + worth reinterpreting the proxy-tier history over?""" + if not old or old.get("source") != "calibrated": + return True + ref = AMBIENT_REF_C + old_at_ref = old["ref_c"] + old["slope"] * (ref - old["ambient_ref_c"]) + new_at_ref = new["ref_c"] + new["slope"] * (ref - new["ambient_ref_c"]) + if abs(new_at_ref - old_at_ref) > ADOPT_DELTA_REF_C: + return True + if abs(new["slope"] - old["slope"]) > ADOPT_DELTA_SLOPE: + return True + olo, ohi = old["ambient_range_c"] + nlo, nhi = new["ambient_range_c"] + return (olo - nlo) > ADOPT_RANGE_EXTEND_C or (nhi - ohi) > ADOPT_RANGE_EXTEND_C + + +def maybe_adopt(db, now: float, setting_key: str, lookback_days: float = 30.0) -> tuple[dict | None, dict | None, str | None]: + """Run a calibration and adopt it if it passes the gates and moves the + model materially. Returns (old_model, new_model, reason): new_model is + None when nothing was adopted and reason says why (or None when there + simply wasn't enough data).""" + cal = calibrate(db, now, lookback_days) + if cal is None: + return None, None, None + why = gate(cal) + if why: + return None, None, why + new = proposed_model(cal, now) + raw = db.get_setting(setting_key) + try: + old = json.loads(raw) if raw else None + except ValueError: + old = None + if not material_change(old, new): + return old, None, "no material change" + db.set_setting(setting_key, json.dumps(new)) + return old, new, None diff --git a/wallmonitor/db.py b/wallmonitor/db.py index 3304517..5481803 100644 --- a/wallmonitor/db.py +++ b/wallmonitor/db.py @@ -615,6 +615,23 @@ def latest_ambient(self) -> dict | None: ) return rows[0] if rows else None + def ambient_series(self, t_from: float, t_to: float, exclude_source: str | None = None) -> list[tuple[float, float]]: + """(ts, temp_c) for every ambient sample in the window, oldest + first, unbucketed and unlimited — calibration needs the real + series. exclude_source drops one tag (the car's roaming sensor).""" + from .calibration import AMBIENT_SQL + + rows = self._rows(AMBIENT_SQL, (t_from, t_to, exclude_source if exclude_source is not None else "")) + return [(row["ts"], row["temp_c"]) for row in rows] + + def idle_calibration_rows(self, t_from: float, t_to: float) -> list[dict]: + """Raw (unbucketed) vitals for the idle-offset calibration, in the + exact shape contrib/calibrate_idle_offset.py reads — same SQL, so + the two can never disagree. Callers chunk by day.""" + from .calibration import VITALS_SQL + + return self._rows(VITALS_SQL, (t_from, t_to)) + def set_setting(self, key: str, value: str) -> None: """Upsert into the string key/value settings table (currently only the thermal baseline anchor lives here).""" diff --git a/wallmonitor/poller.py b/wallmonitor/poller.py index 6c469c0..cde9dce 100644 --- a/wallmonitor/poller.py +++ b/wallmonitor/poller.py @@ -495,6 +495,11 @@ async def _check_identity(self, ts: float, serial: str | None) -> None: await asyncio.to_thread(self.db.clear_alert, ts, IDENTITY_ALERT, "monitor") await self._event(ts, "device_restored", {"serial": serial, "host": self.cfg.host}) + def invalidate_thermal(self) -> None: + """Drop the cached forecast params so the next tick refits under a + newly adopted idle-offset model (proxy-tier fits reinterpret).""" + self._params = None + async def recheck_thermal_drift(self, ts: float) -> None: """Refit history and raise/clear the degradation alert. diff --git a/wallmonitor/static/app.js b/wallmonitor/static/app.js index b59125f..a8d4ee2 100644 --- a/wallmonitor/static/app.js +++ b/wallmonitor/static/app.js @@ -1060,7 +1060,8 @@ async function viewLive(root) { const ambLabel = data.ambient_source === "measured" ? "Garage ambient (sensor)" : data.ambient_source === "measured_car" - ? "Garage ambient (car sensor)" : "Ambient at the charger ≈"; + ? "Garage ambient (car sensor)" + : `Ambient at the charger ≈ (from the idle handle, ±${fmtNum(data.ambient_se_c || 1.5, 1)} °C)`; if (forecast.will_trip) { chip = chipFor("warning", "hot enough to derate"); lines.push(`${ambLabel} ${amb}. A full-rate (${fmtNum(model.ref_current_a, 0)} A) charge started now ` + @@ -1118,11 +1119,22 @@ async function viewLive(root) { (dev.tau_frac < -0.3 ? `, and with a τ this fast only charges of ≥ ${fmtNum(1.8 * dev.default_tau_min, 0)} min ` + "at steady current teach the model." : "."); } + // The handle-proxy ambient rests on the idle-offset model; say whose + // it is. Calibrated from this install's own sensor history, or the + // built-in seed from one install — and, without a sensor, how far a + // proxy read may sit from the truth. + const io = model.idle_offset; + const idleNote = !io ? "" : io.source === "calibrated" + ? ` Idle-offset model calibrated to this install (${fmtNum(io.ref_c, 2)} °C at ${fmtNum(io.ambient_ref_c, 0)} °C from ` + + `${io.segments} idle segments over ${io.days} days, ±${fmtNum(io.ambient_se_c, 1)} °C).` + : ` Idle-offset model is the built-in seed from one install (±${fmtNum(io.ambient_se_c, 1)} °C on handle-derived ambient); ` + + "a stationary sensor posting to /api/ambient calibrates it here automatically."; const modelNote = `Model: τ ≈ ${fmtNum(model.tau_min, 1)} min, +${fmtNum(model.rise_ref_c, 0)} °C at ${fmtNum(model.ref_current_a, 0)} A — ` + (model.fitted ? `fitted from ${model.tau_fits} recorded session ramp${model.tau_fits === 1 ? "" : "s"}.` + priorNote : "defaults from one verified install, used until this charger has fits of its own; refits automatically as sessions accumulate.") + (drift && !drift.drifting && !drift.lead ? ` Heat rise stable across the last ${drift.recent_n + drift.baseline_n} fitted sessions` + - `${drift.off_current_n ? ` (${drift.off_current_n} off-current session${drift.off_current_n === 1 ? "" : "s"} excluded)` : ""}.` : ""); + `${drift.off_current_n ? ` (${drift.off_current_n} off-current session${drift.off_current_n === 1 ? "" : "s"} excluded)` : ""}.` : "") + + idleNote; thermalCard.append(el("div", { class: "chart-card" }, el("div", { class: "chart-title" }, "Thermal derate forecast", chip ? " " : null, chip), ...lines.map((text) => el("div", { class: "note" }, text)), @@ -1683,9 +1695,10 @@ async function viewAlerts(root, rangeKey = "7d") { const now = Date.now() / 1000; const from = now - rangeSeconds(rangeKey); root.append(el("h2", {}, "Alerts")); - const [data, thermalData] = await Promise.all([ + const [data, thermalData, calEvents] = await Promise.all([ getJSON(`/api/alerts?from=${from}&to=${now}`), getJSON("/api/thermal").catch(() => null), + getJSON(`/api/events?from=${now - 120 * 86400}&to=${now}&kinds=idle_offset_calibrated`).catch(() => null), loadAlertCodes(), ]); @@ -1724,9 +1737,13 @@ async function viewAlerts(root, rangeKey = "7d") { `Fitted steady-state rise above ambient, normalized to ${fmtNum(thermalData.model.ref_current_a, 0)} A. ` + "A sustained climb at the same current means added resistance in the current path — inspect before it becomes heat."); root.append(rise.card); + // An adopted idle-offset calibration reinterprets every proxy-tier fit + // at once; mark it, so a step here carries its explanation instead of + // reading as the connector changing. + const calMarks = ((calEvents && calEvents.events) || []).map((ev) => ({ ts: ev.ts, label: "idle-offset recalibrated" })); lineChart(rise.box, { series: [{ name: "Rise (°C)", color: colors.s1, points: fits.map((fit) => [fit.start_ts, fit.rise_ref_c]) }], - unit: "°C", digits: 1, height: 180, + unit: "°C", digits: 1, height: 180, vlines: calMarks, }); if (drift) { // The verdict carries its own uncertainty — a delta from a handful of diff --git a/wallmonitor/thermal.py b/wallmonitor/thermal.py index 2f4e980..d327419 100644 --- a/wallmonitor/thermal.py +++ b/wallmonitor/thermal.py @@ -31,6 +31,7 @@ from __future__ import annotations import math +import json from dataclasses import dataclass from statistics import median @@ -63,35 +64,122 @@ IDLE_OFFSET_AMBIENT_REF_C = 30.0 IDLE_OFFSET_AMBIENT_RANGE_C = (23.0, 38.5) # calibration coverage; clamp outside +# These constants are the *seed*, not the calibration: when a stationary +# ambient sensor gives this install ground truth, calibration.maybe_adopt +# refits the same linear model from its own settled-idle history and +# stores it under this settings key; every proxy read then goes through +# the install's model. Without a sensor the seed stands, labelled as such. +IDLE_OFFSET_SETTING = "idle_offset_model" +# Proxy ambient uncertainty when no calibration exists: the built-in fit's +# own segment scatter was ~0.5 C on its install; a different garage can be +# off by a constant this large without any way to know. +IDLE_OFFSET_UNCALIBRATED_SE_C = 1.5 + + +@dataclass(frozen=True) +class IdleOffset: + """The idle-offset model: offset = ref_c + slope * (ambient - ambient_ref_c), + linear inside ambient_range_c and held at the boundary value outside it + (continuous at both edges), so out-of-range readings degrade to a + constant-offset proxy instead of extrapolating the slope.""" + + ref_c: float = IDLE_OFFSET_REF_C + slope: float = IDLE_OFFSET_SLOPE + ambient_ref_c: float = IDLE_OFFSET_AMBIENT_REF_C + ambient_range_c: tuple[float, float] = IDLE_OFFSET_AMBIENT_RANGE_C + source: str = "built-in" + segments: int = 0 + days: int = 0 + residual_sd_c: float | None = None + calibrated_ts: float | None = None + + def offset_c(self, ambient_c: float) -> float: + lo, hi = self.ambient_range_c + clamped = min(max(ambient_c, lo), hi) + return self.ref_c + self.slope * (clamped - self.ambient_ref_c) + + def handle_c(self, ambient_c: float) -> float: + return ambient_c + self.offset_c(ambient_c) + + def ambient_from_handle(self, handle_c: float) -> float: + ta = (handle_c - self.ref_c + self.slope * self.ambient_ref_c) / (1.0 + self.slope) + lo, hi = self.ambient_range_c + if ta < lo: + return handle_c - self.offset_c(lo) + if ta > hi: + return handle_c - self.offset_c(hi) + return ta -def idle_offset_c(ambient_c: float) -> float: + @property + def ambient_se_c(self) -> float: + """How far a proxy ambient read may sit from the truth (1 sigma): + the calibration's own segment scatter, or the uncalibrated default.""" + if self.source == "calibrated" and self.residual_sd_c is not None: + return max(self.residual_sd_c, 0.2) + return IDLE_OFFSET_UNCALIBRATED_SE_C + + def as_dict(self) -> dict: + return { + "source": self.source, + "ref_c": round(self.ref_c, 3), + "slope": round(self.slope, 4), + "ambient_ref_c": self.ambient_ref_c, + "ambient_range_c": [self.ambient_range_c[0], self.ambient_range_c[1]], + "segments": self.segments, + "days": self.days, + "residual_sd_c": round(self.residual_sd_c, 3) if self.residual_sd_c is not None else None, + "ambient_se_c": round(self.ambient_se_c, 2), + "calibrated_ts": self.calibrated_ts, + } + + @classmethod + def from_dict(cls, data: dict) -> "IdleOffset": + lo, hi = data["ambient_range_c"] + return cls( + ref_c=float(data["ref_c"]), + slope=float(data["slope"]), + ambient_ref_c=float(data["ambient_ref_c"]), + ambient_range_c=(float(lo), float(hi)), + source=str(data.get("source", "calibrated")), + segments=int(data.get("segments", 0)), + days=int(data.get("days", 0)), + residual_sd_c=data.get("residual_sd_c"), + calibrated_ts=data.get("calibrated_ts"), + ) + + +BUILTIN_IDLE_OFFSET = IdleOffset() + + +def load_idle_offset(db: Database) -> IdleOffset: + """This install's idle-offset model: the calibrated one when a sane one + is stored, else the built-in seed. Never raises — a corrupt setting + falls back to the seed.""" + raw = db.get_setting(IDLE_OFFSET_SETTING) + if not raw: + return BUILTIN_IDLE_OFFSET + try: + model = IdleOffset.from_dict(json.loads(raw)) + except (ValueError, KeyError, TypeError): + return BUILTIN_IDLE_OFFSET + if not (-0.95 < model.slope < 0.95) or model.ambient_range_c[0] >= model.ambient_range_c[1]: + return BUILTIN_IDLE_OFFSET + return model + + +def idle_offset_c(ambient_c: float, model: IdleOffset = BUILTIN_IDLE_OFFSET) -> float: """How far above garage air the idle handle settles, at this ambient.""" - lo, hi = IDLE_OFFSET_AMBIENT_RANGE_C - clamped = min(max(ambient_c, lo), hi) - return IDLE_OFFSET_REF_C + IDLE_OFFSET_SLOPE * (clamped - IDLE_OFFSET_AMBIENT_REF_C) + return model.offset_c(ambient_c) -def idle_handle_c(ambient_c: float) -> float: +def idle_handle_c(ambient_c: float, model: IdleOffset = BUILTIN_IDLE_OFFSET) -> float: """Idle handle temperature expected at a given garage air temperature.""" - return ambient_c + idle_offset_c(ambient_c) - + return model.handle_c(ambient_c) -def ambient_from_idle_handle(handle_c: float) -> float: - """Garage air implied by a settled idle handle: inverse of idle_handle_c. - Linear inside the calibrated ambient band; beyond it the offset holds at - the boundary value (continuous at both edges), so out-of-range readings - degrade to a constant-offset proxy instead of extrapolating the slope. - """ - ta = (handle_c - IDLE_OFFSET_REF_C + IDLE_OFFSET_SLOPE * IDLE_OFFSET_AMBIENT_REF_C) / ( - 1.0 + IDLE_OFFSET_SLOPE - ) - lo, hi = IDLE_OFFSET_AMBIENT_RANGE_C - if ta < lo: - return handle_c - idle_offset_c(lo) - if ta > hi: - return handle_c - idle_offset_c(hi) - return ta +def ambient_from_idle_handle(handle_c: float, model: IdleOffset = BUILTIN_IDLE_OFFSET) -> float: + """Garage air implied by a settled idle handle: inverse of idle_handle_c.""" + return model.ambient_from_handle(handle_c) REF_CURRENT_A = 48.0 # rise_ref_c is normalized to this charge current DEFAULT_TAU_MIN = 12.0 @@ -336,7 +424,7 @@ def _latest_measured_ambient(db: Database, now: float) -> tuple[float, str] | No return rows[-1]["temp_c"], tag -def _ambient_before(db: Database, start_ts: float) -> float | None: +def _ambient_before(db: Database, start_ts: float, model: IdleOffset = BUILTIN_IDLE_OFFSET) -> float | None: """Ambient estimate from the idle handle temperature before a session.""" rows = db.vitals_range(start_ts - 2400, start_ts - 30, 5000) idle = [ @@ -348,7 +436,7 @@ def _ambient_before(db: Database, start_ts: float) -> float | None: ] if len(idle) < 5 or max(idle) - min(idle) > 2.0: return None - return ambient_from_idle_handle(median(idle)) + return ambient_from_idle_handle(median(idle), model) # Cool-down-tail ambient: gates for reading ambient from a still-warm @@ -359,7 +447,7 @@ def _ambient_before(db: Database, start_ts: float) -> float | None: COOLDOWN_MAX_RMSE_C = 0.5 -def _decay_asymptote(tail: list[dict], tau_min: float) -> float | None: +def _decay_asymptote(tail: list[dict], tau_min: float, model: IdleOffset = BUILTIN_IDLE_OFFSET) -> float | None: """Ambient from a cooling handle's decay: least-squares asymptote of a first-order lag at a known tau, minus the idle offset. Gated on span, visible drop, and fit quality; None when the tail can't be trusted.""" @@ -386,7 +474,7 @@ def _decay_asymptote(tail: list[dict], tau_min: float) -> float | None: ) if rmse > COOLDOWN_MAX_RMSE_C: return None # not a clean single-exponential decay at this tau - ambient = ambient_from_idle_handle(asymptote) + ambient = ambient_from_idle_handle(asymptote, model) if not (-30.0 <= ambient <= TRIP_HANDLE_C): return None return ambient @@ -406,7 +494,8 @@ def _idle_rows(rows: list[dict]) -> list[dict]: ] -def _ambient_from_cooldown(db: Database, start_ts: float, tau_min: float) -> float | None: +def _ambient_from_cooldown(db: Database, start_ts: float, tau_min: float, + model: IdleOffset = BUILTIN_IDLE_OFFSET) -> float | None: """Ambient from the cool-down tail before a segment that starts warm. A stop/resume or post-derate segment begins before the handle has @@ -430,10 +519,11 @@ def _ambient_from_cooldown(db: Database, start_ts: float, tau_min: float) -> flo break tail.append(row) tail.reverse() - return _decay_asymptote(tail, tau_min) + return _decay_asymptote(tail, tau_min, model) -def _ambient_after(db: Database, end_ts: float, tau_min: float) -> float | None: +def _ambient_after(db: Database, end_ts: float, tau_min: float, + model: IdleOffset = BUILTIN_IDLE_OFFSET) -> float | None: """Ambient at the end of a load window, from the cool-down that follows. The moment current stops, the handle decays from its working temperature @@ -454,7 +544,7 @@ def _ambient_after(db: Database, end_ts: float, tau_min: float) -> float | None: if row["ts"] - tail[-1]["ts"] > 120.0: break tail.append(row) - return _decay_asymptote(tail, tau_min) + return _decay_asymptote(tail, tau_min, model) def fit_sessions(db: Database, now: float, lookback_days: float = 120.0) -> list[dict]: @@ -486,6 +576,7 @@ def fit_sessions(db: Database, now: float, lookback_days: float = 120.0) -> list if session.get("end_ts") and (session.get("charging_s") or 0) >= MIN_SEGMENT_S ][:40] fits: list[dict] = [] + idle_model = load_idle_offset(db) for sess in sessions: # Coarse pass over the whole session to locate charging segments — # bucket-averaged is fine here (and keeps a multi-day session cheap); @@ -539,14 +630,14 @@ def fit_sessions(db: Database, now: float, lookback_days: float = 120.0) -> list measured = _measured_ambient(db, seg_start - MEASURED_AMBIENT_WINDOW_S, seg_start + 60) ambient, ambient_source = measured if measured is not None else (None, None) if ambient is None: - ambient = _ambient_before(db, seg_start) + ambient = _ambient_before(db, seg_start, idle_model) ambient_source = "pre_idle" if ambient is not None else None if ambient is None: # Hot-handle start (stop/resume, post-derate): read ambient # from the previous charge's cool-down tail instead, using # this install's fitted tau (this segment's own plus any # earlier fits this pass). - ambient = _ambient_from_cooldown(db, seg_start, tau_est) + ambient = _ambient_from_cooldown(db, seg_start, tau_est, idle_model) if ambient is not None: ambient_source = "cooldown_tail" ambient_end = None @@ -554,7 +645,7 @@ def fit_sessions(db: Database, now: float, lookback_days: float = 120.0) -> list measured_end = _measured_ambient(db, seg_end - 60, seg_end + MEASURED_AMBIENT_WINDOW_S) ambient_end = measured_end[0] if measured_end is not None else None if ambient_end is None: - ambient_end = _ambient_after(db, seg_end, tau_est) + ambient_end = _ambient_after(db, seg_end, tau_est, idle_model) if ambient_end is not None: # Bracketed: de-trend the samples against the linear ambient # ramp across the load window and refit. With ambient @@ -883,7 +974,9 @@ def _recent_steady_ambient(recent: list[dict], params: ThermalParams) -> float | def predict(db: Database, now: float, params: ThermalParams) -> dict: """Forecast alert-40 for the current state (live session or idle).""" - out: dict = {"model": params.as_dict(), "state": "no_data", "forecast": None} + idle_model = load_idle_offset(db) + out: dict = {"model": {**params.as_dict(), "idle_offset": idle_model.as_dict()}, + "state": "no_data", "forecast": None} recent = [ row for row in db.vitals_range(now - 900, now, 2000) if row.get("handle_temp_c") is not None ] @@ -942,7 +1035,7 @@ def predict(db: Database, now: float, params: ThermalParams) -> dict: if ambient is None: sid = last.get("session_id") sess = db.session(int(sid)) if sid else None - ambient = _ambient_before(db, sess["start_ts"]) if sess else None + ambient = _ambient_before(db, sess["start_ts"], idle_model) if sess else None source = "pre_session" if ambient is None: ambient = _recent_steady_ambient(recent, params) @@ -957,6 +1050,10 @@ def predict(db: Database, now: float, params: ThermalParams) -> dict: t_inf = ambient + params.rise_ref_c * (current / REF_CURRENT_A) ** 2 forecast["basis"] = "model" forecast["ambient_source"] = source + # A model-basis plateau is only as good as its ambient: a sensor + # reads air directly; every handle-derived route carries the + # idle-offset model's own uncertainty, 1:1 into the plateau. + forecast["ambient_se_c"] = 0.3 if source in ("measured", "measured_car") else idle_model.ambient_se_c # No flooring of t_inf at the current temperature: a steady state # below the handle is real, not noise — it's what cooling toward a # lower equilibrium looks like after a current cut or a derate. @@ -989,10 +1086,11 @@ def predict(db: Database, now: float, params: ThermalParams) -> dict: ambient, ambient_source = measured stable = True else: - ambient = ambient_from_idle_handle(last["handle_temp_c"]) + ambient = ambient_from_idle_handle(last["handle_temp_c"], idle_model) ambient_source = "idle_handle" out["ambient_c"] = round(ambient, 1) out["ambient_source"] = ambient_source + out["ambient_se_c"] = 0.3 if measured is not None else idle_model.ambient_se_c out["ambient_stable"] = stable # Hypothetical: a full-rate session started right now. t_inf = ambient + params.rise_ref_c diff --git a/wallmonitor/web.py b/wallmonitor/web.py index d5f48cc..1763f86 100644 --- a/wallmonitor/web.py +++ b/wallmonitor/web.py @@ -224,22 +224,27 @@ async def api_event_ingest(request: web.Request) -> web.Response: # Model parameters change only as new sessions land, so the (SQLite-heavy) # history fit is cached; the live prediction is computed on every call. - thermal_fit: dict = {"params": None, "fits": [], "ts": 0.0} + thermal_fit: dict = {"params": None, "fits": [], "ts": 0.0, "idle_offset": None} async def api_thermal(request: web.Request) -> web.Response: """The full thermal picture: fitted model, live forecast, per-segment fits, drift verdict, baseline anchor. Also what the BLE amp controller polls every 30 s. ?refit busts the 6 h fit cache.""" now = time.time() + # A newly adopted idle-offset model reinterprets every proxy-tier + # fit, so the cache is keyed on the stored model too. + idle_offset = await asyncio.to_thread(db.get_setting, thermal.IDLE_OFFSET_SETTING) if ( thermal_fit["params"] is None or now - thermal_fit["ts"] > 6 * 3600 + or thermal_fit["idle_offset"] != idle_offset or "refit" in request.query ): fits = await asyncio.to_thread(thermal.fit_sessions, db, now) thermal_fit["fits"] = fits thermal_fit["params"] = thermal.fit_history(db, now, fits=fits) thermal_fit["ts"] = now + thermal_fit["idle_offset"] = idle_offset result = await asyncio.to_thread(thermal.predict, db, now, thermal_fit["params"]) anchor = await asyncio.to_thread(thermal.baseline_anchor, db) return web.json_response(