From 4055c952f2bc04fb702351f7232e6da04d5202e5 Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:26:04 +0000 Subject: [PATCH 01/14] =?UTF-8?q?test:=20US1=20failing=20tests=20=E2=80=94?= =?UTF-8?q?=20baseline=20step,=20classification=20fields,=20chronic=20narr?= =?UTF-8?q?ative?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Red phase: 3 new tests assert domain specialists include Baseline step (pcp_fetch_timeseries + pcp_detect_anomalies + 7-day window), classification fields (ANOMALY/RECURRING/BASELINE), and chronic problem narrative guidance. Cross-cutting correctly excluded. --- tests/unit/test_prompts_specialist.py | 92 +++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/unit/test_prompts_specialist.py b/tests/unit/test_prompts_specialist.py index dcee6a6..b524a47 100644 --- a/tests/unit/test_prompts_specialist.py +++ b/tests/unit/test_prompts_specialist.py @@ -183,3 +183,95 @@ def test_specialist_impl_interpolates_lookback(): text = _specialist_investigate_impl("cpu", lookback="4hours")[0]["content"] assert "4hours" in text + + +# --------------------------------------------------------------------------- +# T002: Baseline step presence in domain subsystems +# --------------------------------------------------------------------------- + +_DOMAIN_SUBSYSTEMS = ("cpu", "memory", "disk", "network", "process") + + +def test_baseline_step_present_in_domain_subsystems(): + """T002: Domain specialists include a Baseline step referencing + pcp_fetch_timeseries, pcp_detect_anomalies, and 7-day window.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + for sub in _DOMAIN_SUBSYSTEMS: + text = _specialist_investigate_impl(sub)[0]["content"] + assert "baseline" in text.lower(), f"{sub}: missing Baseline step" + assert "pcp_fetch_timeseries" in text, f"{sub}: Baseline must reference pcp_fetch_timeseries" + assert "pcp_detect_anomalies" in text, f"{sub}: Baseline must reference pcp_detect_anomalies" + assert "7-day" in text.lower() or "7 day" in text.lower(), ( + f"{sub}: Baseline must reference 7-day window" + ) + + +# --------------------------------------------------------------------------- +# T003: Cross-cutting does NOT include Baseline step +# --------------------------------------------------------------------------- + + +def test_baseline_step_absent_from_crosscutting(): + """T003: Cross-cutting specialist does NOT have a Baseline workflow step.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + text = _specialist_investigate_impl("crosscutting")[0]["content"] + # Cross-cutting should not have a numbered "Baseline" step in its workflow + # (it may reference the word "baseline" in domain knowledge for classification, + # but should NOT have a "## Baseline" or "**Baseline**" workflow step) + workflow_section = text.split("## Workflow")[1] if "## Workflow" in text else "" + assert "baseline" not in workflow_section.lower(), ( + "Cross-cutting must NOT have a Baseline workflow step" + ) + + +# --------------------------------------------------------------------------- +# T004: Classification fields in report structure +# --------------------------------------------------------------------------- + + +def test_classification_fields_in_domain_report_guidance(): + """T004: Domain specialist output includes classification, ANOMALY, + RECURRING, BASELINE, baseline_context, severity_despite_baseline.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + required_terms = [ + "classification", + "ANOMALY", + "RECURRING", + "BASELINE", + "baseline_context", + "severity_despite_baseline", + ] + for sub in _DOMAIN_SUBSYSTEMS: + text = _specialist_investigate_impl(sub)[0]["content"] + for term in required_terms: + assert term in text, f"{sub}: missing '{term}' in report guidance" + + +# --------------------------------------------------------------------------- +# T005: Narrative guidance for chronic problems +# --------------------------------------------------------------------------- + + +def test_chronic_problem_narrative_guidance(): + """T005: Domain specialists include narrative guidance for BASELINE-classified + findings — explaining chronic problems in human terms.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + for sub in _DOMAIN_SUBSYSTEMS: + text = _specialist_investigate_impl(sub)[0]["content"].lower() + has_chronic_language = any( + phrase in text + for phrase in ( + "not a new problem", + "your normal", + "chronic", + "historically typical", + "been this way", + ) + ) + assert has_chronic_language, ( + f"{sub}: missing narrative guidance for chronic/baseline findings" + ) From 94eba6f4dde9973488898af5aaeba1b65755ad35 Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:27:11 +0000 Subject: [PATCH 02/14] =?UTF-8?q?feat:=20US1=20=E2=80=94=20baseline=20step?= =?UTF-8?q?=20+=20classification=20fields=20in=20domain=20specialist=20pro?= =?UTF-8?q?mpts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Domain subsystems (cpu, memory, disk, network, process) now include a Baseline step that instructs the agent to fetch 7-day history and run pcp_detect_anomalies. Report guidance includes classification (ANOMALY/ RECURRING/BASELINE), baseline_context, severity_despite_baseline, and chronic problem narrative. Cross-cutting workflow unchanged. --- src/pmmcp/prompts/specialist.py | 70 +++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/src/pmmcp/prompts/specialist.py b/src/pmmcp/prompts/specialist.py index 77e2fa9..1c4f2b0 100644 --- a/src/pmmcp/prompts/specialist.py +++ b/src/pmmcp/prompts/specialist.py @@ -141,6 +141,7 @@ def _specialist_investigate_impl( entry = _SPECIALIST_KNOWLEDGE[subsystem] display = entry["display_name"] prefix = entry["prefix"] + is_domain = prefix is not None # Build context clauses host_clause = f" on host **{host}**" if host else " across all monitored hosts" @@ -149,7 +150,7 @@ def _specialist_investigate_impl( request_clause = f"\n\n**Investigation request**: {request}" if request else "" # Discovery instruction — prefix-based for subsystems with a prefix - if prefix: + if is_domain: discovery = ( f'Use `pcp_discover_metrics(prefix="{prefix}")` as your **primary discovery** ' f"mechanism to enumerate all available {display} metrics. Do not rely solely on " @@ -161,6 +162,65 @@ def _specialist_investigate_impl( "ALL namespaces for anomalies, then drill into the subsystems that surface problems." ) + # Report guidance — domain subsystems get classification fields + report_guidance = entry["report_guidance"] + if is_domain: + report_guidance += """ + +### Finding Classification + +For **each** finding, assign a classification based on the 7-day baseline: + +- **classification**: One of ANOMALY, RECURRING, or BASELINE + - **ANOMALY**: `pcp_detect_anomalies` reports a significant z-score AND the pattern \ +does not recur at consistent times in the 7-day timeseries + - **RECURRING**: The 7-day timeseries shows repeated spikes at consistent times of day \ +(batch jobs, log rotation, backups, cron jobs) — look for time-of-day correlation + - **BASELINE**: Current values are within normal range based on 7-day history (low z-score \ +or no anomaly detected) +- **baseline_context**: Human-readable comparison to the 7-day baseline \ +(e.g., "CPU idle has been below 15% for the past 7 days") +- **severity_despite_baseline**: Threshold-based severity independent of classification \ +(critical/warning/info/none). A BASELINE finding with severity=warning means the host's \ +normal operating state is degraded — this is not a new problem, but it is still a problem. + +### Chronic Problem Articulation + +When a finding is classified as BASELINE but has non-trivial severity_despite_baseline, \ +articulate this clearly: the condition is chronic — it has historically been this way — \ +but "your normal is sick." For example: "CPU idle has been below 10% for a week — this is \ +not a new problem, but the host is chronically saturated."\ +""" + + # Workflow — domain subsystems get a Baseline step after Discover + if is_domain: + workflow = """\ +1. **Discover** available metrics using the approach above. +2. **Baseline** — establish the 7-day historical context: + a. Fetch 7-day historical data at 1hour interval using `pcp_fetch_timeseries` for your \ +key metrics. + b. Run `pcp_detect_anomalies` comparing the current investigation window against the \ +7-day baseline to identify statistically significant deviations. + c. Note the anomaly results — you will use them in the Analyse step to classify findings. + d. **Graceful degradation**: If `pcp_detect_anomalies` returns insufficient data (few or \ +no results — common for new hosts, recent PCP deployments, or archive gaps), fall back to \ +threshold-only analysis. Note "insufficient baseline data, falling back to threshold-only \ +analysis" in your report. If data is sparse (gaps from PCP restarts), attempt detection but \ +note reduced confidence. If 0 days of history are available, skip this Baseline step entirely. +3. **Fetch** key metrics with `pcp_fetch_timeseries` at an appropriate interval for the \ +current investigation window. +4. **Analyse** using the domain knowledge heuristics — check thresholds, correlations, \ +trends. Use the baseline results from step 2 to classify each finding. +5. **Report** each finding in the structured format described above, including classification. +6. **Recommend** next steps — immediate actions, further investigation, or escalation.""" + else: + workflow = """\ +1. **Discover** available metrics using the approach above. +2. **Fetch** key metrics with `pcp_fetch_timeseries` at an appropriate interval. +3. **Analyse** using the domain knowledge heuristics — check thresholds, correlations, trends. +4. **Report** each finding in the structured format described above. +5. **Recommend** next steps — immediate actions, further investigation, or escalation.""" + content = f"""\ You are a **{display} specialist** conducting a focused performance investigation\ {host_clause}{time_clause}{lookback_clause}.{request_clause} @@ -177,15 +237,11 @@ def _specialist_investigate_impl( ## Reporting Structure -{entry["report_guidance"]} +{report_guidance} ## Workflow -1. **Discover** available metrics using the approach above. -2. **Fetch** key metrics with `pcp_fetch_timeseries` at an appropriate interval. -3. **Analyse** using the domain knowledge heuristics — check thresholds, correlations, trends. -4. **Report** each finding in the structured format described above. -5. **Recommend** next steps — immediate actions, further investigation, or escalation. +{workflow} If you find no anomalies, say so explicitly — "no anomalies found in {display}" is a valid result. """ From e3f13c33e702a1275358a315dc37adab903f747c Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:27:51 +0000 Subject: [PATCH 03/14] =?UTF-8?q?test:=20US2=20failing=20tests=20=E2=80=94?= =?UTF-8?q?=20baseline-aware=20heuristics=20in=20domain=20knowledge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Red phase: 5 tests assert each domain specialist's domain_knowledge contains at least one baseline-aware heuristic referencing the 7-day baseline, time-of-day patterns, or scheduled job detection. --- tests/unit/test_prompts_specialist.py | 64 +++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/tests/unit/test_prompts_specialist.py b/tests/unit/test_prompts_specialist.py index b524a47..4b9b72a 100644 --- a/tests/unit/test_prompts_specialist.py +++ b/tests/unit/test_prompts_specialist.py @@ -275,3 +275,67 @@ def test_chronic_problem_narrative_guidance(): assert has_chronic_language, ( f"{sub}: missing narrative guidance for chronic/baseline findings" ) + + +# --------------------------------------------------------------------------- +# T010-T013: Baseline-aware heuristics in domain knowledge +# --------------------------------------------------------------------------- + + +def test_cpu_domain_knowledge_baseline_heuristic(): + """T010: CPU domain_knowledge includes guidance to check whether current + CPU levels are typical for this time of day over the past week.""" + from pmmcp.prompts.specialist import _SPECIALIST_KNOWLEDGE + + dk = _SPECIALIST_KNOWLEDGE["cpu"]["domain_knowledge"].lower() + assert "time of day" in dk or "past week" in dk or "7-day" in dk or "baseline" in dk, ( + "CPU domain_knowledge missing baseline-aware heuristic" + ) + + +def test_memory_domain_knowledge_baseline_heuristic(): + """T011: Memory domain_knowledge includes guidance to compare memory growth + against the 7-day baseline to distinguish leaks from normal working-set growth.""" + from pmmcp.prompts.specialist import _SPECIALIST_KNOWLEDGE + + dk = _SPECIALIST_KNOWLEDGE["memory"]["domain_knowledge"].lower() + has_baseline = any( + phrase in dk + for phrase in ("7-day", "baseline", "working-set growth", "normal growth", "past week") + ) + assert has_baseline, "Memory domain_knowledge missing baseline-aware leak heuristic" + + +def test_disk_domain_knowledge_baseline_heuristic(): + """T012: Disk domain_knowledge includes guidance to check whether I/O spikes + recur at the same time daily (scheduled jobs).""" + from pmmcp.prompts.specialist import _SPECIALIST_KNOWLEDGE + + dk = _SPECIALIST_KNOWLEDGE["disk"]["domain_knowledge"].lower() + has_schedule = any( + phrase in dk + for phrase in ("same time daily", "scheduled job", "recur", "backup", "log rotation") + ) + assert has_schedule, "Disk domain_knowledge missing baseline-aware scheduled job heuristic" + + +def test_network_domain_knowledge_baseline_heuristic(): + """T013a: Network domain_knowledge contains at least one baseline-aware heuristic.""" + from pmmcp.prompts.specialist import _SPECIALIST_KNOWLEDGE + + dk = _SPECIALIST_KNOWLEDGE["network"]["domain_knowledge"].lower() + has_baseline = any( + phrase in dk for phrase in ("baseline", "7-day", "past week", "normal variance") + ) + assert has_baseline, "Network domain_knowledge missing baseline-aware heuristic" + + +def test_process_domain_knowledge_baseline_heuristic(): + """T013b: Process domain_knowledge contains at least one baseline-aware heuristic.""" + from pmmcp.prompts.specialist import _SPECIALIST_KNOWLEDGE + + dk = _SPECIALIST_KNOWLEDGE["process"]["domain_knowledge"].lower() + has_baseline = any( + phrase in dk for phrase in ("baseline", "7-day", "past week", "7-day pattern") + ) + assert has_baseline, "Process domain_knowledge missing baseline-aware heuristic" From 43c76d1e89e9235109bf6dec8da4fbf9d5b7f500 Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:28:46 +0000 Subject: [PATCH 04/14] =?UTF-8?q?feat:=20US2=20=E2=80=94=20baseline-aware?= =?UTF-8?q?=20heuristics=20in=20all=205=20domain=20specialist=20knowledge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each domain specialist now has a 9th heuristic referencing the 7-day baseline: CPU (time-of-day saturation), Memory (leak vs working-set), Disk (scheduled job I/O), Network (drop rate variance), Process (count/context-switch patterns). --- src/pmmcp/prompts/specialist.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pmmcp/prompts/specialist.py b/src/pmmcp/prompts/specialist.py index 1c4f2b0..f377ccb 100644 --- a/src/pmmcp/prompts/specialist.py +++ b/src/pmmcp/prompts/specialist.py @@ -21,7 +21,8 @@ 5. Runqueue depth (`kernel.all.runnable`) sustained > ncpu signals CPU starvation; correlate with load average. 6. Per-CPU imbalance: if one CPU is pegged at 100% while others idle, suspect single-threaded bottleneck or IRQ affinity. 7. High iowait (`kernel.all.cpu.wait.total`) with low user/sys → the CPU is waiting on I/O, investigate disk/network. -8. Sudden CPU spike with no workload change → check for runaway process, cron job, or garbage collection storm.""", +8. Sudden CPU spike with no workload change → check for runaway process, cron job, or garbage collection storm. +9. Before flagging saturation, check the 7-day baseline — is this CPU level typical for this time of day over the past week? A host that always runs hot at 2pm (batch processing) is different from a sudden spike.""", "report_guidance": """\ For each finding report: metric name, observed value (as %), baseline comparison, \ affected time window, and severity (critical/warning/info). Express CPU values as \ @@ -38,7 +39,8 @@ 5. Slab growth (`mem.vmstat.nr_slab_reclaimable`, `nr_slab_unreclaimable`) — unreclaimable slab bloat is a kernel memory leak. 6. Huge page usage (`mem.util.hugepagesTotalBytes` vs `mem.util.hugepagesFreeBytes`) — misconfig wastes reserved memory. 7. Buffer/cache ratio: high `mem.util.bufmem` + `mem.util.cached` with low `mem.util.free` is normal — Linux aggressively caches. -8. Memory pressure trend: plot `mem.util.available` over time — a steady decline indicates a leak even if current usage looks OK.""", +8. Memory pressure trend: plot `mem.util.available` over time — a steady decline indicates a leak even if current usage looks OK. +9. Compare memory growth against the 7-day baseline to distinguish genuine leaks from normal working-set growth — if `mem.util.available` has been declining at the same rate all week, it is the baseline, not a new leak.""", "report_guidance": """\ For each finding report: metric name, observed value in human units (MB/GB), \ percentage of total memory, trend direction (stable/rising/falling), and severity. \ @@ -55,7 +57,8 @@ 5. I/O latency = avactive / (reads + writes) — > 10ms for SSD or > 20ms for HDD is slow. 6. Read vs write ratio: heavy writes with journaling FS (ext4, xfs) amplify actual I/O — check for write-behind flush storms. 7. Correlate disk saturation with CPU iowait (`kernel.all.cpu.wait.total`) — if both high, disk is the bottleneck. -8. Per-device breakdown matters: one saturated device with others idle → workload imbalance or partition misplacement.""", +8. Per-device breakdown matters: one saturated device with others idle → workload imbalance or partition misplacement. +9. Check whether I/O spikes recur at the same time daily — scheduled jobs like backups, log rotation, or cron-driven ETL cause predictable bursts that are not anomalies.""", "report_guidance": """\ For each finding report: device name, metric, observed value in human units \ (IOPS, MB/s, ms latency), device utilisation %, and severity. Always identify \ @@ -72,7 +75,8 @@ 5. Connection states: `network.tcp.currestab` for active connections — sudden spike may indicate connection storm or DDoS. 6. Per-interface breakdown: aggregate numbers hide problems — a saturated eth0 with idle eth1 suggests missing bonding or routing issues. 7. Dropped packets with no errors → buffer exhaustion (ring buffer too small) or CPU too slow to process incoming packets. -8. Compare inbound vs outbound — asymmetric traffic patterns help identify whether the host is a client, server, or relay.""", +8. Compare inbound vs outbound — asymmetric traffic patterns help identify whether the host is a client, server, or relay. +9. Check whether the current packet drop rate is within normal variance over the past week — a host that always drops 0.01% of packets at peak hours is not the same as a sudden 5% drop rate.""", "report_guidance": """\ For each finding report: interface name, metric, observed rate in human units \ (KB/s, MB/s, packets/s), percentage of link capacity if known, and severity. \ @@ -89,7 +93,8 @@ 5. Per-process CPU/memory via hotproc (if available) — identifies the specific process consuming resources. 6. Blocked processes (`proc.runq.blocked`) — processes stuck in uninterruptible sleep, usually waiting on I/O. 7. Thread count trends — growing thread count over time without corresponding workload increase suggests thread pool leak. -8. New process creation rate (`rate(proc.nprocs)`) — high churn (many short-lived processes) wastes fork/exec overhead.""", +8. New process creation rate (`rate(proc.nprocs)`) — high churn (many short-lived processes) wastes fork/exec overhead. +9. Check whether process count and context switch rate match the 7-day pattern before flagging runaway processes — some hosts legitimately run 500+ processes at baseline.""", "report_guidance": """\ For each finding report: process metric, observed value, comparison to healthy \ baseline (e.g., normal process count), trend direction, and severity. Identify \ From 00fda6e1f4052f9d37ff81d185c0c43af642ad2c Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:30:01 +0000 Subject: [PATCH 05/14] =?UTF-8?q?test:=20US5=20=E2=80=94=20graceful=20degr?= =?UTF-8?q?adation=20tests=20(pass=20immediately,=20covered=20by=20US1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Degradation fallback was included in the Baseline step text from US1. Tests validate the contract: threshold-only fallback instruction and "insufficient baseline" limitation note are present for all domain subsystems. --- tests/unit/test_prompts_specialist.py | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/test_prompts_specialist.py b/tests/unit/test_prompts_specialist.py index 4b9b72a..eb4efee 100644 --- a/tests/unit/test_prompts_specialist.py +++ b/tests/unit/test_prompts_specialist.py @@ -339,3 +339,35 @@ def test_process_domain_knowledge_baseline_heuristic(): phrase in dk for phrase in ("baseline", "7-day", "past week", "7-day pattern") ) assert has_baseline, "Process domain_knowledge missing baseline-aware heuristic" + + +# --------------------------------------------------------------------------- +# T020-T021: Graceful degradation when baseline data is insufficient +# --------------------------------------------------------------------------- + + +def test_graceful_degradation_fallback_instruction(): + """T020: Domain specialists include instructions to fall back to + threshold-only analysis if pcp_detect_anomalies returns insufficient data.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + for sub in _DOMAIN_SUBSYSTEMS: + text = _specialist_investigate_impl(sub)[0]["content"].lower() + assert "threshold-only" in text or "threshold only" in text, ( + f"{sub}: missing threshold-only fallback instruction" + ) + assert "insufficient" in text or "fall back" in text or "fallback" in text, ( + f"{sub}: missing fallback trigger language" + ) + + +def test_graceful_degradation_report_limitation(): + """T021: Domain specialists include instructions to note 'insufficient baseline' + or similar limitation wording in the report when degraded.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + for sub in _DOMAIN_SUBSYSTEMS: + text = _specialist_investigate_impl(sub)[0]["content"].lower() + assert "insufficient baseline" in text, ( + f"{sub}: missing 'insufficient baseline' limitation note" + ) From 62e2ca844bf678fbaa6834f97436b05d4017f0a0 Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:30:40 +0000 Subject: [PATCH 06/14] =?UTF-8?q?fix:=20lint=20=E2=80=94=20line=20length?= =?UTF-8?q?=20in=20baseline=20step=20test=20assertions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/unit/test_prompts_specialist.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_prompts_specialist.py b/tests/unit/test_prompts_specialist.py index eb4efee..fddccc9 100644 --- a/tests/unit/test_prompts_specialist.py +++ b/tests/unit/test_prompts_specialist.py @@ -200,8 +200,12 @@ def test_baseline_step_present_in_domain_subsystems(): for sub in _DOMAIN_SUBSYSTEMS: text = _specialist_investigate_impl(sub)[0]["content"] assert "baseline" in text.lower(), f"{sub}: missing Baseline step" - assert "pcp_fetch_timeseries" in text, f"{sub}: Baseline must reference pcp_fetch_timeseries" - assert "pcp_detect_anomalies" in text, f"{sub}: Baseline must reference pcp_detect_anomalies" + assert "pcp_fetch_timeseries" in text, ( + f"{sub}: Baseline must reference pcp_fetch_timeseries" + ) + assert "pcp_detect_anomalies" in text, ( + f"{sub}: Baseline must reference pcp_detect_anomalies" + ) assert "7-day" in text.lower() or "7 day" in text.lower(), ( f"{sub}: Baseline must reference 7-day window" ) From 5335c51de2ccdc534d137dd82a12dbe89272d1e8 Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:31:09 +0000 Subject: [PATCH 07/14] =?UTF-8?q?test:=20US3=20failing=20tests=20=E2=80=94?= =?UTF-8?q?=20cross-cutting=20classification=20prioritisation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Red phase: 3 tests assert cross-cutting specialist prioritises ANOMALY over RECURRING/BASELINE, flags correlated anomalies across subsystems, and notes mixed classification scenarios. --- tests/unit/test_prompts_specialist.py | 50 +++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/unit/test_prompts_specialist.py b/tests/unit/test_prompts_specialist.py index fddccc9..52723fc 100644 --- a/tests/unit/test_prompts_specialist.py +++ b/tests/unit/test_prompts_specialist.py @@ -345,6 +345,56 @@ def test_process_domain_knowledge_baseline_heuristic(): assert has_baseline, "Process domain_knowledge missing baseline-aware heuristic" +# --------------------------------------------------------------------------- +# T026-T028: Cross-cutting classification prioritisation +# --------------------------------------------------------------------------- + + +def test_crosscutting_classification_prioritisation(): + """T026: Cross-cutting output includes guidance to prioritise + ANOMALY-classified findings over RECURRING or BASELINE.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + text = _specialist_investigate_impl("crosscutting")[0]["content"] + assert "ANOMALY" in text, "Cross-cutting missing ANOMALY reference" + has_priority = any( + phrase in text.lower() + for phrase in ("prioriti", "rank", "above", "higher") + ) + assert has_priority, ( + "Cross-cutting missing prioritisation guidance for ANOMALY" + ) + + +def test_crosscutting_correlated_anomalies(): + """T027: Cross-cutting output includes guidance to flag correlated + anomalies across multiple subsystems at the same timestamp.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + text = _specialist_investigate_impl("crosscutting")[0]["content"].lower() + assert "correlat" in text, ( + "Cross-cutting missing correlated anomaly guidance" + ) + has_multi = any( + phrase in text + for phrase in ("multiple subsystem", "across subsystem", "same timestamp") + ) + assert has_multi, ( + "Cross-cutting missing multi-subsystem anomaly correlation" + ) + + +def test_crosscutting_mixed_classification_guidance(): + """T028: Cross-cutting output includes guidance to note when one + subsystem reports BASELINE while another reports ANOMALY.""" + from pmmcp.prompts.specialist import _specialist_investigate_impl + + text = _specialist_investigate_impl("crosscutting")[0]["content"] + assert "BASELINE" in text and "ANOMALY" in text, ( + "Cross-cutting missing BASELINE/ANOMALY mixed classification ref" + ) + + # --------------------------------------------------------------------------- # T020-T021: Graceful degradation when baseline data is insufficient # --------------------------------------------------------------------------- From 72f51f58acee66e605e30c1d88d755a592d12058 Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:31:26 +0000 Subject: [PATCH 08/14] =?UTF-8?q?feat:=20US3=20=E2=80=94=20cross-cutting?= =?UTF-8?q?=20specialist=20prioritises=20by=20classification?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-cutting domain knowledge now instructs the agent to prioritise ANOMALY over RECURRING/BASELINE, flag correlated anomalies across subsystems, and note mixed classification scenarios. --- src/pmmcp/prompts/specialist.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pmmcp/prompts/specialist.py b/src/pmmcp/prompts/specialist.py index f377ccb..cc9edac 100644 --- a/src/pmmcp/prompts/specialist.py +++ b/src/pmmcp/prompts/specialist.py @@ -111,7 +111,10 @@ 5. Load average vs individual subsystems: high load with low CPU user% → the load is I/O-bound or memory-bound, not compute-bound. 6. Time correlation: find the exact moment things changed, then look at ALL subsystems at that timestamp. 7. Use `pcp_compare_windows` to quantify before/after — "it got 3× worse" is more useful than "it's bad." -8. Check derived metrics (derived.cpu.utilisation, derived.mem.utilisation, derived.disk.utilisation) for quick triage.""", +8. Check derived metrics (derived.cpu.utilisation, derived.mem.utilisation, derived.disk.utilisation) for quick triage. +9. Prioritise ANOMALY-classified findings above RECURRING or BASELINE — what changed is more actionable than what has always been wrong. +10. Flag correlated anomalies across multiple subsystems at the same timestamp with higher confidence — if disk and CPU both spike simultaneously, the root cause is likely upstream of both. +11. When one subsystem reports BASELINE while another reports ANOMALY at the same time, the ANOMALY subsystem is more likely root cause — the BASELINE subsystem's chronic condition is not the trigger.""", "report_guidance": """\ For each finding report: the originating subsystem, metric, observed value, \ cross-subsystem correlation (e.g., "disk saturation causing CPU iowait"), \ From 4906195c61e676126607d71cc34a2aec1bc9c8cd Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:32:14 +0000 Subject: [PATCH 09/14] =?UTF-8?q?test:=20US4=20failing=20tests=20=E2=80=94?= =?UTF-8?q?=20coordinator=20classification-based=20ranking?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Red phase: 3 tests assert coordinator synthesis ranks ANOMALY above BASELINE/RECURRING, calls out normal behaviour, and highlights recurring pattern matches. --- tests/unit/test_prompts_coordinator.py | 51 ++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/unit/test_prompts_coordinator.py b/tests/unit/test_prompts_coordinator.py index dcca88d..be61321 100644 --- a/tests/unit/test_prompts_coordinator.py +++ b/tests/unit/test_prompts_coordinator.py @@ -119,3 +119,54 @@ def test_coordinator_impl_interpolates_lookback(): text = _coordinate_investigation_impl(request="issue", lookback="6hours")[0]["content"] assert "6hours" in text + + +# --------------------------------------------------------------------------- +# T032-T034: Classification-based ranking in coordinator synthesis +# --------------------------------------------------------------------------- + + +def test_coordinator_classification_ranking(): + """T032: Coordinator ranks ANOMALY above BASELINE/RECURRING regardless + of severity, with severity as secondary sort within each tier.""" + from pmmcp.prompts.coordinator import _coordinate_investigation_impl + + text = _coordinate_investigation_impl(request="app is slow")[0]["content"] + assert "ANOMALY" in text, "Coordinator missing ANOMALY reference" + has_ranking = any( + phrase in text.lower() + for phrase in ("rank", "prioriti", "above", "tier") + ) + assert has_ranking, "Coordinator missing classification ranking guidance" + + +def test_coordinator_baseline_callout(): + """T033: Coordinator explicitly calls out findings that are normal + behaviour for the host.""" + from pmmcp.prompts.coordinator import _coordinate_investigation_impl + + text = _coordinate_investigation_impl(request="app is slow")[0]["content"] + has_callout = any( + phrase in text.lower() + for phrase in ( + "normal behaviour", + "normal behavior", + "baseline behaviour", + "baseline behavior", + "chronic", + ) + ) + assert has_callout, ( + "Coordinator missing baseline/normal behaviour callout" + ) + + +def test_coordinator_recurring_pattern_highlight(): + """T034: Coordinator highlights when an apparent anomaly matches a + known recurring pattern.""" + from pmmcp.prompts.coordinator import _coordinate_investigation_impl + + text = _coordinate_investigation_impl(request="app is slow")[0]["content"] + assert "RECURRING" in text or "recurring" in text, ( + "Coordinator missing recurring pattern reference" + ) From 35bbb1521e4a9b9a7eb39fdbbd5cea75fe245ea3 Mon Sep 17 00:00:00 2001 From: Paul Smith Date: Tue, 10 Mar 2026 02:32:51 +0000 Subject: [PATCH 10/14] =?UTF-8?q?feat:=20US4=20=E2=80=94=20coordinator=20s?= =?UTF-8?q?ynthesis=20ranks=20by=20classification=20then=20severity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Synthesis phase now ranks ANOMALY > RECURRING > BASELINE (severity as secondary sort within each tier). Output structure updated from "Findings by Severity" to "Findings by Classification & Severity" with sections for New Anomalies, Recurring Patterns, Baseline Behaviour, and Normal Operation. --- src/pmmcp/prompts/coordinator.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/pmmcp/prompts/coordinator.py b/src/pmmcp/prompts/coordinator.py index b6760ae..69d9161 100644 --- a/src/pmmcp/prompts/coordinator.py +++ b/src/pmmcp/prompts/coordinator.py @@ -95,9 +95,21 @@ def _coordinate_investigation_impl( story of what happened. "At 14:32, memory utilisation crossed 95%, triggering swap \ activity, which caused disk I/O to spike, which manifested as CPU iowait." -4. **Rank by impact**: Order findings by severity and blast radius, not by subsystem. +4. **Rank by classification, then severity**: Group findings by classification tier, \ + not by subsystem. ANOMALY findings rank above RECURRING, which rank above BASELINE — \ + regardless of severity. Within each tier, sort by severity (critical → warning → info). \ + What changed (ANOMALY) is more actionable than what has always been wrong (BASELINE). -5. **Recommend actions**: Concrete next steps — not "investigate further" but \ +5. **Call out normal behaviour**: Explicitly identify findings that are baseline \ + behaviour — chronic conditions that are normal for this host. These still matter \ + (a host whose "normal" is CPU-saturated is sick), but they are not the trigger \ + for the current incident. + +6. **Highlight recurring patterns**: When an apparent anomaly matches a known \ + recurring pattern (RECURRING classification from specialists), call this out — \ + "this spike looks alarming but occurs daily at 2am during the backup window." + +7. **Recommend actions**: Concrete next steps — not "investigate further" but \ "check process X for memory leak" or "increase swap space as immediate mitigation." ## Output Structure @@ -109,10 +121,20 @@ def _coordinate_investigation_impl( ## Root Cause Analysis -## Findings by Severity +## Findings by Classification & Severity + +### New Anomalies 1. [CRITICAL] ... 2. [WARNING] ... -3. [INFO] ... + +### Recurring Patterns +1. [WARNING] ... (occurs daily at