From f62b82a438765e051da7d25400d54d3652f0436d Mon Sep 17 00:00:00 2001 From: Praveen Mittal Date: Thu, 3 Sep 2026 18:27:00 +0200 Subject: [PATCH] fix: scope system-instruction change detection to a single agent updateSystemInstructionFlags() compared consecutive LLM spans across the entire session, with no partitioning by agent. In any multi-agent / sub-agent flow, crossing an agent boundary makes the system instruction legitimately differ (each agent has its own instruction and cache context), which incorrectly tripped the "System Instruction Performance Analysis" warning and showed a diff between two unrelated agents' prompts. Group the time-sorted LLM spans by agent (event author, falling back to the span's own attrAgentName) before comparing consecutive turns, so the comparison only ever runs within a single agent's own sequence of calls. Fixes #462 --- .../components/chat/chat.component.spec.ts | 73 +++++++++++++++++++ src/app/components/chat/chat.component.ts | 52 +++++++++---- 2 files changed, 109 insertions(+), 16 deletions(-) diff --git a/src/app/components/chat/chat.component.spec.ts b/src/app/components/chat/chat.component.spec.ts index 6b4f165d..de44945b 100644 --- a/src/app/components/chat/chat.component.spec.ts +++ b/src/app/components/chat/chat.component.spec.ts @@ -28,6 +28,7 @@ import {ActivatedRoute, NavigationEnd, Router, UrlTree} from '@angular/router'; import {BehaviorSubject, NEVER, of, ReplaySubject, Subject, throwError} from 'rxjs'; import {EvalCase} from '../../core/models/Eval'; +import {OPERATION_GENERATE_CONTENT} from '../../core/models/Trace'; import {Session} from '../../core/models/Session'; import {UiEvent} from '../../core/models/UiEvent'; import {AGENT_SERVICE, AgentService} from '../../core/services/interfaces/agent'; @@ -2149,4 +2150,76 @@ describe('ChatComponent', () => { }); }); }); + + describe('updateSystemInstructionFlags', () => { + function llmSpan( + eventId: string, startTime: number, systemInstruction: string): any { + return { + attrEventId: eventId, + attrOperationName: OPERATION_GENERATE_CONTENT, + start_time: startTime, + io: {inputs: {system_instruction: systemInstruction}}, + }; + } + + it('should not flag a system instruction change across an agent boundary', + () => { + component.eventData.set( + 'event-root-1', {id: 'event-root-1', author: 'root_agent'}); + component.eventData.set( + 'event-sub-1', {id: 'event-sub-1', author: 'sub_agent'}); + component.eventData.set( + 'event-root-2', {id: 'event-root-2', author: 'root_agent'}); + component.traceData = [ + llmSpan('event-root-1', 1, 'root instruction'), + llmSpan('event-sub-1', 2, 'sub instruction'), + llmSpan('event-root-2', 3, 'root instruction'), + ]; + + component['updateSystemInstructionFlags'](); + + expect(component.eventData.get('event-sub-1').systemInstructionChanged) + .toBeFalsy(); + expect(component.eventData.get('event-root-2').systemInstructionChanged) + .toBeFalsy(); + }); + + it('should still flag a real system instruction change within the same agent', + () => { + component.eventData.set( + 'event-root-1', {id: 'event-root-1', author: 'root_agent'}); + component.eventData.set( + 'event-root-2', {id: 'event-root-2', author: 'root_agent'}); + component.traceData = [ + llmSpan('event-root-1', 1, 'instruction v1'), + llmSpan('event-root-2', 2, 'instruction v2'), + ]; + + component['updateSystemInstructionFlags'](); + + const event = component.eventData.get('event-root-2'); + expect(event.systemInstructionChanged).toBeTrue(); + expect(event.precedingSystemInstruction).toBe('instruction v1'); + expect(event.currentSystemInstruction).toBe('instruction v2'); + }); + + it('should fall back to the span\'s own attrAgentName when the event has no author', + () => { + component.eventData.set('event-root-1', {id: 'event-root-1'}); + component.eventData.set('event-sub-1', {id: 'event-sub-1'}); + component.eventData.set('event-root-2', {id: 'event-root-2'}); + component.traceData = [ + {...llmSpan('event-root-1', 1, 'root instruction'), attrAgentName: 'root_agent'}, + {...llmSpan('event-sub-1', 2, 'sub instruction'), attrAgentName: 'sub_agent'}, + {...llmSpan('event-root-2', 3, 'root instruction'), attrAgentName: 'root_agent'}, + ]; + + component['updateSystemInstructionFlags'](); + + expect(component.eventData.get('event-sub-1').systemInstructionChanged) + .toBeFalsy(); + expect(component.eventData.get('event-root-2').systemInstructionChanged) + .toBeFalsy(); + }); + }); }); diff --git a/src/app/components/chat/chat.component.ts b/src/app/components/chat/chat.component.ts index 7b19e8cb..bfaa5e03 100644 --- a/src/app/components/chat/chat.component.ts +++ b/src/app/components/chat/chat.component.ts @@ -2694,22 +2694,42 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { event.currentSystemInstruction = undefined; } - // Compare consecutive LLM turns - for (let i = 1; i < llmSpans.length; i++) { - const currentSpan = llmSpans[i]; - const precedingSpan = llmSpans[i - 1]; - - const currentSys = extractSystemInstruction(currentSpan.io?.inputs); - const precedingSys = extractSystemInstruction(precedingSpan.io?.inputs); - - if (currentSys && precedingSys && currentSys !== precedingSys) { - const eventId = currentSpan.attrEventId; - if (eventId) { - const event = this.eventData.get(eventId); - if (event) { - event.systemInstructionChanged = true; - event.precedingSystemInstruction = precedingSys; - event.currentSystemInstruction = currentSys; + // A system instruction is only meaningful *within* a single agent's own + // sequence of calls: distinct agents never shared a cache context to + // begin with, so crossing an agent boundary is not a real "change". + // Group the (already time-sorted) spans by agent before comparing + // consecutive turns, instead of comparing across the whole session. + const spansByAgent = new Map(); + for (const span of llmSpans) { + const eventId = span.attrEventId; + const event = eventId ? this.eventData.get(eventId) : undefined; + const agentKey = event?.author || span.attrAgentName || ''; + const bucket = spansByAgent.get(agentKey); + if (bucket) { + bucket.push(span); + } else { + spansByAgent.set(agentKey, [span]); + } + } + + // Compare consecutive LLM turns within each agent's own sequence. + for (const agentSpans of spansByAgent.values()) { + for (let i = 1; i < agentSpans.length; i++) { + const currentSpan = agentSpans[i]; + const precedingSpan = agentSpans[i - 1]; + + const currentSys = extractSystemInstruction(currentSpan.io?.inputs); + const precedingSys = extractSystemInstruction(precedingSpan.io?.inputs); + + if (currentSys && precedingSys && currentSys !== precedingSys) { + const eventId = currentSpan.attrEventId; + if (eventId) { + const event = this.eventData.get(eventId); + if (event) { + event.systemInstructionChanged = true; + event.precedingSystemInstruction = precedingSys; + event.currentSystemInstruction = currentSys; + } } } }