Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions src/app/components/chat/chat.component.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {ActivatedRoute, NavigationEnd, Router, UrlTree} from '@angular/router';
import {BehaviorSubject, NEVER, of, ReplaySubject, Subject, throwError} from 'rxjs';

import {EvalCase} from '../../core/models/Eval';
import {OPERATION_GENERATE_CONTENT} from '../../core/models/Trace';
import {Session} from '../../core/models/Session';
import {UiEvent} from '../../core/models/UiEvent';
import {AGENT_SERVICE, AgentService} from '../../core/services/interfaces/agent';
Expand Down Expand Up @@ -2149,4 +2150,76 @@ describe('ChatComponent', () => {
});
});
});

describe('updateSystemInstructionFlags', () => {
function llmSpan(
eventId: string, startTime: number, systemInstruction: string): any {
return {
attrEventId: eventId,
attrOperationName: OPERATION_GENERATE_CONTENT,
start_time: startTime,
io: {inputs: {system_instruction: systemInstruction}},
};
}

it('should not flag a system instruction change across an agent boundary',
() => {
component.eventData.set(
'event-root-1', {id: 'event-root-1', author: 'root_agent'});
component.eventData.set(
'event-sub-1', {id: 'event-sub-1', author: 'sub_agent'});
component.eventData.set(
'event-root-2', {id: 'event-root-2', author: 'root_agent'});
component.traceData = [
llmSpan('event-root-1', 1, 'root instruction'),
llmSpan('event-sub-1', 2, 'sub instruction'),
llmSpan('event-root-2', 3, 'root instruction'),
];

component['updateSystemInstructionFlags']();

expect(component.eventData.get('event-sub-1').systemInstructionChanged)
.toBeFalsy();
expect(component.eventData.get('event-root-2').systemInstructionChanged)
.toBeFalsy();
});

it('should still flag a real system instruction change within the same agent',
() => {
component.eventData.set(
'event-root-1', {id: 'event-root-1', author: 'root_agent'});
component.eventData.set(
'event-root-2', {id: 'event-root-2', author: 'root_agent'});
component.traceData = [
llmSpan('event-root-1', 1, 'instruction v1'),
llmSpan('event-root-2', 2, 'instruction v2'),
];

component['updateSystemInstructionFlags']();

const event = component.eventData.get('event-root-2');
expect(event.systemInstructionChanged).toBeTrue();
expect(event.precedingSystemInstruction).toBe('instruction v1');
expect(event.currentSystemInstruction).toBe('instruction v2');
});

it('should fall back to the span\'s own attrAgentName when the event has no author',
() => {
component.eventData.set('event-root-1', {id: 'event-root-1'});
component.eventData.set('event-sub-1', {id: 'event-sub-1'});
component.eventData.set('event-root-2', {id: 'event-root-2'});
component.traceData = [
{...llmSpan('event-root-1', 1, 'root instruction'), attrAgentName: 'root_agent'},
{...llmSpan('event-sub-1', 2, 'sub instruction'), attrAgentName: 'sub_agent'},
{...llmSpan('event-root-2', 3, 'root instruction'), attrAgentName: 'root_agent'},
];

component['updateSystemInstructionFlags']();

expect(component.eventData.get('event-sub-1').systemInstructionChanged)
.toBeFalsy();
expect(component.eventData.get('event-root-2').systemInstructionChanged)
.toBeFalsy();
});
});
});
52 changes: 36 additions & 16 deletions src/app/components/chat/chat.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2694,22 +2694,42 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy {
event.currentSystemInstruction = undefined;
}

// Compare consecutive LLM turns
for (let i = 1; i < llmSpans.length; i++) {
const currentSpan = llmSpans[i];
const precedingSpan = llmSpans[i - 1];

const currentSys = extractSystemInstruction(currentSpan.io?.inputs);
const precedingSys = extractSystemInstruction(precedingSpan.io?.inputs);

if (currentSys && precedingSys && currentSys !== precedingSys) {
const eventId = currentSpan.attrEventId;
if (eventId) {
const event = this.eventData.get(eventId);
if (event) {
event.systemInstructionChanged = true;
event.precedingSystemInstruction = precedingSys;
event.currentSystemInstruction = currentSys;
// A system instruction is only meaningful *within* a single agent's own
// sequence of calls: distinct agents never shared a cache context to
// begin with, so crossing an agent boundary is not a real "change".
// Group the (already time-sorted) spans by agent before comparing
// consecutive turns, instead of comparing across the whole session.
const spansByAgent = new Map<string, Span[]>();
for (const span of llmSpans) {
const eventId = span.attrEventId;
const event = eventId ? this.eventData.get(eventId) : undefined;
const agentKey = event?.author || span.attrAgentName || '';
const bucket = spansByAgent.get(agentKey);
if (bucket) {
bucket.push(span);
} else {
spansByAgent.set(agentKey, [span]);
}
}

// Compare consecutive LLM turns within each agent's own sequence.
for (const agentSpans of spansByAgent.values()) {
for (let i = 1; i < agentSpans.length; i++) {
const currentSpan = agentSpans[i];
const precedingSpan = agentSpans[i - 1];

const currentSys = extractSystemInstruction(currentSpan.io?.inputs);
const precedingSys = extractSystemInstruction(precedingSpan.io?.inputs);

if (currentSys && precedingSys && currentSys !== precedingSys) {
const eventId = currentSpan.attrEventId;
if (eventId) {
const event = this.eventData.get(eventId);
if (event) {
event.systemInstructionChanged = true;
event.precedingSystemInstruction = precedingSys;
event.currentSystemInstruction = currentSys;
}
}
}
}
Expand Down