From 1d6223b9034a7cd977aefc79d508828745e2a9bf Mon Sep 17 00:00:00 2001 From: Andrew Klatzke Date: Fri, 26 Jun 2026 12:03:58 -0800 Subject: [PATCH] chore: reverts judge stripping behavior --- .../__tests__/LangChainModelRunner.test.ts | 21 +++++- .../src/LangChainModelRunner.ts | 39 ++++++---- .../__tests__/OpenAIModelRunner.test.ts | 21 ++++++ .../server-ai-openai/src/OpenAIModelRunner.ts | 36 ++++++---- .../__tests__/VercelModelRunner.test.ts | 24 +++++++ .../server-ai-vercel/src/VercelModelRunner.ts | 38 ++++++---- .../sdk/server-ai/__tests__/Judge.test.ts | 66 +---------------- .../__tests__/LDAIClientImpl.test.ts | 72 ++----------------- packages/sdk/server-ai/src/LDAIClientImpl.ts | 32 +-------- packages/sdk/server-ai/src/api/judge/Judge.ts | 21 ------ .../sdk/server-ai/src/api/providers/Runner.ts | 10 ++- 11 files changed, 153 insertions(+), 227 deletions(-) diff --git a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts index 2ae5a3e018..e9d3e5a6b9 100644 --- a/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts +++ b/packages/ai-providers/server-ai-langchain/__tests__/LangChainModelRunner.test.ts @@ -1,6 +1,6 @@ import { AIMessage } from '@langchain/core/messages'; -import type { LDAICompletionConfig } from '@launchdarkly/server-sdk-ai'; +import type { LDAICompletionConfig, LDMessage } from '@launchdarkly/server-sdk-ai'; import { LangChainModelRunner } from '../src/LangChainModelRunner'; @@ -63,6 +63,25 @@ describe('LangChainModelRunner', () => { expect(passed[1].content).toBe('hi'); }); + it('uses a LDMessage[] as-is without prepending config messages', async () => { + const response = new AIMessage('direct reply'); + mockLLM.invoke.mockResolvedValue(response); + + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'You are X' }], + }; + const r = new LangChainModelRunner(mockLLM, configWithMessages, mockLogger); + const inputMessages: LDMessage[] = [ + { role: 'user', content: 'direct question' }, + ]; + await r.run(inputMessages); + + const passed = mockLLM.invoke.mock.calls[0][0]; + expect(passed).toHaveLength(1); + expect(passed[0].content).toBe('direct question'); + }); + it('marks success=false and warns when content is non-string (multimodal)', async () => { mockLLM.invoke.mockResolvedValue(new AIMessage([{ type: 'image' }] as any)); diff --git a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts index 5f86191f81..de7696fd14 100644 --- a/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts +++ b/packages/ai-providers/server-ai-langchain/src/LangChainModelRunner.ts @@ -5,6 +5,7 @@ import { AIMessage, BaseMessage, HumanMessage } from '@langchain/core/messages'; import type { LDAICompletionConfig, LDLogger, + LDMessage, Runner, RunnerResult, } from '@launchdarkly/server-sdk-ai'; @@ -38,25 +39,35 @@ export class LangChainModelRunner implements Runner { } /** - * Run the LangChain model with the given user prompt. + * Run the LangChain model with the given user prompt or message array. * - * The runner maintains a LangChain `InMemoryChatMessageHistory` that is - * initialized from any messages on the AI config (system prompt, etc.). On - * every invocation the user prompt is appended to the existing history - * before being sent to the model. When `multiTurn` is `true` (the default) - * and the call succeeds with non-empty content, the user prompt and the - * assistant's reply are persisted to the history so subsequent calls - * continue the conversation. When `multiTurn` is `false`, history is - * treated as read-only — useful for stateless runners (e.g. judges) where - * every call should see only the initial config messages plus the current - * input. Failed calls leave the history unchanged so the next call can - * retry cleanly. + * When `input` is a string, the runner maintains a LangChain + * `InMemoryChatMessageHistory` initialized from any messages on the AI + * config. The user prompt is appended to the existing history before being + * sent to the model. When `multiTurn` is `true` (the default) and the call + * succeeds with non-empty content, the user prompt and the assistant's reply + * are persisted to the history so subsequent calls continue the + * conversation. When `multiTurn` is `false`, history is treated as + * read-only — useful for stateless runners (e.g. judges) where every call + * should see only the initial config messages plus the current input. + * Failed calls leave the history unchanged so the next call can retry + * cleanly. * - * @param input The user prompt string. + * When `input` is a pre-built `LDMessage[]` it is used as-is — config + * messages are not prepended and history is not updated. + * + * @param input The user prompt string, or a pre-built message array. * @param outputType Optional JSON schema for structured output. When provided, * the parsed result is exposed via {@link RunnerResult.parsed}. */ - async run(input: string, outputType?: Record): Promise { + async run(input: string | LDMessage[], outputType?: Record): Promise { + if (Array.isArray(input)) { + const langchainMessages = convertMessagesToLangChain(input); + return outputType !== undefined + ? this._runStructured(langchainMessages, outputType) + : this._runCompletion(langchainMessages); + } + const langchainMessages: BaseMessage[] = [ ...(await this._chatHistory.getMessages()), new HumanMessage(input), diff --git a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts index f0b20be095..ef2c80335c 100644 --- a/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts +++ b/packages/ai-providers/server-ai-openai/__tests__/OpenAIModelRunner.test.ts @@ -75,6 +75,27 @@ describe('OpenAIModelRunner', () => { }); }); + it('passes a LDMessage[] input directly without prepending config messages', async () => { + const mockResponse = { + choices: [{ message: { content: 'Evaluation result' } }], + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }; + (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue(mockResponse as any); + + const messages = [ + { role: 'system' as const, content: 'You are a judge' }, + { role: 'user' as const, content: 'Rate this: hello' }, + ]; + const result = await runner.run(messages); + + expect(mockOpenAI.chat.completions.create).toHaveBeenCalledWith({ + model: 'gpt-3.5-turbo', + messages, + }); + expect(result.content).toBe('Evaluation result'); + expect(result.metrics.success).toBe(true); + }); + it('marks the result unsuccessful when response has no content', async () => { const mockResponse = { choices: [{ message: {} }] }; (mockOpenAI.chat.completions.create as jest.Mock).mockResolvedValue(mockResponse as any); diff --git a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts index e87b83d52e..e4d66ec6f5 100644 --- a/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts +++ b/packages/ai-providers/server-ai-openai/src/OpenAIModelRunner.ts @@ -39,25 +39,33 @@ export class OpenAIModelRunner implements Runner { } /** - * Run the OpenAI model with the given user prompt. + * Run the OpenAI model with the given user prompt or message array. * - * The runner maintains a conversation history that is initialized from any - * messages on the AI config (system prompt, instructions, etc.). On every - * invocation the user prompt is appended to the existing history before - * being sent to the model. When `multiTurn` is `true` (the default) and the - * call succeeds with non-empty content, the user prompt and the assistant's - * reply are persisted to the history so subsequent calls continue the - * conversation. When `multiTurn` is `false`, history is treated as - * read-only — useful for stateless runners (e.g. judges) where every call - * should see only the initial config messages plus the current input. - * Failed calls leave the history unchanged so the next call can retry - * cleanly. + * When `input` is a string, the runner maintains a conversation history + * initialized from any messages on the AI config. The user prompt is + * appended to the existing history before being sent to the model. When + * `multiTurn` is `true` (the default) and the call succeeds with non-empty + * content, the user prompt and the assistant's reply are persisted to the + * history so subsequent calls continue the conversation. When `multiTurn` + * is `false`, history is treated as read-only — useful for stateless runners + * (e.g. judges) where every call should see only the initial config messages + * plus the current input. Failed calls leave the history unchanged so the + * next call can retry cleanly. * - * @param input The user prompt string. + * When `input` is a pre-built `LDMessage[]` it is used as-is — config + * messages are not prepended and history is not updated. + * + * @param input The user prompt string, or a pre-built message array. * @param outputType Optional JSON schema for structured output. When provided, * the response is parsed and exposed via {@link RunnerResult.parsed}. */ - async run(input: string, outputType?: Record): Promise { + async run(input: string | LDMessage[], outputType?: Record): Promise { + if (Array.isArray(input)) { + return outputType !== undefined + ? this._runStructured(input, outputType) + : this._runCompletion(input); + } + const userMessage: LDMessage = { role: 'user', content: input }; const messages: LDMessage[] = [...this._history, userMessage]; diff --git a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts index d78d28edc6..014075333c 100644 --- a/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts +++ b/packages/ai-providers/server-ai-vercel/__tests__/VercelModelRunner.test.ts @@ -90,6 +90,30 @@ describe('VercelModelRunner', () => { expect(out.metrics.tokens).toEqual({ total: 100, input: 40, output: 60 }); }); + it('uses a LDMessage[] directly without prepending config messages', async () => { + (generateText as jest.Mock).mockResolvedValue({ + text: 'direct', + usage: { totalTokens: 5, promptTokens: 2, completionTokens: 3 }, + }); + + const configWithMessages: LDAICompletionConfig = { + ...baseConfig, + messages: [{ role: 'system', content: 'Should not appear' }], + }; + const r = new VercelModelRunner(fakeModel as any, configWithMessages, {}, mockLogger); + const prebuilt = [ + { role: 'system' as const, content: 'Custom system' }, + { role: 'user' as const, content: 'Direct input' }, + ]; + await r.run(prebuilt); + + expect(generateText).toHaveBeenCalledWith({ + model: fakeModel, + messages: prebuilt, + experimental_telemetry: { isEnabled: true }, + }); + }); + it('returns success=false when generateText throws', async () => { const err = new Error('boom'); (generateText as jest.Mock).mockRejectedValue(err); diff --git a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts index ccadc3d6ba..d0e2044696 100644 --- a/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts +++ b/packages/ai-providers/server-ai-vercel/src/VercelModelRunner.ts @@ -3,6 +3,7 @@ import { generateObject, generateText, jsonSchema, LanguageModel, ModelMessage } import type { LDAICompletionConfig, LDLogger, + LDMessage, Runner, RunnerResult, } from '@launchdarkly/server-sdk-ai'; @@ -38,25 +39,34 @@ export class VercelModelRunner implements Runner { } /** - * Run the Vercel AI model with the given user prompt. + * Run the Vercel AI model with the given user prompt or message array. * - * The runner maintains a conversation history (as Vercel AI SDK - * `ModelMessage`s) that is initialized from any messages on the AI config - * (system prompt, etc.). On every invocation the user prompt is appended to - * the existing history before being sent to the model. When `multiTurn` is - * `true` (the default) and the call succeeds with non-empty content, the - * user prompt and the assistant's reply are persisted to the history so - * subsequent calls continue the conversation. When `multiTurn` is `false`, - * history is treated as read-only — useful for stateless runners (e.g. - * judges) where every call should see only the initial config messages - * plus the current input. Failed calls leave the history unchanged so the - * next call can retry cleanly. + * When `input` is a string, the runner maintains a conversation history (as + * Vercel AI SDK `ModelMessage`s) initialized from any messages on the AI + * config. The user prompt is appended to the existing history before being + * sent to the model. When `multiTurn` is `true` (the default) and the call + * succeeds with non-empty content, the user prompt and the assistant's reply + * are persisted to the history so subsequent calls continue the + * conversation. When `multiTurn` is `false`, history is treated as + * read-only — useful for stateless runners (e.g. judges) where every call + * should see only the initial config messages plus the current input. Failed + * calls leave the history unchanged so the next call can retry cleanly. * - * @param input The user prompt string. + * When `input` is a pre-built `LDMessage[]` it is used as-is — config + * messages are not prepended and history is not updated. + * + * @param input The user prompt string, or a pre-built message array. * @param outputType Optional JSON schema for structured output. When provided, * the parsed object is exposed via {@link RunnerResult.parsed}. */ - async run(input: string, outputType?: Record): Promise { + async run(input: string | LDMessage[], outputType?: Record): Promise { + if (Array.isArray(input)) { + const vercelMessages = convertMessagesToVercel(input) as ModelMessage[]; + return outputType !== undefined + ? this._runStructured(vercelMessages, outputType) + : this._runCompletion(vercelMessages); + } + const userMessage: ModelMessage = { role: 'user', content: input }; const messages: ModelMessage[] = [...this._history, userMessage]; diff --git a/packages/sdk/server-ai/__tests__/Judge.test.ts b/packages/sdk/server-ai/__tests__/Judge.test.ts index 75a1d33a89..577b4249ff 100644 --- a/packages/sdk/server-ai/__tests__/Judge.test.ts +++ b/packages/sdk/server-ai/__tests__/Judge.test.ts @@ -2,74 +2,10 @@ import { LDLogger } from '@launchdarkly/js-server-sdk-common'; import { LDAIConfigTracker } from '../src/api/config/LDAIConfigTracker'; import { LDAIJudgeConfig, LDMessage } from '../src/api/config/types'; -import { Judge, stripLegacyJudgeMessages } from '../src/api/judge/Judge'; +import { Judge } from '../src/api/judge/Judge'; import { RunnerResult } from '../src/api/model/types'; import { Runner } from '../src/api/providers/Runner'; -describe('stripLegacyJudgeMessages', () => { - it('strips assistant messages containing {{message_history}}', () => { - const messages: LDMessage[] = [ - { role: 'system', content: 'You are a judge.' }, - { role: 'assistant', content: 'Here is the history: {{message_history}}' }, - ]; - const result = stripLegacyJudgeMessages(messages); - expect(result).toHaveLength(1); - expect(result[0].role).toBe('system'); - }); - - it('strips user messages containing {{response_to_evaluate}}', () => { - const messages: LDMessage[] = [ - { role: 'system', content: 'You are a judge.' }, - { role: 'user', content: 'Evaluate: {{response_to_evaluate}}' }, - ]; - const result = stripLegacyJudgeMessages(messages); - expect(result).toHaveLength(1); - expect(result[0].role).toBe('system'); - }); - - it('strips all legacy template messages from a typical legacy config', () => { - const messages: LDMessage[] = [ - { role: 'system', content: 'You are a judge.' }, - { role: 'assistant', content: '{{message_history}}' }, - { role: 'user', content: '{{response_to_evaluate}}' }, - ]; - const result = stripLegacyJudgeMessages(messages); - expect(result).toHaveLength(1); - expect(result[0].role).toBe('system'); - }); - - it('does not strip system messages even when they contain template variables', () => { - const messages: LDMessage[] = [ - { - role: 'system', - content: 'Judge using {{message_history}} and {{response_to_evaluate}}.', - }, - ]; - const result = stripLegacyJudgeMessages(messages); - expect(result).toHaveLength(1); - expect(result[0].role).toBe('system'); - }); - - it('leaves non-system messages without template variables untouched', () => { - const messages: LDMessage[] = [ - { role: 'system', content: 'You are a judge.' }, - { role: 'user', content: 'This is a regular message.' }, - ]; - const result = stripLegacyJudgeMessages(messages); - expect(result).toHaveLength(2); - }); - - it('returns an empty array for an empty input', () => { - expect(stripLegacyJudgeMessages([])).toEqual([]); - }); - - it('passes a new-style system-only config through unchanged', () => { - const messages: LDMessage[] = [{ role: 'system', content: 'You are a judge.' }]; - const result = stripLegacyJudgeMessages(messages); - expect(result).toEqual(messages); - }); -}); - describe('Judge', () => { let mockRunner: jest.Mocked; let mockTracker: jest.Mocked; diff --git a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts index d7db880263..6aafe5bc55 100644 --- a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts @@ -12,15 +12,7 @@ import { LDAIClientImpl } from '../src/LDAIClientImpl'; import { LDClientMin } from '../src/LDClientMin'; import { aiSdkLanguage, aiSdkName, aiSdkVersion } from '../src/sdkInfo'; -// Mock Judge and RunnerFactory. Preserve the real `stripLegacyJudgeMessages` -// helper so the real `_judgeConfig` strip path can be exercised by tests. -jest.mock('../src/api/judge/Judge', () => { - const actual = jest.requireActual('../src/api/judge/Judge'); - return { - ...actual, - Judge: jest.fn(), - }; -}); +jest.mock('../src/api/judge/Judge'); jest.mock('../src/api/providers/RunnerFactory'); const mockLdClient: jest.Mocked = { @@ -191,10 +183,7 @@ describe('config evaluation', () => { const evaluateSpy = jest.spyOn(client as any, '_evaluate'); const result = await client.judgeConfig(key, testContext, defaultValue); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', { - message_history: '{{message_history}}', - response_to_evaluate: '{{response_to_evaluate}}', - }); + expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', undefined); // Should use first value from evaluationMetricKeys expect(result.evaluationMetricKey).toBe('relevance'); expect(result.createTracker).toBeDefined(); @@ -227,10 +216,7 @@ describe('config evaluation', () => { const evaluateSpy = jest.spyOn(client as any, '_evaluate'); const result = await client.judgeConfig(key, testContext, defaultValue); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', { - message_history: '{{message_history}}', - response_to_evaluate: '{{response_to_evaluate}}', - }); + expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', undefined); expect(result.evaluationMetricKey).toBe('relevance'); expect(result.createTracker).toBeDefined(); expect(result.enabled).toBe(true); @@ -263,10 +249,7 @@ describe('config evaluation', () => { const evaluateSpy = jest.spyOn(client as any, '_evaluate'); const result = await client.judgeConfig(key, testContext, defaultValue); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', { - message_history: '{{message_history}}', - response_to_evaluate: '{{response_to_evaluate}}', - }); + expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', undefined); expect(result.evaluationMetricKey).toBe('helpfulness'); expect(result.createTracker).toBeDefined(); expect(result.enabled).toBe(true); @@ -299,10 +282,7 @@ describe('config evaluation', () => { const evaluateSpy = jest.spyOn(client as any, '_evaluate'); const result = await client.judgeConfig(key, testContext, defaultValue); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', { - message_history: '{{message_history}}', - response_to_evaluate: '{{response_to_evaluate}}', - }); + expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', undefined); // Empty string should be treated as invalid, so should fall back to first value in evaluationMetricKeys expect(result.evaluationMetricKey).toBe('relevance'); expect(result.createTracker).toBeDefined(); @@ -335,10 +315,7 @@ describe('config evaluation', () => { const evaluateSpy = jest.spyOn(client as any, '_evaluate'); const result = await client.judgeConfig(key, testContext, defaultValue); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', { - message_history: '{{message_history}}', - response_to_evaluate: '{{response_to_evaluate}}', - }); + expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', undefined); // Should skip empty and whitespace strings, use first valid value expect(result.evaluationMetricKey).toBe('relevance'); expect(result.createTracker).toBeDefined(); @@ -644,46 +621,11 @@ describe('judgeConfig method', () => { key, 1, ); - expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', { - ...variables, - message_history: '{{message_history}}', - response_to_evaluate: '{{response_to_evaluate}}', - }); - // System messages without legacy template variables pass through unchanged. + expect(evaluateSpy).toHaveBeenCalledWith(key, testContext, defaultValue, 'judge', variables); expect(result).toMatchObject(mockJudgeConfig); expect(result.messages).toEqual(mockJudgeConfig.messages); evaluateSpy.mockRestore(); }); - - it('strips legacy judge template messages from the returned config', async () => { - const client = new LDAIClientImpl(mockLdClient); - const key = 'test-judge'; - const defaultValue: LDAIJudgeConfigDefault = { - enabled: false, - }; - - const mockJudgeConfig = { - enabled: true, - model: { name: 'gpt-4' }, - provider: { name: 'openai' }, - evaluationMetricKey: 'relevance', - messages: [ - { role: 'system' as const, content: 'You are a judge.' }, - { role: 'assistant' as const, content: '{{message_history}}' }, - { role: 'user' as const, content: 'Evaluate: {{response_to_evaluate}}' }, - ], - createTracker: () => ({}) as any, - toVercelAISDK: jest.fn(), - }; - - const evaluateSpy = jest.spyOn(client as any, '_evaluate'); - evaluateSpy.mockResolvedValue(mockJudgeConfig); - - const result = await client.judgeConfig(key, testContext, defaultValue); - - expect(result.messages).toEqual([{ role: 'system', content: 'You are a judge.' }]); - evaluateSpy.mockRestore(); - }); }); describe('createJudge method', () => { diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 075f950789..50baab8ad6 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -24,7 +24,7 @@ import { import { LDAIConfigFlagValue, LDAIConfigUtils } from './api/config/LDAIConfigUtils'; import { AgentGraphDefinition, LDAgentGraphFlagValue, LDGraphTracker } from './api/graph'; import { Evaluator } from './api/judge/Evaluator'; -import { Judge, stripLegacyJudgeMessages } from './api/judge/Judge'; +import { Judge } from './api/judge/Judge'; import { LDAIClient } from './api/LDAIClient'; import { RunnerFactory, SupportedAIProvider } from './api/providers'; import { LDAIConfigTrackerImpl } from './LDAIConfigTrackerImpl'; @@ -227,42 +227,14 @@ export class LDAIClientImpl implements LDAIClient { defaultValue: LDAIJudgeConfigDefault, variables?: Record, ): Promise { - if (variables?.message_history !== undefined) { - this._logger?.warn( - "The variable 'message_history' is reserved by the judge and will be ignored.", - ); - } - if (variables?.response_to_evaluate !== undefined) { - this._logger?.warn( - "The variable 'response_to_evaluate' is reserved by the judge and will be ignored.", - ); - } - - // Re-inject the reserved variables as their literal placeholders so they - // survive Mustache interpolation in `_evaluate`. Without this, legacy - // templates like `{{message_history}}` get rendered to empty strings and - // `stripLegacyJudgeMessages` below cannot detect them. - const extendedVariables = { - ...variables, - message_history: '{{message_history}}', - response_to_evaluate: '{{response_to_evaluate}}', - }; - const config = (await this._evaluate( key, context, defaultValue, 'judge', - extendedVariables, + variables, )) as LDAIJudgeConfig; - // Strip legacy judge template messages (containing {{message_history}} or - // {{response_to_evaluate}}) before returning the config. New-style configs - // omit these and rely on Judge._buildEvaluationInput. - if (config.messages) { - return { ...config, messages: stripLegacyJudgeMessages(config.messages) }; - } - return config; } diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index 129eca7472..c42508d4ec 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -23,27 +23,6 @@ const EVALUATION_SCHEMA = { additionalProperties: false, } as const; -/** - * Remove legacy judge template messages from a message list. - * - * Strips any non-system message whose content contains `{{message_history}}` - * or `{{response_to_evaluate}}`. These were used by older judge configs to - * indicate where the SDK should interpolate the evaluated conversation; new - * configs omit them entirely and rely on the string input built by - * `Judge._buildEvaluationInput`. - * - * @param messages The raw message list from the judge AI config. - * @returns A new list with legacy template messages removed. - */ -export function stripLegacyJudgeMessages(messages: LDMessage[]): LDMessage[] { - return messages.filter( - (msg) => - msg.role === 'system' || - (!msg.content.includes('{{message_history}}') && - !msg.content.includes('{{response_to_evaluate}}')), - ); -} - /** * Judge implementation that handles evaluation functionality and conversation management. * diff --git a/packages/sdk/server-ai/src/api/providers/Runner.ts b/packages/sdk/server-ai/src/api/providers/Runner.ts index 7a642275e8..e0356bd9af 100644 --- a/packages/sdk/server-ai/src/api/providers/Runner.ts +++ b/packages/sdk/server-ai/src/api/providers/Runner.ts @@ -1,3 +1,4 @@ +import { LDMessage } from '../config/types'; import { AgentGraphRunnerResult } from '../graph/types'; import { RunnerResult } from '../model/types'; @@ -10,14 +11,17 @@ import { RunnerResult } from '../model/types'; */ export interface Runner { /** - * Invoke the model with the given input string. + * Invoke the model with the given input. * - * @param input The string input to the model. + * @param input The user prompt string, or a pre-built message array. When a + * string is provided, config messages are prepended automatically. When an + * {@link LDMessage} array is provided, it is used as-is (config messages + * are not prepended — the caller is responsible for the full message list). * @param outputType Optional JSON schema for structured output. When provided, * the model should return structured data accessible via `RunnerResult.parsed`. * @returns Promise resolving to a RunnerResult. */ - run(input: string, outputType?: Record): Promise; + run(input: string | LDMessage[], outputType?: Record): Promise; } /**