diff --git a/docs/model_server_rest_api_responses.md b/docs/model_server_rest_api_responses.md index 2ff2f2c246..f40374f9a3 100644 --- a/docs/model_server_rest_api_responses.md +++ b/docs/model_server_rest_api_responses.md @@ -100,7 +100,7 @@ curl http://localhost/v3/responses \ | ignore_eos | ✅ | ❌ | bool (default: `false`) | Whether to ignore the `EOS` token and continue generating tokens after the `EOS` token is generated. | | include_stop_str_in_output | ✅ | ❌ | bool (default: `false` if `stream=false`, `true` if `stream=true`) | Whether to include matched stop string in output. Setting it to false when `stream=true` is invalid configuration and will result in error. | | logprobs | ⚠️ | ❌ | bool (default: `false`) | Include the log probabilities on the logprob of the returned output token. **_In stream mode logprobs are not supported._** | -| response_format | ✅ | ❌ | object (optional) | An object specifying the format that the model must output. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs. Additionally accepts [XGrammar structural tags format](https://github.com/mlc-ai/xgrammar/blob/v0.1.26/docs/tutorials/structural_tag.md#format-types). OpenAI Responses API uses `text.format` instead (not supported in OVMS). | +| response_format | ✅ | ❌ | object (optional) | OVMS extension alias for structured output format. Setting to `{ "type": "json_schema", "json_schema": {...} }` enables Structured Outputs. Additionally accepts [XGrammar structural tags format](https://github.com/mlc-ai/xgrammar/blob/v0.1.26/docs/tutorials/structural_tag.md#format-types). For OpenAI-compatible payloads on `/v3/responses`, prefer `text.format` (supported by OVMS and internally mapped to `response_format`). | | tools | ⚠️ | ✅ | array (optional) | A list of tools the model may call. Currently, only **function** tools are supported. OpenAI also supports built-in tools (web_search, file_search, code_interpreter, etc.) and MCP tools. OVMS additionally accepts a flat `{type, name, parameters}` format alongside the nested `{type, function: {name, parameters}}` format. See [OpenAI API reference](https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools) for more details. | | tool_choice | ✅ | ✅ | string or object (optional) | Controls which (if any) tool is called by the model. `none` means the model will not call any tool and instead generates a message. `auto` means the model can pick between generating a message or calling one or more tools. `required` means that model should call at least one tool. Specifying a particular function via `{"type": "function", "function": {"name": "my_function"}}` forces the model to call that tool. | | reasoning | ⚠️ | ✅ | object (optional) | Configuration for reasoning/thinking mode. The `effort` field accepts `"low"`, `"medium"`, or `"high"` — any value enables thinking mode (`enable_thinking: true` is injected into chat template kwargs). The `summary` field is accepted but ignored. | diff --git a/src/llm/apis/openai_responses.cpp b/src/llm/apis/openai_responses.cpp index 3e7ef45d22..c1d54eb0df 100644 --- a/src/llm/apis/openai_responses.cpp +++ b/src/llm/apis/openai_responses.cpp @@ -45,6 +45,110 @@ namespace ovms { static constexpr const char* OUTPUT_ITEM_ID = "msg-0"; static constexpr const char* REASONING_ITEM_ID = "rs-0"; +rapidjson::Document buildResponsesTextFormat(const rapidjson::Document& doc) { + rapidjson::Document formatDoc; + formatDoc.SetObject(); + auto& alloc = formatDoc.GetAllocator(); + + auto textIt = doc.FindMember("text"); + if (textIt != doc.MemberEnd() && textIt->value.IsObject()) { + auto formatIt = textIt->value.GetObject().FindMember("format"); + if (formatIt != textIt->value.GetObject().MemberEnd() && formatIt->value.IsObject()) { + formatDoc.CopyFrom(formatIt->value, alloc); + return formatDoc; + } + } + + auto responseFormatIt = doc.FindMember("response_format"); + if (responseFormatIt != doc.MemberEnd() && responseFormatIt->value.IsObject()) { + const auto& responseFormat = responseFormatIt->value.GetObject(); + auto typeIt = responseFormat.FindMember("type"); + if (typeIt != responseFormat.MemberEnd() && typeIt->value.IsString()) { + const std::string responseFormatType = typeIt->value.GetString(); + if (responseFormatType == "json_schema") { + formatDoc.AddMember("type", rapidjson::Value("json_schema", alloc), alloc); + auto jsonSchemaIt = responseFormat.FindMember("json_schema"); + if (jsonSchemaIt != responseFormat.MemberEnd() && jsonSchemaIt->value.IsObject()) { + for (auto it = jsonSchemaIt->value.MemberBegin(); it != jsonSchemaIt->value.MemberEnd(); ++it) { + rapidjson::Value key(it->name, alloc); + rapidjson::Value value(it->value, alloc); + formatDoc.AddMember(key, value, alloc); + } + return formatDoc; + } + } + } + formatDoc.CopyFrom(responseFormatIt->value, alloc); + return formatDoc; + } + + formatDoc.AddMember("type", rapidjson::Value("text", alloc), alloc); + return formatDoc; +} + +absl::Status injectResponseFormatFromResponsesTextFormat(rapidjson::Document& doc) { + auto responseFormatIt = doc.FindMember("response_format"); + const bool responseFormatProvided = + responseFormatIt != doc.MemberEnd() && !responseFormatIt->value.IsNull(); + + auto textIt = doc.FindMember("text"); + if (textIt == doc.MemberEnd() || textIt->value.IsNull()) { + return absl::OkStatus(); + } + if (!textIt->value.IsObject()) { + return absl::InvalidArgumentError("text is not an object"); + } + const auto textObj = textIt->value.GetObject(); + auto formatIt = textObj.FindMember("format"); + if (formatIt == textObj.MemberEnd() || formatIt->value.IsNull()) { + return absl::OkStatus(); + } + if (!formatIt->value.IsObject()) { + return absl::InvalidArgumentError("text.format is not an object"); + } + if (responseFormatProvided) { + return absl::InvalidArgumentError("Provide only one of response_format or text.format"); + } + + const auto formatObj = formatIt->value.GetObject(); + auto typeIt = formatObj.FindMember("type"); + if (typeIt == formatObj.MemberEnd() || !typeIt->value.IsString()) { + return absl::InvalidArgumentError("text.format.type is not a valid string"); + } + + const std::string formatType = typeIt->value.GetString(); + if (formatType == "text") { + return absl::OkStatus(); + } + + rapidjson::Value normalizedFormat(rapidjson::kObjectType); + auto& allocator = doc.GetAllocator(); + + if (formatType == "json_schema" && !formatObj.HasMember("json_schema")) { + normalizedFormat.AddMember("type", rapidjson::Value("json_schema", allocator), allocator); + rapidjson::Value jsonSchema(rapidjson::kObjectType); + for (auto it = formatObj.MemberBegin(); it != formatObj.MemberEnd(); ++it) { + if (!it->name.IsString() || std::string(it->name.GetString()) == "type") { + continue; + } + rapidjson::Value key(it->name, allocator); + rapidjson::Value value(it->value, allocator); + jsonSchema.AddMember(key, value, allocator); + } + normalizedFormat.AddMember("json_schema", jsonSchema, allocator); + } else { + normalizedFormat.CopyFrom(formatIt->value, allocator); + } + + if (responseFormatIt != doc.MemberEnd()) { + responseFormatIt->value.CopyFrom(normalizedFormat, allocator); + } else { + rapidjson::Value key("response_format", allocator); + doc.AddMember(key, normalizedFormat, allocator); + } + return absl::OkStatus(); +} + static std::string joinServerSideEvents(const std::vector& events) { if (events.empty()) { return ""; @@ -255,6 +359,7 @@ static absl::StatusOr classifyInputItem(const rapidjson: // rapidjson messages array) is provided by the Sink template parameter, which // must implement: // absl::Status extractContent(itemObj, index, std::string& outText); +// absl::Status extractToolOutput(itemObj, std::string& outText); // void emitToolMessage(callId, output); // void emitMessage(role, contentText, reasoning); // reasoning empty -> skip // void emitAssistantWithToolCalls(contentText, reasoning, toolCalls); @@ -322,9 +427,9 @@ class ResponsesInputBuilder { if (callIdIt != itemObj.MemberEnd() && callIdIt->value.IsString()) callId = callIdIt->value.GetString(); std::string output; - auto outputIt = itemObj.FindMember("output"); - if (outputIt != itemObj.MemberEnd() && outputIt->value.IsString()) - output = outputIt->value.GetString(); + auto status = sink.extractToolOutput(itemObj, output); + if (!status.ok()) + return status; sink.emitToolMessage(callId, output); return absl::OkStatus(); } @@ -453,12 +558,60 @@ class ChatHistorySink { return absl::OkStatus(); } + // Extract the `output` field of a function_call_output item into either a + // plain string or a text-typed content array preserved in pendingContentArray. + // The text-typed array is left for TextContentNormalizationProcessor to flatten, + // matching how extractContent handles user/system input content arrays. + absl::Status extractToolOutput(const rapidjson::Value::ConstObject& itemObj, + std::string& outText) { + outText.clear(); + hasPendingContent = false; + pendingContentArray.SetArray(); + auto outputIt = itemObj.FindMember("output"); + if (outputIt == itemObj.MemberEnd()) + return absl::InvalidArgumentError("function_call_output item is missing required output field"); + if (outputIt->value.IsString()) { + outText = outputIt->value.GetString(); + return absl::OkStatus(); + } + if (!outputIt->value.IsArray()) + return absl::InvalidArgumentError("function_call_output.output must be a string or array"); + if (outputIt->value.Empty()) + return absl::InvalidArgumentError("function_call_output.output array must not be empty"); + for (const auto& contentItem : outputIt->value.GetArray()) { + if (!contentItem.IsObject()) + return absl::InvalidArgumentError("function_call_output.output items must be objects"); + auto contentObj = contentItem.GetObject(); + auto typeIt = contentObj.FindMember("type"); + if (typeIt == contentObj.MemberEnd() || !typeIt->value.IsString()) + return absl::InvalidArgumentError("function_call_output.output item type is missing or invalid"); + const std::string type = typeIt->value.GetString(); + if (type != "input_text" && type != "output_text") + return absl::InvalidArgumentError(absl::StrCat("unsupported function_call_output.output item type: ", type)); + auto textIt = contentObj.FindMember("text"); + if (textIt == contentObj.MemberEnd() || !textIt->value.IsString()) + return absl::InvalidArgumentError(absl::StrCat(type, " requires a valid text field")); + rapidjson::Value textEntry(rapidjson::kObjectType); + textEntry.AddMember("type", rapidjson::Value("text", scratchDoc.GetAllocator()), scratchDoc.GetAllocator()); + textEntry.AddMember("text", rapidjson::Value(textIt->value.GetString(), scratchDoc.GetAllocator()), scratchDoc.GetAllocator()); + pendingContentArray.PushBack(textEntry, scratchDoc.GetAllocator()); + } + hasPendingContent = true; + return absl::OkStatus(); + } + void emitToolMessage(const std::string& callId, const std::string& output) { chatHistory.push_back({}); chatHistory.last()["role"] = "tool"; if (!callId.empty()) chatHistory.last()["tool_call_id"] = callId; - chatHistory.last()["content"] = output; + if (hasPendingContent) { + // Preserve content array for TextContentNormalizationProcessor to flatten. + chatHistory.last()["content"] = rapidJsonValueToJsonContainer(pendingContentArray); + hasPendingContent = false; + } else { + chatHistory.last()["content"] = output; + } } void emitMessage(const std::string& role, const std::string& contentText, const std::string& reasoning) { @@ -710,6 +863,11 @@ absl::Status OpenAIResponsesHandler::parseResponsesPart(std::optional return absl::InvalidArgumentError("max_output_tokens value should be greater than 0"); } + auto textFormatStatus = injectResponseFormatFromResponsesTextFormat(doc); + if (!textFormatStatus.ok()) { + return textFormatStatus; + } + return parseResponseFormat(); } @@ -795,13 +953,12 @@ void OpenAIResponsesHandler::serializeCommonResponseParameters(Writer(request.temperature.value())); } + + rapidjson::Document textFormat = buildResponsesTextFormat(doc); writer.String("text"); writer.StartObject(); writer.String("format"); - writer.StartObject(); - writer.String("type"); - writer.String("text"); - writer.EndObject(); + textFormat.Accept(writer); writer.EndObject(); serializeToolChoice(writer); serializeTools(writer); diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 65ea56336c..7c08fe7ba2 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -1313,6 +1313,51 @@ TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesContainsO ASSERT_NE(serialized.find("\"text\":"), std::string::npos) << serialized; } +TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesEchoesTextFormat) { + std::string json = R"({ + "model": "llama", + "input": "Say hello world", + "max_output_tokens": 10, + "text": { + "format": { + "type": "json_schema", + "name": "IntBox", + "strict": true, + "schema": { + "type": "object", + "properties": { + "value": { + "type": "integer" + } + }, + "required": ["value"] + } + } + } + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + + auto apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + std::optional maxTokensLimit; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + ASSERT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + + ov::genai::EncodedResults results; + ov::Tensor outputIds = tokenizer->encode("{\"value\":1}", ov::genai::add_special_tokens(false)).input_ids; + ASSERT_EQ(outputIds.get_shape().size(), 2); + ASSERT_EQ(outputIds.get_shape()[0], 1); + ASSERT_EQ(outputIds.get_element_type(), ov::element::i64); + int64_t* outputIdsData = reinterpret_cast(outputIds.data()); + results.tokens = {std::vector(outputIdsData, outputIdsData + outputIds.get_shape()[1])}; + + std::string serialized = apiHandler->serializeUnaryResponse(results); + ASSERT_NE(serialized.find("\"text\":{\"format\":{\"type\":\"json_schema\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"name\":\"IntBox\""), std::string::npos) << serialized; + ASSERT_NE(serialized.find("\"strict\":true"), std::string::npos) << serialized; +} + TEST_F(HttpOpenAIHandlerParsingTest, serializeUnaryResponseForResponsesContainsReasoningOutputItem) { std::string json = R"({ "model": "llama", @@ -2533,6 +2578,91 @@ TEST_F(HttpOpenAIHandlerParsingTest, ParsingResponsesNUnaryIsAccepted) { EXPECT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); } +TEST_F(HttpOpenAIHandlerParsingTest, ParsingResponsesTextFormatJsonSchemaSetsResponseFormat) { + std::string json = R"({ + "model": "llama", + "input": "valid prompt", + "text": { + "format": { + "type": "json_schema", + "name": "IntBox", + "strict": true, + "schema": { + "type": "object", + "properties": { + "value": { + "type": "integer" + } + }, + "required": ["value"] + } + } + } + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + std::optional maxTokensLimit; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + std::shared_ptr apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + EXPECT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::OkStatus()); + ASSERT_TRUE(apiHandler->getResponseFormat().has_value()); + + std::string expectedResponseFormat = R"({"type":"structural_tag","format":{"type":"json_schema","json_schema":{"name":"IntBox","strict":true,"type":"object","properties":{"value":{"type":"integer"}},"required":["value"]}}})"; + rapidjson::Document expectedDoc; + expectedDoc.Parse(expectedResponseFormat.c_str()); + ASSERT_FALSE(expectedDoc.HasParseError()); + + rapidjson::Document actualDoc; + actualDoc.Parse(apiHandler->getResponseFormat().value().c_str()); + ASSERT_FALSE(actualDoc.HasParseError()); + EXPECT_TRUE(expectedDoc == actualDoc); +} + +TEST_F(HttpOpenAIHandlerParsingTest, ParsingResponsesTextFormatAndResponseFormatConflictFails) { + std::string json = R"({ + "model": "llama", + "input": "valid prompt", + "response_format": { + "type": "json_schema", + "json_schema": { + "schema": { + "type": "object", + "properties": { + "a": { + "type": "string" + } + }, + "required": ["a"] + } + } + }, + "text": { + "format": { + "type": "json_schema", + "name": "IntBox", + "strict": true, + "schema": { + "type": "object", + "properties": { + "value": { + "type": "integer" + } + }, + "required": ["value"] + } + } + } + })"; + doc.Parse(json.c_str()); + ASSERT_FALSE(doc.HasParseError()); + std::optional maxTokensLimit; + uint32_t bestOfLimit = 0; + std::optional maxModelLength; + std::shared_ptr apiHandler = std::make_shared(doc, ovms::Endpoint::RESPONSES, std::chrono::system_clock::now(), *tokenizer); + EXPECT_EQ(apiHandler->parseRequest(maxTokensLimit, bestOfLimit, maxModelLength), absl::InvalidArgumentError("Provide only one of response_format or text.format")); +} + TEST_F(HttpOpenAIHandlerParsingTest, ParsingResponsesFlatFunctionToolsSucceeds) { std::string json = R"({ "model": "llama", @@ -4096,6 +4226,39 @@ TEST_F(HttpOpenAIHandlerParsingTest, ResponsesFunctionCallMergedIntoAssistantToo })"); } +TEST_F(HttpOpenAIHandlerParsingTest, ResponsesFunctionCallOutputAsTextArrayPassesThrough) { + // function_call_output.output given as an array of input_text/output_text + // parts is preserved as a text-typed chat/completions content array. The + // downstream TextContentNormalizationProcessor is responsible for + // flattening it into a single string; this translator no longer does that + // eagerly. + expectResponsesEquivalentToChatCompletions(doc, *tokenizer, + R"({ + "model": "llama", + "input": [ + {"role": "user", "content": [{"type":"input_text","text":"weather?"}]}, + {"type": "function_call", "id": "call_1", "call_id": "call_1", + "name": "get_weather", "arguments": "{\"city\":\"Paris\"}"}, + {"type": "function_call_output", "call_id": "call_1", "output": [ + {"type": "input_text", "text": "part1"}, + {"type": "output_text", "text": "part2"} + ]} + ] + })", + R"({ + "messages": [ + {"role":"user","content":[{"type":"text","text":"weather?"}]}, + {"role":"assistant","content":"","tool_calls":[ + {"id":"call_1","type":"function","function":{"name":"get_weather","arguments":"{\"city\":\"Paris\"}"}} + ]}, + {"role":"tool","tool_call_id":"call_1","content":[ + {"type":"text","text":"part1"}, + {"type":"text","text":"part2"} + ]} + ] + })"); +} + TEST_F(HttpOpenAIHandlerParsingTest, ResponsesReasoningPlusFunctionCallRidesOnAssistant) { // reasoning + function_call should both attach to the synthesised assistant // turn that owns the tool_calls.