diff --git a/demos/common/export_models/export_model.py b/demos/common/export_models/export_model.py index b16694eb90..d2813fee33 100644 --- a/demos/common/export_models/export_model.py +++ b/demos/common/export_models/export_model.py @@ -64,6 +64,7 @@ def add_common_arguments(parser): parser_embeddings_ov.add_argument('--pooling', default="CLS", choices=["CLS", "LAST", "MEAN"], help='Embeddings pooling mode', dest='pooling') parser_embeddings_ov.add_argument('--truncate', default=False, action='store_true', help='Truncate the prompts to fit to the embeddings model', dest='truncate') parser_embeddings_ov.add_argument('--num_streams', default=1,type=int, help='The number of parallel execution streams to use for the model. Use at least 2 on 2 socket CPU systems.', dest='num_streams') +parser_embeddings_ov.add_argument('--max_length', default=None, type=int, help='Maximum length of the embeddings input', dest='max_length') parser_rerank_ov = subparsers.add_parser('rerank_ov', help='export model for rerank endpoint with directory structure aligned with OpenVINO tools') add_common_arguments(parser_rerank_ov) @@ -187,6 +188,8 @@ def add_common_arguments(parser): {%- if truncate %} truncate: true,{% endif %} target_device: "{{target_device|default("CPU", true)}}" + {%- if max_length is not none %} + max_length: {{max_length}},{% endif %} } } } diff --git a/demos/embeddings/README.md b/demos/embeddings/README.md index 519963b2ed..5e756ab768 100644 --- a/demos/embeddings/README.md +++ b/demos/embeddings/README.md @@ -250,7 +250,7 @@ python export_model.py embeddings_ov --source_model sentence-transformers/all-mp :::{tab-item} Qwen/Qwen3-Embedding-0.6B :sync: Qwen3-Embedding-0.6B-int8 ```bash -docker run --user $(id -u):$(id -g) --rm -v $(pwd)/models:/models:rw openvino/model_server:latest --pull --model_repository_path /models --source_model OpenVINO/Qwen3-Embedding-0.6B-int8-ov --pooling LAST --task embeddings --target_device NPU +docker run --user $(id -u):$(id -g) --rm -v $(pwd)/models:/models:rw openvino/model_server:latest --pull --model_repository_path /models --source_model OpenVINO/Qwen3-Embedding-0.6B-int8-ov --pooling LAST --task embeddings --target_device NPU --max_length 1000 docker run --user $(id -u):$(id -g) --rm -v $(pwd)/models:/models:rw openvino/model_server:latest --add_to_config --config_path /models/config.json --model_name Qwen3-Embedding-0.6B-int8-ov --model_path OpenVINO/Qwen3-Embedding-0.6B-int8-ov ``` @@ -258,11 +258,11 @@ docker run --user $(id -u):$(id -g) --rm -v $(pwd)/models:/models:rw openvino/mo :::{tab-item} BAAI/bge-large-en-v1.5 :sync: BAAI/bge-large-en-v1.5-fp16 ```console -python export_model.py embeddings_ov --source_model BAAI/bge-large-en-v1.5 --pooling CLS --weight-format fp16 --target_device NPU --config_file_path models/config.json --model_repository_path models +python export_model.py embeddings_ov --source_model BAAI/bge-large-en-v1.5 --pooling CLS --weight-format fp16 --target_device NPU --config_file_path models/config.json --model_repository_path models --max_length 1000 ``` ::: :::: - +> **Note** For NPU Set `--max_length` as low as possible to speed up model loading and inference. > **Note** For NPU Change the `--weight-format` to quantize the model to `fp16`, `int8` or `int4` precision. For int4 precisions, add required extra parameter `--extra_quantization_params "--sym --ratio 1.0 --group-size -1"` > **Note** For NPU the pooling mode --pooling LAST has the best accuracy. > **Note** For NPU and the weight-format int4, use `--extra_quantization_params "--sym --ratio 1.0 --group-size -1"` @@ -611,7 +611,7 @@ The `tokenize` endpoint provides a simple API for tokenizing input text using th Example usage: ```console -curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"BAAI/bge-large-en-v1.5\", \"text\": \"hello world\" }" +curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"bge-large-en-v1.5\", \"text\": \"hello world\" }" ``` Response: ```json @@ -628,7 +628,7 @@ It's possible to use additional parameters: Example usage: ```console -curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"BAAI/bge-large-en-v1.5\", \"text\": \"hello world\", \"max_length\": 10, \"pad_to_max_length\": true, \"padding_side\": \"left\", \"add_special_tokens\": true }" +curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"bge-large-en-v1.5\", \"text\": \"hello world\", \"max_length\": 10, \"pad_to_max_length\": true, \"padding_side\": \"left\", \"add_special_tokens\": true }" ``` Response: diff --git a/docs/parameters.md b/docs/parameters.md index c07dd0484e..a05f236b01 100644 --- a/docs/parameters.md +++ b/docs/parameters.md @@ -181,6 +181,7 @@ Task specific parameters for different tasks (text generation/image generation/e | `--normalize` | `bool` | Normalize the embeddings. Default: true. | | `--truncate` | `bool` | Truncate input when it exceeds model context length. Default: false | | `--pooling` | `string` | Pooling option. One of: CLS, LAST, MEAN. Default: CLS. | +| `--max_legth` | `integer` | Maximum input length in tokens. If omitted, OVMS will detect it from the model's config.json. | ### Rerank | option | Value format | Description | diff --git a/src/capi_frontend/server_settings.hpp b/src/capi_frontend/server_settings.hpp index 4eaa34ac98..26d4eb95ca 100644 --- a/src/capi_frontend/server_settings.hpp +++ b/src/capi_frontend/server_settings.hpp @@ -135,6 +135,7 @@ struct EmbeddingsGraphSettingsImpl { std::string normalize = "true"; std::string truncate = "false"; std::optional pooling; + std::optional maxLength; }; struct TextToSpeechGraphSettingsImpl { diff --git a/src/embeddings/embeddings_calculator_ov.proto b/src/embeddings/embeddings_calculator_ov.proto index 1836dba6da..0ca5282a96 100644 --- a/src/embeddings/embeddings_calculator_ov.proto +++ b/src/embeddings/embeddings_calculator_ov.proto @@ -36,4 +36,5 @@ message EmbeddingsCalculatorOVOptions { } optional Pooling pooling = 5; optional bool truncate = 6 [default = false]; + optional uint32 max_length = 7; } diff --git a/src/embeddings/embeddings_node_initializer.cpp b/src/embeddings/embeddings_node_initializer.cpp index 658d7e99a8..f54745c83a 100644 --- a/src/embeddings/embeddings_node_initializer.cpp +++ b/src/embeddings/embeddings_node_initializer.cpp @@ -74,13 +74,22 @@ class EmbeddingsNodeInitializer : public NodeInitializer { nodeOptions.has_pooling() ? std::make_optional(nodeOptions.pooling()) : std::nullopt; const auto pooling = resolveEmbeddingsPooling(modelsPath, graphPooling); + const std::optional configuredMaxLength = + nodeOptions.has_max_length() ? std::make_optional(nodeOptions.max_length()) : std::nullopt; + + if (configuredMaxLength.has_value() && configuredMaxLength.value() == 0) { + SPDLOG_ERROR("Embeddings node name: {} invalid max_length (0) in graph: {}.", nodeName, graphName); + return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID; + } + auto servable = std::make_shared( nodeOptions.models_path(), nodeOptions.target_device(), nodeOptions.plugin_config(), basePath, pooling, - nodeOptions.normalize_embeddings()); + nodeOptions.normalize_embeddings(), + configuredMaxLength); servable->initialize( nodeOptions.models_path(), nodeOptions.target_device(), diff --git a/src/embeddings/embeddings_servable.cpp b/src/embeddings/embeddings_servable.cpp index 7adcba6a8e..fac2eca433 100644 --- a/src/embeddings/embeddings_servable.cpp +++ b/src/embeddings/embeddings_servable.cpp @@ -370,6 +370,10 @@ void reshapeModel(std::shared_ptr& model, // End code from OpenVINO GenAI repository std::shared_ptr EmbeddingsServable::applyPrePostProcessing(ov::Core& core, std::shared_ptr model, ov::AnyMap& properties) { + if (this->configuredMaxLength.has_value()) { + SPDLOG_DEBUG("Overriding detected max model length {} with configured value {}", this->maxModelLength.value_or(0), this->configuredMaxLength.value()); + this->maxModelLength = this->configuredMaxLength; + } if (this->targetDevice == "NPU" && model->is_dynamic()) { TextEmbeddingPipeline::Config config; switch (this->pooling) { diff --git a/src/embeddings/embeddings_servable.hpp b/src/embeddings/embeddings_servable.hpp index 6f0c8b8543..663eb6e478 100644 --- a/src/embeddings/embeddings_servable.hpp +++ b/src/embeddings/embeddings_servable.hpp @@ -20,7 +20,10 @@ #include "src/filesystem/filesystem.hpp" #include "src/port/rapidjson_istreamwrapper.hpp" #include "src/port/rapidjson_error.hpp" + +#include #include +#include #include #include @@ -34,10 +37,12 @@ struct EmbeddingsServable : public SidepacketServable { const std::string& pluginConfig, const std::string& graphPath, mediapipe::EmbeddingsCalculatorOVOptions_Pooling pooling, - bool normalizeEmbeddings) : + bool normalizeEmbeddings, + std::optional configuredMaxLength = std::nullopt) : SidepacketServable(modelDir, targetDevice, pluginConfig, graphPath), pooling(pooling), - normalizeEmbeddings(normalizeEmbeddings) {} + normalizeEmbeddings(normalizeEmbeddings), + configuredMaxLength(configuredMaxLength) {} int getTargetOutputIndex() const { return targetOutputIndex; @@ -61,6 +66,7 @@ struct EmbeddingsServable : public SidepacketServable { private: mediapipe::EmbeddingsCalculatorOVOptions_Pooling pooling; bool normalizeEmbeddings; + std::optional configuredMaxLength; bool npuPostprocessingRequired = false; ov::CompiledModel postProcCompiledModel; std::unique_ptr postProcInferRequestsQueue; diff --git a/src/graph_export/embeddings_graph_cli_parser.cpp b/src/graph_export/embeddings_graph_cli_parser.cpp index 908e4c2607..b0fec5a2cc 100644 --- a/src/graph_export/embeddings_graph_cli_parser.cpp +++ b/src/graph_export/embeddings_graph_cli_parser.cpp @@ -55,7 +55,11 @@ void EmbeddingsGraphCLIParser::createOptions() { ("pooling", "Pooling option. One of: CLS, LAST, MEAN. If omitted, OVMS will detect pooling automatically.", cxxopts::value(), - "POOLING"); + "POOLING") + ("max_length", + "Maximum input length in tokens. If omitted, OVMS will detect it from the model's config.json.", + cxxopts::value(), + "MAX_LENGTH"); } void EmbeddingsGraphCLIParser::printHelp() { @@ -99,6 +103,13 @@ void EmbeddingsGraphCLIParser::prepare(OvmsServerMode serverMode, HFSettingsImpl if (result->count("pooling") > 0) { embeddingsGraphSettings.pooling = result->operator[]("pooling").as(); } + if (result->count("max_length") > 0) { + const auto maxLength = result->operator[]("max_length").as(); + if (maxLength == 0) { + throw std::invalid_argument("max_length must be greater than 0"); + } + embeddingsGraphSettings.maxLength = maxLength; + } } if (embeddingsGraphSettings.pooling.has_value() && !(embeddingsGraphSettings.pooling.value() == "CLS" || embeddingsGraphSettings.pooling.value() == "LAST" || embeddingsGraphSettings.pooling.value() == "MEAN")) { diff --git a/src/graph_export/graph_export.cpp b/src/graph_export/graph_export.cpp index 8c6c68c942..d3488884a4 100644 --- a/src/graph_export/graph_export.cpp +++ b/src/graph_export/graph_export.cpp @@ -337,6 +337,10 @@ node { oss << R"( pooling: )" << graphSettings.pooling.value() << R"(,)"; } + if (graphSettings.maxLength.has_value()) { + oss << R"( + max_length: )" << graphSettings.maxLength.value() << R"(,)"; + } if (!exportSettings.targetDevice.empty()) { oss << R"( target_device: ")" << exportSettings.targetDevice << R"(",)"; diff --git a/src/test/embeddings/config_embeddings.json b/src/test/embeddings/config_embeddings.json index ef4edd5267..6851e6dabd 100644 --- a/src/test/embeddings/config_embeddings.json +++ b/src/test/embeddings/config_embeddings.json @@ -25,6 +25,11 @@ "name":"embeddings_ov_pooling_last", "base_path":"/ovms/src/test/embeddings/", "graph_path":"/ovms/src/test/embeddings/graph_ov_pooling_last.pbtxt" + }, + { + "name":"embeddings_ov_max_length", + "base_path":"/ovms/src/test/embeddings/", + "graph_path":"/ovms/src/test/embeddings/graph_ov_max_length.pbtxt" } ] } diff --git a/src/test/embeddings/graph_ov_max_length.pbtxt b/src/test/embeddings/graph_ov_max_length.pbtxt new file mode 100644 index 0000000000..e9e0dcfeb2 --- /dev/null +++ b/src/test/embeddings/graph_ov_max_length.pbtxt @@ -0,0 +1,30 @@ +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +input_stream: "REQUEST_PAYLOAD:input" +output_stream: "RESPONSE_PAYLOAD:output" + +node { + name: "embeddingsNode1" + calculator: "EmbeddingsCalculatorOV" + input_stream: "REQUEST_PAYLOAD:input" + output_stream: "RESPONSE_PAYLOAD:output" + input_side_packet: "EMBEDDINGS_NODE_RESOURCES:embeddings_servable" + node_options: { + [type.googleapis.com/mediapipe.EmbeddingsCalculatorOVOptions]: { + models_path: "/ovms/src/test/llm_testing/thenlper/gte-small/ov" + max_length: 32 + } + } +} diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 1b9f1e1313..834b561db4 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -561,6 +561,38 @@ TEST_F(EmbeddingsHttpTest, positivePoolingLast) { ASSERT_EQ(d["data"][0]["embedding"].Size(), EMBEDDING_OUTPUT_SIZE); } +TEST_F(EmbeddingsHttpTest, positiveWithinConfiguredMaxLength) { + std::string words; + for (int i = 0; i < 20; i++) { + words += "hello "; + } + std::string requestBody = "{ \"model\": \"embeddings_ov_max_length\", \"input\": \"" + words + " \"}"; + + Status status = handler->dispatchToProcessor(endpoint, requestBody, &response, comp, responseComponents, writer, multiPartParser); + ASSERT_EQ(status, + ovms::StatusCode::OK) + << status.string(); + rapidjson::Document d; + rapidjson::ParseResult ok = d.Parse(response.c_str()); + ASSERT_EQ(ok.Code(), 0); + ASSERT_TRUE(d["data"][0]["embedding"].IsArray()); + ASSERT_EQ(d["data"][0]["embedding"].Size(), EMBEDDING_OUTPUT_SIZE); +} + +TEST_F(EmbeddingsHttpTest, negativeExceedsConfiguredMaxLength) { + std::string words; + for (int i = 0; i < 35; i++) { + words += "hello "; + } + std::string requestBody = "{ \"model\": \"embeddings_ov_max_length\", \"input\": \"" + words + " \"}"; + + Status status = handler->dispatchToProcessor(endpoint, requestBody, &response, comp, responseComponents, writer, multiPartParser); + ASSERT_EQ(status, + ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR) + << status.string(); + ASSERT_THAT(status.string(), ::testing::HasSubstr("longer than allowed")); +} + TEST_F(EmbeddingsHttpTest, accessingCalculatorWithInvalidJson) { std::string requestBody = R"( { diff --git a/src/test/graph_export_test.cpp b/src/test/graph_export_test.cpp index 6f45dfe8fe..fe5bfc1551 100644 --- a/src/test/graph_export_test.cpp +++ b/src/test/graph_export_test.cpp @@ -351,6 +351,7 @@ node { normalize_embeddings: false, truncate: true, pooling: LAST, + max_length: 512, target_device: "GPU", plugin_config: '{"NUM_STREAMS":"2"}', } @@ -785,6 +786,7 @@ TEST_F(GraphCreationTest, embeddingsPositiveNonDefault) { embeddingsGraphSettings.normalize = "false"; embeddingsGraphSettings.truncate = "true"; embeddingsGraphSettings.pooling = "LAST"; + embeddingsGraphSettings.maxLength = 512; hfSettings.graphSettings = std::move(embeddingsGraphSettings); assertCreatedGraphEquals(hfSettings, expectedEmbeddingsGraphContents); } diff --git a/src/test/ovmsconfig_test.cpp b/src/test/ovmsconfig_test.cpp index c0d218009c..ff631afc3c 100644 --- a/src/test/ovmsconfig_test.cpp +++ b/src/test/ovmsconfig_test.cpp @@ -27,6 +27,7 @@ #include "../utils/env_guard.hpp" #include "../config.hpp" #include "src/filesystem/filesystem.hpp" +#include "../graph_export/embeddings_graph_cli_parser.hpp" #include "../graph_export/graph_cli_parser.hpp" #include "../ovms_exit_codes.hpp" #include "../systeminfo.hpp" @@ -560,6 +561,24 @@ TEST_F(OvmsConfigDeathTest, negativeImageGenerationGraph_MaxNumInferenceStepsZer EXPECT_THROW(ovms::Config::instance().parse(arg_count, n_argv), std::invalid_argument); } +TEST(OvmsGraphConfigTest, negativeEmbeddingsGraph_MaxLengthZero) { + char* n_argv[] = { + (char*)"ovms", + (char*)"--pull", + (char*)"--source_model", + (char*)"some/model", + (char*)"--model_repository_path", + (char*)"/some/path", + (char*)"--task", + (char*)"embeddings", + (char*)"--max_length", + (char*)"0", + }; + int arg_count = 10; + ConstructorEnabledConfig config; + EXPECT_THROW(config.parse(arg_count, n_argv), std::invalid_argument); +} + TEST(OvmsGraphConfigTest, negativeImageGenerationGraph_SourceLorasEmptyAlias) { char* n_argv[] = { (char*)"ovms", @@ -1934,9 +1953,11 @@ TEST(OvmsGraphConfigTest, positiveAllChangedEmbeddings) { (char*)"--plugin_config", (char*)"{\"SOME_KEY\":\"SOME_VALUE\"}", (char*)"--cache_dir", - (char*)"/tmp/cache_dir_with_emptiness"}; + (char*)"/tmp/cache_dir_with_emptiness", + (char*)"--max_length", + (char*)"512"}; - int arg_count = 24; + int arg_count = 26; ConstructorEnabledConfig config; config.parse(arg_count, n_argv); @@ -1951,6 +1972,8 @@ TEST(OvmsGraphConfigTest, positiveAllChangedEmbeddings) { ASSERT_EQ(embeddingsGraphSettings.truncate, "true"); ASSERT_TRUE(embeddingsGraphSettings.pooling.has_value()); ASSERT_EQ(embeddingsGraphSettings.pooling.value(), "CLS"); + ASSERT_TRUE(embeddingsGraphSettings.maxLength.has_value()); + ASSERT_EQ(embeddingsGraphSettings.maxLength.value(), 512); ASSERT_EQ(exportSettings.pluginConfig.numStreams, 2); ASSERT_EQ(exportSettings.targetDevice, "GPU"); ASSERT_EQ(exportSettings.modelName, servingName); @@ -2036,6 +2059,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultEmbeddings) { ASSERT_EQ(embeddingsGraphSettings.normalize, "true"); ASSERT_EQ(embeddingsGraphSettings.truncate, "false"); ASSERT_FALSE(embeddingsGraphSettings.pooling.has_value()); + ASSERT_FALSE(embeddingsGraphSettings.maxLength.has_value()); ASSERT_EQ(exportSettings.pluginConfig.numStreams, 1); ASSERT_EQ(exportSettings.targetDevice, ""); ASSERT_EQ(exportSettings.modelName, modelName); @@ -3116,6 +3140,33 @@ TEST(OvmsGraphCliParserTest, invalidReasoningParserNameThrowsInvalidArgument) { std::invalid_argument); } +TEST(OvmsGraphCliParserTest, embeddingsMaxLengthZeroThrowsInvalidArgument) { + ovms::HFSettingsImpl hfSettings; + ovms::EmbeddingsGraphCLIParser parser; + std::vector args = {"--max_length", "0"}; + parser.parse(args); + EXPECT_THROW({ + try { + parser.prepare(ovms::HF_PULL_MODE, hfSettings, "test_model"); + } catch (const std::invalid_argument& e) { + EXPECT_NE(std::string(e.what()).find("max_length must be greater than 0"), std::string::npos); + throw; + } + }, + std::invalid_argument); +} + +TEST(OvmsGraphCliParserTest, embeddingsMaxLengthNonZeroIsAccepted) { + ovms::HFSettingsImpl hfSettings; + ovms::EmbeddingsGraphCLIParser parser; + std::vector args = {"--max_length", "1"}; + parser.parse(args); + EXPECT_NO_THROW(parser.prepare(ovms::HF_PULL_MODE, hfSettings, "test_model")); + auto& embeddingsGraphSettings = std::get(hfSettings.graphSettings); + ASSERT_TRUE(embeddingsGraphSettings.maxLength.has_value()); + ASSERT_EQ(embeddingsGraphSettings.maxLength.value(), 1u); +} + TEST(OvmsGraphCliParserTest, validParserNamesAreAccepted) { ovms::HFSettingsImpl hfSettings; ovms::GraphCLIParser parser;