Adding --max_length parameter for embeddings

demos/common/export_models/export_model.py

@@ -64,6 +64,7 @@ def add_common_arguments(parser):
 parser_embeddings_ov.add_argument('--pooling', default="CLS", choices=["CLS", "LAST", "MEAN"], help='Embeddings pooling mode', dest='pooling')
 parser_embeddings_ov.add_argument('--truncate', default=False, action='store_true', help='Truncate the prompts to fit to the embeddings model', dest='truncate')
 parser_embeddings_ov.add_argument('--num_streams', default=1,type=int, help='The number of parallel execution streams to use for the model. Use at least 2 on 2 socket CPU systems.', dest='num_streams')
+parser_embeddings_ov.add_argument('--max_length', default=None, type=int, help='Maximum length of the embeddings input', dest='max_length')
 
 parser_rerank_ov = subparsers.add_parser('rerank_ov', help='export model for rerank endpoint with directory structure aligned with OpenVINO tools')
 add_common_arguments(parser_rerank_ov)
@@ -187,6 +188,8 @@ def add_common_arguments(parser):
       {%- if truncate %}
       truncate: true,{% endif %}
       target_device: "{{target_device|default("CPU", true)}}"
+      {%- if max_length is not none %}
+      max_length: {{max_length}},{% endif %}
     }
   }
 }

demos/embeddings/README.md

@@ -250,19 +250,19 @@ python export_model.py embeddings_ov --source_model sentence-transformers/all-mp
 :::{tab-item} Qwen/Qwen3-Embedding-0.6B
 :sync: Qwen3-Embedding-0.6B-int8
 ```bash
-docker run --user $(id -u):$(id -g) --rm -v $(pwd)/models:/models:rw openvino/model_server:latest --pull --model_repository_path /models --source_model OpenVINO/Qwen3-Embedding-0.6B-int8-ov --pooling LAST --task embeddings --target_device NPU
+docker run --user $(id -u):$(id -g) --rm -v $(pwd)/models:/models:rw openvino/model_server:latest --pull --model_repository_path /models --source_model OpenVINO/Qwen3-Embedding-0.6B-int8-ov --pooling LAST --task embeddings --target_device NPU --max_length 1000
 
 docker run --user $(id -u):$(id -g) --rm -v $(pwd)/models:/models:rw openvino/model_server:latest --add_to_config --config_path /models/config.json --model_name Qwen3-Embedding-0.6B-int8-ov --model_path OpenVINO/Qwen3-Embedding-0.6B-int8-ov
 ```
 :::
 :::{tab-item} BAAI/bge-large-en-v1.5
 :sync: BAAI/bge-large-en-v1.5-fp16
 ```console
-python export_model.py embeddings_ov --source_model BAAI/bge-large-en-v1.5 --pooling CLS --weight-format fp16 --target_device NPU --config_file_path models/config.json --model_repository_path models
+python export_model.py embeddings_ov --source_model BAAI/bge-large-en-v1.5 --pooling CLS --weight-format fp16 --target_device NPU --config_file_path models/config.json --model_repository_path models --max_length 1000
 ```
 :::
 ::::
-
+> **Note** For NPU Set `--max_length` as low as possible to speed up model loading and inference.
 > **Note** For NPU Change the `--weight-format` to quantize the model to `fp16`, `int8` or `int4` precision. For int4 precisions, add required extra parameter `--extra_quantization_params "--sym --ratio 1.0 --group-size -1"`
 > **Note** For NPU the pooling mode --pooling LAST has the best accuracy.
 > **Note** For NPU and the weight-format int4, use `--extra_quantization_params "--sym --ratio 1.0 --group-size -1"`
@@ -611,7 +611,7 @@ The `tokenize` endpoint provides a simple API for tokenizing input text using th
 
 Example usage:
 ```console
-curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"BAAI/bge-large-en-v1.5\", \"text\": \"hello world\" }"
+curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"bge-large-en-v1.5\", \"text\": \"hello world\" }"
 ```
 Response:
 ```json
@@ -628,7 +628,7 @@ It's possible to use additional parameters:
 
  Example usage:
 ```console
-curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"BAAI/bge-large-en-v1.5\", \"text\": \"hello world\", \"max_length\": 10, \"pad_to_max_length\": true, \"padding_side\": \"left\", \"add_special_tokens\": true }"
+curl http://localhost:8000/v3/tokenize -H "Content-Type: application/json" -d "{ \"model\": \"bge-large-en-v1.5\", \"text\": \"hello world\", \"max_length\": 10, \"pad_to_max_length\": true, \"padding_side\": \"left\", \"add_special_tokens\": true }"
 ```
 
 Response:

docs/parameters.md

@@ -181,6 +181,7 @@ Task specific parameters for different tasks (text generation/image generation/e
 | `--normalize`             | `bool`       | Normalize the embeddings. Default: true.                                       |
 | `--truncate`              | `bool`       | Truncate input when it exceeds model context length. Default: false            |
 | `--pooling`          | `string`       | Pooling option. One of: CLS, LAST, MEAN. Default: CLS.                                           |
+| `--max_legth`        | `integer`      | Maximum input length in tokens. If omitted, OVMS will detect it from the model's config.json.            |
 
 ### Rerank
 | option                    | Value format | Description                                                                    |

src/capi_frontend/server_settings.hpp

@@ -135,6 +135,7 @@ struct EmbeddingsGraphSettingsImpl {
     std::string normalize = "true";
     std::string truncate = "false";
     std::optional<std::string> pooling;
+    std::optional<uint32_t> maxLength;
 };
 
 struct TextToSpeechGraphSettingsImpl {

src/embeddings/embeddings_calculator_ov.proto

@@ -36,4 +36,5 @@ message EmbeddingsCalculatorOVOptions {
     }
     optional Pooling pooling = 5;
     optional bool truncate = 6 [default = false];
+    optional uint32 max_length = 7;
 }

src/embeddings/embeddings_node_initializer.cpp

@@ -74,13 +74,22 @@ class EmbeddingsNodeInitializer : public NodeInitializer {
             nodeOptions.has_pooling() ? std::make_optional(nodeOptions.pooling()) : std::nullopt;
         const auto pooling = resolveEmbeddingsPooling(modelsPath, graphPooling);
 
+        const std::optional<uint32_t> configuredMaxLength =
+            nodeOptions.has_max_length() ? std::make_optional(nodeOptions.max_length()) : std::nullopt;
+
+        if (configuredMaxLength.has_value() && configuredMaxLength.value() == 0) {
+            SPDLOG_ERROR("Embeddings node name: {} invalid max_length (0) in graph: {}.", nodeName, graphName);
+            return StatusCode::MEDIAPIPE_GRAPH_CONFIG_FILE_INVALID;
+        }
+
         auto servable = std::make_shared<EmbeddingsServable>(
             nodeOptions.models_path(),
             nodeOptions.target_device(),
             nodeOptions.plugin_config(),
             basePath,
             pooling,
-            nodeOptions.normalize_embeddings());
+            nodeOptions.normalize_embeddings(),
+            configuredMaxLength);
         servable->initialize(
             nodeOptions.models_path(),
             nodeOptions.target_device(),

src/embeddings/embeddings_servable.cpp

@@ -370,6 +370,10 @@ void reshapeModel(std::shared_ptr<Model>& model,
 // End code from OpenVINO GenAI repository
 
 std::shared_ptr<ov::Model> EmbeddingsServable::applyPrePostProcessing(ov::Core& core, std::shared_ptr<ov::Model> model, ov::AnyMap& properties) {
+    if (this->configuredMaxLength.has_value()) {
+        SPDLOG_DEBUG("Overriding detected max model length {} with configured value {}", this->maxModelLength.value_or(0), this->configuredMaxLength.value());
+        this->maxModelLength = this->configuredMaxLength;
+    }
     if (this->targetDevice == "NPU" && model->is_dynamic()) {
         TextEmbeddingPipeline::Config config;
         switch (this->pooling) {

src/embeddings/embeddings_servable.hpp

@@ -20,7 +20,10 @@
 #include "src/filesystem/filesystem.hpp"
 #include "src/port/rapidjson_istreamwrapper.hpp"
 #include "src/port/rapidjson_error.hpp"
+
+#include <cstdint>
 #include <memory>
+#include <optional>
 #include <string>
 #include <unordered_map>
 
@@ -34,10 +37,12 @@ struct EmbeddingsServable : public SidepacketServable {
         const std::string& pluginConfig,
         const std::string& graphPath,
         mediapipe::EmbeddingsCalculatorOVOptions_Pooling pooling,
-        bool normalizeEmbeddings) :
+        bool normalizeEmbeddings,
+        std::optional<uint32_t> configuredMaxLength = std::nullopt) :
         SidepacketServable(modelDir, targetDevice, pluginConfig, graphPath),
         pooling(pooling),
-        normalizeEmbeddings(normalizeEmbeddings) {}
+        normalizeEmbeddings(normalizeEmbeddings),
+        configuredMaxLength(configuredMaxLength) {}
 
     int getTargetOutputIndex() const {
         return targetOutputIndex;
@@ -61,6 +66,7 @@ struct EmbeddingsServable : public SidepacketServable {
 private:
     mediapipe::EmbeddingsCalculatorOVOptions_Pooling pooling;
     bool normalizeEmbeddings;
+    std::optional<uint32_t> configuredMaxLength;
     bool npuPostprocessingRequired = false;
     ov::CompiledModel postProcCompiledModel;
     std::unique_ptr<OVInferRequestsQueue> postProcInferRequestsQueue;

src/graph_export/embeddings_graph_cli_parser.cpp

@@ -55,7 +55,11 @@ void EmbeddingsGraphCLIParser::createOptions() {
         ("pooling",
             "Pooling option. One of: CLS, LAST, MEAN. If omitted, OVMS will detect pooling automatically.",
             cxxopts::value<std::string>(),
-            "POOLING");
+            "POOLING")
+        ("max_length",
+            "Maximum input length in tokens. If omitted, OVMS will detect it from the model's config.json.",
+            cxxopts::value<uint32_t>(),
+            "MAX_LENGTH");
 }
 
 void EmbeddingsGraphCLIParser::printHelp() {
@@ -99,6 +103,13 @@ void EmbeddingsGraphCLIParser::prepare(OvmsServerMode serverMode, HFSettingsImpl
         if (result->count("pooling") > 0) {
             embeddingsGraphSettings.pooling = result->operator[]("pooling").as<std::string>();
         }
+        if (result->count("max_length") > 0) {
+            const auto maxLength = result->operator[]("max_length").as<uint32_t>();
+             if (maxLength == 0) {
+                 throw std::invalid_argument("max_length must be greater than 0");
+             }
+             embeddingsGraphSettings.maxLength = maxLength;
+        }
     }
     if (embeddingsGraphSettings.pooling.has_value() &&
         !(embeddingsGraphSettings.pooling.value() == "CLS" || embeddingsGraphSettings.pooling.value() == "LAST" || embeddingsGraphSettings.pooling.value() == "MEAN")) {

src/graph_export/graph_export.cpp

@@ -337,6 +337,10 @@ node {
         oss << R"(
             pooling: )" << graphSettings.pooling.value() << R"(,)";
     }
+    if (graphSettings.maxLength.has_value()) {
+        oss << R"(
+            max_length: )" << graphSettings.maxLength.value() << R"(,)";
+    }
     if (!exportSettings.targetDevice.empty()) {
         oss << R"(
             target_device: ")" << exportSettings.targetDevice << R"(",)";

src/test/embeddings/config_embeddings.json

@@ -25,6 +25,11 @@
         "name":"embeddings_ov_pooling_last",
         "base_path":"/ovms/src/test/embeddings/",
         "graph_path":"/ovms/src/test/embeddings/graph_ov_pooling_last.pbtxt"
+    },
+    {
+        "name":"embeddings_ov_max_length",
+        "base_path":"/ovms/src/test/embeddings/",
+        "graph_path":"/ovms/src/test/embeddings/graph_ov_max_length.pbtxt"
     }
     ]
 }

src/test/embeddings/graph_ov_max_length.pbtxt

@@ -0,0 +1,30 @@
+# Copyright 2026 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+input_stream: "REQUEST_PAYLOAD:input"
+output_stream: "RESPONSE_PAYLOAD:output"
+
+node {
+    name: "embeddingsNode1"
+    calculator: "EmbeddingsCalculatorOV"
+    input_stream: "REQUEST_PAYLOAD:input"
+    output_stream: "RESPONSE_PAYLOAD:output"
+    input_side_packet: "EMBEDDINGS_NODE_RESOURCES:embeddings_servable"
+      node_options: {
+      [type.googleapis.com/mediapipe.EmbeddingsCalculatorOVOptions]: {
+        models_path: "/ovms/src/test/llm_testing/thenlper/gte-small/ov"
+        max_length: 32
+      }
+    }
+}

src/test/embeddingsnode_test.cpp

@@ -561,6 +561,38 @@ TEST_F(EmbeddingsHttpTest, positivePoolingLast) {
     ASSERT_EQ(d["data"][0]["embedding"].Size(), EMBEDDING_OUTPUT_SIZE);
 }
 
+TEST_F(EmbeddingsHttpTest, positiveWithinConfiguredMaxLength) {
+    std::string words;
+    for (int i = 0; i < 20; i++) {
+        words += "hello ";
+    }
+    std::string requestBody = "{ \"model\": \"embeddings_ov_max_length\", \"input\": \"" + words + " \"}";
+
+    Status status = handler->dispatchToProcessor(endpoint, requestBody, &response, comp, responseComponents, writer, multiPartParser);
+    ASSERT_EQ(status,
+        ovms::StatusCode::OK)
+        << status.string();
+    rapidjson::Document d;
+    rapidjson::ParseResult ok = d.Parse(response.c_str());
+    ASSERT_EQ(ok.Code(), 0);
+    ASSERT_TRUE(d["data"][0]["embedding"].IsArray());
+    ASSERT_EQ(d["data"][0]["embedding"].Size(), EMBEDDING_OUTPUT_SIZE);
+}
+
+TEST_F(EmbeddingsHttpTest, negativeExceedsConfiguredMaxLength) {
+    std::string words;
+    for (int i = 0; i < 35; i++) {
+        words += "hello ";
+    }
+    std::string requestBody = "{ \"model\": \"embeddings_ov_max_length\", \"input\": \"" + words + " \"}";
+
+    Status status = handler->dispatchToProcessor(endpoint, requestBody, &response, comp, responseComponents, writer, multiPartParser);
+    ASSERT_EQ(status,
+        ovms::StatusCode::MEDIAPIPE_EXECUTION_ERROR)
+        << status.string();
+    ASSERT_THAT(status.string(), ::testing::HasSubstr("longer than allowed"));
+}
+
 TEST_F(EmbeddingsHttpTest, accessingCalculatorWithInvalidJson) {
     std::string requestBody = R"(
         {

src/test/graph_export_test.cpp

@@ -351,6 +351,7 @@ node {
             normalize_embeddings: false,
             truncate: true,
             pooling: LAST,
+            max_length: 512,
             target_device: "GPU",
             plugin_config: '{"NUM_STREAMS":"2"}',
         }
@@ -785,6 +786,7 @@ TEST_F(GraphCreationTest, embeddingsPositiveNonDefault) {
     embeddingsGraphSettings.normalize = "false";
     embeddingsGraphSettings.truncate = "true";
     embeddingsGraphSettings.pooling = "LAST";
+    embeddingsGraphSettings.maxLength = 512;
     hfSettings.graphSettings = std::move(embeddingsGraphSettings);
     assertCreatedGraphEquals(hfSettings, expectedEmbeddingsGraphContents);
 }

src/test/ovmsconfig_test.cpp

@@ -27,6 +27,7 @@
 #include "../utils/env_guard.hpp"
 #include "../config.hpp"
 #include "src/filesystem/filesystem.hpp"
+#include "../graph_export/embeddings_graph_cli_parser.hpp"
 #include "../graph_export/graph_cli_parser.hpp"
 #include "../ovms_exit_codes.hpp"
 #include "../systeminfo.hpp"
@@ -560,6 +561,24 @@ TEST_F(OvmsConfigDeathTest, negativeImageGenerationGraph_MaxNumInferenceStepsZer
     EXPECT_THROW(ovms::Config::instance().parse(arg_count, n_argv), std::invalid_argument);
 }
 
+TEST(OvmsGraphConfigTest, negativeEmbeddingsGraph_MaxLengthZero) {
+    char* n_argv[] = {
+        (char*)"ovms",
+        (char*)"--pull",
+        (char*)"--source_model",
+        (char*)"some/model",
+        (char*)"--model_repository_path",
+        (char*)"/some/path",
+        (char*)"--task",
+        (char*)"embeddings",
+        (char*)"--max_length",
+        (char*)"0",
+    };
+    int arg_count = 10;
+    ConstructorEnabledConfig config;
+    EXPECT_THROW(config.parse(arg_count, n_argv), std::invalid_argument);
+}
+
 TEST(OvmsGraphConfigTest, negativeImageGenerationGraph_SourceLorasEmptyAlias) {
     char* n_argv[] = {
         (char*)"ovms",
@@ -1934,9 +1953,11 @@ TEST(OvmsGraphConfigTest, positiveAllChangedEmbeddings) {
         (char*)"--plugin_config",
         (char*)"{\"SOME_KEY\":\"SOME_VALUE\"}",
         (char*)"--cache_dir",
-        (char*)"/tmp/cache_dir_with_emptiness"};
+        (char*)"/tmp/cache_dir_with_emptiness",
+        (char*)"--max_length",
+        (char*)"512"};
 
-    int arg_count = 24;
+    int arg_count = 26;
     ConstructorEnabledConfig config;
     config.parse(arg_count, n_argv);
 
@@ -1951,6 +1972,8 @@ TEST(OvmsGraphConfigTest, positiveAllChangedEmbeddings) {
     ASSERT_EQ(embeddingsGraphSettings.truncate, "true");
     ASSERT_TRUE(embeddingsGraphSettings.pooling.has_value());
     ASSERT_EQ(embeddingsGraphSettings.pooling.value(), "CLS");
+    ASSERT_TRUE(embeddingsGraphSettings.maxLength.has_value());
+    ASSERT_EQ(embeddingsGraphSettings.maxLength.value(), 512);
     ASSERT_EQ(exportSettings.pluginConfig.numStreams, 2);
     ASSERT_EQ(exportSettings.targetDevice, "GPU");
     ASSERT_EQ(exportSettings.modelName, servingName);
@@ -2036,6 +2059,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultEmbeddings) {
     ASSERT_EQ(embeddingsGraphSettings.normalize, "true");
     ASSERT_EQ(embeddingsGraphSettings.truncate, "false");
     ASSERT_FALSE(embeddingsGraphSettings.pooling.has_value());
+    ASSERT_FALSE(embeddingsGraphSettings.maxLength.has_value());
     ASSERT_EQ(exportSettings.pluginConfig.numStreams, 1);
     ASSERT_EQ(exportSettings.targetDevice, "");
     ASSERT_EQ(exportSettings.modelName, modelName);
@@ -3116,6 +3140,33 @@ TEST(OvmsGraphCliParserTest, invalidReasoningParserNameThrowsInvalidArgument) {
         std::invalid_argument);
 }
 
+TEST(OvmsGraphCliParserTest, embeddingsMaxLengthZeroThrowsInvalidArgument) {
+    ovms::HFSettingsImpl hfSettings;
+    ovms::EmbeddingsGraphCLIParser parser;
+    std::vector<std::string> args = {"--max_length", "0"};
+    parser.parse(args);
+    EXPECT_THROW({
+        try {
+            parser.prepare(ovms::HF_PULL_MODE, hfSettings, "test_model");
+        } catch (const std::invalid_argument& e) {
+            EXPECT_NE(std::string(e.what()).find("max_length must be greater than 0"), std::string::npos);
+            throw;
+        }
+    },
+        std::invalid_argument);
+}
+
+TEST(OvmsGraphCliParserTest, embeddingsMaxLengthNonZeroIsAccepted) {
+    ovms::HFSettingsImpl hfSettings;
+    ovms::EmbeddingsGraphCLIParser parser;
+    std::vector<std::string> args = {"--max_length", "1"};
+    parser.parse(args);
+    EXPECT_NO_THROW(parser.prepare(ovms::HF_PULL_MODE, hfSettings, "test_model"));
+    auto& embeddingsGraphSettings = std::get<ovms::EmbeddingsGraphSettingsImpl>(hfSettings.graphSettings);
+    ASSERT_TRUE(embeddingsGraphSettings.maxLength.has_value());
+    ASSERT_EQ(embeddingsGraphSettings.maxLength.value(), 1u);
+}
+
 TEST(OvmsGraphCliParserTest, validParserNamesAreAccepted) {
     ovms::HFSettingsImpl hfSettings;
     ovms::GraphCLIParser parser;