Add idle model unload for mediapipe LLM graphs

docs/llm/reference.md

@@ -112,6 +112,16 @@ The calculator supports the following `node_options` for tuning the pipeline con
 -    `optional string tool_parser` - name of the parser to use for tool calls extraction from model output before creating a response;
 -    `optional bool enable_tool_guided_generation` - enable enforcing tool schema during generation. Requires setting response parser. [default = false];
 -    `optional SparseAttentionConfig sparse_attention_config` - Sparse attention configuration. Disabled if not specified.
+-    `optional int64 idle_unload_timeout_seconds` - unload the graph's model resources after this many seconds with no inference requests, freeing GPU/CPU memory; the model is reloaded automatically on the next request. `0` disables the feature [default = 0]. See [Idle model unload](#idle-model-unload).
+
+### Idle model unload
+When `idle_unload_timeout_seconds` is set to a positive value, the model server unloads the LLM graph's heavy resources (the continuous batching pipeline, freeing GPU VRAM / host memory) after the configured period without any inference requests. The first request after an unload transparently reloads the model and is served once it is ready, so the GPU can be used by other workloads while a model is idle.
+
+Notes:
+- Only inference requests reset the idle timer; status/metrics/health endpoints do not keep a model loaded.
+- The first request after an idle unload pays the reload latency. Combine with [model caching](../model_cache.md) (`--cache_dir`) so the reload is a fast cache import rather than a full recompile.
+- The graph reports as `AVAILABLE` while idle-unloaded (it auto-reloads on demand). The `ovms_graph_loaded` metric reports `1` when loaded and `0` when idle-unloaded.
+- Supported for LLM continuous-batching graphs. Graphs containing Python nodes are not supported with this setting.
 
 ### Caching settings
 The value of `cache_size` might have performance and stability implications. It is used for storing LLM model KV cache data. Adjust it based on your environment capabilities, model size and expected level of concurrency.

spelling-whitelist.txt

@@ -42,3 +42,8 @@ src/test/llm/output_parsers/gemma4_output_parser_test.cpp
 src/test/llm/output_parsers/qwen3_output_parser_test.cpp:697: thi ==> the, this
 extras/chat_template_examples/chat_template_onyx.jinja
 src/test/llm/chat_templates/chat_template_onyx.jinja
+src/mediapipe_internal/mediapipegraphdefinition.cpp
+src/mediapipe_internal/mediapipegraphdefinition.hpp
+src/model_group_manager.cpp
+src/test/llm/llmnode_test.cpp
+nowNs ==> knowns, nouns

src/BUILD

@@ -158,6 +158,7 @@ ovms_cc_library(
 ovms_cc_library(
     name = "libovms_servable_definition",
     hdrs = ["servable_definition.hpp"],
+    visibility = ["//visibility:public"],
 )
 ovms_cc_library(
     name = "libovms_single_version_servable_definition",
@@ -575,48 +576,6 @@ ovms_cc_library(
     ],
     visibility = ["//visibility:public"],
 )
-ovms_cc_library(
-    name = "modelmanager",
-    hdrs = ["modelmanager.hpp"],
-    srcs = ["modelmanager.cpp"],
-    deps = select({
-        "//conditions:default": [],
-        "//:not_disable_mediapipe" : [
-            "//src/mediapipe_internal:libovms_mediapipe",
-        ],
-    }) + [
-        "cleaner_utils",
-        "customloaders",
-        "libovms_config",
-        "libovms_model_instance_provider",
-        "libovms_ov_utils",
-        "libovms_servable_definition",
-        "libovms_servable_name_checker",
-        "libovmslogging",
-        "libovmsschema",
-        "libovmsstring_utils",
-        "libovmsstatus",
-        "model",
-        "modelconfig",
-        "modelinstance",
-        "modelinstanceunloadguard",
-        "resources_cleaner",
-        "//src/dags:custom_node_library_manager",
-        "//src/dags:dag_resource_manager",
-        "//src/dags:pipeline_config_parser",
-        "//src/dags:pipeline_factory",
-        "//src/dags:pipelinedefinition",
-        "//src/filesystem:libovmsfilesystem",
-        "//src/filesystem:libovmsfilesystemfactory",
-        "//src/graph_export:graph_export",
-        "//src/metrics:libovms_metric_provider",
-        "//src/metrics:libovmsmetrics",
-        "@com_github_tencent_rapidjson//:rapidjson",
-        "//src/port:rapidjson_stringbuffer",
-        "//src/port:rapidjson_writer",
-    ],
-    visibility = ["//visibility:public"],
-)
 ovms_cc_library(
     name = "rest_parser_utils",
     hdrs = [
@@ -701,24 +660,6 @@ ovms_cc_library(
     ],
     visibility = ["//visibility:public"],
 )
-ovms_cc_library(
-    name = "servablemanagermodule",
-    hdrs = ["servablemanagermodule.hpp"],
-    srcs = ["servablemanagermodule.cpp"],
-    deps = select({
-        "//:not_disable_python": [
-            "//src/python:libovmspythonmodule",
-        ],
-        "//:disable_python": []
-    }) + [
-        "cpp_headers",
-        "libovms_module",
-        "libovmslogging",
-        "modelmanager",
-        "//src/metrics:libovmsmetrics",
-    ],
-    visibility = ["//visibility:public"],
-)
 ovms_cc_library(
     name = "ovms_lib",
     hdrs = [
@@ -832,8 +773,8 @@ ovms_cc_library(
             "libovms_kfs_utils",
             "libovms_kfs_grpc_inference_service_h",
             "modelchangesubscription",
-            "modelmanager",
-            "servablemanagermodule",
+            "//src/servable_management:modelmanager",
+            "//src/servable_management:servablemanagermodule",
             "//src/filesystem:libovmslocalfilesystem", # indirectly & directly through factory
             "libovmslogging",
             "//src/metrics:libovmsmetrics",
@@ -2258,6 +2199,9 @@ cc_test(
             ":test_test_models_configs",
             ":test_cmd_exec",
             ":test_modelinstance_test",
+            ":servable_loading_queue_test",
+            ":test_idle_model_test",
+            ":test_servable_group_manager",
             ] + select({
                 "//conditions:default": [
                     ":openvino_remote_tensors_tests",
@@ -2282,6 +2226,7 @@ cc_test(
                 ":text2image_test",
                 "//src/rerank:rerank_api_handler",
                 ":embeddings_handler_tests",
+                ":test_idle_mediapipe_test",
                 "libovms_mediapipe_kfs_executor",
                 "//src/mediapipe_internal:mediapipe_utils",
                 "tensorflow_type_utils",
@@ -2370,6 +2315,55 @@ cc_library(
     linkopts = COMMON_STATIC_LIBS_LINKOPTS,
 )
 
+ovms_cc_test_library(
+    name = "servable_loading_queue_test",
+    srcs = ["test/servable_loading_queue_test.cpp"],
+    deps = [
+        "//src/servable_management:servable_loading_queue",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+ovms_cc_test_library(
+    name = "test_idle_model_test",
+    srcs = ["test/idle_model_test.cpp"],
+    deps = [
+        ":test_constructor_enabled_model_manager",
+        ":test_test_models",
+        ":test_test_models_configs",
+        ":test_test_with_temp_dir",
+        ":test_utils",
+        "//third_party:openvino",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+ovms_cc_test_library(
+    name = "test_servable_group_manager",
+    srcs = ["test/servable_group_manager_test.cpp"],
+    deps = [
+        ":test_constructor_enabled_model_manager",
+        ":test_test_models",
+        ":test_test_with_temp_dir",
+        "//src/servable_management:modelmanager",
+        "//src:modelconfig",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+ovms_cc_test_library(
+    name = "test_idle_mediapipe_test",
+    srcs = ["test/idle_mediapipe_test.cpp"],
+    deps = [
+        ":test_constructor_enabled_model_manager",
+        ":test_utils",
+        "//src/dags:pipelinedefinitionstatus",
+        "//src/mediapipe_internal:libovms_mediapipe",
+        "//src/mediapipe_internal:mediapipegraphconfig",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 cc_library(
     name = "test_utils",
     linkstatic = 1,

src/capi_frontend/capi.cpp

@@ -40,13 +40,13 @@
 #include "../deserialization_main.hpp"
 #include "../inference_executor.hpp"
 #include "../modelinstanceunloadguard.hpp"
-#include "../modelmanager.hpp"
+#include "src/servable_management/modelmanager.hpp"
 #include "../module_names.hpp"
 #include "../ovms.h"  // NOLINT
 #include "../profiler.hpp"
 #include "../dags/pipelinedefinitionstatus.hpp"
 #include "../servable_definition.hpp"
-#include "../servablemanagermodule.hpp"
+#include "src/servable_management/servablemanagermodule.hpp"
 #include "../server.hpp"
 #include "../single_version_servable_definition.hpp"
 #include "../status.hpp"

src/capi_frontend/capi_dag_utils.cpp

@@ -41,7 +41,9 @@ OVMS_ServableState convertToServableState(ovms::PipelineDefinitionStateCode code
     case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED:
     case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION:
         return OVMS_ServableState::OVMS_STATE_LOADING_FAILED;
-    }
+    case ovms::PipelineDefinitionStateCode::SLEEPING:
+        return OVMS_ServableState::OVMS_STATE_AVAILABLE;
+    }  // TODO #atobiszei idle management C-API change - new value in enum?
     throw new std::exception();
 }
 

src/capi_frontend/server_settings.hpp

@@ -244,6 +244,7 @@ struct ServerSettingsImpl {
     std::string grpcChannelArguments;
     uint32_t filesystemPollWaitMilliseconds = 1000;
     uint32_t resourcesCleanerPollWaitSeconds = 300;
+    uint32_t idleUnloadTimeoutSeconds = 0;
     std::string cacheDir;
     bool withPython = false;
     bool startedWithCLI = false;
@@ -266,6 +267,7 @@ struct ModelsSettingsImpl {
     uint32_t nireq = 0;
     std::string targetDevice;
     std::string pluginConfig;
+    std::optional<std::string> groupName;
     std::vector<std::string> userSetSingleModelArguments;
 
     std::string configPath;

src/cli_parser.cpp

@@ -72,7 +72,7 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
     std::stringstream ss;
     try {
         options = std::make_unique<cxxopts::Options>(argv[0], "OpenVINO Model Server");
-        auto configOptions = std::make_unique<cxxopts::Options>("ovms --add_to_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME> --model_repository_path <MODEL_REPO_PATH> \n  ovms --add_to_config --config_path <CONFIG_PATH> --model_path <MODEL_PATH> --model_name <MODEL_NAME>  \n  ovms --remove_from_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME>", "config management commands:");
+        auto configOptions = std::make_unique<cxxopts::Options>("ovms --add_to_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME> --model_repository_path <MODEL_REPO_PATH> \n  ovms --add_to_config --config_path <CONFIG_PATH> --model_path <MODEL_PATH> --model_name <MODEL_NAME> --group_name <GROUP> \n  ovms --remove_from_config --config_path <CONFIG_PATH> --model_name <MODEL_NAME>", "config management commands:");
         // Adding this option to parse unrecognised options in another parser
         options->allow_unrecognised_options();
 
@@ -137,6 +137,10 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
                 "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.",
                 cxxopts::value<uint32_t>()->default_value("1"),
                 "FILE_SYSTEM_POLL_WAIT_SECONDS")
+            ("idle_unload_timeout_seconds",
+                "Idle timeout in seconds for model group unloading. When > 0, models not in the 'permanent' group are loaded on demand and unloaded after this idle period. Only effective with config.json multi-model setup. Default is 0 (disabled).",
+                cxxopts::value<uint32_t>()->default_value("0"),
+                "IDLE_UNLOAD_TIMEOUT_SECONDS")
             ("custom_node_resources_cleaner_interval_seconds",
                 "Time interval between two consecutive resources cleanup scans. Default is 300. Zero value disables resources cleaner.",
                 cxxopts::value<uint32_t>()->default_value("300"),
@@ -209,7 +213,11 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
             ("remove_from_config",
                 "Directive to remove a model from configuration file. This parameter should be executed with --config_path and --model_name to specify which model to remove.",
                 cxxopts::value<bool>()->default_value("false"),
-                "REMOVE_FROM_CONFIG");
+                "REMOVE_FROM_CONFIG")
+            ("group_name",
+                "Optional group name for idle model group management. Used with --add_to_config.",
+                cxxopts::value<std::string>(),
+                "GROUP_NAME");
 
         // Set default value for model_repository_path from environment variable if it exists and is not empty
         std::string defaultModelRepoPath = "";
@@ -343,6 +351,10 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
                 "Name of the model",
                 cxxopts::value<std::string>(),
                 "MODEL_NAME")
+            ("group_name",
+                "Optional group name for idle model group management",
+                cxxopts::value<std::string>(),
+                "GROUP_NAME")
             ("config_path",
                 "Path to json configuration file",
                 cxxopts::value<std::string>()->default_value(defaultConfigPath),
@@ -569,6 +581,7 @@ void CLIParser::prepareServer(ServerSettingsImpl& serverSettings) {
     serverSettings.filesystemPollWaitMilliseconds = result->operator[]("file_system_poll_wait_seconds").as<uint32_t>() * 1000;
 
     serverSettings.resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as<uint32_t>();
+    serverSettings.idleUnloadTimeoutSeconds = result->operator[]("idle_unload_timeout_seconds").as<uint32_t>();
     serverSettings.grpcWorkers = result->operator[]("grpc_workers").as<uint32_t>();
 
     if (result->count("log_level"))
@@ -921,6 +934,10 @@ void CLIParser::prepareConfigExport(ModelsSettingsImpl& modelsSettings) {
     } else if (!result->operator[]("model_repository_path").as<std::string>().empty() && result->count("model_name")) {
         modelsSettings.modelPath = FileSystem::joinPath({result->operator[]("model_repository_path").as<std::string>(), modelsSettings.modelName});
     }
+    if (result->count("group_name")) {
+        modelsSettings.groupName = result->operator[]("group_name").as<std::string>();
+        modelsSettings.userSetSingleModelArguments.push_back("group_name");
+    }
     std::string defaultConfigPath = "";
     const char* envModelRepoPath = std::getenv("OVMS_MODEL_REPOSITORY_PATH");
     if (envModelRepoPath != nullptr && std::string(envModelRepoPath).length() > 0) {

src/config.cpp

@@ -141,7 +141,7 @@ bool Config::validateUserSettingsInConfigAddRemoveModel(const ModelsSettingsImpl
     static const std::vector<std::string> allowedForRemove = {"model_name", "config_path"};
     static const std::vector<std::string> allowedForAdd = {"model_name", "model_path", "config_path",
         "batch_size", "shape", "layout", "mean", "scale", "color_format", "precision",
-        "model_version_policy", "nireq", "target_device", "plugin_config"};
+        "model_version_policy", "nireq", "target_device", "plugin_config", "group_name"};
 
     const auto& allowedUserSettings = (exportType == ENABLE_MODEL) ? allowedForAdd : allowedForRemove;
     std::vector<std::string> usedButDisallowedUserSettings;
@@ -432,6 +432,7 @@ const std::string& Config::tracePath() const { return this->serverSettings.trace
 const std::string& Config::grpcChannelArguments() const { return this->serverSettings.grpcChannelArguments; }
 uint32_t Config::filesystemPollWaitMilliseconds() const { return this->serverSettings.filesystemPollWaitMilliseconds; }
 uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->serverSettings.resourcesCleanerPollWaitSeconds; }
+uint32_t Config::idleUnloadTimeoutSeconds() const { return this->serverSettings.idleUnloadTimeoutSeconds; }
 bool Config::allowCredentials() const { return this->serverSettings.allowCredentials; }
 const std::string& Config::allowedOrigins() const { return this->serverSettings.allowedOrigins; }
 const std::string& Config::allowedMethods() const { return this->serverSettings.allowedMethods; }

src/config.hpp

@@ -312,6 +312,13 @@ class Config {
      */
     uint32_t resourcesCleanerPollWaitSeconds() const;
 
+    /**
+     * @brief Get the idle unload timeout in seconds (0 = disabled)
+     *
+     * @return uint32_t
+     */
+    uint32_t idleUnloadTimeoutSeconds() const;
+
     bool allowCredentials() const;
     const std::string& allowedOrigins() const;
     const std::string& allowedMethods() const;

src/config_export_module/config_export.cpp

@@ -56,6 +56,8 @@ static void addOptionalModelFields(rapidjson::Value& configObj, const ModelsSett
         configObj.AddMember("target_device", rapidjson::Value(modelSettings.targetDevice.c_str(), alloc), alloc);
     if (!modelSettings.pluginConfig.empty())
         addJsonOrStringMember(configObj, "plugin_config", modelSettings.pluginConfig, alloc);
+    if (modelSettings.groupName.has_value())
+        configObj.AddMember("group_name", rapidjson::Value(modelSettings.groupName.value().c_str(), alloc), alloc);
 }
 
 Status loadJsonConfig(const std::string& jsonFilename, rapidjson::Document& configJson) {

src/dags/pipelinedefinition.cpp

@@ -138,7 +138,7 @@ Status PipelineDefinition::reload(ModelInstanceProvider& modelInstanceProvider,
     // block creating new unloadGuards
     this->status.handle(ReloadEvent());
     resetSubscriptions(modelInstanceProvider);
-    while (requestsHandlesCounter > 0) {
+    while (pendingCreateExecutorCount > 0) {
         std::this_thread::sleep_for(std::chrono::microseconds(1));
     }
     // deinitialize all resources that are associated with nodes that are currently in PipelineDefinition, but not in nodeInfos
@@ -153,7 +153,7 @@ Status PipelineDefinition::reload(ModelInstanceProvider& modelInstanceProvider,
 void PipelineDefinition::retire(ModelInstanceProvider& modelInstanceProvider) {
     resetSubscriptions(modelInstanceProvider);
     this->status.handle(RetireEvent());
-    while (requestsHandlesCounter > 0) {
+    while (pendingCreateExecutorCount > 0) {
         std::this_thread::sleep_for(std::chrono::microseconds(1));
     }
     // deinitalize all resources