diff --git a/docs/llm/reference.md b/docs/llm/reference.md index edb999cad9..f582ef4134 100644 --- a/docs/llm/reference.md +++ b/docs/llm/reference.md @@ -112,6 +112,16 @@ The calculator supports the following `node_options` for tuning the pipeline con - `optional string tool_parser` - name of the parser to use for tool calls extraction from model output before creating a response; - `optional bool enable_tool_guided_generation` - enable enforcing tool schema during generation. Requires setting response parser. [default = false]; - `optional SparseAttentionConfig sparse_attention_config` - Sparse attention configuration. Disabled if not specified. +- `optional int64 idle_unload_timeout_seconds` - unload the graph's model resources after this many seconds with no inference requests, freeing GPU/CPU memory; the model is reloaded automatically on the next request. `0` disables the feature [default = 0]. See [Idle model unload](#idle-model-unload). + +### Idle model unload +When `idle_unload_timeout_seconds` is set to a positive value, the model server unloads the LLM graph's heavy resources (the continuous batching pipeline, freeing GPU VRAM / host memory) after the configured period without any inference requests. The first request after an unload transparently reloads the model and is served once it is ready, so the GPU can be used by other workloads while a model is idle. + +Notes: +- Only inference requests reset the idle timer; status/metrics/health endpoints do not keep a model loaded. +- The first request after an idle unload pays the reload latency. Combine with [model caching](../model_cache.md) (`--cache_dir`) so the reload is a fast cache import rather than a full recompile. +- The graph reports as `AVAILABLE` while idle-unloaded (it auto-reloads on demand). The `ovms_graph_loaded` metric reports `1` when loaded and `0` when idle-unloaded. +- Supported for LLM continuous-batching graphs. Graphs containing Python nodes are not supported with this setting. ### Caching settings The value of `cache_size` might have performance and stability implications. It is used for storing LLM model KV cache data. Adjust it based on your environment capabilities, model size and expected level of concurrency. diff --git a/spelling-whitelist.txt b/spelling-whitelist.txt index 8d2ea40cfe..6c8cd12e57 100644 --- a/spelling-whitelist.txt +++ b/spelling-whitelist.txt @@ -42,3 +42,8 @@ src/test/llm/output_parsers/gemma4_output_parser_test.cpp src/test/llm/output_parsers/qwen3_output_parser_test.cpp:697: thi ==> the, this extras/chat_template_examples/chat_template_onyx.jinja src/test/llm/chat_templates/chat_template_onyx.jinja +src/mediapipe_internal/mediapipegraphdefinition.cpp +src/mediapipe_internal/mediapipegraphdefinition.hpp +src/model_group_manager.cpp +src/test/llm/llmnode_test.cpp +nowNs ==> knowns, nouns diff --git a/src/BUILD b/src/BUILD index 1e2368327b..aaeb438378 100644 --- a/src/BUILD +++ b/src/BUILD @@ -158,6 +158,7 @@ ovms_cc_library( ovms_cc_library( name = "libovms_servable_definition", hdrs = ["servable_definition.hpp"], + visibility = ["//visibility:public"], ) ovms_cc_library( name = "libovms_single_version_servable_definition", @@ -575,48 +576,6 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "modelmanager", - hdrs = ["modelmanager.hpp"], - srcs = ["modelmanager.cpp"], - deps = select({ - "//conditions:default": [], - "//:not_disable_mediapipe" : [ - "//src/mediapipe_internal:libovms_mediapipe", - ], - }) + [ - "cleaner_utils", - "customloaders", - "libovms_config", - "libovms_model_instance_provider", - "libovms_ov_utils", - "libovms_servable_definition", - "libovms_servable_name_checker", - "libovmslogging", - "libovmsschema", - "libovmsstring_utils", - "libovmsstatus", - "model", - "modelconfig", - "modelinstance", - "modelinstanceunloadguard", - "resources_cleaner", - "//src/dags:custom_node_library_manager", - "//src/dags:dag_resource_manager", - "//src/dags:pipeline_config_parser", - "//src/dags:pipeline_factory", - "//src/dags:pipelinedefinition", - "//src/filesystem:libovmsfilesystem", - "//src/filesystem:libovmsfilesystemfactory", - "//src/graph_export:graph_export", - "//src/metrics:libovms_metric_provider", - "//src/metrics:libovmsmetrics", - "@com_github_tencent_rapidjson//:rapidjson", - "//src/port:rapidjson_stringbuffer", - "//src/port:rapidjson_writer", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "rest_parser_utils", hdrs = [ @@ -701,24 +660,6 @@ ovms_cc_library( ], visibility = ["//visibility:public"], ) -ovms_cc_library( - name = "servablemanagermodule", - hdrs = ["servablemanagermodule.hpp"], - srcs = ["servablemanagermodule.cpp"], - deps = select({ - "//:not_disable_python": [ - "//src/python:libovmspythonmodule", - ], - "//:disable_python": [] - }) + [ - "cpp_headers", - "libovms_module", - "libovmslogging", - "modelmanager", - "//src/metrics:libovmsmetrics", - ], - visibility = ["//visibility:public"], -) ovms_cc_library( name = "ovms_lib", hdrs = [ @@ -832,8 +773,8 @@ ovms_cc_library( "libovms_kfs_utils", "libovms_kfs_grpc_inference_service_h", "modelchangesubscription", - "modelmanager", - "servablemanagermodule", + "//src/servable_management:modelmanager", + "//src/servable_management:servablemanagermodule", "//src/filesystem:libovmslocalfilesystem", # indirectly & directly through factory "libovmslogging", "//src/metrics:libovmsmetrics", @@ -2258,6 +2199,9 @@ cc_test( ":test_test_models_configs", ":test_cmd_exec", ":test_modelinstance_test", + ":servable_loading_queue_test", + ":test_idle_model_test", + ":test_servable_group_manager", ] + select({ "//conditions:default": [ ":openvino_remote_tensors_tests", @@ -2282,6 +2226,7 @@ cc_test( ":text2image_test", "//src/rerank:rerank_api_handler", ":embeddings_handler_tests", + ":test_idle_mediapipe_test", "libovms_mediapipe_kfs_executor", "//src/mediapipe_internal:mediapipe_utils", "tensorflow_type_utils", @@ -2370,6 +2315,55 @@ cc_library( linkopts = COMMON_STATIC_LIBS_LINKOPTS, ) +ovms_cc_test_library( + name = "servable_loading_queue_test", + srcs = ["test/servable_loading_queue_test.cpp"], + deps = [ + "//src/servable_management:servable_loading_queue", + "@com_google_googletest//:gtest", + ], +) + +ovms_cc_test_library( + name = "test_idle_model_test", + srcs = ["test/idle_model_test.cpp"], + deps = [ + ":test_constructor_enabled_model_manager", + ":test_test_models", + ":test_test_models_configs", + ":test_test_with_temp_dir", + ":test_utils", + "//third_party:openvino", + "@com_google_googletest//:gtest", + ], +) + +ovms_cc_test_library( + name = "test_servable_group_manager", + srcs = ["test/servable_group_manager_test.cpp"], + deps = [ + ":test_constructor_enabled_model_manager", + ":test_test_models", + ":test_test_with_temp_dir", + "//src/servable_management:modelmanager", + "//src:modelconfig", + "@com_google_googletest//:gtest", + ], +) + +ovms_cc_test_library( + name = "test_idle_mediapipe_test", + srcs = ["test/idle_mediapipe_test.cpp"], + deps = [ + ":test_constructor_enabled_model_manager", + ":test_utils", + "//src/dags:pipelinedefinitionstatus", + "//src/mediapipe_internal:libovms_mediapipe", + "//src/mediapipe_internal:mediapipegraphconfig", + "@com_google_googletest//:gtest", + ], +) + cc_library( name = "test_utils", linkstatic = 1, diff --git a/src/capi_frontend/capi.cpp b/src/capi_frontend/capi.cpp index 5ff98510d2..bbd16d07eb 100644 --- a/src/capi_frontend/capi.cpp +++ b/src/capi_frontend/capi.cpp @@ -40,13 +40,13 @@ #include "../deserialization_main.hpp" #include "../inference_executor.hpp" #include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "../module_names.hpp" #include "../ovms.h" // NOLINT #include "../profiler.hpp" #include "../dags/pipelinedefinitionstatus.hpp" #include "../servable_definition.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../single_version_servable_definition.hpp" #include "../status.hpp" diff --git a/src/capi_frontend/capi_dag_utils.cpp b/src/capi_frontend/capi_dag_utils.cpp index c2b13d4eaa..fbbfd0e344 100644 --- a/src/capi_frontend/capi_dag_utils.cpp +++ b/src/capi_frontend/capi_dag_utils.cpp @@ -41,7 +41,9 @@ OVMS_ServableState convertToServableState(ovms::PipelineDefinitionStateCode code case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED: case ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION: return OVMS_ServableState::OVMS_STATE_LOADING_FAILED; - } + case ovms::PipelineDefinitionStateCode::SLEEPING: + return OVMS_ServableState::OVMS_STATE_AVAILABLE; + } // TODO #atobiszei idle management C-API change - new value in enum? throw new std::exception(); } diff --git a/src/capi_frontend/server_settings.hpp b/src/capi_frontend/server_settings.hpp index 4eaa34ac98..768a4f5f38 100644 --- a/src/capi_frontend/server_settings.hpp +++ b/src/capi_frontend/server_settings.hpp @@ -244,6 +244,7 @@ struct ServerSettingsImpl { std::string grpcChannelArguments; uint32_t filesystemPollWaitMilliseconds = 1000; uint32_t resourcesCleanerPollWaitSeconds = 300; + uint32_t idleUnloadTimeoutSeconds = 0; std::string cacheDir; bool withPython = false; bool startedWithCLI = false; @@ -266,6 +267,7 @@ struct ModelsSettingsImpl { uint32_t nireq = 0; std::string targetDevice; std::string pluginConfig; + std::optional groupName; std::vector userSetSingleModelArguments; std::string configPath; diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index d7125abc87..131cf3277b 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -72,7 +72,7 @@ std::variant> CLIParser::parse(int argc, char* std::stringstream ss; try { options = std::make_unique(argv[0], "OpenVINO Model Server"); - auto configOptions = std::make_unique("ovms --add_to_config --config_path --model_name --model_repository_path \n ovms --add_to_config --config_path --model_path --model_name \n ovms --remove_from_config --config_path --model_name ", "config management commands:"); + auto configOptions = std::make_unique("ovms --add_to_config --config_path --model_name --model_repository_path \n ovms --add_to_config --config_path --model_path --model_name --group_name \n ovms --remove_from_config --config_path --model_name ", "config management commands:"); // Adding this option to parse unrecognised options in another parser options->allow_unrecognised_options(); @@ -137,6 +137,10 @@ std::variant> CLIParser::parse(int argc, char* "Time interval between config and model versions changes detection. Default is 1. Zero or negative value disables changes monitoring.", cxxopts::value()->default_value("1"), "FILE_SYSTEM_POLL_WAIT_SECONDS") + ("idle_unload_timeout_seconds", + "Idle timeout in seconds for model group unloading. When > 0, models not in the 'permanent' group are loaded on demand and unloaded after this idle period. Only effective with config.json multi-model setup. Default is 0 (disabled).", + cxxopts::value()->default_value("0"), + "IDLE_UNLOAD_TIMEOUT_SECONDS") ("custom_node_resources_cleaner_interval_seconds", "Time interval between two consecutive resources cleanup scans. Default is 300. Zero value disables resources cleaner.", cxxopts::value()->default_value("300"), @@ -209,7 +213,11 @@ std::variant> CLIParser::parse(int argc, char* ("remove_from_config", "Directive to remove a model from configuration file. This parameter should be executed with --config_path and --model_name to specify which model to remove.", cxxopts::value()->default_value("false"), - "REMOVE_FROM_CONFIG"); + "REMOVE_FROM_CONFIG") + ("group_name", + "Optional group name for idle model group management. Used with --add_to_config.", + cxxopts::value(), + "GROUP_NAME"); // Set default value for model_repository_path from environment variable if it exists and is not empty std::string defaultModelRepoPath = ""; @@ -343,6 +351,10 @@ std::variant> CLIParser::parse(int argc, char* "Name of the model", cxxopts::value(), "MODEL_NAME") + ("group_name", + "Optional group name for idle model group management", + cxxopts::value(), + "GROUP_NAME") ("config_path", "Path to json configuration file", cxxopts::value()->default_value(defaultConfigPath), @@ -569,6 +581,7 @@ void CLIParser::prepareServer(ServerSettingsImpl& serverSettings) { serverSettings.filesystemPollWaitMilliseconds = result->operator[]("file_system_poll_wait_seconds").as() * 1000; serverSettings.resourcesCleanerPollWaitSeconds = result->operator[]("custom_node_resources_cleaner_interval_seconds").as(); + serverSettings.idleUnloadTimeoutSeconds = result->operator[]("idle_unload_timeout_seconds").as(); serverSettings.grpcWorkers = result->operator[]("grpc_workers").as(); if (result->count("log_level")) @@ -921,6 +934,10 @@ void CLIParser::prepareConfigExport(ModelsSettingsImpl& modelsSettings) { } else if (!result->operator[]("model_repository_path").as().empty() && result->count("model_name")) { modelsSettings.modelPath = FileSystem::joinPath({result->operator[]("model_repository_path").as(), modelsSettings.modelName}); } + if (result->count("group_name")) { + modelsSettings.groupName = result->operator[]("group_name").as(); + modelsSettings.userSetSingleModelArguments.push_back("group_name"); + } std::string defaultConfigPath = ""; const char* envModelRepoPath = std::getenv("OVMS_MODEL_REPOSITORY_PATH"); if (envModelRepoPath != nullptr && std::string(envModelRepoPath).length() > 0) { diff --git a/src/config.cpp b/src/config.cpp index cdbd47c6f0..986ea7bb94 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -141,7 +141,7 @@ bool Config::validateUserSettingsInConfigAddRemoveModel(const ModelsSettingsImpl static const std::vector allowedForRemove = {"model_name", "config_path"}; static const std::vector allowedForAdd = {"model_name", "model_path", "config_path", "batch_size", "shape", "layout", "mean", "scale", "color_format", "precision", - "model_version_policy", "nireq", "target_device", "plugin_config"}; + "model_version_policy", "nireq", "target_device", "plugin_config", "group_name"}; const auto& allowedUserSettings = (exportType == ENABLE_MODEL) ? allowedForAdd : allowedForRemove; std::vector usedButDisallowedUserSettings; @@ -432,6 +432,7 @@ const std::string& Config::tracePath() const { return this->serverSettings.trace const std::string& Config::grpcChannelArguments() const { return this->serverSettings.grpcChannelArguments; } uint32_t Config::filesystemPollWaitMilliseconds() const { return this->serverSettings.filesystemPollWaitMilliseconds; } uint32_t Config::resourcesCleanerPollWaitSeconds() const { return this->serverSettings.resourcesCleanerPollWaitSeconds; } +uint32_t Config::idleUnloadTimeoutSeconds() const { return this->serverSettings.idleUnloadTimeoutSeconds; } bool Config::allowCredentials() const { return this->serverSettings.allowCredentials; } const std::string& Config::allowedOrigins() const { return this->serverSettings.allowedOrigins; } const std::string& Config::allowedMethods() const { return this->serverSettings.allowedMethods; } diff --git a/src/config.hpp b/src/config.hpp index d710bc4e9a..4f4709c11b 100644 --- a/src/config.hpp +++ b/src/config.hpp @@ -312,6 +312,13 @@ class Config { */ uint32_t resourcesCleanerPollWaitSeconds() const; + /** + * @brief Get the idle unload timeout in seconds (0 = disabled) + * + * @return uint32_t + */ + uint32_t idleUnloadTimeoutSeconds() const; + bool allowCredentials() const; const std::string& allowedOrigins() const; const std::string& allowedMethods() const; diff --git a/src/config_export_module/config_export.cpp b/src/config_export_module/config_export.cpp index f2f5ecc39c..27474ba587 100644 --- a/src/config_export_module/config_export.cpp +++ b/src/config_export_module/config_export.cpp @@ -56,6 +56,8 @@ static void addOptionalModelFields(rapidjson::Value& configObj, const ModelsSett configObj.AddMember("target_device", rapidjson::Value(modelSettings.targetDevice.c_str(), alloc), alloc); if (!modelSettings.pluginConfig.empty()) addJsonOrStringMember(configObj, "plugin_config", modelSettings.pluginConfig, alloc); + if (modelSettings.groupName.has_value()) + configObj.AddMember("group_name", rapidjson::Value(modelSettings.groupName.value().c_str(), alloc), alloc); } Status loadJsonConfig(const std::string& jsonFilename, rapidjson::Document& configJson) { diff --git a/src/dags/pipelinedefinition.cpp b/src/dags/pipelinedefinition.cpp index 2acdc768d4..3a6940183d 100644 --- a/src/dags/pipelinedefinition.cpp +++ b/src/dags/pipelinedefinition.cpp @@ -138,7 +138,7 @@ Status PipelineDefinition::reload(ModelInstanceProvider& modelInstanceProvider, // block creating new unloadGuards this->status.handle(ReloadEvent()); resetSubscriptions(modelInstanceProvider); - while (requestsHandlesCounter > 0) { + while (pendingCreateExecutorCount > 0) { std::this_thread::sleep_for(std::chrono::microseconds(1)); } // deinitialize all resources that are associated with nodes that are currently in PipelineDefinition, but not in nodeInfos @@ -153,7 +153,7 @@ Status PipelineDefinition::reload(ModelInstanceProvider& modelInstanceProvider, void PipelineDefinition::retire(ModelInstanceProvider& modelInstanceProvider) { resetSubscriptions(modelInstanceProvider); this->status.handle(RetireEvent()); - while (requestsHandlesCounter > 0) { + while (pendingCreateExecutorCount > 0) { std::this_thread::sleep_for(std::chrono::microseconds(1)); } // deinitalize all resources diff --git a/src/dags/pipelinedefinitionstatus.cpp b/src/dags/pipelinedefinitionstatus.cpp index 5fb27479b0..856ef4f616 100644 --- a/src/dags/pipelinedefinitionstatus.cpp +++ b/src/dags/pipelinedefinitionstatus.cpp @@ -35,7 +35,8 @@ const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCo {PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, "LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION"}, {PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION, "AVAILABLE_REQUIRED_REVALIDATION"}, {PipelineDefinitionStateCode::AVAILABLE, "AVAILABLE"}, - {PipelineDefinitionStateCode::RETIRED, "RETIRED"}}; + {PipelineDefinitionStateCode::RETIRED, "RETIRED"}, + {PipelineDefinitionStateCode::SLEEPING, "SLEEPING"}}; return names.at(code); } @@ -62,6 +63,9 @@ StateKeeper BeginState::handle(const RetireEvent& e) const { throw std::logic_error(INVALID_TRANSITION_MESSAGE); return {}; } +StateChanger BeginState::handle(const SleepEvent& e) const { + return {}; +} PipelineDefinitionStateCode ReloadState::getStateCode() const { return code; @@ -84,6 +88,10 @@ StateKeeper ReloadState::handle(const RetireEvent& e) const { throw std::logic_error(INVALID_TRANSITION_MESSAGE); return {}; } +StateKeeper ReloadState::handle(const SleepEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} PipelineDefinitionStateCode AvailableState::getStateCode() const { return code; @@ -105,6 +113,9 @@ StateChanger AvailableState::handle(const UsedMod StateChanger AvailableState::handle(const RetireEvent& e) const { return {}; } +StateChanger AvailableState::handle(const SleepEvent& e) const { + return {}; +} PipelineDefinitionStateCode AvailableRequiredRevalidation::getStateCode() const { return code; @@ -124,6 +135,9 @@ StateKeeper AvailableRequiredRevalidation::handle(const UsedModelChangedEvent& e StateChanger AvailableRequiredRevalidation::handle(const RetireEvent& e) const { return {}; } +StateChanger AvailableRequiredRevalidation::handle(const SleepEvent& e) const { + return {}; +} PipelineDefinitionStateCode LoadingPreconditionFailedState::getStateCode() const { return code; @@ -145,6 +159,10 @@ StateChanger LoadingPreconditio StateChanger LoadingPreconditionFailedState::handle(const RetireEvent& e) const { return {}; } +StateChanger LoadingPreconditionFailedState::handle(const SleepEvent& e) const { + // Revert a failed wake-up reload back to SLEEPING so the next request retries. + return {}; +} PipelineDefinitionStateCode LoadingFailedLastValidationRequiredRevalidation::getStateCode() const { return code; @@ -164,6 +182,9 @@ StateKeeper LoadingFailedLastValidationRequiredRevalidation::handle(const UsedMo StateChanger LoadingFailedLastValidationRequiredRevalidation::handle(const RetireEvent& e) const { return {}; } +StateKeeper LoadingFailedLastValidationRequiredRevalidation::handle(const SleepEvent& e) const { + return {}; +} PipelineDefinitionStateCode RetiredState::getStateCode() const { return code; @@ -187,6 +208,33 @@ StateKeeper RetiredState::handle(const RetireEvent& e) const { throw std::logic_error(INVALID_TRANSITION_MESSAGE); return {}; } +StateKeeper RetiredState::handle(const SleepEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} + +PipelineDefinitionStateCode SleepingState::getStateCode() const { + return code; +} +StateChanger SleepingState::handle(const ReloadEvent& e) const { + return {}; // wake-up: transition through reload path +} +StateChanger SleepingState::handle(const RetireEvent& e) const { + return {}; // config removal while unloaded +} +StateChanger SleepingState::handle(const ValidationPassedEvent& e) const { + return {}; // defensive: if validation passes directly, go available +} +StateKeeper SleepingState::handle(const ValidationFailedEvent& e) const { + return {}; +} +StateKeeper SleepingState::handle(const UsedModelChangedEvent& e) const { + return {}; +} +StateKeeper SleepingState::handle(const SleepEvent& e) const { + throw std::logic_error(INVALID_TRANSITION_MESSAGE); + return {}; +} PipelineDefinitionStatus::PipelineDefinitionStatus(const std::string& type, const std::string& name) : MachineState(type, name) {} @@ -195,12 +243,19 @@ bool PipelineDefinitionStatus::isAvailable() const { return (state == PipelineDefinitionStateCode::AVAILABLE) || (state == PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); } +bool PipelineDefinitionStatus::isSleeping() const { + return getStateCode() == PipelineDefinitionStateCode::SLEEPING; +} +bool PipelineDefinitionStatus::appearsAvailable() const { + return isAvailable() || isSleeping(); +} bool PipelineDefinitionStatus::canEndLoaded() const { auto state = getStateCode(); return isAvailable() || (state == PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION) || (state == PipelineDefinitionStateCode::BEGIN) || - (state == PipelineDefinitionStateCode::RELOADING); + (state == PipelineDefinitionStateCode::RELOADING) || + (state == PipelineDefinitionStateCode::SLEEPING); } bool PipelineDefinitionStatus::isRevalidationRequired() const { auto state = getStateCode(); @@ -233,6 +288,15 @@ std::tuple PipelineDefinitionSta ModelVersionState::END, ModelVersionStatusErrorCode::OK}; + case PipelineDefinitionStateCode::SLEEPING: + // Report AVAILABLE: the graph auto-reloads on the next inference request, + // so health checks and routing should treat it as available. Reporting END + // or UNLOADING would cause clients and load-balancers to permanently + // exclude this servable from their pools. + return { + ModelVersionState::AVAILABLE, + ModelVersionStatusErrorCode::OK}; + default: return {}; } diff --git a/src/dags/pipelinedefinitionstatus.hpp b/src/dags/pipelinedefinitionstatus.hpp index 59039f08a0..ca400118d0 100644 --- a/src/dags/pipelinedefinitionstatus.hpp +++ b/src/dags/pipelinedefinitionstatus.hpp @@ -34,7 +34,8 @@ enum class PipelineDefinitionStateCode { LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION, AVAILABLE_REQUIRED_REVALIDATION, AVAILABLE, - RETIRED + RETIRED, + SLEEPING }; const std::string& pipelineDefinitionStateCodeToString(PipelineDefinitionStateCode code); @@ -112,6 +113,11 @@ struct LoadingFailedLastValidationRequiredRevalidation; * State in which pipeline is retired - removed from config */ struct RetiredState; +/** + * State in which pipeline is idle-unloaded (resources freed) but not retired. + * Auto-reloads on the next inference request. + */ +struct SleepingState; #define EVENT_STRUCT_WITH_NAME(x) \ struct x { \ @@ -131,6 +137,7 @@ EVENT_STRUCT_WITH_NAME(ValidationFailedEvent); EVENT_STRUCT_WITH_NAME(ValidationPassedEvent); EVENT_STRUCT_WITH_NAME(UsedModelChangedEvent); EVENT_STRUCT_WITH_NAME(RetireEvent); +EVENT_STRUCT_WITH_NAME(SleepEvent); template struct StateChanger { @@ -155,6 +162,7 @@ struct BeginState { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateKeeper handle(const RetireEvent& e) const; + StateChanger handle(const SleepEvent& e) const; }; struct ReloadState { @@ -165,6 +173,7 @@ struct ReloadState { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateKeeper handle(const RetireEvent& e) const; + StateKeeper handle(const SleepEvent& e) const; }; struct AvailableState { @@ -175,6 +184,7 @@ struct AvailableState { StateKeeper handle(const ValidationFailedEvent& e) const; StateChanger handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + StateChanger handle(const SleepEvent& e) const; }; struct AvailableRequiredRevalidation { @@ -185,6 +195,7 @@ struct AvailableRequiredRevalidation { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + StateChanger handle(const SleepEvent& e) const; }; struct LoadingPreconditionFailedState { @@ -195,6 +206,11 @@ struct LoadingPreconditionFailedState { StateKeeper handle(const ValidationFailedEvent& e) const; StateChanger handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + // A failed wake-up reload of an idle graph reverts to SLEEPING so the next + // inference request can retry the wake (self-healing once the underlying issue + // is resolved). Only wakeUpIfSleeping() sends SleepEvent from this state; + // the watcher's unload() only does so from AVAILABLE. + StateChanger handle(const SleepEvent& e) const; }; struct LoadingFailedLastValidationRequiredRevalidation { @@ -205,6 +221,7 @@ struct LoadingFailedLastValidationRequiredRevalidation { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateChanger handle(const RetireEvent& e) const; + StateKeeper handle(const SleepEvent& e) const; }; struct RetiredState { @@ -215,12 +232,30 @@ struct RetiredState { StateChanger handle(const ValidationFailedEvent& e) const; StateKeeper handle(const UsedModelChangedEvent& e) const; StateKeeper handle(const RetireEvent& e) const; + StateKeeper handle(const SleepEvent& e) const; +}; + +struct SleepingState { + static const PipelineDefinitionStateCode code = PipelineDefinitionStateCode::SLEEPING; + PipelineDefinitionStateCode getStateCode() const; + // Wake-up: reuse the reload path + StateChanger handle(const ReloadEvent& e) const; + // Config removal while unloaded + StateChanger handle(const RetireEvent& e) const; + // Defensive: if validation somehow passes after an unload, go back to AVAILABLE + StateChanger handle(const ValidationPassedEvent& e) const; + // All other events are no-ops in SLEEPING + StateKeeper handle(const ValidationFailedEvent& e) const; + StateKeeper handle(const UsedModelChangedEvent& e) const; + StateKeeper handle(const SleepEvent& e) const; }; -class PipelineDefinitionStatus : public MachineState { +class PipelineDefinitionStatus : public MachineState { public: PipelineDefinitionStatus(const std::string& type, const std::string& name); bool isAvailable() const; + bool isSleeping() const; + bool appearsAvailable() const; bool canEndLoaded() const; bool isRevalidationRequired() const; std::tuple convertToModelStatus() const; diff --git a/src/grpc_utils.cpp b/src/grpc_utils.cpp index a9e41eb4eb..c75a8ca838 100644 --- a/src/grpc_utils.cpp +++ b/src/grpc_utils.cpp @@ -102,6 +102,7 @@ const grpc::Status grpc(const Status& status) { {StatusCode::MODEL_VERSION_NOT_LOADED_YET, grpc::StatusCode::UNAVAILABLE}, {StatusCode::PIPELINE_DEFINITION_NOT_LOADED_YET, grpc::StatusCode::UNAVAILABLE}, {StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_YET, grpc::StatusCode::UNAVAILABLE}, + {StatusCode::SERVER_SHUTTING_DOWN, grpc::StatusCode::UNAVAILABLE}, // UNKNOWN }; auto it = grpcStatusMap.find(status.getCode()); diff --git a/src/grpcservermodule.cpp b/src/grpcservermodule.cpp index a0e1c4d657..f0725db68b 100644 --- a/src/grpcservermodule.cpp +++ b/src/grpcservermodule.cpp @@ -33,9 +33,9 @@ #include "config.hpp" #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" -#include "modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "network_utils.hpp" -#include "servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "server.hpp" #include "stringutils.hpp" #include "systeminfo.hpp" diff --git a/src/http_rest_api_handler.cpp b/src/http_rest_api_handler.cpp index db544b9130..f69b81db3d 100644 --- a/src/http_rest_api_handler.cpp +++ b/src/http_rest_api_handler.cpp @@ -53,11 +53,11 @@ #include "model_metric_reporter.hpp" #include "modelinstance.hpp" #include "modelinstanceunloadguard.hpp" -#include "modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "profiler.hpp" #include "rest_parser.hpp" #include "rest_utils.hpp" -#include "servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "server.hpp" #include "status.hpp" #include "stringutils.hpp" @@ -73,6 +73,7 @@ #include "mediapipe_internal/mediapipegraphexecutor.hpp" #endif +#include "src/servable_management/servable_group_manager.hpp" #include "kfs_frontend/kfs_request_utils.hpp" #include "predict_request_validation_utils.hpp" #include "deserialization_main.hpp" diff --git a/src/http_server.cpp b/src/http_server.cpp index 747187d873..291f8f518b 100644 --- a/src/http_server.cpp +++ b/src/http_server.cpp @@ -82,6 +82,7 @@ static const ovms::HTTPStatusCode http(const ovms::Status& status) { {StatusCode::NO_MODEL_VERSION_AVAILABLE, ovms::HTTPStatusCode::ERROR}, {StatusCode::MODEL_NOT_LOADED, ovms::HTTPStatusCode::ERROR}, {StatusCode::SERVER_NOT_READY, ovms::HTTPStatusCode::SERVICE_UNAV}, + {StatusCode::SERVER_SHUTTING_DOWN, ovms::HTTPStatusCode::SERVICE_UNAV}, {StatusCode::JSON_INVALID, ovms::HTTPStatusCode::PRECOND_FAILED}, {StatusCode::MODELINSTANCE_NOT_FOUND, ovms::HTTPStatusCode::ERROR}, {StatusCode::SHAPE_WRONG_FORMAT, ovms::HTTPStatusCode::ERROR}, @@ -97,6 +98,9 @@ static const ovms::HTTPStatusCode http(const ovms::Status& status) { {StatusCode::MODEL_VERSION_MISSING, ovms::HTTPStatusCode::NOT_FOUND}, {StatusCode::MEDIAPIPE_EXECUTION_ERROR, ovms::HTTPStatusCode::BAD_REQUEST}, {StatusCode::MEDIAPIPE_PRECONDITION_FAILED, ovms::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::MEDIAPIPE_PUT_TO_SLEEP_STATE_NOT_AVAILABLE, ovms::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::MEDIAPIPE_PUT_TO_SLEEP_REQUESTS_IN_FLIGHT, ovms::HTTPStatusCode::PRECOND_FAILED}, + {StatusCode::MEDIAPIPE_PUT_TO_SLEEP_ACTIVE_INFERENCES, ovms::HTTPStatusCode::PRECOND_FAILED}, {StatusCode::MEDIAPIPE_GRAPH_ADD_PACKET_INPUT_STREAM, ovms::HTTPStatusCode::PRECOND_FAILED}, {StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE, ovms::HTTPStatusCode::NOT_FOUND}, {StatusCode::MODEL_VERSION_NOT_LOADED_YET, ovms::HTTPStatusCode::NOT_FOUND}, diff --git a/src/kfs_frontend/kfs_grpc_inference_service.cpp b/src/kfs_frontend/kfs_grpc_inference_service.cpp index e6ebb2bb0d..510964b5aa 100644 --- a/src/kfs_frontend/kfs_grpc_inference_service.cpp +++ b/src/kfs_frontend/kfs_grpc_inference_service.cpp @@ -44,11 +44,11 @@ #include "../deserialization_main.hpp" #include "../inference_executor.hpp" #include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "../ovinferrequestsqueue.hpp" #include "../servable_definition.hpp" #include "../servable_definition_unload_guard.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../single_version_servable_definition.hpp" #include "../status.hpp" @@ -129,7 +129,7 @@ Status KFSInferenceServiceImpl::getModelReady(const KFSGetModelStatusRequest* re if (!svsd) { return StatusCode::MODEL_NAME_MISSING; } - response->set_ready(svsd->isAvailable()); + response->set_ready(svsd->getStatus().appearsAvailable()); INCREMENT_IF_ENABLED(svsd->getMetricReporter().getModelReadyMetric(executionContext, true)); return StatusCode::OK; } @@ -359,7 +359,7 @@ Status KFSInferenceServiceImpl::buildResponse( Status KFSInferenceServiceImpl::buildResponse( SingleVersionServableDefinition& definition, KFSGetModelStatusResponse* response) { - bool isReady = definition.getStatus().isAvailable(); + bool isReady = definition.getStatus().appearsAvailable(); SPDLOG_DEBUG("Creating ModelReady response for definition: {}; ready: {}", definition.getName(), isReady); response->set_ready(isReady); return StatusCode::OK; diff --git a/src/mediapipe_internal/mediapipefactory.cpp b/src/mediapipe_internal/mediapipefactory.cpp index 0be6d4b3bf..c5757ec0a1 100644 --- a/src/mediapipe_internal/mediapipefactory.cpp +++ b/src/mediapipe_internal/mediapipefactory.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -56,24 +55,25 @@ MediapipeFactory::MediapipeFactory(PythonBackend* pythonBackend) { Status MediapipeFactory::createDefinition(const std::string& pipelineName, const MediapipeGraphConfig& config, MetricProvider& metrics, - const ServableNameChecker& checker) { + const ServableNameChecker& checker, + bool lazyLoad) { if (definitionExists(pipelineName)) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Mediapipe graph definition: {} is already created", pipelineName); return StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST; } std::shared_ptr graphDefinition = std::make_shared( - pipelineName, config, metrics.getMetricRegistry(), &metrics.getMetricConfig(), pythonBackend); - auto stat = graphDefinition->validate(checker); - if (stat.getCode() == StatusCode::MEDIAPIPE_GRAPH_NAME_OCCUPIED) { - return stat; + pipelineName, config, metrics.getMetricRegistry(), &metrics.getMetricConfig(), pythonBackend, lazyLoad); + Status stat = StatusCode::OK; + if (!lazyLoad) { + stat = graphDefinition->validate(checker); + if (stat.getCode() == StatusCode::MEDIAPIPE_GRAPH_NAME_OCCUPIED) { + return stat; + } } std::unique_lock lock(definitionsMtx); definitions.insert({pipelineName, std::move(graphDefinition)}); - // Register LoRA aliases discovered during validation (image gen graphs) - const auto& def = definitions[pipelineName]; - for (const auto& alias : def->getLoraAliases()) { - loraAliases[alias] = pipelineName; - SPDLOG_LOGGER_INFO(modelmanager_logger, "Registered LoRA alias: {} -> {}", alias, pipelineName); + if (!lazyLoad) { + registerLoraAliasesForUnlocked(pipelineName); } return stat; } @@ -116,11 +116,7 @@ Status MediapipeFactory::reloadDefinition(const std::string& name, clearLoraAliases(name); auto status = mgd->reload(checker, config); if (status.ok()) { - std::unique_lock lock(definitionsMtx); - for (const auto& alias : mgd->getLoraAliases()) { - loraAliases[alias] = name; - SPDLOG_LOGGER_INFO(modelmanager_logger, "Registered LoRA alias: {} -> {}", alias, name); - } + registerLoraAliasesFor(name); } return status; } @@ -146,14 +142,37 @@ Status MediapipeFactory::create(std::unique_ptr& pipelin return definition.create(pipeline); } -void MediapipeFactory::retireOtherThan(std::set&& graphsInConfigFile) { - std::for_each(definitions.begin(), - definitions.end(), - [&graphsInConfigFile](auto& nameDefinitionPair) { - if (graphsInConfigFile.find(nameDefinitionPair.second->getName()) == graphsInConfigFile.end() && nameDefinitionPair.second->getStateCode() != PipelineDefinitionStateCode::RETIRED) { - nameDefinitionPair.second->retire(); - } - }); +[[nodiscard]] Status MediapipeFactory::wakeUpDefinition(const std::string& graphName, const ServableNameChecker& checker) { + MediapipeGraphDefinition* definition = findDefinitionByName(graphName); + if (definition == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested to wake up mediapipe graph definition but it does not exist: {}", graphName); + return StatusCode::INTERNAL_ERROR; + } + // Rejects every non-SLEEPING state, so a wake-up scheduled off a stale group snapshot + // cannot resurrect a graph removed from the config. + auto status = definition->wakeUpIfSleeping(checker); + if (status.ok()) { + registerLoraAliasesFor(graphName); + } + return status; +} + + [[nodiscard]] Status MediapipeFactory::putToSleepDefinition(const std::string& graphName) { + MediapipeGraphDefinition* definition = findDefinitionByName(graphName); + if (definition == nullptr) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Requested to put to sleep mediapipe graph definition but it does not exist: {}", graphName); + return StatusCode::INTERNAL_ERROR; + } + return definition->putToSleep(); +} + +Status MediapipeFactory::retireDefinition(const std::string& graphName) { + MediapipeGraphDefinition* definition = findDefinitionByName(graphName); + if (definition == nullptr) { + return StatusCode::MEDIAPIPE_DEFINITION_NAME_MISSING; + } + definition->retire(); + return StatusCode::OK; } Status MediapipeFactory::revalidatePipelines() { @@ -174,24 +193,33 @@ const std::vector MediapipeFactory::getNamesOfAvailableMediapipePip std::vector names; std::shared_lock lock(definitionsMtx); for (auto& [name, definition] : definitions) { - if (definition->getStatus().isAvailable() && !definition->shouldHideBaseModelInRouting()) { + if (definition->getStatus().appearsAvailable() && !definition->shouldHideBaseModelInRouting()) { names.push_back(definition->getName()); } } // Add LoRA aliases that point to available definitions for (const auto& [alias, graphName] : loraAliases) { auto it = definitions.find(graphName); - if (it != definitions.end() && it->second->getStatus().isAvailable()) { + if (it != definitions.end() && it->second->getStatus().appearsAvailable()) { names.push_back(alias); } } return names; } -void MediapipeFactory::registerLoraAlias(const std::string& alias, const std::string& graphName) { +void MediapipeFactory::registerLoraAliasesForUnlocked(const std::string& graphName) { + auto it = definitions.find(graphName); + if (it == definitions.end()) + return; + for (const auto& alias : it->second->getLoraAliases()) { + loraAliases[alias] = graphName; + SPDLOG_LOGGER_INFO(modelmanager_logger, "Registered LoRA alias: {} -> {}", alias, graphName); + } +} + +void MediapipeFactory::registerLoraAliasesFor(const std::string& graphName) { std::unique_lock lock(definitionsMtx); - loraAliases[alias] = graphName; - SPDLOG_LOGGER_INFO(modelmanager_logger, "Registered LoRA alias: {} -> {}", alias, graphName); + registerLoraAliasesForUnlocked(graphName); } void MediapipeFactory::clearLoraAliases(const std::string& graphName) { diff --git a/src/mediapipe_internal/mediapipefactory.hpp b/src/mediapipe_internal/mediapipefactory.hpp index a9ac8ae9b0..a60d36e04f 100644 --- a/src/mediapipe_internal/mediapipefactory.hpp +++ b/src/mediapipe_internal/mediapipefactory.hpp @@ -17,7 +17,6 @@ #include #include -#include #include #include #include @@ -40,6 +39,7 @@ class MediapipeFactory { std::map loraAliases; // alias -> real graph definition name mutable std::shared_mutex definitionsMtx; PythonBackend* pythonBackend{nullptr}; + void registerLoraAliasesForUnlocked(const std::string& graphName); public: MediapipeFactory() = delete; @@ -47,7 +47,8 @@ class MediapipeFactory { Status createDefinition(const std::string& pipelineName, const MediapipeGraphConfig& config, MetricProvider& metrics, - const ServableNameChecker& checker); + const ServableNameChecker& checker, + bool lazyLoad = false); bool definitionExists(const std::string& name) const; @@ -56,14 +57,16 @@ class MediapipeFactory { const std::string& name) const; MediapipeGraphDefinition* findDefinitionByName(const std::string& name) const; - void registerLoraAlias(const std::string& alias, const std::string& graphName); + void registerLoraAliasesFor(const std::string& graphName); void clearLoraAliases(const std::string& graphName); bool aliasesConflictExcluding(const std::vector& aliases, const std::string& ownGraphName) const; Status reloadDefinition(const std::string& pipelineName, const MediapipeGraphConfig& config, const ServableNameChecker& checker); - void retireOtherThan(std::set&& pipelinesInConfigFile); + [[nodiscard]] Status wakeUpDefinition(const std::string& pipelineName, const ServableNameChecker& checker); + [[nodiscard]] Status putToSleepDefinition(const std::string& pipelineName); + Status retireDefinition(const std::string& pipelineName); Status revalidatePipelines(); const std::vector getMediapipePipelinesNames() const; const std::vector getNamesOfAvailableMediapipePipelines() const; diff --git a/src/mediapipe_internal/mediapipegraphconfig.cpp b/src/mediapipe_internal/mediapipegraphconfig.cpp index 200de9c289..62748784be 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.cpp +++ b/src/mediapipe_internal/mediapipegraphconfig.cpp @@ -119,6 +119,21 @@ Status MediapipeGraphConfig::parseNode(const rapidjson::Value& v) { this->setSubconfigPath(DEFAULT_SUBCONFIG_FILENAME); this->setModelMeshSubconfigPath(DEFAULT_MODELMESH_SUBCONFIG_FILENAME); } + if (v.HasMember("idle_unload_timeout_seconds")) { + int timeoutSeconds = v["idle_unload_timeout_seconds"].GetInt(); + if (timeoutSeconds < 0) { + SPDLOG_ERROR("idle_unload_timeout_seconds must be >= 0 for mediapipe graph: {}", this->getGraphName()); + return StatusCode::JSON_INVALID; + } + this->setIdleUnloadTimeoutSeconds(timeoutSeconds); + SPDLOG_DEBUG("Mediapipe graph {} idle_unload_timeout_seconds set to {}", this->getGraphName(), timeoutSeconds); + } + if (v.HasMember("group_name")) { + this->setGroupName(v["group_name"].GetString()); + } else { + this->setGroupName(this->getGraphName()); + } + SPDLOG_DEBUG("Mediapipe graph {} group_name set to {}", this->getGraphName(), this->getGroupName()); } catch (std::logic_error& e) { SPDLOG_DEBUG("Relative path error: {}", e.what()); return StatusCode::INTERNAL_ERROR; diff --git a/src/mediapipe_internal/mediapipegraphconfig.hpp b/src/mediapipe_internal/mediapipegraphconfig.hpp index 88ed1d9f25..c15dc5ed84 100644 --- a/src/mediapipe_internal/mediapipegraphconfig.hpp +++ b/src/mediapipe_internal/mediapipegraphconfig.hpp @@ -65,6 +65,19 @@ class MediapipeGraphConfig { */ std::optional graphQueueSize; + /** + * @brief Idle unload timeout in seconds. + * 0 (default) = feature disabled. + * When > 0, the graph's heavy resources are freed after this many seconds + * of zero in-flight requests, and lazily reloaded on the next inference. + */ + int idleUnloadTimeoutSeconds = 0; + + /** + * @brief Group name for idle model management. Defaults to graph name. + */ + std::string groupName; + public: MediapipeGraphConfig(const std::string& graphName = "", const std::string& basePath = "", @@ -170,6 +183,22 @@ class MediapipeGraphConfig { return this->graphQueueSize.value_or(0); } + int getIdleUnloadTimeoutSeconds() const { + return this->idleUnloadTimeoutSeconds; + } + + void setIdleUnloadTimeoutSeconds(int seconds) { + this->idleUnloadTimeoutSeconds = seconds; + } + + const std::string& getGroupName() const { + return this->groupName; + } + + void setGroupName(const std::string& groupName) { + this->groupName = groupName; + } + bool isReloadRequired(const MediapipeGraphConfig& rhs) const; /** diff --git a/src/mediapipe_internal/mediapipegraphdefinition.cpp b/src/mediapipe_internal/mediapipegraphdefinition.cpp index 5d2cd21bbe..5df6b6584b 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.cpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.cpp @@ -246,6 +246,7 @@ Status MediapipeGraphDefinition::validate(const ServableNameChecker& checker) { if (!validationResult.ok()) { return validationResult; } + validationResult = resolveGraphQueueSize(); if (!validationResult.ok()) { return validationResult; @@ -293,6 +294,8 @@ Status MediapipeGraphDefinition::validate(const ServableNameChecker& checker) { lock.unlock(); notifier.passed = true; + // Graph resources are now loaded (covers both initial load and wake-up reload). + SET_IF_ENABLED(this->reporter->graphLoaded, 1); SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Finished validation of mediapipe: {}", getName()); SPDLOG_LOGGER_INFO(modelmanager_logger, "Mediapipe: {} inputs: {}", getName(), getTensorMapString(inputsInfo)); SPDLOG_LOGGER_INFO(modelmanager_logger, "Mediapipe: {} outputs: {}", getName(), getTensorMapString(outputsInfo)); @@ -323,14 +326,22 @@ MediapipeGraphDefinition::MediapipeGraphDefinition(const std::string name, const MediapipeGraphConfig& config, MetricRegistry* registry, const MetricConfig* metricConfig, - PythonBackend* pythonBackend) : + PythonBackend* pythonBackend, + bool lazyLoad) : SingleVersionServableDefinition(name), sidePacketMaps(std::make_shared()), status(SCHEDULER_CLASS_NAME, getName()), pythonBackend(pythonBackend), reporter(std::make_unique(metricConfig, registry, name)) { mgconfig = config; + idleUnloadTimeoutSecondsCache.store(mgconfig.getIdleUnloadTimeoutSeconds(), std::memory_order_relaxed); passKfsRequestFlag = false; + lastActivityTimeNs = std::make_shared>(0); + recordActivity(); + activeInferenceCount = std::make_shared>(0); + if (lazyLoad) { + this->status.handle(SleepEvent()); + } } Status MediapipeGraphDefinition::createInputsInfo() { @@ -387,6 +398,11 @@ Status MediapipeGraphDefinition::createOutputsInfo() { } Status MediapipeGraphDefinition::create(std::unique_ptr& pipeline) { + // Update idle-tracking timestamp on every inference acquisition path. + // Status endpoints / health checks do not reach this method, so idle + // tracking is automatically inference-only. + recordActivity(); + std::unique_ptr unloadGuard; Status status = waitForLoaded(unloadGuard); if (!status.ok()) { @@ -399,12 +415,14 @@ Status MediapipeGraphDefinition::create(std::unique_ptr& pipeline = std::make_unique(getName(), std::to_string(getVersion()), this->config, this->inputTypes, this->outputTypes, this->inputNames, this->outputNames, *this->sidePacketMaps, - this->pythonBackend, this->reporter.get(), std::move(graphIdGuard)); + this->pythonBackend, this->reporter.get(), std::move(graphIdGuard), + this->activeInferenceCount, this->lastActivityTimeNs); } else { pipeline = std::make_unique(getName(), std::to_string(getVersion()), this->config, this->inputTypes, this->outputTypes, this->inputNames, this->outputNames, *this->sidePacketMaps, - this->pythonBackend, this->reporter.get()); + this->pythonBackend, this->reporter.get(), + this->activeInferenceCount, this->lastActivityTimeNs); } SPDLOG_DEBUG("Created Mediapipe graph executor: {}", getName()); return status; @@ -474,25 +492,127 @@ Status MediapipeGraphDefinition::setStreamTypes() { } Status MediapipeGraphDefinition::reload(const ServableNameChecker& checker, const MediapipeGraphConfig& config) { + // Serialize against unload()/wakeUp() on the watcher/request threads. + // Recursive: wakeUpIfSleeping() already holds this and calls reload(). + std::lock_guard lock(lifecycleMtx); // block creating new unloadGuards this->status.handle(ReloadEvent()); - while (requestsHandlesCounter > 0) { + while (pendingCreateExecutorCount > 0) { std::this_thread::sleep_for(std::chrono::microseconds(1)); } this->mgconfig = config; + // Refresh the lock-free cache while we still hold lifecycleMtx. + idleUnloadTimeoutSecondsCache.store(this->mgconfig.getIdleUnloadTimeoutSeconds(), std::memory_order_relaxed); this->queue.reset(); this->sidePacketMaps = std::make_shared(); return validate(checker); } void MediapipeGraphDefinition::retire() { + std::lock_guard lock(lifecycleMtx); // Block creating new unloadGuards this->status.handle(RetireEvent()); - while (requestsHandlesCounter > 0) { - std::this_thread::sleep_for(std::chrono::microseconds(1)); + unloadComponentsAfterPendingExecutorsAreCreated(); +} + +bool MediapipeGraphDefinition::isIdleUnloadEnabled() const { + // Lock-free read of the cached timeout (mgconfig is only safe under lifecycleMtx). + return idleUnloadTimeoutSecondsCache.load(std::memory_order_relaxed) > 0; +} + +bool MediapipeGraphDefinition::shouldUnloadDueToIdle() const { + // Advisory pre-filter ONLY — reads no unsynchronized per-definition state. + // It must NOT read this->status (the state-machine variant) without the lock, + // since the config thread can mutate it concurrently. putToSleep() performs the + // authoritative state==AVAILABLE check under lifecycleMtx. + // pendingCreateExecutorCount, lastActivityTimeNs and idleUnloadTimeoutSecondsCache + // are all atomics, so every read here is data-race-free. We never read mgconfig + // (only safe under lifecycleMtx) on this advisory path. + int64_t timeoutSeconds = idleUnloadTimeoutSecondsCache.load(std::memory_order_relaxed); + if (timeoutSeconds <= 0) { + return false; + } + if (pendingCreateExecutorCount.load(std::memory_order_relaxed) != 0) { + return false; + } + // Guard: if inferences are actively executing, never report idle. + // activeInferenceCount is bumped when a MediapipeGraphExecutor is created (in + // create()) by its RAII ActiveInferenceGuard, held for the executor's lifetime + // (which spans the inference), and decremented (with a lastActivityTimeNs refresh) + // when the executor is destroyed after the inference completes or throws. + if (activeInferenceCount && activeInferenceCount->load(std::memory_order_acquire) > 0) { + return false; + } + int64_t lastActivity = lastActivityTimeNs->load(std::memory_order_relaxed); + int64_t nowNs = std::chrono::steady_clock::now().time_since_epoch().count(); + int64_t timeoutNs = timeoutSeconds * 1'000'000'000LL; + return (nowNs - lastActivity) >= timeoutNs; +} + +[[nodiscard]] Status MediapipeGraphDefinition::putToSleep() { + std::lock_guard lock(lifecycleMtx); + if (status.getStateCode() == PipelineDefinitionStateCode::SLEEPING) { + return StatusCode::OK; } - this->queue.reset(); - this->sidePacketMaps.reset(); + if (status.getStateCode() != PipelineDefinitionStateCode::AVAILABLE) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Skipping idle-unload of mediapipe graph {}: state is no longer AVAILABLE", getName()); + return Status(StatusCode::MEDIAPIPE_PUT_TO_SLEEP_STATE_NOT_AVAILABLE, + "Cannot put mediapipe graph to sleep: state is not AVAILABLE"); + } + if (pendingCreateExecutorCount.load(std::memory_order_acquire) != 0) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Skipping idle-unload of mediapipe graph {}: requests in flight", getName()); + return Status(StatusCode::MEDIAPIPE_PUT_TO_SLEEP_REQUESTS_IN_FLIGHT, + "Cannot put mediapipe graph to sleep: requests are in flight"); + } + if (activeInferenceCount && activeInferenceCount->load(std::memory_order_acquire) > 0) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Skipping idle-unload of mediapipe graph {}: active inferences in progress", getName()); + return Status(StatusCode::MEDIAPIPE_PUT_TO_SLEEP_ACTIVE_INFERENCES, + "Cannot put mediapipe graph to sleep: active inferences in progress"); + } + + this->status.handle(SleepEvent()); + + unloadComponentsAfterPendingExecutorsAreCreated(); + + SET_IF_ENABLED(this->reporter->graphLoaded, 0); + SPDLOG_LOGGER_INFO(modelmanager_logger, + "Mediapipe graph {} idle-unloaded (freed GPU/CPU resources after {}s idle timeout)", + getName(), mgconfig.getIdleUnloadTimeoutSeconds()); + return StatusCode::OK; +} + +Status MediapipeGraphDefinition::wakeUpIfSleeping(const ServableNameChecker& checker) { + std::lock_guard lock(lifecycleMtx); + auto state = status.getStateCode(); + if (state == PipelineDefinitionStateCode::AVAILABLE || state == PipelineDefinitionStateCode::RELOADING) + return StatusCode::OK; + if (state != PipelineDefinitionStateCode::SLEEPING) + return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE; + SPDLOG_LOGGER_INFO(modelmanager_logger, + "Mediapipe graph {} is SLEEPING; triggering lazy wake-up reload", getName()); + auto start = std::chrono::steady_clock::now(); + Status reloadStatus = reload(checker, this->mgconfig); + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + if (reloadStatus.ok()) { + // Only reset the idle timer on a successful wake; on failure leave it so + // the existing failure-state handling applies and we don't mask the error. + recordActivity(); + SPDLOG_LOGGER_INFO(modelmanager_logger, + "Mediapipe graph {} wake-up completed in {}ms", + getName(), elapsed.count()); + } else { + // Wake-up reload failed. For now sleeping the graph again. So that new request will try to load again. + this->status.handle(SleepEvent()); + SPDLOG_LOGGER_ERROR(modelmanager_logger, + "Mediapipe graph {} wake-up failed after {}ms: {}. Reverted to SLEEPING; " + "next request will retry the wake.", + getName(), elapsed.count(), reloadStatus.string()); + } + return reloadStatus; } bool MediapipeGraphDefinition::isReloadRequired(const MediapipeGraphConfig& config) const { @@ -511,6 +631,14 @@ StatusCode MediapipeGraphDefinition::notLoadedAnymoreCode() const { return StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE; } +void MediapipeGraphDefinition::unloadComponentsAfterPendingExecutorsAreCreated() { + while (pendingCreateExecutorCount > 0) { + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + this->queue.reset(); + this->sidePacketMaps.reset(); +} + Status MediapipeGraphDefinition::initializeNodes() { SPDLOG_INFO("MediapipeGraphDefinition initializing graph nodes"); bool success = false; diff --git a/src/mediapipe_internal/mediapipegraphdefinition.hpp b/src/mediapipe_internal/mediapipegraphdefinition.hpp index 30c1f06a7b..566484d3c8 100644 --- a/src/mediapipe_internal/mediapipegraphdefinition.hpp +++ b/src/mediapipe_internal/mediapipegraphdefinition.hpp @@ -14,8 +14,11 @@ // limitations under the License. //***************************************************************************** #pragma once +#include +#include #include #include +#include #include #include #include @@ -56,7 +59,8 @@ class MediapipeGraphDefinition : public SingleVersionServableDefinition { const MediapipeGraphConfig& config = MGC, MetricRegistry* registry = nullptr, const MetricConfig* metricConfig = nullptr, - PythonBackend* pythonBackend = nullptr); + PythonBackend* pythonBackend = nullptr, + bool lazyLoad = false); const PipelineDefinitionStatus& getStatus() const override { return this->status; @@ -78,6 +82,23 @@ class MediapipeGraphDefinition : public SingleVersionServableDefinition { Status initializeNodes(); bool isReloadRequired(const MediapipeGraphConfig& config) const; + // Idle unload feature + Status putToSleep(); + Status wakeUpIfSleeping(const ServableNameChecker& checker); + bool isIdleUnloadEnabled() const; + bool shouldUnloadDueToIdle() const; + + // Record inference activity. Defaults to now; tests pass an explicit timestamp. + void recordActivity(int64_t timestampNs = std::chrono::steady_clock::now().time_since_epoch().count()) { + lastActivityTimeNs->store(timestampNs, std::memory_order_relaxed); + } + + // Returns the shared active-inference counter so create() can hand it to the executor. + // Not exposed in tests directly — use shouldUnloadDueToIdle() to observe the effect. + const std::shared_ptr>& getActiveInferenceCount() const { + return activeInferenceCount; + } + static const std::string SCHEDULER_CLASS_NAME; protected: @@ -132,6 +153,7 @@ class MediapipeGraphDefinition : public SingleVersionServableDefinition { private: StatusCode notLoadedYetCode() const override; StatusCode notLoadedAnymoreCode() const override; + void unloadComponentsAfterPendingExecutorsAreCreated(); tensor_map_t inputsInfo; tensor_map_t outputsInfo; @@ -149,5 +171,41 @@ class MediapipeGraphDefinition : public SingleVersionServableDefinition { std::unique_ptr reporter; std::shared_ptr queue; + + // Idle unload: timestamp (nanoseconds from steady_clock epoch) of the last + // inference activity. Updated in create() on every inference acquisition and + // when an in-flight inference finishes (via ActiveInferenceGuard destructor). + // Held as shared_ptr so executors can safely write to it even after the + // definition is retired/destroyed — the atomic outlives the definition. + std::shared_ptr> lastActivityTimeNs; + + // Count of inferences currently executing on this graph. Incremented when a + // MediapipeGraphExecutor is created (in create()) via the executor's RAII + // ActiveInferenceGuard, and decremented when that executor is destroyed (after + // the caller finishes infer()/inferStream()). The count is therefore held for + // the executor's lifetime, which spans the inference. A non-zero value prevents + // shouldUnloadDueToIdle()/unload() from tearing down the definition. + // Shared_ptr so MediapipeGraphExecutor can hold a copy safely beyond the + // create() call — the executor owns the counter reference for its lifetime. + std::shared_ptr> activeInferenceCount; + + // Cached copy of mgconfig.getIdleUnloadTimeoutSeconds() so the watcher thread can + // read it lock-free. mgconfig itself is only safe to read under lifecycleMtx + // (reload() reassigns it). Updated in the constructor and in reload() (under the + // lock) whenever mgconfig is assigned. + std::atomic idleUnloadTimeoutSecondsCache{0}; + + // TODO FIXME (@atobiszei): revisit whether lifecycleMtx is still needed + // now that ServableLoadingQueue serializes task dispatch. + // Serializes ALL per-definition lifecycle mutations (reload/retire/unload/wakeUp) + // so they are mutually exclusive regardless of which thread runs them or which + // outer lock (ModelManager::configMtx) is held by the caller. This is required + // because unload() runs on the watcher thread (no configMtx) while reload()/retire() + // run on the config thread (under configMtx) and they mutate the same per-definition + // state (this->status variant, this->sidePacketMaps). + // Recursive because wakeUpIfSleeping() holds it and calls reload(), which also takes it. + // Lock ordering is one-directional: configMtx -> lifecycleMtx. Nothing here ever + // acquires configMtx, so no deadlock is possible. + mutable std::recursive_mutex lifecycleMtx; }; } // namespace ovms diff --git a/src/mediapipe_internal/mediapipegraphexecutor.cpp b/src/mediapipe_internal/mediapipegraphexecutor.cpp index 26757de401..330f9b2705 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.cpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.cpp @@ -47,7 +47,9 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, MediapipeServableMetricReporter* mediapipeServableMetricReporter, - GraphIdGuard&& guard) : + GraphIdGuard&& guard, + std::shared_ptr> activeInferenceCount, + std::shared_ptr> lastActivityTimeNs) : name(name), version(version), config(config), @@ -59,7 +61,10 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( pythonBackend(pythonBackend), currentStreamTimestamp(::mediapipe::Timestamp(STARTING_TIMESTAMP_VALUE)), mediapipeServableMetricReporter(mediapipeServableMetricReporter), - guard(std::move(guard)) {} + guard(std::move(guard)), + activeInferenceGuard(activeInferenceCount + ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) + : std::nullopt) {} MediapipeGraphExecutor::MediapipeGraphExecutor( const std::string& name, const std::string& version, @@ -70,7 +75,9 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( std::vector outputNames, const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, - MediapipeServableMetricReporter* mediapipeServableMetricReporter) : + MediapipeServableMetricReporter* mediapipeServableMetricReporter, + std::shared_ptr> activeInferenceCount, + std::shared_ptr> lastActivityTimeNs) : name(name), version(version), config(config), @@ -81,6 +88,9 @@ MediapipeGraphExecutor::MediapipeGraphExecutor( sidePacketMaps(sidePacketMaps), pythonBackend(pythonBackend), currentStreamTimestamp(::mediapipe::Timestamp(STARTING_TIMESTAMP_VALUE)), - mediapipeServableMetricReporter(mediapipeServableMetricReporter) {} + mediapipeServableMetricReporter(mediapipeServableMetricReporter), + activeInferenceGuard(activeInferenceCount + ? std::optional(ActiveInferenceGuard(std::move(activeInferenceCount), std::move(lastActivityTimeNs))) + : std::nullopt) {} } // namespace ovms diff --git a/src/mediapipe_internal/mediapipegraphexecutor.hpp b/src/mediapipe_internal/mediapipegraphexecutor.hpp index 8dae471191..3948c6d722 100644 --- a/src/mediapipe_internal/mediapipegraphexecutor.hpp +++ b/src/mediapipe_internal/mediapipegraphexecutor.hpp @@ -14,6 +14,8 @@ // limitations under the License. //***************************************************************************** #pragma once +#include +#include #include #include #include @@ -48,6 +50,47 @@ namespace ovms { class PythonBackend; class ServableMetricReporter; + +// RAII guard that tracks an in-flight inference on a MediapipeGraphDefinition. +// Increments the counter on construction; decrements it and refreshes +// lastActivityTimeNs on destruction (even if the inference threw). Both are held as +// shared_ptrs so they remain valid even if the definition is reloaded or retired +// while the inference (and thus the owning executor) is still alive. +// The lastActivityTimeNs refresh on decrement ensures that completing a long +// generation resets the idle timer — preventing an immediate re-unload on the next +// watcher cycle. +struct ActiveInferenceGuard { + std::shared_ptr> counter; + std::shared_ptr> lastActivityTimeNs; + + ActiveInferenceGuard(std::shared_ptr> counter, + std::shared_ptr> lastActivityTimeNs) : + counter(std::move(counter)), + lastActivityTimeNs(std::move(lastActivityTimeNs)) { + if (this->counter) { + this->counter->fetch_add(1, std::memory_order_acq_rel); + } + } + + ~ActiveInferenceGuard() { + if (counter) { + // Refresh activity timestamp BEFORE decrementing so the watcher sees a + // recent activity time if it samples between the refresh and the decrement. + if (lastActivityTimeNs) { + lastActivityTimeNs->store( + std::chrono::steady_clock::now().time_since_epoch().count(), + std::memory_order_relaxed); + } + counter->fetch_sub(1, std::memory_order_acq_rel); + } + } + + // Non-copyable, movable. + ActiveInferenceGuard(const ActiveInferenceGuard&) = delete; + ActiveInferenceGuard& operator=(const ActiveInferenceGuard&) = delete; + ActiveInferenceGuard(ActiveInferenceGuard&&) = default; + ActiveInferenceGuard& operator=(ActiveInferenceGuard&&) = default; +}; class MediapipeGraphExecutor; inline StatusCode mediapipeAbslToOvmsStatus(absl::StatusCode code) { @@ -140,6 +183,13 @@ class MediapipeGraphExecutor { MediapipeServableMetricReporter* mediapipeServableMetricReporter; std::optional guard; + // RAII guard tracking this executor's active inference on the parent definition. + // Held for the entire lifetime of the executor so that the in-flight-inference + // check in shouldUnloadDueToIdle() / unload() sees a non-zero count while any + // inference method (infer / inferStream) is executing. On destruction (when the + // executor goes out of scope after inference completes) the counter decrements + // and lastActivityTimeNs is refreshed. + std::optional activeInferenceGuard; public: MediapipeGraphExecutor(const std::string& name, @@ -150,7 +200,9 @@ class MediapipeGraphExecutor { std::vector inputNames, std::vector outputNames, const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, - MediapipeServableMetricReporter* mediapipeServableMetricReporter, GraphIdGuard&& guard); + MediapipeServableMetricReporter* mediapipeServableMetricReporter, GraphIdGuard&& guard, + std::shared_ptr> activeInferenceCount = nullptr, + std::shared_ptr> lastActivityTimeNs = nullptr); // Constructor without graph queue (old path - graph created per-request) MediapipeGraphExecutor(const std::string& name, const std::string& version, @@ -160,7 +212,9 @@ class MediapipeGraphExecutor { std::vector inputNames, std::vector outputNames, const GraphSidePackets& sidePacketMaps, PythonBackend* pythonBackend, - MediapipeServableMetricReporter* mediapipeServableMetricReporter); + MediapipeServableMetricReporter* mediapipeServableMetricReporter, + std::shared_ptr> activeInferenceCount = nullptr, + std::shared_ptr> lastActivityTimeNs = nullptr); template Status infer(const RequestType* request, ResponseType* response, ExecutionContext executionContext) { diff --git a/src/metrics/metric_config.cpp b/src/metrics/metric_config.cpp index fedfb4a806..95997a4ab0 100644 --- a/src/metrics/metric_config.cpp +++ b/src/metrics/metric_config.cpp @@ -55,6 +55,7 @@ const std::string METRIC_NAME_WAIT_FOR_INFER_REQ_TIME = "ovms_wait_for_infer_req // MediaPipe const std::string METRIC_NAME_CURRENT_GRAPHS = "ovms_current_graphs"; +const std::string METRIC_NAME_GRAPH_LOADED = "ovms_graph_loaded"; const std::string METRIC_NAME_RESPONSES = "ovms_responses"; const std::string METRIC_NAME_REQUESTS_ACCEPTED = "ovms_requests_accepted"; diff --git a/src/metrics/metric_config.hpp b/src/metrics/metric_config.hpp index bf1430f9c2..365a5cdb67 100644 --- a/src/metrics/metric_config.hpp +++ b/src/metrics/metric_config.hpp @@ -41,6 +41,7 @@ extern const std::string METRIC_NAME_WAIT_FOR_INFER_REQ_TIME; // MediaPipe extern const std::string METRIC_NAME_CURRENT_GRAPHS; +extern const std::string METRIC_NAME_GRAPH_LOADED; extern const std::string METRIC_NAME_RESPONSES; extern const std::string METRIC_NAME_REQUESTS_ACCEPTED; @@ -100,6 +101,7 @@ class MetricConfig { {METRIC_NAME_INFERENCE_TIME}, {METRIC_NAME_WAIT_FOR_INFER_REQ_TIME}, {METRIC_NAME_CURRENT_GRAPHS}, + {METRIC_NAME_GRAPH_LOADED}, {METRIC_NAME_REQUESTS_ACCEPTED}, {METRIC_NAME_REQUESTS_REJECTED}, {METRIC_NAME_GRAPH_ERROR}, diff --git a/src/model.cpp b/src/model.cpp index d9dadbdb13..454b1459cc 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -88,7 +88,7 @@ void Model::updateDefaultVersion(int ignoredVersion) { for (const auto& [version, versionInstance] : modelVersions) { if (version != ignoredVersion && version > newDefaultVersion && - ModelVersionState::AVAILABLE == versionInstance->getStatus().getState()) { + versionInstance->getStatus().appearsAvailable()) { newDefaultVersion = version; } } @@ -117,14 +117,14 @@ std::shared_ptr Model::modelInstanceFactory(const std::stri return std::make_shared(modelName, modelVersion, ieCore, registry, metricConfig); } -Status Model::addVersion(const ModelConfig& config, ov::Core& ieCore, MetricRegistry* registry, const MetricConfig* metricConfig) { +Status Model::addVersion(const ModelConfig& config, ov::Core& ieCore, MetricRegistry* registry, const MetricConfig* metricConfig, bool lazyLoad) { const auto& version = config.getVersion(); std::shared_ptr modelInstance = modelInstanceFactory(config.getName(), version, ieCore, registry, metricConfig); std::unique_lock lock(modelVersionsMtx); modelVersions.emplace(version, modelInstance); lock.unlock(); - auto status = modelInstance->loadModel(config); + auto status = modelInstance->loadModel(config, lazyLoad); if (!status.ok()) { return status; } @@ -133,7 +133,7 @@ Status Model::addVersion(const ModelConfig& config, ov::Core& ieCore, MetricRegi return StatusCode::OK; } -Status Model::addVersions(std::shared_ptr& versionsToStart, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr& versionsFailed, MetricRegistry* registry, const MetricConfig* metricConfig) { +Status Model::addVersions(std::shared_ptr& versionsToStart, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr& versionsFailed, MetricRegistry* registry, const MetricConfig* metricConfig, bool lazyLoad) { Status result = StatusCode::OK; downloadModels(fs, config, versionsToStart); versionsFailed->clear(); @@ -141,7 +141,7 @@ Status Model::addVersions(std::shared_ptr& versionsToStart, ov SPDLOG_INFO("Will add model: {}; version: {} ...", getName(), version); config.setVersion(version); config.parseModelMapping(); - auto status = addVersion(config, ieCore, registry, metricConfig); + auto status = addVersion(config, ieCore, registry, metricConfig, lazyLoad); if (!status.ok()) { SPDLOG_ERROR("Error occurred while loading model: {}; version: {}; error: {}", getName(), @@ -161,6 +161,17 @@ const std::shared_ptr Model::getModelInstanceByVersion(const mode return it != modelVersions.end() ? it->second : nullptr; } +Status Model::wakeUpIfSleeping() { + std::shared_lock lock(modelVersionsMtx); + for (auto& [version, instance] : modelVersions) { + auto status = instance->wakeUpIfSleeping(); + if (!status.ok()) { + return status; + } + } + return StatusCode::OK; +} + Status Model::retireVersions(std::shared_ptr& versionsToRetire) { Status result = StatusCode::OK; for (const auto version : *versionsToRetire) { @@ -225,6 +236,13 @@ void Model::retireAllVersions() { subscriptionManager.notifySubscribers(); } +void Model::putToSleepAllVersions() { + std::shared_lock lock(modelVersionsMtx); + for (const auto& [version, instance] : modelVersions) { + instance->putToSleep(); + } +} + void Model::cleanupAllVersions() { if (!(customLoaderName.empty())) { auto& customloaders = ovms::CustomLoaders::instance(); diff --git a/src/model.hpp b/src/model.hpp index bd31b73a0b..f26b456907 100644 --- a/src/model.hpp +++ b/src/model.hpp @@ -90,7 +90,7 @@ class Model : public ServableDefinition { * * @return status */ - virtual Status addVersion(const ModelConfig& config, ov::Core& ieCore, MetricRegistry* registry = nullptr, const MetricConfig* metricConfig = nullptr); + virtual Status addVersion(const ModelConfig& config, ov::Core& ieCore, MetricRegistry* registry = nullptr, const MetricConfig* metricConfig = nullptr, bool lazyLoad = false); /** * @brief ModelInstances factory @@ -161,6 +161,8 @@ class Model : public ServableDefinition { */ const std::shared_ptr getModelInstanceByVersion(const model_version_t& version) const; + Status wakeUpIfSleeping(); + /** * @brief Adds new versions of ModelInstance * @@ -168,7 +170,7 @@ class Model : public ServableDefinition { * * @return status */ - Status addVersions(std::shared_ptr& versions, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr& versionsFailed, MetricRegistry* registry = nullptr, const MetricConfig* metricConfig = nullptr); + Status addVersions(std::shared_ptr& versions, ovms::ModelConfig& config, std::shared_ptr& fs, ov::Core& ieCore, std::shared_ptr& versionsFailed, MetricRegistry* registry = nullptr, const MetricConfig* metricConfig = nullptr, bool lazyLoad = false); /** * @brief Retires versions of Model @@ -193,6 +195,8 @@ class Model : public ServableDefinition { */ void retireAllVersions(); + void putToSleepAllVersions(); + /** * @brief Cleans up all versions of Model */ diff --git a/src/model_metric_reporter.cpp b/src/model_metric_reporter.cpp index 1e60689957..841d55c594 100644 --- a/src/model_metric_reporter.cpp +++ b/src/model_metric_reporter.cpp @@ -267,6 +267,18 @@ MediapipeServableMetricReporter::MediapipeServableMetricReporter(const MetricCon SPDLOG_INFO("DISABLED {}", METRIC_NAME_CURRENT_GRAPHS); } + familyName = METRIC_NAME_GRAPH_LOADED; + if (metricConfig->isFamilyEnabled(familyName)) { + auto family = registry->createFamily(familyName, + "Whether the MediaPipe graph resources are loaded (1) or idle-unloaded (0)."); + THROW_IF_NULL(family, "cannot create family"); + this->graphLoaded = family->addMetric( + {{"name", graphName}}); + THROW_IF_NULL(this->graphLoaded, "cannot create metric"); + } else { + SPDLOG_INFO("DISABLED {}", METRIC_NAME_GRAPH_LOADED); + } + familyName = METRIC_NAME_REQUESTS_ACCEPTED; if (metricConfig->isFamilyEnabled(familyName)) { auto family = registry->createFamily(familyName, diff --git a/src/model_metric_reporter.hpp b/src/model_metric_reporter.hpp index 4ca4cbbedc..4a721b442c 100644 --- a/src/model_metric_reporter.hpp +++ b/src/model_metric_reporter.hpp @@ -112,6 +112,9 @@ class MediapipeServableMetricReporter : public StatusMetricReporter { public: std::unique_ptr currentGraphs; + // 1 = graph resources loaded, 0 = idle-unloaded. Always 1 after a successful + // load for graphs that never enable idle unload. + std::unique_ptr graphLoaded; // KFS std::unique_ptr requestAcceptedGrpcModelInfer; diff --git a/src/modelconfig.cpp b/src/modelconfig.cpp index c02d22c199..8a0aed41ce 100644 --- a/src/modelconfig.cpp +++ b/src/modelconfig.cpp @@ -115,6 +115,10 @@ bool ModelConfig::isReloadRequired(const ModelConfig& rhs) const { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to plugin config mismatch", this->name); return true; } + if (this->groupName != rhs.groupName) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to group name mismatch", this->name); + return true; + } if (!isLayoutConfigurationEqual(rhs)) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "ModelConfig {} reload required due to named layout mismatch", this->name); return true; @@ -720,6 +724,14 @@ Status ModelConfig::parseNode(const rapidjson::Value& v) { SPDLOG_DEBUG("allow_cache: {}", v["allow_cache"].GetBool()); } + // Group name for idle model management + if (v.HasMember("group_name")) { + setGroupName(v["group_name"].GetString()); + } else { + setGroupName(getName()); + } + SPDLOG_DEBUG("group_name: {}", getGroupName()); + // if the config has models which require custom loader to be used, then load the same here if (v.HasMember("custom_loader_options")) { if (!parseCustomLoaderOptionsConfig(v["custom_loader_options"]).ok()) { diff --git a/src/modelconfig.hpp b/src/modelconfig.hpp index 353715f651..055750835c 100644 --- a/src/modelconfig.hpp +++ b/src/modelconfig.hpp @@ -199,6 +199,11 @@ class ModelConfig { */ std::optional precision; + /** + * @brief Group name for idle model management. Defaults to model name. + */ + std::string groupName; + public: /** * @brief Construct a new Model Config object @@ -281,6 +286,24 @@ class ModelConfig { this->name = name; } + /** + * @brief Get the group name + * + * @return const std::string& + */ + const std::string& getGroupName() const { + return this->groupName; + } + + /** + * @brief Set the group name + * + * @param groupName + */ + void setGroupName(const std::string& groupName) { + this->groupName = groupName; + } + /** * @brief Get local path to specific model version where .xml and .bin is located for loading * diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index b4842deac8..a571b018a9 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1327,7 +1327,7 @@ Status ModelInstance::setCacheOptions(const ModelConfig& config) { return StatusCode::OK; } -Status ModelInstance::loadModel(const ModelConfig& config) { +Status ModelInstance::loadModel(const ModelConfig& config, bool lazyLoad) { std::lock_guard loadingLock(loadingMutex); SPDLOG_INFO("Loading model: {}, version: {}, from path: {}, with target device: {} ...", config.getName(), config.getVersion(), config.getPath(), config.getTargetDevice()); @@ -1337,10 +1337,39 @@ Status ModelInstance::loadModel(const ModelConfig& config) { SPDLOG_INFO("Some inputs shapes for model {} are set to auto", config.getName()); } this->status = ModelVersionStatus(config.getName(), config.getVersion()); + if (lazyLoad) { + this->config = config; + this->path = config.getPath(); + this->status.setSleeping(); + return StatusCode::OK; + } this->status.setLoading(); return loadModelImpl(config); } +Status ModelInstance::wakeUpIfSleeping() { + std::lock_guard loadingLock(loadingMutex); + auto state = status.getState(); + if (state == ModelVersionState::AVAILABLE || state == ModelVersionState::LOADING) + return StatusCode::OK; + if (state != ModelVersionState::SLEEPING) + return StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE; + SPDLOG_INFO("Waking up model: {}, version: {} ...", getName(), getVersion()); + auto loadStatus = loadModelImpl(this->config); + if (!loadStatus.ok()) { + // Keep failed wake-ups retryable from inference path, same as mediapipe. + this->status.setSleeping(); + } + return loadStatus; +} + +void ModelInstance::putToSleep() { + std::lock_guard loadingLock(loadingMutex); + SPDLOG_INFO("Putting model to sleep: {}, version: {} ...", getName(), getVersion()); + unloadModelComponents(); + this->status.setSleeping(); +} + Status ModelInstance::reloadModel(const ModelConfig& config, const DynamicModelParameter& parameter) { std::lock_guard loadingLock(loadingMutex); this->status.setLoading(); diff --git a/src/modelinstance.hpp b/src/modelinstance.hpp index 76941e480c..e2ff492480 100644 --- a/src/modelinstance.hpp +++ b/src/modelinstance.hpp @@ -452,7 +452,6 @@ class ModelInstance : public Servable { const ModelVersionStatus& getStatus() const { return status; } - /** * @brief Internal method for setting cache options */ @@ -547,8 +546,10 @@ class ModelInstance : public Servable { * * @return Status */ - virtual Status loadModel(const ModelConfig& config); + virtual Status loadModel(const ModelConfig& config, bool lazyLoad = false); + Status wakeUpIfSleeping(); + void putToSleep(); /** * @brief Reloads model version * diff --git a/src/modelversionstatus.cpp b/src/modelversionstatus.cpp index 35fe807aee..2f8a8bf256 100644 --- a/src/modelversionstatus.cpp +++ b/src/modelversionstatus.cpp @@ -30,6 +30,7 @@ static const std::unordered_map versionStatesStr {ModelVersionState::START, "START"}, {ModelVersionState::LOADING, "LOADING"}, {ModelVersionState::AVAILABLE, "AVAILABLE"}, + {ModelVersionState::SLEEPING, "SLEEPING"}, {ModelVersionState::UNLOADING, "UNLOADING"}, {ModelVersionState::END, "END"}}; const std::string& ModelVersionStateToString(ModelVersionState state) { @@ -77,6 +78,14 @@ bool ModelVersionStatus::isFailedLoading() const { return this->state == ovms::ModelVersionState::LOADING && this->errorCode == ovms::ModelVersionStatusErrorCode::UNKNOWN; } +bool ModelVersionStatus::isSleeping() const { + return this->state == ovms::ModelVersionState::SLEEPING; +} + +bool ModelVersionStatus::appearsAvailable() const { + return this->state == ovms::ModelVersionState::AVAILABLE || this->state == ovms::ModelVersionState::SLEEPING; +} + void ModelVersionStatus::setLoading(ModelVersionStatusErrorCode error_code) { SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); state = ModelVersionState::LOADING; @@ -91,6 +100,13 @@ void ModelVersionStatus::setAvailable(ModelVersionStatusErrorCode error_code) { logStatus(); } +void ModelVersionStatus::setSleeping(ModelVersionStatusErrorCode error_code) { + SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); + state = ModelVersionState::SLEEPING; + errorCode = error_code; + logStatus(); +} + void ModelVersionStatus::setUnloading(ModelVersionStatusErrorCode error_code) { SPDLOG_DEBUG("{}: {} - {} (previous state: {}) -> error: {}", __func__, this->modelName, this->version, ModelVersionStateToString(this->state), ModelVersionStatusErrorCodeToString(error_code)); state = ModelVersionState::UNLOADING; diff --git a/src/modelversionstatus.hpp b/src/modelversionstatus.hpp index 87dded0350..e82665ee9d 100644 --- a/src/modelversionstatus.hpp +++ b/src/modelversionstatus.hpp @@ -32,6 +32,7 @@ enum class ModelVersionState : int { START = 10, LOADING = 20, AVAILABLE = 30, + SLEEPING = 35, UNLOADING = 40, END = 50 }; @@ -89,10 +90,14 @@ class ModelVersionStatus { bool willEndUnloaded() const; bool isFailedLoading() const; + bool isSleeping() const; + bool appearsAvailable() const; void setLoading(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); void setAvailable(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); + void setSleeping(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); + void setUnloading(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); void setEnd(ModelVersionStatusErrorCode error_code = ModelVersionStatusErrorCode::OK); diff --git a/src/schema.cpp b/src/schema.cpp index 7d76980dca..23da14985f 100644 --- a/src/schema.cpp +++ b/src/schema.cpp @@ -241,6 +241,9 @@ const std::string MODEL_CONFIG_DEFINITION = R"( } }, "minProperties": 1 + }, + "group_name": { + "type": "string" } }, "additionalProperties": false @@ -357,6 +360,13 @@ const std::string MODELS_CONFIG_SCHEMA = R"({ }, "subconfig": { "type": "string" + }, + "idle_unload_timeout_seconds": { + "type": "integer", + "minimum": 0 + }, + "group_name": { + "type": "string" } }, "additionalProperties": false diff --git a/src/servable_management/BUILD b/src/servable_management/BUILD new file mode 100644 index 0000000000..e6c06a0b6d --- /dev/null +++ b/src/servable_management/BUILD @@ -0,0 +1,103 @@ +# +# Copyright (c) 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +load("//:common_settings.bzl", "ovms_cc_library") + +ovms_cc_library( + name = "servable_loading_task", + hdrs = ["servable_loading_task.hpp"], + deps = select({ + "//conditions:default": [], + "//:not_disable_mediapipe": [ + "//src/mediapipe_internal:mediapipegraphconfig", + ], + }) + [ + "//src:modelconfig", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "servable_loading_queue", + hdrs = ["servable_loading_queue.hpp"], + srcs = ["servable_loading_queue.cpp"], + deps = [ + "servable_loading_task", + "//src:libovmslogging", + ], + visibility = ["//visibility:public"], +) + +ovms_cc_library( + name = "modelmanager", + hdrs = ["modelmanager.hpp", "servable_group_manager.hpp"], + srcs = ["modelmanager.cpp", "servable_group_manager.cpp"], + deps = select({ + "//conditions:default": [], + "//:not_disable_mediapipe" : [ + "//src/mediapipe_internal:libovms_mediapipe", + ], + }) + [ + "servable_loading_queue", + "//src:cleaner_utils", + "//src:customloaders", + "//src:libovms_config", + "//src:libovms_model_instance_provider", + "//src:libovms_ov_utils", + "//src:libovms_servable_definition", + "//src:libovms_servable_name_checker", + "//src:libovmslogging", + "//src:libovmsschema", + "//src:libovmsstring_utils", + "//src:libovmsstatus", + "//src:model", + "//src:modelconfig", + "//src:modelinstance", + "//src:modelinstanceunloadguard", + "//src:resources_cleaner", + "//src/dags:custom_node_library_manager", + "//src/dags:dag_resource_manager", + "//src/dags:pipeline_config_parser", + "//src/dags:pipeline_factory", + "//src/dags:pipelinedefinition", + "//src/filesystem:libovmsfilesystem", + "//src/filesystem:libovmsfilesystemfactory", + "//src/graph_export:graph_export", + "//src/metrics:libovms_metric_provider", + "//src/metrics:libovmsmetrics", + "@com_github_tencent_rapidjson//:rapidjson", + "//src/port:rapidjson_stringbuffer", + "//src/port:rapidjson_writer", + ], + visibility = ["//visibility:public"], +) +ovms_cc_library( + name = "servablemanagermodule", + hdrs = ["servablemanagermodule.hpp"], + srcs = ["servablemanagermodule.cpp"], + deps = select({ + "//:not_disable_python": [ + "//src/python:libovmspythonmodule", + ], + "//:disable_python": [] + }) + [ + "//src:cpp_headers", + "//src:libovms_module", + "//src:libovmslogging", + "modelmanager", + "//src/metrics:libovmsmetrics", + ], + visibility = ["//visibility:public"], +) diff --git a/src/modelmanager.cpp b/src/servable_management/modelmanager.cpp similarity index 85% rename from src/modelmanager.cpp rename to src/servable_management/modelmanager.cpp index c14c1efe4e..7c48d88582 100644 --- a/src/modelmanager.cpp +++ b/src/servable_management/modelmanager.cpp @@ -45,33 +45,35 @@ #pragma warning(pop) #include -#include "cleaner_utils.hpp" -#include "config.hpp" -#include "customloaderconfig.hpp" -#include "customloaderinterface.hpp" -#include "customloaders.hpp" -#include "dags/custom_node_library_manager.hpp" -#include "dags/pipeline_config_parser.hpp" -#include "dags/pipeline_factory.hpp" -#include "dags/pipelinedefinition.hpp" -#include "filesystem/filesystem.hpp" -#include "filesystem/filesystemfactory.hpp" -#include "graph_export/graph_export.hpp" -#include "logging.hpp" +#include "src/cleaner_utils.hpp" +#include "src/config.hpp" +#include "src/customloaderconfig.hpp" +#include "src/customloaderinterface.hpp" +#include "src/customloaders.hpp" +#include "src/dags/custom_node_library_manager.hpp" +#include "src/dags/pipeline_config_parser.hpp" +#include "src/dags/pipeline_factory.hpp" +#include "src/dags/pipelinedefinition.hpp" +#include "src/filesystem/filesystem.hpp" +#include "src/filesystem/filesystemfactory.hpp" +#include "src/graph_export/graph_export.hpp" +#include "src/logging.hpp" +#include "servable_group_manager.hpp" +#include "servable_loading_queue.hpp" #if (MEDIAPIPE_DISABLE == 0) -#include "mediapipe_internal/mediapipefactory.hpp" -#include "mediapipe_internal/mediapipegraphdefinition.hpp" +#include "src/mediapipe_internal/mediapipefactory.hpp" +#include "src/mediapipe_internal/mediapipegraphdefinition.hpp" #endif -#include "metrics/metric_config.hpp" -#include "metrics/metric_registry.hpp" -#include "model.hpp" -#include "modelinstance.hpp" // for logging -#include "modelinstanceunloadguard.hpp" -#include "ov_utils.hpp" -#include "schema.hpp" -#include "servable_definition.hpp" -#include "stringutils.hpp" -#include "systeminfo.hpp" +#include "src/metrics/metric_config.hpp" +#include "src/metrics/metric_registry.hpp" +#include "src/model.hpp" +#include "src/modelinstance.hpp" // for logging +#include "src/modelinstanceunloadguard.hpp" +#include "src/ov_utils.hpp" +#include "src/schema.hpp" +#include "src/servable_definition.hpp" +#include "src/stringutils.hpp" +#include "src/systeminfo.hpp" namespace ovms { @@ -82,6 +84,7 @@ const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "c:\\Intel\\openvino_cache"; const std::string DEFAULT_MODEL_CACHE_DIRECTORY = "/opt/cache"; #endif ModelManager::ModelManager(const std::string& modelCacheDirectory, MetricRegistry* registry, PythonBackend* pythonBackend) : + loadingQueue(std::make_unique()), pipelineFactory(std::make_unique()), #if (MEDIAPIPE_DISABLE == 0) mediapipeFactory(std::make_unique(pythonBackend)), @@ -92,6 +95,86 @@ ModelManager::ModelManager(const std::string& modelCacheDirectory, MetricRegistr metricRegistry(registry), pythonBackend(pythonBackend) { this->ieCore = std::make_unique(); + loadingQueue->start([this](ServableLoadingTask& task) -> Status { + switch (task.type) { + case ServableLoadingTaskType::LoadModel: { + if (!task.modelConfig.has_value()) { + return StatusCode::INTERNAL_ERROR; + } + return reloadModelWithVersions(task.modelConfig.value()); + } + case ServableLoadingTaskType::WakeUpModel: { + auto model = findModelByName(task.name); + if (model) { + return model->wakeUpIfSleeping(); + } + // Model was never instantiated (e.g. added to config while its group was idle). + auto it = servedModelConfigs.find(task.name); + if (it == servedModelConfigs.end()) + return StatusCode::MODEL_NAME_MISSING; + return reloadModelWithVersions(it->second); + } + case ServableLoadingTaskType::PutToSleepModel: { + auto model = findModelByName(task.name); + if (!model) { + return StatusCode::MODEL_NAME_MISSING; + } + model->putToSleepAllVersions(); + return StatusCode::OK; + } + case ServableLoadingTaskType::RetireModel: { + auto model = findModelByName(task.name); + if (!model) { + return StatusCode::MODEL_NAME_MISSING; + } + model->retireAllVersions(); + return StatusCode::OK; + } +#if (MEDIAPIPE_DISABLE == 0) + case ServableLoadingTaskType::LoadMediapipe: { + if (!task.graphConfig.has_value()) { + return StatusCode::INTERNAL_ERROR; + } + const auto& config = task.graphConfig.value(); + auto* def = mediapipeFactory->findDefinitionByName(task.name); + if (!def) { + // Non-permanent idle groups: create as SLEEPING to skip expensive loading + if (servableGroupManager && servableGroupManager->isEnabled() && + !config.getGroupName().empty() && config.getGroupName() != "permanent") { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, + "Mediapipe graph:{} belongs to non-permanent group '{}'; creating as SLEEPING", + task.name, config.getGroupName()); + bool lazyLoad = true; + return mediapipeFactory->createDefinition(task.name, config, *this, *this, lazyLoad); + } + return mediapipeFactory->createDefinition(task.name, config, *this, *this); + } + if (def->isReloadRequired(config)) { + return mediapipeFactory->reloadDefinition(task.name, config, *this); + } + return StatusCode::OK; + } + case ServableLoadingTaskType::WakeUpMediapipe: { + // TODO consider moving whole part as an interface to ServableContainer so that we + // could just call servableContainer->wakeUp(A). However we would need to to expose scheduler then + return mediapipeFactory->wakeUpDefinition(task.name, *this); + } + case ServableLoadingTaskType::PutToSleepMediapipe: { + return mediapipeFactory->putToSleepDefinition(task.name); + } + case ServableLoadingTaskType::RetireMediapipe: { + return mediapipeFactory->retireDefinition(task.name); + } +#else + case ServableLoadingTaskType::LoadMediapipe: + case ServableLoadingTaskType::RetireMediapipe: + case ServableLoadingTaskType::WakeUpMediapipe: + case ServableLoadingTaskType::PutToSleepMediapipe: + return StatusCode::INTERNAL_ERROR; +#endif + } + return StatusCode::INTERNAL_ERROR; + }); OV_LOGGER("ov::Core(): {}", reinterpret_cast(this->ieCore.get())); // Take --cache_dir from CLI @@ -187,6 +270,13 @@ Status ModelManager::start(const Config& config) { resourcesCleanupIntervalMillisec = config.resourcesCleanerPollWaitSeconds() * 1000; Status status; this->startedWithConfigFile = (config.configPath() != ""); + + // Initialize model group manager if idle unload is enabled and using config file + if (this->startedWithConfigFile && config.idleUnloadTimeoutSeconds() > 0) { + servableGroupManager = std::make_unique(static_cast(config.idleUnloadTimeoutSeconds()) * 1'000'000ULL); + SPDLOG_INFO("Model group idle management enabled with {}s timeout", config.idleUnloadTimeoutSeconds()); + } + if (isStartedWithConfigFile()) { status = startFromFile(config.configPath()); } else { @@ -449,28 +539,6 @@ Status ModelManager::validateUserSettingsInSingleModelCliGraphStart(const Models return StatusCode::OK; } -Status ModelManager::processMediapipeConfig(const MediapipeGraphConfig& config, std::set& mediapipesInConfigFile, MediapipeFactory& factory) { - if (mediapipesInConfigFile.find(config.getGraphName()) != mediapipesInConfigFile.end()) { - SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated mediapipe names: {} defined in config file. Only first graph will be loaded.", config.getGraphName()); - return StatusCode::OK; - } - mediapipesInConfigFile.insert(config.getGraphName()); - MediapipeGraphDefinition* mediapipeGraphDefinition = factory.findDefinitionByName(config.getGraphName()); - if (mediapipeGraphDefinition == nullptr) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Mediapipe graph:{} was not loaded so far. Triggering load", config.getGraphName()); - auto status = factory.createDefinition(config.getGraphName(), config, *this, *this); - return status; - } - if (mediapipeGraphDefinition->isReloadRequired(config)) { - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Mediapipe graph:{} triggering reload", config.getGraphName()); - auto status = factory.reloadDefinition(config.getGraphName(), - config, - *this); - return status; - } - SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Mediapipe graph:{} already loaded and reload is not required", config.getGraphName()); - return StatusCode::OK; -} #endif #if (MEDIAPIPE_DISABLE == 0) @@ -532,11 +600,35 @@ Status ModelManager::ConfigLoader::loadCustomNodeLibrariesConfig(ModelManager& m } #if (MEDIAPIPE_DISABLE == 0) +[[nodiscard]] Status ModelManager::retireMediapipesOtherThan(const std::set& graphsInConfigFile) { + std::vector>> futures; + for (const auto& graphName : mediapipeFactory->getMediapipePipelinesNames()) { + if (graphsInConfigFile.find(graphName) != graphsInConfigFile.end()) { + continue; + } + auto* definition = mediapipeFactory->findDefinitionByName(graphName); + if (definition == nullptr || definition->getStateCode() == PipelineDefinitionStateCode::RETIRED) { + continue; + } + ServableLoadingTask task{ServableLoadingTaskType::RetireMediapipe, graphName, /*urgent=*/false}; + futures.emplace_back(graphName, loadingQueue->scheduleTask(std::move(task))); + } + Status firstErrorStatus = StatusCode::OK; + // Config reload must not return before removed graphs stopped serving. + for (auto& [graphName, future] : futures) { + auto status = future.get(); + if (status != StatusCode::OK) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to retire mediapipe graph:{} - {}", graphName, status.string()); + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); + } + } + return firstErrorStatus; +} + Status ModelManager::loadMediapipeGraphsConfig(std::vector& mediapipesInConfigFile) { if (mediapipesInConfigFile.size() == 0) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Configuration file doesn't have mediapipe property."); - mediapipeFactory->retireOtherThan({}); - return StatusCode::OK; + return retireMediapipesOtherThan({}); } std::set mediapipesInConfigFileNames; Status firstErrorStatus = StatusCode::OK; @@ -544,13 +636,22 @@ Status ModelManager::loadMediapipeGraphsConfig(std::vector for (const auto& mediapipeGraphConfig : mediapipesInConfigFile) { mediapipesInConfigFileNames.insert(mediapipeGraphConfig.getGraphName()); } - mediapipeFactory->retireOtherThan(std::move(mediapipesInConfigFileNames)); - std::set mediapipesAlreadyLoaded; + auto retireStatus = retireMediapipesOtherThan(mediapipesInConfigFileNames); + if (retireStatus != StatusCode::OK) { + IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(retireStatus); + } + std::set alreadyScheduled; for (const auto& mediapipeGraphConfig : mediapipesInConfigFile) { + if (!alreadyScheduled.insert(mediapipeGraphConfig.getGraphName()).second) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Duplicated mediapipe names: {} defined in config file. Only first graph will be loaded.", mediapipeGraphConfig.getGraphName()); + continue; + } if (spdlog::default_logger_raw()->level() <= spdlog::level::debug) { mediapipeGraphConfig.logGraphConfigContent(); } - auto status = processMediapipeConfig(mediapipeGraphConfig, mediapipesAlreadyLoaded, *mediapipeFactory); + ServableLoadingTask task{ServableLoadingTaskType::LoadMediapipe, mediapipeGraphConfig.getGraphName(), mediapipeGraphConfig}; + auto future = loadingQueue->scheduleTask(std::move(task)); + auto status = future.get(); if (status != StatusCode::OK) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); } @@ -751,7 +852,9 @@ Status ModelManager::ConfigLoader::loadModels(ModelManager& modelManager, const continue; } - status = modelManager.reloadModelWithVersions(modelConfig); + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, modelName, modelConfig}; + auto future = modelManager.loadingQueue->scheduleTask(std::move(task)); + status = future.get(); IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); modelsInConfigFile.emplace(modelName); @@ -867,7 +970,9 @@ Status ModelManager::tryReloadGatedModelConfigs(std::vector& gatedM Status firstErrorStatus = StatusCode::OK; for (auto& modelConfig : gatedModelConfigs) { SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Trying to reload model({}) configuration", modelConfig.getName()); - auto status = reloadModelWithVersions(modelConfig); + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, modelConfig.getName(), modelConfig}; + auto future = loadingQueue->scheduleTask(std::move(task)); + auto status = future.get(); if (!status.ok()) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); continue; @@ -885,7 +990,7 @@ Status ModelManager::tryReloadGatedModelConfigs(std::vector& gatedM Status ModelManager::loadConfig() { rapidjson::Document configJson; - std::lock_guard loadingLock(configMtx); + std::lock_guard loadingLock(configMtx); // TODO(idle-unload): @atobiszei narrow scope to parsing-only after queue refactoring Status status = parseConfig(this->configFilename, configJson, this->lastConfigFileMD5, WRONG_CONFIG_FILE_RETRY_DELAY_MS, MAX_CONFIG_JSON_READ_RETRY_COUNT); if (!status.ok()) { this->lastLoadConfigStatus = status; @@ -956,6 +1061,11 @@ Status ModelManager::loadConfig() { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); } + // Build model groups (non-permanent servables start SLEEPING via lazyLoad) + if (servableGroupManager && servableGroupManager->isEnabled()) { + servableGroupManager->buildGroups(this->servedModelConfigs, *this); + } + this->lastLoadConfigStatus = firstErrorStatus; return firstErrorStatus; } @@ -991,13 +1101,15 @@ void ModelManager::retireModelsRemovedFromConfigFile(const std::set } Status ModelManager::updateConfigurationWithoutConfigFile() { - std::lock_guard loadingLock(configMtx); + std::lock_guard loadingLock(configMtx); // TODO(idle-unload): @atobiszei narrow scope to parsing-only after queue refactoring SPDLOG_LOGGER_TRACE(modelmanager_logger, "Checking if something changed with model versions"); bool reloadNeeded = false; Status firstErrorStatus = StatusCode::OK; Status status; for (auto& [name, config] : servedModelConfigs) { - status = reloadModelWithVersions(config); + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, name, config}; + auto future = loadingQueue->scheduleTask(std::move(task)); + status = future.get(); if (!status.ok()) { IF_ERROR_NOT_OCCURRED_EARLIER_THEN_SET_FIRST_ERROR(status); } else if (status == StatusCode::OK_RELOADED) { @@ -1044,11 +1156,35 @@ Status ModelManager::configFileReloadNeeded(bool& isNeeded) { return StatusCode::OK; } +void ModelManager::unloadIdleGraphs() { +#if (MEDIAPIPE_DISABLE == 0) + std::vector toUnload; + { + const auto& names = mediapipeFactory->getMediapipePipelinesNames(); + for (const auto& name : names) { + MediapipeGraphDefinition* def = mediapipeFactory->findDefinitionByName(name); + if (def && def->shouldUnloadDueToIdle()) { + toUnload.push_back(name); + } + } + } + for (const auto& name : toUnload) { + bool urgentUnload = false; + auto future = requestServablePutToSleep(name, urgentUnload); + auto status = future.get(); + if (!status.ok()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, + "Failed to idle-unload mediapipe graph {}: {}", name, status.string()); + } + } +#endif +} + void ModelManager::watcher(std::future exitSignal, bool watchConfigFile) { SPDLOG_LOGGER_INFO(modelmanager_logger, "Started model manager thread"); while (exitSignal.wait_for(std::chrono::milliseconds(this->watcherIntervalMillisec)) == std::future_status::timeout) { SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle begin"); - std::unique_lock loadingLock(configMtx); + std::unique_lock loadingLock(configMtx); // TODO(idle-unload): @atobiszei narrow scope to parsing-only after queue refactoring if (watchConfigFile) { bool isNeeded; configFileReloadNeeded(isNeeded); @@ -1058,6 +1194,16 @@ void ModelManager::watcher(std::future exitSignal, bool watchConfigFile) { } updateConfigurationWithoutConfigFile(); loadingLock.unlock(); + // Idle-unload sweep: free resources of graphs idle past their timeout. + // Done AFTER releasing configMtx — unload() only needs the factory's + // definitions lock and the per-definition lifecycleMtx, and is + // non-blocking (it skips graphs with in-flight requests rather than + // draining). This keeps configMtx hold time minimal. + unloadIdleGraphs(); + // Model group idle unload: unload the active non-permanent group if idle + if (servableGroupManager && servableGroupManager->isEnabled()) { + servableGroupManager->unloadActiveGroupIfIdle(*this); + } SPDLOG_LOGGER_TRACE(modelmanager_logger, "Models configuration and filesystem check cycle end"); } SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped model manager thread"); @@ -1107,6 +1253,7 @@ void ModelManager::join() { if (cleanerStarted) { cleanerExitTrigger.set_value(); } + loadingQueue->requestStop(); if (watcherStarted) { if (monitor.joinable()) { @@ -1123,6 +1270,7 @@ void ModelManager::join() { SPDLOG_INFO("Shutdown cleaner thread"); } } + loadingQueue->stop(); } void ModelManager::getVersionsToChange( @@ -1315,9 +1463,11 @@ Status ModelManager::readAvailableVersions(std::shared_ptr& fs, cons } Status ModelManager::addModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToStart, std::shared_ptr& versionsFailed) { + bool lazyLoad = servableGroupManager && servableGroupManager->isEnabled() && + config.getGroupName() != "permanent"; Status status = StatusCode::OK; try { - status = model->addVersions(versionsToStart, config, fs, *ieCore, versionsFailed, this->metricRegistry, this->metricConfig.get()); + status = model->addVersions(versionsToStart, config, fs, *ieCore, versionsFailed, this->metricRegistry, this->metricConfig.get(), lazyLoad); if (!status.ok()) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Error occurred while loading model: {} versions; error: {}", config.getName(), @@ -1459,6 +1609,28 @@ Status ModelManager::reloadModelWithVersions(ModelConfig& config) { return blocking_status; } +std::future ModelManager::requestServableWakeUp(const std::string& name, bool urgent) { +#if (MEDIAPIPE_DISABLE == 0) + if (mediapipeFactory->findDefinitionByName(name)) { + ServableLoadingTask task{ServableLoadingTaskType::WakeUpMediapipe, name, urgent}; + return loadingQueue->scheduleTask(std::move(task)); + } +#endif + ServableLoadingTask task{ServableLoadingTaskType::WakeUpModel, name, urgent}; + return loadingQueue->scheduleTask(std::move(task)); +} + +std::future ModelManager::requestServablePutToSleep(const std::string& name, bool urgent) { +#if (MEDIAPIPE_DISABLE == 0) + if (mediapipeFactory->findDefinitionByName(name)) { + ServableLoadingTask task{ServableLoadingTaskType::PutToSleepMediapipe, name, urgent}; + return loadingQueue->scheduleTask(std::move(task)); + } +#endif + ServableLoadingTask task{ServableLoadingTaskType::PutToSleepModel, name, urgent}; + return loadingQueue->scheduleTask(std::move(task)); +} + const std::shared_ptr ModelManager::findModelInstance(const std::string& name, model_version_t version) const { auto model = findModelByName(name); if (!model) { @@ -1477,6 +1649,27 @@ const std::shared_ptr ModelManager::findModelByName(const std::string& na return it != models.end() ? it->second : nullptr; } +bool ModelManager::isServableAvailable(const std::string& name) const { + // TODO @atobiszei idle add version option + auto model = findModelByName(name); + if (model) { + // Version policy is not considered here - any servable version is enough to answer a request. + for (const auto& [version, instance] : model->getModelVersions()) { + if (instance->getStatus().getState() == ModelVersionState::AVAILABLE) { + return true; + } + } + return false; + } +#if (MEDIAPIPE_DISABLE == 0) + auto* def = mediapipeFactory->findDefinitionByName(name); + if (def) { + return def->getStateCode() == PipelineDefinitionStateCode::AVAILABLE; + } +#endif + return false; +} + bool ModelManager::subscribeToModel(const std::string& name, model_version_t version, NotifyReceiver& receiver) { auto model = findModelByName(name); if (!model) { @@ -1550,6 +1743,14 @@ Status ModelManager::getModelInstance(const std::string& modelName, std::unique_ptr& modelInstanceUnloadGuardPtr) const { SPDLOG_DEBUG("Requesting model: {}; version: {}.", modelName, modelVersionId); + if (servableGroupManager && servableGroupManager->isEnabled()) { + auto status = servableGroupManager->ensureServableLoaded(modelName, const_cast(*this)); + if (!status.ok()) { + SPDLOG_ERROR("Failed to load servable '{}': {}", modelName, status.string()); + return status; + } + } + auto model = findModelByName(modelName); if (model == nullptr) { return StatusCode::MODEL_NAME_MISSING; @@ -1574,10 +1775,15 @@ const CustomNodeLibraryManager& ModelManager::getCustomNodeLibraryManager() cons } const std::vector ModelManager::getNamesOfAvailableModels() const { + // In idle management mode, report all configured models as available + if (servableGroupManager && servableGroupManager->isEnabled()) { + return servableGroupManager->getAllConfiguredServableNames(); + } std::vector names; std::shared_lock lock(modelsMtx); for (auto& [name, model] : models) { - if (model->getDefaultModelInstance() && model->getDefaultModelInstance()->getStatus().getState() == ModelVersionState::AVAILABLE) { + auto instance = model->getDefaultModelInstance(); + if (instance && instance->getStatus().appearsAvailable()) { names.push_back(model->getName()); } } @@ -1587,6 +1793,14 @@ const std::vector ModelManager::getNamesOfAvailableModels() const { Status ModelManager::createPipeline(std::unique_ptr& graph, const std::string& name) { #if (MEDIAPIPE_DISABLE == 0) + if (servableGroupManager && servableGroupManager->isEnabled()) { + // TODO current preview limitation -> we wait for whole group to load + auto status = servableGroupManager->ensureServableLoaded(name, *this); + if (!status.ok()) { + SPDLOG_ERROR("Failed to load servable '{}': {}", name, status.string()); + return status; + } + } return this->mediapipeFactory->create(graph, name); #else SPDLOG_ERROR("Mediapipe support was disabled during build process..."); diff --git a/src/modelmanager.hpp b/src/servable_management/modelmanager.hpp similarity index 92% rename from src/modelmanager.hpp rename to src/servable_management/modelmanager.hpp index 21f8dbad0e..132c921015 100644 --- a/src/modelmanager.hpp +++ b/src/servable_management/modelmanager.hpp @@ -26,13 +26,13 @@ #include #include -#include "dags/dag_resource_manager.hpp" -#include "metrics/metric_provider.hpp" -#include "model_instance_provider.hpp" -#include "modelconfig.hpp" -#include "resources_cleaner.hpp" -#include "servable_name_checker.hpp" -#include "status.hpp" +#include "src/dags/dag_resource_manager.hpp" +#include "src/metrics/metric_provider.hpp" +#include "src/model_instance_provider.hpp" +#include "src/modelconfig.hpp" +#include "src/resources_cleaner.hpp" +#include "src/servable_name_checker.hpp" +#include "src/status.hpp" namespace ov { class Core; @@ -57,9 +57,11 @@ class MediapipeFactory; class MediapipeGraphConfig; class MediapipeGraphExecutor; class ModelInstance; +class ServableGroupManager; class ServableDefinition; class ModelInstanceUnloadGuard; class Pipeline; +class ServableLoadingQueue; class PipelineFactory; struct FunctorResourcesCleaner; class PythonBackend; @@ -85,6 +87,7 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M std::map> models; std::unique_ptr ieCore; + std::unique_ptr loadingQueue; std::unique_ptr pipelineFactory; #if (MEDIAPIPE_DISABLE == 0) std::unique_ptr mediapipeFactory; @@ -109,7 +112,7 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M Status addModelVersions(std::shared_ptr& model, std::shared_ptr& fs, ModelConfig& config, std::shared_ptr& versionsToStart, std::shared_ptr& versionsFailed); #if (MEDIAPIPE_DISABLE == 0) - Status processMediapipeConfig(const MediapipeGraphConfig& config, std::set& mediapipesInConfigFile, MediapipeFactory& factory); + [[nodiscard]] Status retireMediapipesOtherThan(const std::set& graphsInConfigFile); Status loadMediapipeGraphsConfig(std::vector& mediapipesInConfigFile); Status loadMediapipeSubConfigModels(std::vector& gatedModelConfigs, std::set& modelsInConfigFile, std::set& modelsWithInvalidConfig, std::unordered_map& newModelConfigs, std::vector& mediapipesInConfigFile); @@ -128,6 +131,12 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M */ void watcher(std::future exitSignal, bool watchConfigFile); + /** + * @brief Sweep mediapipe graph definitions and unload any that have been + * idle past their configured idle_unload_timeout_seconds. + */ + void unloadIdleGraphs(); + /** * @brief Cleaner thread for resources cleanup */ @@ -206,6 +215,8 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M */ uint32_t resourcesCleanupIntervalMillisec = 1000; + std::unique_ptr servableGroupManager; + private: /** * @brief last md5sum of configfile @@ -231,6 +242,7 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M */ std::string rootDirectoryPath; bool startedWithConfigFile = false; + /** * @brief Set json config directory path * @@ -301,6 +313,10 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M return models; } + ServableGroupManager* getGroupManager() const { + return servableGroupManager.get(); + } + const std::vector getNamesOfAvailableModels() const; /** @@ -314,6 +330,9 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M const MediapipeFactory& getMediapipeFactory() const { return *mediapipeFactory; } + MediapipeFactory& getMediapipeFactory() { + return *mediapipeFactory; + } #endif const CustomNodeLibraryManager& getCustomNodeLibraryManager() const; @@ -339,6 +358,8 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M return true; } + bool isServableAvailable(const std::string& name) const; + /** * @brief Finds model instance with specific name and version, returns default if version not specified * @@ -394,6 +415,10 @@ class ModelManager : public ServableNameChecker, public MetricProvider, public M */ Status reloadModelWithVersions(ModelConfig& config); + // Enqueue an urgent servable load request (for inference threads). + std::future requestServableWakeUp(const std::string& name, bool urgent); + std::future requestServablePutToSleep(const std::string& name, bool urgent); + /** * @brief Starts model manager using ovms::Config * diff --git a/src/servable_management/servable_group_manager.cpp b/src/servable_management/servable_group_manager.cpp new file mode 100644 index 0000000000..90705c1c7c --- /dev/null +++ b/src/servable_management/servable_group_manager.cpp @@ -0,0 +1,410 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "servable_group_manager.hpp" + +#include +#include +#include +#include +#include +#include + +#include "src/logging.hpp" +#include "src/model.hpp" +#include "src/modelconfig.hpp" +#include "src/modelinstance.hpp" +#include "modelmanager.hpp" +#if (MEDIAPIPE_DISABLE == 0) +#include "src/mediapipe_internal/mediapipefactory.hpp" +#include "src/mediapipe_internal/mediapipegraphdefinition.hpp" +#endif + +namespace ovms { + +ServableGroupManager::ServableGroupManager(uint64_t idleTimeoutMicroseconds) : + idleTimeoutMicroseconds(idleTimeoutMicroseconds), + lastActivityTimeNs(std::make_shared>( + std::chrono::steady_clock::now().time_since_epoch().count())) { +} + +void ServableGroupManager::buildGroups(const std::unordered_map& modelConfigs, + ModelManager& mm) { + std::unique_lock lock(groupsMtx); + groups.clear(); + servableToGroup.clear(); + + for (const auto& [name, config] : modelConfigs) { + const std::string& groupName = config.getGroupName(); + groups[groupName].groupName = groupName; + groups[groupName].modelNames.insert(name); + servableToGroup[name] = groupName; + } + +#if (MEDIAPIPE_DISABLE == 0) + // Also process mediapipe graph definitions + for (const auto& graphName : mm.getMediapipeFactory().getMediapipePipelinesNames()) { + MediapipeGraphDefinition* def = mm.getMediapipeFactory().findDefinitionByName(graphName); + if (def == nullptr) { + continue; + } + // Retired definitions stay in the factory after config removal; registering them + // here would let a later wake-up resurrect a graph the user deleted. + if (def->getStateCode() == PipelineDefinitionStateCode::RETIRED) { + continue; + } + const std::string& groupName = def->getMediapipeGraphConfig().getGroupName(); + if (groupName.empty()) { + // No group_name set — treat graph name as its own group + groups[graphName].groupName = graphName; + groups[graphName].mediapipeNames.insert(graphName); + servableToGroup[graphName] = graphName; + } else { + groups[groupName].groupName = groupName; + groups[groupName].mediapipeNames.insert(graphName); + servableToGroup[graphName] = groupName; + } + } +#endif + + size_t totalServables = modelConfigs.size(); +#if (MEDIAPIPE_DISABLE == 0) + totalServables += mm.getMediapipeFactory().getMediapipePipelinesNames().size(); +#endif + SPDLOG_LOGGER_INFO(modelmanager_logger, "Model group manager built {} groups from {} servables", groups.size(), totalServables); + for (const auto& [gname, ginfo] : groups) { + SPDLOG_LOGGER_INFO(modelmanager_logger, " Group '{}': {} models, {} mediapipe graphs{}", + gname, ginfo.modelNames.size(), ginfo.mediapipeNames.size(), + ginfo.isPermanent() ? " (permanent)" : ""); + } +} + +std::string ServableGroupManager::getGroupForServable(const std::string& servableName) const { + std::shared_lock lock(groupsMtx); + auto it = servableToGroup.find(servableName); + if (it != servableToGroup.end()) { + return it->second; + } + return ""; +} + +bool ServableGroupManager::isGroupLoaded(const std::string& groupName) const { + if (groupName.empty()) { + return false; + } + { + std::shared_lock lock(groupsMtx); + auto it = groups.find(groupName); + if (it != groups.end() && it->second.isPermanent()) { + return true; + } + } + return isActiveGroup(groupName); +} + +bool ServableGroupManager::isActiveGroup(const std::string& groupName) const { + std::shared_lock lock(activeGroupNameMtx); + return activeGroupName == groupName; +} + +void ServableGroupManager::setActiveGroup(const std::string& groupName) { + std::unique_lock lock(activeGroupNameMtx); + activeGroupName = groupName; +} + +std::string ServableGroupManager::getActiveGroupName() const { + std::shared_lock lock(activeGroupNameMtx); + return activeGroupName; +} + +void ServableGroupManager::recordActivity() { + lastActivityTimeNs->store( + std::chrono::steady_clock::now().time_since_epoch().count(), + std::memory_order_relaxed); +} + +std::vector ServableGroupManager::getAllConfiguredServableNames() const { + std::shared_lock lock(groupsMtx); + std::vector names; + for (const auto& [servableName, groupName] : servableToGroup) { + names.push_back(servableName); + } + return names; +} + +std::unordered_map ServableGroupManager::getGroups() const { + std::shared_lock lock(groupsMtx); + return groups; +} + +bool ServableGroupManager::canUnloadActiveGroup(ModelManager& mm) const { + // TODO @atobiszei this is vulnerable to TOCTOU + const std::string groupName = getActiveGroupName(); + std::shared_lock lock(groupsMtx); + auto it = groups.find(groupName); + if (it == groups.end()) { + return true; + } + const auto& groupInfo = it->second; + + // Check all classic models in the group + for (const auto& modelName : groupInfo.modelNames) { + auto model = mm.findModelByName(modelName); + if (model == nullptr) { + continue; + } + for (const auto& [version, instance] : model->getModelVersions()) { + if (!instance->canUnloadInstance()) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Cannot unload group '{}': model {} version {} has active requests", + groupName, modelName, version); + return false; + } + if (instance->getStatus().getState() == ModelVersionState::LOADING) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Cannot unload group '{}': model {} version {} is loading", + groupName, modelName, version); + return false; + } + } + } + +#if (MEDIAPIPE_DISABLE == 0) + // Check all mediapipe graphs in the group + for (const auto& graphName : groupInfo.mediapipeNames) { + MediapipeGraphDefinition* def = mm.getMediapipeFactory().findDefinitionByName(graphName); + if (def == nullptr) { + continue; + } + auto activeCount = def->getActiveInferenceCount(); + if (activeCount && activeCount->load(std::memory_order_acquire) > 0) { + SPDLOG_LOGGER_TRACE(modelmanager_logger, "Cannot unload group '{}': mediapipe graph {} has active inferences", + groupName, graphName); + return false; + } + } +#endif + + return true; +} + +Status ServableGroupManager::loadGroup(const std::string& groupName, ModelManager& mm, + const std::string& requestedServable) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Loading model group '{}'", groupName); + + std::shared_lock lock(groupsMtx); + auto it = groups.find(groupName); + if (it == groups.end()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Model group '{}' not found", groupName); + return StatusCode::GROUP_LOAD_FAILED; + } + const auto& groupInfo = it->second; + lock.unlock(); + + // The servable that triggered the wake-up is loaded first and preempts queued + // work, so time-to-first-response does not depend on group member ordering. + std::vector>> futures; + const bool isRequestedInGroup = groupInfo.modelNames.count(requestedServable) > 0 || + groupInfo.mediapipeNames.count(requestedServable) > 0; + if (isRequestedInGroup) { + futures.emplace_back(requestedServable, mm.requestServableWakeUp(requestedServable, /*urgent=*/true)); + } + for (const auto& modelName : groupInfo.modelNames) { + if (modelName == requestedServable) { + continue; + } + futures.emplace_back(modelName, mm.requestServableWakeUp(modelName, /*urgent=*/false)); + } +#if (MEDIAPIPE_DISABLE == 0) + for (const auto& graphName : groupInfo.mediapipeNames) { + if (graphName == requestedServable) { + continue; + } + futures.emplace_back(graphName, mm.requestServableWakeUp(graphName, /*urgent=*/false)); + } +#endif + + // Caller waits on the servable it asked for; the rest of the group only produces logs. + Status requestedStatus = isRequestedInGroup ? Status(StatusCode::GROUP_LOAD_FAILED) : Status(StatusCode::OK); + for (auto& [name, future] : futures) { + auto status = future.get(); + if (name == requestedServable) { + requestedStatus = status; + } + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to load '{}' in group '{}': {}", name, groupName, status.string()); + } else { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Loaded '{}' in group '{}'", name, groupName); + } + } + + // Set even on partial failure - loaded members must stay tracked so they can be unloaded later. + setActiveGroup(groupName); + recordActivity(); + + if (!requestedStatus.ok()) { + return requestedStatus; + } + SPDLOG_LOGGER_INFO(modelmanager_logger, "Model group '{}' loaded successfully", groupName); + return StatusCode::OK; +} + +Status ServableGroupManager::unloadGroup(const std::string& groupName, ModelManager& mm, bool urgent) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Unloading model group '{}'", groupName); + + std::shared_lock lock(groupsMtx); + auto it = groups.find(groupName); + if (it == groups.end()) { + return StatusCode::OK; + } + const auto& groupInfo = it->second; + lock.unlock(); + + // Enqueue put-to-sleep tasks via queue and collect futures + std::vector>> futures; + for (const auto& modelName : groupInfo.modelNames) { + futures.emplace_back(modelName, mm.requestServablePutToSleep(modelName, urgent)); + } +#if (MEDIAPIPE_DISABLE == 0) + for (const auto& graphName : groupInfo.mediapipeNames) { + futures.emplace_back(graphName, mm.requestServablePutToSleep(graphName, urgent)); + } +#endif + + for (auto& [name, future] : futures) { + auto status = future.get(); + if (!status.ok()) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Failed to unload '{}' in group '{}': {}", name, groupName, status.string()); + } else { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Unloaded '{}' in group '{}'", name, groupName); + } + } + + if (isActiveGroup(groupName)) { + setActiveGroup(""); + } + SPDLOG_LOGGER_INFO(modelmanager_logger, "Model group '{}' unloaded successfully", groupName); + return StatusCode::OK; +} + +[[nodiscard]] Status ServableGroupManager::swapToGroup(const std::string& groupName, ModelManager& mm, + const std::string& requestedServable) { + const std::string previousGroup = getActiveGroupName(); + if (!previousGroup.empty()) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Swapping model group from '{}' to '{}'", previousGroup, groupName); + // Wait for active requests to drain with bounded retry + constexpr int kMaxRetries = 300; // 30 seconds at 100ms intervals + constexpr int kRetryIntervalMs = 100; + for (int i = 0; i < kMaxRetries; ++i) { + if (canUnloadActiveGroup(mm)) { + break; + } + if (i == kMaxRetries - 1) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Timed out waiting for group '{}' to drain requests before swap to '{}'", + previousGroup, groupName); + return StatusCode::GROUP_UNLOAD_BLOCKED; + } + std::this_thread::sleep_for(std::chrono::milliseconds(kRetryIntervalMs)); + } + auto unloadStatus = unloadGroup(previousGroup, mm, /*urgent=*/true); + if (!unloadStatus.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to unload group '{}': {}", previousGroup, unloadStatus.string()); + return unloadStatus; + } + } + return loadGroup(groupName, mm, requestedServable); +} + + [[nodiscard]] Status ServableGroupManager::ensureServableLoaded(const std::string& servableName, ModelManager& mm) { + const std::string groupName = getGroupForServable(servableName); + if (groupName.empty()) { + // Not managed by group manager - let normal flow handle it + return StatusCode::OK; + } + if (isGroupLoaded(groupName) && mm.isServableAvailable(servableName)) { + recordActivity(); + return StatusCode::OK; + } + + // Serialize group swaps + std::lock_guard swapLock(loadUnloadMtx); + + // Double-check after acquiring the lock + if (isGroupLoaded(groupName) && mm.isServableAvailable(servableName)) { + recordActivity(); + return StatusCode::OK; + } + + // Group is resident but this member is not - e.g. it was slept on its own per-graph + // timeout, or it failed during the group load. No reason to swap the whole group. + if (isGroupLoaded(groupName)) { + auto status = mm.requestServableWakeUp(servableName, /*urgent=*/true).get(); + if (!status.ok()) { + return status; + } + recordActivity(); + return StatusCode::OK; + } + + return swapToGroup(groupName, mm, servableName); +} + +void ServableGroupManager::unloadActiveGroupIfIdle(ModelManager& mm) { + if (!isEnabled()) { + return; + } + std::string groupToUnload = getActiveGroupName(); + if (groupToUnload.empty()) { + return; + } + + // Check if we have a permanent group as active (should not happen, but safety) + { + std::shared_lock lock(groupsMtx); + auto it = groups.find(groupToUnload); + if (it != groups.end() && it->second.isPermanent()) { + return; + } + } + + // Check idle timeout + int64_t lastActivity = lastActivityTimeNs->load(std::memory_order_relaxed); + int64_t nowNs = std::chrono::steady_clock::now().time_since_epoch().count(); + int64_t timeoutNs = static_cast(idleTimeoutMicroseconds) * 1'000LL; + if ((nowNs - lastActivity) < timeoutNs) { + return; + } + + // Check if we can safely unload (no active requests) + if (!canUnloadActiveGroup(mm)) { + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Skipping idle unload of group '{}': active requests in flight", groupToUnload); + return; + } + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Idle unloading model group '{}' after {}us timeout", groupToUnload, idleTimeoutMicroseconds); + std::lock_guard swapLock(loadUnloadMtx); + // Re-check after acquiring lock + groupToUnload = getActiveGroupName(); + if (groupToUnload.empty()) { + return; + } + if (!canUnloadActiveGroup(mm)) { + return; + } + auto status = unloadGroup(groupToUnload, mm, /*urgent=*/false); + if (!status.ok()) { + SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to idle unload group '{}': {}", groupToUnload, status.string()); + } +} + +} // namespace ovms diff --git a/src/servable_management/servable_group_manager.hpp b/src/servable_management/servable_group_manager.hpp new file mode 100644 index 0000000000..9b2f5273e1 --- /dev/null +++ b/src/servable_management/servable_group_manager.hpp @@ -0,0 +1,88 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/status.hpp" + +namespace ovms { + +class ModelConfig; +class ModelManager; + +struct ModelGroupInfo { + std::string groupName; + std::set modelNames; + std::set mediapipeNames; + bool isPermanent() const { return groupName == "permanent"; } +}; + +class ServableGroupManager { +public: + explicit ServableGroupManager(uint64_t idleTimeoutMicroseconds); + + bool isEnabled() const { return idleTimeoutMicroseconds > 0; } + uint64_t getIdleTimeoutMicroseconds() const { return idleTimeoutMicroseconds; } + + void buildGroups(const std::unordered_map& modelConfigs, + ModelManager& mm); + + std::string getGroupForServable(const std::string& servableName) const; + + [[nodiscard]] Status ensureServableLoaded(const std::string& servableName, ModelManager& mm); + + void unloadActiveGroupIfIdle(ModelManager& mm); + + void recordActivity(); + + std::vector getAllConfiguredServableNames() const; + // needed only for tests + std::unordered_map getGroups() const; + std::string getActiveGroupName() const; + bool isGroupLoaded(const std::string& groupName) const; + +private: + bool canUnloadActiveGroup(ModelManager& mm) const; + [[nodiscard]] Status loadGroup(const std::string& groupName, ModelManager& mm, const std::string& requestedServable); + [[nodiscard]] Status unloadGroup(const std::string& groupName, ModelManager& mm, bool urgent); + [[nodiscard]] Status swapToGroup(const std::string& groupName, ModelManager& mm, const std::string& requestedServable); + bool isActiveGroup(const std::string& groupName) const; + void setActiveGroup(const std::string& groupName); + + uint64_t idleTimeoutMicroseconds; + + mutable std::shared_mutex groupsMtx; + std::unordered_map groups; + std::unordered_map servableToGroup; + + mutable std::mutex loadUnloadMtx; + // Read on every inference request, written only on group swaps. + mutable std::shared_mutex activeGroupNameMtx; + std::string activeGroupName; + + std::shared_ptr> lastActivityTimeNs; +}; + +} // namespace ovms diff --git a/src/servable_management/servable_loading_queue.cpp b/src/servable_management/servable_loading_queue.cpp new file mode 100644 index 0000000000..50a3e5905b --- /dev/null +++ b/src/servable_management/servable_loading_queue.cpp @@ -0,0 +1,112 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include "servable_loading_queue.hpp" + +#include + +#include "src/logging.hpp" + +namespace ovms { + +ServableLoadingQueue::~ServableLoadingQueue() { + stop(); +} + +void ServableLoadingQueue::start(TaskProcessor processor) { + std::lock_guard lock(this->mutex); + if (this->running) { + return; + } + this->processor = std::move(processor); + this->running = true; + this->worker = std::thread(&ServableLoadingQueue::workerLoop, this); +} + +void ServableLoadingQueue::requestStop() { + { + std::lock_guard lock(this->mutex); + if (!this->running) { + return; + } + this->running = false; + } + this->cv.notify_one(); +} + +void ServableLoadingQueue::setTaskObserver(TaskObserver observer) { + std::lock_guard lock(this->mutex); + this->taskObserver = std::move(observer); +} + +void ServableLoadingQueue::stop() { + requestStop(); + if (this->worker.joinable()) { + this->worker.join(); + } + std::lock_guard lock(this->mutex); + while (!this->queue.empty()) { + auto& task = this->queue.front(); + task.completion.set_value(StatusCode::SERVER_SHUTTING_DOWN); + this->queue.pop_front(); + } +} + +std::future ServableLoadingQueue::scheduleTask(ServableLoadingTask task) { + auto future = task.completion.get_future(); + { + std::lock_guard lock(this->mutex); + if (this->taskObserver) { + this->taskObserver(TaskEvent::Scheduled, task); + } + if (task.urgent) { + this->queue.push_front(std::move(task)); + } else { + this->queue.push_back(std::move(task)); + } + } + this->cv.notify_one(); + return future; +} + +void ServableLoadingQueue::workerLoop() { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Started servable loading queue thread"); + while (true) { + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, ""}; + TaskObserver observer; + { + std::unique_lock lock(this->mutex); + this->cv.wait(lock, [this] { return !this->queue.empty() || !this->running; }); + if (!this->running) { + break; + } + task = std::move(this->queue.front()); + this->queue.pop_front(); + // Copy under the lock: reading it unlocked would race with setTaskObserver(), + // and calling it locked would deadlock an observer that re-enters the queue. + observer = this->taskObserver; + } + if (observer) { + observer(TaskEvent::Executed, task); + } + SPDLOG_LOGGER_DEBUG(modelmanager_logger, "Processing {} task for: {}", + static_cast(task.type), task.name); + Status status = this->processor(task); + task.completion.set_value(status); + } + SPDLOG_LOGGER_INFO(modelmanager_logger, "Stopped servable loading queue thread"); +} + +} // namespace ovms diff --git a/src/servable_management/servable_loading_queue.hpp b/src/servable_management/servable_loading_queue.hpp new file mode 100644 index 0000000000..7032894d86 --- /dev/null +++ b/src/servable_management/servable_loading_queue.hpp @@ -0,0 +1,67 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "servable_loading_task.hpp" + +namespace ovms { + +using TaskProcessor = std::function; + +enum class TaskEvent { + Scheduled, + Executed +}; +using TaskObserver = std::function; + +class ServableLoadingQueue { +public: + ServableLoadingQueue() = default; + ~ServableLoadingQueue(); + + ServableLoadingQueue(const ServableLoadingQueue&) = delete; + ServableLoadingQueue& operator=(const ServableLoadingQueue&) = delete; + + void start(TaskProcessor processor); + // Blocks until worker joins, then drains pending tasks. Must always be called. + void stop(); + // Signals worker to stop without blocking. stop() must still be called after. + void requestStop(); + + void setTaskObserver(TaskObserver observer); + + std::future scheduleTask(ServableLoadingTask task); + +private: + void workerLoop(); + + TaskProcessor processor; + TaskObserver taskObserver; + std::thread worker; + std::deque queue; + std::mutex mutex; + std::condition_variable cv; + bool running = false; +}; + +} // namespace ovms diff --git a/src/servable_management/servable_loading_task.hpp b/src/servable_management/servable_loading_task.hpp new file mode 100644 index 0000000000..5c7ffeabc8 --- /dev/null +++ b/src/servable_management/servable_loading_task.hpp @@ -0,0 +1,73 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#pragma once + +#include +#include +#include + +#include "src/modelconfig.hpp" +#if (MEDIAPIPE_DISABLE == 0) +#include "src/mediapipe_internal/mediapipegraphconfig.hpp" +#endif + +namespace ovms { + +enum class ServableLoadingTaskType { + LoadModel, + RetireModel, + WakeUpModel, + PutToSleepModel, + LoadMediapipe, + RetireMediapipe, + WakeUpMediapipe, + PutToSleepMediapipe +}; + +struct ServableLoadingTask { + ServableLoadingTaskType type; + std::string name; + bool urgent = false; + std::optional modelConfig; +#if (MEDIAPIPE_DISABLE == 0) + std::optional graphConfig; +#endif + std::promise completion; + + ServableLoadingTask(ServableLoadingTaskType type, const std::string& name, const ModelConfig& config) : + type(type), + name(name), + modelConfig(config) {} + +#if (MEDIAPIPE_DISABLE == 0) + ServableLoadingTask(ServableLoadingTaskType type, const std::string& name, const MediapipeGraphConfig& config) : + type(type), + name(name), + graphConfig(config) {} +#endif + + ServableLoadingTask(ServableLoadingTaskType type, const std::string& name, bool urgent = false) : + type(type), + name(name), + urgent(urgent) {} + + ServableLoadingTask(ServableLoadingTask&&) = default; + ServableLoadingTask& operator=(ServableLoadingTask&&) = default; + ServableLoadingTask(const ServableLoadingTask&) = delete; + ServableLoadingTask& operator=(const ServableLoadingTask&) = delete; +}; + +} // namespace ovms diff --git a/src/servablemanagermodule.cpp b/src/servable_management/servablemanagermodule.cpp similarity index 92% rename from src/servablemanagermodule.cpp rename to src/servable_management/servablemanagermodule.cpp index 3c9e8ad291..a6fb908c31 100644 --- a/src/servablemanagermodule.cpp +++ b/src/servable_management/servablemanagermodule.cpp @@ -13,18 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** -#include "servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include #include -#include "config.hpp" -#include "logging.hpp" -#include "metrics/metric_module.hpp" +#include "src/config.hpp" +#include "src/logging.hpp" +#include "src/metrics/metric_module.hpp" #include "modelmanager.hpp" -#include "server.hpp" +#include "src/server.hpp" #if (PYTHON_DISABLE == 0) -#include "python/pythoninterpretermodule.hpp" +#include "src/python/pythoninterpretermodule.hpp" #endif namespace ovms { diff --git a/src/servablemanagermodule.hpp b/src/servable_management/servablemanagermodule.hpp similarity index 97% rename from src/servablemanagermodule.hpp rename to src/servable_management/servablemanagermodule.hpp index 1ccee8862f..1b06ea142f 100644 --- a/src/servablemanagermodule.hpp +++ b/src/servable_management/servablemanagermodule.hpp @@ -16,7 +16,7 @@ #pragma once #include -#include "module.hpp" +#include "src/module.hpp" namespace ovms { class Config; diff --git a/src/server.cpp b/src/server.cpp index f0a27a770b..e6f8cc9b36 100644 --- a/src/server.cpp +++ b/src/server.cpp @@ -60,12 +60,12 @@ #include "kfs_frontend/kfs_grpc_inference_service.hpp" #include "logging.hpp" #include "metrics/metric_module.hpp" -#include "modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "ovms_exit_codes.hpp" #include "profiler.hpp" #include "profilermodule.hpp" #include "pull_module/hf_pull_model_module.hpp" -#include "servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "shutdown_state.hpp" #include "servables_config_manager_module/servablesconfigmanagermodule.hpp" #include "stringutils.hpp" @@ -428,6 +428,14 @@ Status Server::startModules(ovms::Config& config) { if (config.getServerSettings().withPython) { INSERT_MODULE(PYTHON_INTERPRETER_MODULE_NAME, it); START_MODULE(it); + auto pythonModule = dynamic_cast(it->second.get()); + if (pythonModule->ownsPythonInterpreter()) { + // Natively GIL is held by the thread that initialized interpreter, so we only need to release it, if we own the interpreter. + // If it was initialized externally, then the external thread shall release the GIL before launching that module. + // Must happen before ServableManagerModule starts: it synchronously loads the initial config on the + // servable loading queue's worker thread, which needs to acquire the GIL for Python-backed nodes. + pythonModule->releaseGILFromThisThread(); + } } #endif #if MTR_ENABLED @@ -473,17 +481,6 @@ Status Server::startModules(ovms::Config& config) { } GET_MODULE(SERVABLE_MANAGER_MODULE_NAME, it); START_MODULE(it); -#if (PYTHON_DISABLE == 0) - if (config.getServerSettings().withPython) { - GET_MODULE(PYTHON_INTERPRETER_MODULE_NAME, it); - auto pythonModule = dynamic_cast(it->second.get()); - if (pythonModule->ownsPythonInterpreter()) { - // Natively GIL is held by the thread that initialized interpreter, so we only need to release it, if we own the interpreter. - // If it was initialized externally, then the external thread shall release the GIL before launching that module. - pythonModule->releaseGILFromThisThread(); - } - } -#endif return status; } diff --git a/src/single_version_servable_definition.hpp b/src/single_version_servable_definition.hpp index 6bdebd114b..9f7b3744a9 100644 --- a/src/single_version_servable_definition.hpp +++ b/src/single_version_servable_definition.hpp @@ -56,11 +56,11 @@ class SingleVersionServableDefinition : public ServableDefinition, public Servab uint32_t waitForLoadedTimeoutMicroseconds = WAIT_FOR_LOADED_DEFAULT_TIMEOUT_MICROSECONDS); protected: - std::atomic requestsHandlesCounter = 0; + std::atomic pendingCreateExecutorCount = 0; std::condition_variable loadedNotify; - void increaseRequestsHandlesCount() { ++requestsHandlesCounter; } - void decreaseRequestsHandlesCount() { --requestsHandlesCounter; } + void increaseRequestsHandlesCount() { ++pendingCreateExecutorCount; } + void decreaseRequestsHandlesCount() { --pendingCreateExecutorCount; } virtual StatusCode notLoadedYetCode() const = 0; virtual StatusCode notLoadedAnymoreCode() const = 0; diff --git a/src/status.cpp b/src/status.cpp index 0394192e86..c3220fdc71 100644 --- a/src/status.cpp +++ b/src/status.cpp @@ -213,6 +213,9 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::MEDIAPIPE_INCORRECT_SERVABLE_NAME, "Subsequent request with incorrect servable name"}, {StatusCode::MEDIAPIPE_INCORRECT_SERVABLE_VERSION, "Subsequent request with incorrect servable version"}, {StatusCode::MEDIAPIPE_PRECONDITION_FAILED, "Mediapipe graph precondition failed"}, + {StatusCode::MEDIAPIPE_PUT_TO_SLEEP_STATE_NOT_AVAILABLE, "Cannot put mediapipe graph to sleep: state is not AVAILABLE"}, + {StatusCode::MEDIAPIPE_PUT_TO_SLEEP_REQUESTS_IN_FLIGHT, "Cannot put mediapipe graph to sleep: requests are in flight"}, + {StatusCode::MEDIAPIPE_PUT_TO_SLEEP_ACTIVE_INFERENCES, "Cannot put mediapipe graph to sleep: active inferences in progress"}, // Python Nodes {StatusCode::PYTHON_NODE_NAME_ALREADY_EXISTS, "The Python Node name is already present in nodes list"}, @@ -318,6 +321,7 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::NONEXISTENT_LOG_LEVEL, "Tried to use nonexisting log level"}, {StatusCode::NONEXISTENT_PTR, "Tried to use nonexisting pointer"}, {StatusCode::SERVER_NOT_READY, "Server is not ready"}, + {StatusCode::SERVER_SHUTTING_DOWN, "Server is shutting down"}, // Server Start errors {StatusCode::OPTIONS_USAGE_ERROR, "options validation error"}, @@ -346,5 +350,8 @@ const std::unordered_map Status::statusMessageMap = { {StatusCode::DEVICE_WRONG_FORMAT, "Device is in wrong format"}, {StatusCode::SHAPE_DYNAMIC_BUT_NPU_USED, "Shape is dynamic but NPU is used"}, {StatusCode::STATIC_RESOLUTION_MISUSE, "Wrong usage of static resolution"}, + + {StatusCode::GROUP_LOAD_FAILED, "Model group failed to load"}, + {StatusCode::GROUP_UNLOAD_BLOCKED, "Cannot unload model group due to active requests"}, }; } // namespace ovms diff --git a/src/status.hpp b/src/status.hpp index 94be7948cb..166fe99bdb 100644 --- a/src/status.hpp +++ b/src/status.hpp @@ -259,6 +259,9 @@ enum class StatusCode { MEDIAPIPE_INCORRECT_SERVABLE_NAME, MEDIAPIPE_INCORRECT_SERVABLE_VERSION, MEDIAPIPE_PRECONDITION_FAILED, + MEDIAPIPE_PUT_TO_SLEEP_STATE_NOT_AVAILABLE, + MEDIAPIPE_PUT_TO_SLEEP_REQUESTS_IN_FLIGHT, + MEDIAPIPE_PUT_TO_SLEEP_ACTIVE_INFERENCES, // Python Nodes PYTHON_NODE_NAME_ALREADY_EXISTS, @@ -330,6 +333,7 @@ enum class StatusCode { NONEXISTENT_LOG_LEVEL, NONEXISTENT_PTR, SERVER_NOT_READY, + SERVER_SHUTTING_DOWN, // Server Start errors OPTIONS_USAGE_ERROR, @@ -359,6 +363,10 @@ enum class StatusCode { SHAPE_DYNAMIC_BUT_NPU_USED, STATIC_RESOLUTION_MISUSE, + // Model Group Management + GROUP_LOAD_FAILED, + GROUP_UNLOAD_BLOCKED, + STATUS_CODE_END }; diff --git a/src/test/c_api_stress_tests.cpp b/src/test/c_api_stress_tests.cpp index 79cca10323..1a281e71f2 100644 --- a/src/test/c_api_stress_tests.cpp +++ b/src/test/c_api_stress_tests.cpp @@ -31,7 +31,7 @@ #include "../modelconfig.hpp" #include "../modelinstance.hpp" #include "../prediction_service_utils.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../status.hpp" #include "../stringutils.hpp" diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index a1bbd5dbf0..a2f9ac1da2 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -41,7 +41,7 @@ #include "../filesystem/filesystem.hpp" #include "src/metrics/metric_module.hpp" #include "../ovms.h" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../version.hpp" #include "c_api_test_utils.hpp" @@ -1979,7 +1979,7 @@ class MockModelInstanceWithSetOutputInfo : public ovms::ModelInstance { status = ovms::ModelVersionStatus("UNUSED_NAME", UNUSED_MODEL_VERSION, ovms::ModelVersionState::START); } virtual ~MockModelInstanceWithSetOutputInfo() {} - ovms::Status loadModel(const ovms::ModelConfig& config) override { + ovms::Status loadModel(const ovms::ModelConfig& config, bool lazyLoad = false) override { ModelInstance::loadModel(config); return ovms::StatusCode::OK; } diff --git a/src/test/constructor_enabled_model_manager.cpp b/src/test/constructor_enabled_model_manager.cpp index 87d0202b5d..99cee92033 100644 --- a/src/test/constructor_enabled_model_manager.cpp +++ b/src/test/constructor_enabled_model_manager.cpp @@ -17,10 +17,19 @@ #include -#include "../status.hpp" +#include + +#include "src/servable_management/servable_group_manager.hpp" +#include "src/status.hpp" ConstructorEnabledModelManager::ConstructorEnabledModelManager(const std::string& modelCacheDirectory, ovms::PythonBackend* pythonBackend) : ovms::ModelManager(modelCacheDirectory, ®istry, pythonBackend) {} + +ConstructorEnabledModelManager::ConstructorEnabledModelManager(uint64_t idleTimeoutMicroseconds) : + ovms::ModelManager("", ®istry, nullptr) { + servableGroupManager = std::make_unique(idleTimeoutMicroseconds); +} + ConstructorEnabledModelManager::~ConstructorEnabledModelManager() { join(); spdlog::info("Destructor of modelmanager(Enabled one). Models #:{}", models.size()); diff --git a/src/test/constructor_enabled_model_manager.hpp b/src/test/constructor_enabled_model_manager.hpp index 9ceeaf970d..f192b85b2b 100644 --- a/src/test/constructor_enabled_model_manager.hpp +++ b/src/test/constructor_enabled_model_manager.hpp @@ -18,23 +18,21 @@ #include #include "src/metrics/metric_registry.hpp" -#include "src/modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" +#include "src/servable_management/servable_loading_queue.hpp" class ConstructorEnabledModelManager : public ovms::ModelManager { ovms::MetricRegistry registry; public: ConstructorEnabledModelManager(const std::string& modelCacheDirectory = "", ovms::PythonBackend* pythonBackend = nullptr); + ConstructorEnabledModelManager(uint64_t idleTimeoutMicroseconds); ~ConstructorEnabledModelManager(); - /* - * Loads config but resets the config filename to the one provided in the argument. In production server this is only changed once - */ + ovms::Status loadConfig(const std::string& jsonFilename); - /** - * @brief Updates OVMS configuration with cached configuration file. Will check for newly added model versions - */ void updateConfigurationWithoutConfigFile(); void setWaitForModelLoadedTimeoutMs(int value); + ovms::ServableLoadingQueue& getLoadingQueue() { return *loadingQueue; } }; class ResourcesAccessModelManager : public ConstructorEnabledModelManager { public: diff --git a/src/test/embeddingsnode_test.cpp b/src/test/embeddingsnode_test.cpp index 1b9f1e1313..724fe8d852 100644 --- a/src/test/embeddingsnode_test.cpp +++ b/src/test/embeddingsnode_test.cpp @@ -21,7 +21,7 @@ #include "../embeddings/embeddings_node_initializer_utils.hpp" #include "../http_rest_api_handler.hpp" #include "../mediapipe_internal/mediapipefactory.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "rapidjson/document.h" #include "test_http_utils.hpp" diff --git a/src/test/ensemble_config_change_stress.cpp b/src/test/ensemble_config_change_stress.cpp index 9da708dc56..10a3cf594d 100644 --- a/src/test/ensemble_config_change_stress.cpp +++ b/src/test/ensemble_config_change_stress.cpp @@ -28,7 +28,7 @@ #include "../kfs_frontend/kfs_utils.hpp" #include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../status.hpp" #include "../stringutils.hpp" diff --git a/src/test/environment.cpp b/src/test/environment.cpp index d49cd3a8f4..e6ed042e35 100644 --- a/src/test/environment.cpp +++ b/src/test/environment.cpp @@ -33,9 +33,26 @@ void Environment::SetUp() { } else { SPDLOG_INFO("Unstable tests will be skipped since RUN_UNSTABLE env variable was not set to 1. Remember to use bazel test parameter --test_env when triggering tests using bazel."); } + const char* runAllIdleTestsEnv = std::getenv("RUN_ALL_IDLE"); + if (runAllIdleTestsEnv) { + std::string runAllIdleTestsEnvContent(runAllIdleTestsEnv); + if (runAllIdleTestsEnvContent == "1") { + Environment::runAllIdleTests = true; + SPDLOG_INFO("RUN_ALL_IDLE was set to 1. Will run idle servable tests documenting known defects"); + } else { + SPDLOG_WARN("Idle servable tests documenting known defects will be skipped since RUN_ALL_IDLE env variable was not set to 1. It was set to: {}", runAllIdleTestsEnvContent); + } + } else { + SPDLOG_INFO("Idle servable tests documenting known defects will be skipped since RUN_ALL_IDLE env variable was not set to 1. Remember to use bazel test parameter --test_env when triggering tests using bazel."); + } } bool Environment::shouldRunUnstableTests() { return Environment::runUnstableTests; } +bool Environment::shouldRunAllIdleTests() { + return Environment::runAllIdleTests; +} + bool Environment::runUnstableTests = false; +bool Environment::runAllIdleTests = false; diff --git a/src/test/environment.hpp b/src/test/environment.hpp index a3b2b3dc32..6d70c3bc6f 100644 --- a/src/test/environment.hpp +++ b/src/test/environment.hpp @@ -24,9 +24,18 @@ return; \ } +// Gates tests that document known idle servable management defects and are expected to fail until fixed. +#define SKIP_AND_EXIT_IF_NOT_RUNNING_ALL_IDLE(reason) \ + if (!Environment::shouldRunAllIdleTests()) { \ + GTEST_SKIP() << "Skipping idle test since RUN_ALL_IDLE was not set to 1. " << (reason); \ + return; \ + } + class Environment : public testing::Environment { public: void SetUp() override; static bool runUnstableTests; static bool shouldRunUnstableTests(); + static bool runAllIdleTests; + static bool shouldRunAllIdleTests(); }; diff --git a/src/test/get_mediapipe_graph_metadata_response_test.cpp b/src/test/get_mediapipe_graph_metadata_response_test.cpp index ab4c3debd5..f227e8bbb3 100644 --- a/src/test/get_mediapipe_graph_metadata_response_test.cpp +++ b/src/test/get_mediapipe_graph_metadata_response_test.cpp @@ -33,7 +33,7 @@ #include "../model.hpp" #include "../modelinstance.hpp" #include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "../modelversionstatus.hpp" #include "../prediction_service_utils.hpp" #include "../schema.hpp" diff --git a/src/test/http_openai_handler_test.cpp b/src/test/http_openai_handler_test.cpp index 65ea56336c..607658dd10 100644 --- a/src/test/http_openai_handler_test.cpp +++ b/src/test/http_openai_handler_test.cpp @@ -33,7 +33,7 @@ #include "../client_connection.hpp" #include #include "../module_names.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "environment.hpp" #include "src/utils/env_guard.hpp" diff --git a/src/test/http_rest_api_handler_test.cpp b/src/test/http_rest_api_handler_test.cpp index e17ca23b25..ef93da62f6 100644 --- a/src/test/http_rest_api_handler_test.cpp +++ b/src/test/http_rest_api_handler_test.cpp @@ -19,8 +19,8 @@ #include "../http_rest_api_handler.hpp" #include "src/filesystem/localfilesystem.hpp" #include "../logging.hpp" -#include "../modelmanager.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/modelmanager.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "platform_utils.hpp" #include "test_utils.hpp" diff --git a/src/test/idle_mediapipe_test.cpp b/src/test/idle_mediapipe_test.cpp new file mode 100644 index 0000000000..dd2dc932c1 --- /dev/null +++ b/src/test/idle_mediapipe_test.cpp @@ -0,0 +1,135 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include + +#include + +#include "src/dags/pipelinedefinitionstatus.hpp" +#include "src/mediapipe_internal/mediapipegraphconfig.hpp" +#include "src/mediapipe_internal/mediapipegraphdefinition.hpp" +#include "src/status.hpp" +#include "constructor_enabled_model_manager.hpp" +#include "test_utils.hpp" + +using namespace ovms; + +static const std::string kSimplePbtxt = R"( + input_stream: "in" + output_stream: "out" +)"; + +class MediapipeIdleSleepTest : public ::testing::Test { +protected: + ConstructorEnabledModelManager manager; + + std::unique_ptr makeSleepingDef(const std::string& name) { + MediapipeGraphConfig mgc{name, "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + return std::make_unique(name, mgc, kSimplePbtxt, nullptr, true); + } + + std::unique_ptr makeAvailableDef(const std::string& name) { + MediapipeGraphConfig mgc{name, "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + auto def = std::make_unique(name, mgc, kSimplePbtxt, nullptr); + def->forceValidationPassedEventForTest(); + return def; + } +}; + +TEST_F(MediapipeIdleSleepTest, LazyLoadStartsSleeping) { + auto def = makeSleepingDef("graph1"); + EXPECT_EQ(def->getStateCode(), PipelineDefinitionStateCode::SLEEPING); + EXPECT_TRUE(def->getStatus().isSleeping()); +} + +TEST_F(MediapipeIdleSleepTest, UnloadTransitionsAvailableToSleeping) { + auto def = makeAvailableDef("graph1"); + ASSERT_EQ(def->getStateCode(), PipelineDefinitionStateCode::AVAILABLE); + + ASSERT_EQ(def->putToSleep(), StatusCode::OK); + EXPECT_EQ(def->getStateCode(), PipelineDefinitionStateCode::SLEEPING); +} + +TEST_F(MediapipeIdleSleepTest, UnloadOnSleepingIsNoop) { + auto def = makeSleepingDef("graph1"); + ASSERT_EQ(def->putToSleep(), StatusCode::OK); + EXPECT_EQ(def->getStateCode(), PipelineDefinitionStateCode::SLEEPING); +} + +TEST_F(MediapipeIdleSleepTest, WakeUpOnAvailableIsNoop) { + auto def = makeAvailableDef("graph1"); + ASSERT_EQ(def->wakeUpIfSleeping(manager), StatusCode::OK); + EXPECT_EQ(def->getStateCode(), PipelineDefinitionStateCode::AVAILABLE); +} + +TEST_F(MediapipeIdleSleepTest, WakeUpOnRetiredReturnsError) { + auto def = makeAvailableDef("graph1"); + def->retire(); + ASSERT_EQ(def->getStateCode(), PipelineDefinitionStateCode::RETIRED); + + auto status = def->wakeUpIfSleeping(manager); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.getCode(), StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE); +} + +TEST_F(MediapipeIdleSleepTest, WakeUpOnBeginReturnsError) { + MediapipeGraphConfig mgc{"graph1", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("graph1", mgc, kSimplePbtxt, nullptr); + ASSERT_EQ(def.getStateCode(), PipelineDefinitionStateCode::BEGIN); + + auto status = def.wakeUpIfSleeping(manager); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.getCode(), StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE); +} + +TEST_F(MediapipeIdleSleepTest, ConcurrentWakeUpAllSucceed) { + auto def = makeAvailableDef("graph1"); + ASSERT_EQ(def->putToSleep(), StatusCode::OK); + ASSERT_EQ(def->getStateCode(), PipelineDefinitionStateCode::SLEEPING); + + constexpr int numThreads = 8; + std::promise startSignal; + std::shared_future ready = startSignal.get_future().share(); + std::vector> threadReady(numThreads); + std::vector results(numThreads); + std::vector threads; + threads.reserve(numThreads); + for (int i = 0; i < numThreads; ++i) { + threads.emplace_back([&results, &def, &ready, &threadReady, &mgr = manager, i]() { + threadReady[i].set_value(); + ready.wait(); + results[i] = def->wakeUpIfSleeping(mgr); + }); + } + for (int i = 0; i < numThreads; ++i) { + threadReady[i].get_future().wait(); + } + startSignal.set_value(); + for (auto& t : threads) { + t.join(); + } + // With trivial pbtxt, reload may fail (no real graph), so we just verify + // no crash/deadlock and state is consistent. + auto finalState = def->getStateCode(); + EXPECT_TRUE(finalState == PipelineDefinitionStateCode::AVAILABLE || + finalState == PipelineDefinitionStateCode::SLEEPING); +} diff --git a/src/test/idle_model_test.cpp b/src/test/idle_model_test.cpp new file mode 100644 index 0000000000..eced2991d3 --- /dev/null +++ b/src/test/idle_model_test.cpp @@ -0,0 +1,274 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include +#include + +#include + +#include + +#include "src/model.hpp" +#include "src/modelinstance.hpp" +#include "src/modelversionstatus.hpp" +#include "constructor_enabled_model_manager.hpp" +#include "platform_utils.hpp" +#include "test_utils.hpp" +#include "test_models.hpp" +#include "test_models_configs.hpp" +#include "test_with_temp_dir.hpp" + +using namespace ovms; + +static const std::string idleModelConfig = R"({ + "model_config_list": [ + { + "config": { + "name": "dummy", + "base_path": ")" + dummy_model_location + + R"(", + "target_device": "CPU", + "model_version_policy": {"all": {}} + } + } + ] +})"; + +class IdleModelManagementTest : public TestWithTempDir { +protected: + std::string configFilePath; + + void writeConfig(const std::string& content) { + configFilePath = directoryPath + "/config.json"; + std::ofstream ofs(configFilePath); + ofs << content; + } + + void SetUp() override { + TestWithTempDir::SetUp(); + writeConfig(idleModelConfig); + } +}; + +TEST_F(IdleModelManagementTest, NonPermanentModelStartsAsSleepingButAppearsAvailable) { + ConstructorEnabledModelManager manager(30'000'000); + auto status = manager.loadConfig(configFilePath); + ASSERT_TRUE(status.ok()) << status.string(); + auto model = manager.findModelByName("dummy"); + ASSERT_NE(model, nullptr); + auto instance = model->getDefaultModelInstance(); + ASSERT_NE(instance, nullptr); + EXPECT_EQ(instance->getStatus().getState(), ModelVersionState::SLEEPING); + + auto availableNames = manager.getNamesOfAvailableModels(); + EXPECT_NE(std::find(availableNames.begin(), availableNames.end(), "dummy"), + availableNames.end()); +} + +TEST_F(IdleModelManagementTest, PermanentGroupModelIsFullyLoaded) { + std::string permanentConfig = R"({ + "model_config_list": [ + { + "config": { + "name": "dummy", + "base_path": ")" + + dummy_model_location + R"(", + "target_device": "CPU", + "model_version_policy": {"all": {}}, + "group_name": "permanent" + } + } + ] + })"; + writeConfig(permanentConfig); + + ConstructorEnabledModelManager manager(30'000'000); + auto status = manager.loadConfig(configFilePath); + ASSERT_TRUE(status.ok()) << status.string(); + auto model = manager.findModelByName("dummy"); + ASSERT_NE(model, nullptr); + auto instance = model->getDefaultModelInstance(); + ASSERT_NE(instance, nullptr); + EXPECT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); +} + +TEST_F(IdleModelManagementTest, SleepingModelSelectedAsDefaultVersion) { + ConstructorEnabledModelManager manager(30'000'000); + auto status = manager.loadConfig(configFilePath); + ASSERT_TRUE(status.ok()) << status.string(); + + auto model = manager.findModelByName("dummy"); + ASSERT_NE(model, nullptr); + auto instance = model->getDefaultModelInstance(); + ASSERT_NE(instance, nullptr); + EXPECT_TRUE(instance->getStatus().appearsAvailable()); + EXPECT_TRUE(instance->getStatus().isSleeping()); +} + +class ModelInstanceSleepTest : public ::testing::Test { +protected: + std::unique_ptr ieCore; + void SetUp() override { + ieCore = std::make_unique(); + } +}; + +TEST_F(ModelInstanceSleepTest, LazyLoadThenWakeUp) { + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(DUMMY_MODEL_CONFIG, true), StatusCode::OK); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::SLEEPING); + + auto status = instance.wakeUpIfSleeping(); + ASSERT_TRUE(status.ok()) << status.string(); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::AVAILABLE); +} + +TEST_F(ModelInstanceSleepTest, WakeUpThenPutToSleep) { + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(DUMMY_MODEL_CONFIG, true), StatusCode::OK); + ASSERT_TRUE(instance.wakeUpIfSleeping().ok()); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::AVAILABLE); + + instance.putToSleep(); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::SLEEPING); +} + +TEST_F(ModelInstanceSleepTest, WakeUpWithInvalidPathFails) { + const std::string nonexistentPath = getGenericFullPathForTmp("/tmp/idle_model_test_nonexistent_path"); + ModelConfig badConfig = DUMMY_MODEL_CONFIG; + badConfig.setBasePath(nonexistentPath); + badConfig.setLocalPath(nonexistentPath); + + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(badConfig, true), StatusCode::OK); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::SLEEPING); + + auto status = instance.wakeUpIfSleeping(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::SLEEPING); + + // Failed wake-up should remain retryable on every next request. + status = instance.wakeUpIfSleeping(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::SLEEPING); +} + +TEST_F(ModelInstanceSleepTest, WakeUpIfAlreadyAvailableIsNoop) { + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(DUMMY_MODEL_CONFIG, true), StatusCode::OK); + ASSERT_TRUE(instance.wakeUpIfSleeping().ok()); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::AVAILABLE); + + ASSERT_TRUE(instance.wakeUpIfSleeping().ok()); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::AVAILABLE); +} + +TEST_F(ModelInstanceSleepTest, WakeUpOnRetiredModelReturnsError) { + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(DUMMY_MODEL_CONFIG, true), StatusCode::OK); + instance.retireModel(); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::END); + + auto status = instance.wakeUpIfSleeping(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.getCode(), StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE); +} + +TEST_F(ModelInstanceSleepTest, ConcurrentWakeUpAllSucceed) { + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(DUMMY_MODEL_CONFIG, true), StatusCode::OK); + + constexpr int numThreads = 20; + std::promise startSignal; + std::shared_future ready = startSignal.get_future().share(); + std::vector> threadReady(numThreads); + std::vector results(numThreads); + std::vector threads; + threads.reserve(numThreads); + for (int i = 0; i < numThreads; ++i) { + threads.emplace_back([&results, &instance, &ready, &threadReady, i]() { + threadReady[i].set_value(); + ready.wait(); + results[i] = instance.wakeUpIfSleeping(); + }); + } + for (int i = 0; i < numThreads; ++i) { + threadReady[i].get_future().wait(); + } + startSignal.set_value(); + for (auto& t : threads) { + t.join(); + } + for (int i = 0; i < numThreads; ++i) { + EXPECT_TRUE(results[i].ok()) << "Thread " << i << " failed: " << results[i].string(); + } + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::AVAILABLE); +} + +TEST_F(ModelInstanceSleepTest, RetireThenWakeUpReturnsError) { + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(DUMMY_MODEL_CONFIG, true), StatusCode::OK); + ASSERT_TRUE(instance.wakeUpIfSleeping().ok()); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::AVAILABLE); + + instance.retireModel(); + EXPECT_EQ(instance.getStatus().getState(), ModelVersionState::END); + + auto status = instance.wakeUpIfSleeping(); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(status.getCode(), StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE); +} + +TEST_F(ModelInstanceSleepTest, ConcurrentWakeUpAndPutToSleep) { + ModelInstance instance("dummy", 1, *ieCore); + ASSERT_EQ(instance.loadModel(DUMMY_MODEL_CONFIG, true), StatusCode::OK); + ASSERT_EQ(instance.getStatus().getState(), ModelVersionState::SLEEPING); + + constexpr int numWakers = 20; + std::promise startSignal; + std::shared_future ready = startSignal.get_future().share(); + std::vector> threadReady(numWakers + 1); + std::vector wakeResults(numWakers); + std::vector threads; + threads.reserve(numWakers + 1); + + for (int i = 0; i < numWakers; ++i) { + threads.emplace_back([&wakeResults, &instance, &ready, &threadReady, i]() { + threadReady[i].set_value(); + ready.wait(); + wakeResults[i] = instance.wakeUpIfSleeping(); + }); + } + threads.emplace_back([&instance, &ready, &threadReady, numWakers]() { + threadReady[numWakers].set_value(); + ready.wait(); + instance.putToSleep(); + }); + + for (int i = 0; i <= numWakers; ++i) { + threadReady[i].get_future().wait(); + } + startSignal.set_value(); + for (auto& t : threads) { + t.join(); + } + + auto finalState = instance.getStatus().getState(); + EXPECT_TRUE(finalState == ModelVersionState::AVAILABLE || + finalState == ModelVersionState::SLEEPING); +} diff --git a/src/test/kfs_metadata_test.cpp b/src/test/kfs_metadata_test.cpp index 5a69913387..8867c3a4d5 100644 --- a/src/test/kfs_metadata_test.cpp +++ b/src/test/kfs_metadata_test.cpp @@ -63,7 +63,7 @@ class ModelMetadataResponseBuild : public ::testing::Test { } // Keeps the model in loading state forever - ovms::Status loadModel(const ovms::ModelConfig& config) override { + ovms::Status loadModel(const ovms::ModelConfig& config, bool lazyLoad = false) override { status.setLoading(); return ovms::StatusCode::OK; } diff --git a/src/test/kfs_rest_test.cpp b/src/test/kfs_rest_test.cpp index ea58b0dd7b..693aaf12a5 100644 --- a/src/test/kfs_rest_test.cpp +++ b/src/test/kfs_rest_test.cpp @@ -26,7 +26,7 @@ #include "../grpcservermodule.hpp" #include "../http_async_writer_interface.hpp" #include "../http_rest_api_handler.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../status.hpp" #include "../version.hpp" diff --git a/src/test/llm/llmnode_test.cpp b/src/test/llm/llmnode_test.cpp index 8d37a2676e..363bb21d93 100644 --- a/src/test/llm/llmnode_test.cpp +++ b/src/test/llm/llmnode_test.cpp @@ -6193,3 +6193,499 @@ TEST_F(DetectDraftModelStrategyTest, MtpTakesPriorityOverXmlScan) { << "\n\n\n\n\n"; EXPECT_EQ(ovms::detectDraftModelStrategy(directoryPath), DS::MTP); } +// --------------------------------------------------------------------------- +// Idle unload feature: LLM graph lifecycle (issue #4141) +// These tests require the opt-125m model fixture. +// --------------------------------------------------------------------------- + +class LLMIdleUnloadTest : public ::testing::Test { +protected: + // Builds a minimal continuous-batching LLM graph pbtxt pointing at opt-125m. + static std::string buildOptGraphPbtxt() { + std::string modelsPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m"); + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + + node: { + name: "llmNode" + calculator: "HttpLLMCalculator" + input_stream: "LOOPBACK:loopback" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + input_side_packet: "LLM_NODE_RESOURCES:llm" + output_stream: "LOOPBACK:loopback" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + input_stream_info: { + tag_index: 'LOOPBACK:0', + back_edge: true + } + node_options: { + [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { + models_path: ")" + + modelsPath + R"(" + cache_size: 1 + } + } + input_stream_handler { + input_stream_handler: "SyncSetInputStreamHandler", + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { + tag_index: "LOOPBACK:0" + } + } + } + } + } + )"; + adjustConfigForTargetPlatform(testPbtxt); + return testPbtxt; + } +}; + +static int64_t secondsAgo(int64_t seconds) { + return std::chrono::steady_clock::now().time_since_epoch().count() - seconds * 1'000'000'000LL; +} + +// Unload after idle: build LLM graph with small timeout, simulate idle, unload, assert freed. +TEST_F(LLMIdleUnloadTest, UnloadAfterIdleFreesResources) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); + ASSERT_TRUE(def.isIdleUnloadEnabled()); + + // Not yet idle -> should not unload. + ASSERT_FALSE(def.shouldUnloadDueToIdle()); + + // Backdate activity well past the timeout. + def.recordActivity(secondsAgo(60)); + ASSERT_TRUE(def.shouldUnloadDueToIdle()); + + ASSERT_EQ(def.putToSleep(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + // Resources freed: sidePacketMaps is reset. + ASSERT_EQ(def.sidePacketMapsPtrForTest(), nullptr); + ASSERT_FALSE(def.isAvailable()); + ASSERT_TRUE(def.getStatus().isSleeping()); +} + +// Lazy reload: after unload, wakeUpIfSleeping brings it back to AVAILABLE with resources. +TEST_F(LLMIdleUnloadTest, WakeUpReloadsResources) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + def.recordActivity(secondsAgo(60)); + ASSERT_EQ(def.putToSleep(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), nullptr); + + // Wake up. + ASSERT_EQ(def.wakeUpIfSleeping(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_TRUE(def.isAvailable()); + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); + + // Wake-up while already AVAILABLE is a no-op success. + ASSERT_EQ(def.wakeUpIfSleeping(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +// Idle timer reset: acquiring the graph (create) refreshes lastActivity. +TEST_F(LLMIdleUnloadTest, CreateResetsIdleTimer) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + // Make it look idle. + def.recordActivity(secondsAgo(60)); + ASSERT_TRUE(def.shouldUnloadDueToIdle()); + + // Acquiring the graph updates lastActivity, so it is no longer idle. + std::unique_ptr executor; + ASSERT_EQ(def.create(executor), StatusCode::OK); + ASSERT_NE(executor, nullptr); + ASSERT_FALSE(def.shouldUnloadDueToIdle()); +} + +// Disabled by default: timeout 0 -> never idle-unloads. +TEST_F(LLMIdleUnloadTest, DisabledByDefaultNeverUnloads) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + // idle_unload_timeout_seconds not set -> defaults to 0 (disabled) + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + ASSERT_FALSE(def.isIdleUnloadEnabled()); + def.recordActivity(secondsAgo(100000)); + ASSERT_FALSE(def.shouldUnloadDueToIdle()); +} + +// Exactly-one-reload under concurrency: N threads call wakeUpIfSleeping on an SLEEPING def. +// Best-effort: asserts all end AVAILABLE and the graph is loaded exactly once afterwards. +// Note: this verifies the end-state invariant (single AVAILABLE graph, resources present); +// the per-definition mutex guarantees a single reload, but counting reloads deterministically +// from the test would require instrumentation hooks not present, so we assert the observable +// post-condition instead. +TEST_F(LLMIdleUnloadTest, ConcurrentWakeUpEndsAvailable) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + def.recordActivity(secondsAgo(60)); + ASSERT_EQ(def.putToSleep(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + + constexpr int kThreads = 8; + std::vector threads; + std::vector results(kThreads, StatusCode::UNKNOWN_ERROR); + for (int i = 0; i < kThreads; ++i) { + threads.emplace_back([&def, &manager, &results, i]() { + results[i] = def.wakeUpIfSleeping(manager); + }); + } + for (auto& t : threads) { + t.join(); + } + for (int i = 0; i < kThreads; ++i) { + ASSERT_EQ(results[i], StatusCode::OK) << "thread " << i << " status: " << results[i].string(); + } + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); +} + +// Best-effort stress: interleave unload() (watcher role) and wakeUpIfSleeping() +// (request role) repeatedly and assert the graph never ends in a torn state. +// lifecycleMtx makes unload and wake mutually exclusive, so every observed +// settled state must be internally consistent: AVAILABLE with resources, or +// cleanly SLEEPING (empty maps). Determinism is limited by thread scheduling; +// this exercises the FIX 1/FIX 2 serialization rather than asserting an exact +// sequence. +TEST_F(LLMIdleUnloadTest, ConcurrentUnloadWakeNeverTearsState) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + std::atomic stop{false}; + std::atomic errors{0}; + + // Unloader thread: keeps backdating + trying to unload. + std::thread unloader([&]() { + while (!stop.load()) { + def.recordActivity(secondsAgo(60)); + auto s = def.putToSleep(); + if (!s.ok()) + errors.fetch_add(1); + std::this_thread::yield(); + } + }); + + // Several waker threads: keep waking it back up. + constexpr int kWakers = 4; + std::vector wakers; + for (int i = 0; i < kWakers; ++i) { + wakers.emplace_back([&]() { + while (!stop.load()) { + auto s = def.wakeUpIfSleeping(manager); + if (!s.ok()) + errors.fetch_add(1); + std::this_thread::yield(); + } + }); + } + + // Run for a short bounded period. + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + stop.store(true); + unloader.join(); + for (auto& t : wakers) { + t.join(); + } + + ASSERT_EQ(errors.load(), 0); + + // Quiesce: ensure it ends AVAILABLE with resources (no torn RELOADING/null state). + ASSERT_EQ(def.wakeUpIfSleeping(manager), StatusCode::OK); + auto finalState = def.getStateCode(); + // A settled state must be either AVAILABLE (with resources) or SLEEPING (empty). + if (finalState == ovms::PipelineDefinitionStateCode::AVAILABLE) { + ASSERT_NE(def.getGenAiServable("llmNode"), nullptr); + } else { + ASSERT_EQ(finalState, ovms::PipelineDefinitionStateCode::SLEEPING); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), nullptr); + } +} + +// Best-effort: exercise unload() (watcher role) concurrently with reload() and +// retire() (config role) on the same definition. Verifies the lifecycleMtx +// serialization (NEW-1 fix): no crash, and a consistent final state. +// NOTE: data races are not deterministically catchable without TSAN (unavailable +// in this environment), so this is a smoke/stress test, not a proof of absence. +TEST_F(LLMIdleUnloadTest, ConcurrentUnloadReloadRetireNoCrash) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaIdle", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaIdle", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + std::atomic stop{false}; + std::atomic retired{false}; + + // Watcher-role thread: keep trying to idle-unload. + std::thread unloader([&]() { + while (!stop.load()) { + def.recordActivity(secondsAgo(60)); + (void)def.putToSleep(); + std::this_thread::yield(); + } + }); + + // Config-role thread: keep reloading (re-bring it up after unload). + std::thread reloader([&]() { + while (!stop.load()) { + (void)def.reload(manager, def.getMediapipeGraphConfig()); + std::this_thread::yield(); + } + }); + + std::this_thread::sleep_for(std::chrono::milliseconds(400)); + stop.store(true); + unloader.join(); + reloader.join(); + + // Now retire concurrently is not needed for crash-safety beyond above, but + // exercise retire() once after the storm to confirm it serializes cleanly. + def.retire(); + retired.store(true); + ASSERT_TRUE(retired.load()); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); +} + +// ───────────────────────────────────────────────────────────────────────────── +// TASK 1 tests: ActiveInferenceGuard — in-flight inference prevents idle unload +// ───────────────────────────────────────────────────────────────────────────── + +// Model-free unit test: directly exercise the activeInferenceCount atomic that +// shouldUnloadDueToIdle() and unload() consult. No LLM model required. +TEST(MediapipeIdleUnloadGuard, ActiveInferenceCountBlocksShouldUnload) { + // Build a minimal graph definition with idle unload enabled. + // Use buildOptGraphPbtxt() indirectly via LLMIdleUnloadTest helpers is not + // available here — we just need a definition with a non-zero timeout and + // a synthetic counter. We can use the shared_ptr that getActiveInferenceCount() + // returns directly, bypassing the executor machinery. + + // A standalone atomic acts as the counter. + auto counter = std::make_shared>(0); + auto lastActivity = std::make_shared>( + std::chrono::steady_clock::now().time_since_epoch().count() - 60LL * 1'000'000'000LL); + + // Simulate increment (inference start). + { + ovms::ActiveInferenceGuard guard(counter, lastActivity); + EXPECT_EQ(counter->load(), 1); + } + // After destruction, counter back to 0 and lastActivity refreshed. + EXPECT_EQ(counter->load(), 0); + int64_t nowNs = std::chrono::steady_clock::now().time_since_epoch().count(); + // lastActivity should be within 2 seconds of now (generous for slow machines). + EXPECT_GT(lastActivity->load(), nowNs - 2LL * 1'000'000'000LL); +} + +TEST(MediapipeIdleUnloadGuard, ActiveInferenceCountExceptionSafe) { + auto counter = std::make_shared>(0); + auto lastActivity = std::make_shared>(0); + + try { + ovms::ActiveInferenceGuard guard(counter, lastActivity); + EXPECT_EQ(counter->load(), 1); + throw std::runtime_error("simulated inference error"); + } catch (...) { + } + // Must be 0 even after exception path. + EXPECT_EQ(counter->load(), 0); +} + +TEST(MediapipeIdleUnloadGuard, MultipleGuardsNested) { + auto counter = std::make_shared>(0); + auto lastActivity = std::make_shared>(0); + { + ovms::ActiveInferenceGuard g1(counter, lastActivity); + EXPECT_EQ(counter->load(), 1); + { + ovms::ActiveInferenceGuard g2(counter, lastActivity); + EXPECT_EQ(counter->load(), 2); + } + EXPECT_EQ(counter->load(), 1); + } + EXPECT_EQ(counter->load(), 0); +} + +// Integration test: create() on a real definition increments the counter; +// when the executor is destroyed the counter returns to 0. +// Requires the LLM model (opt-125m). Guard under GTEST_SKIP for CI environments. +TEST_F(LLMIdleUnloadTest, ActiveInferenceGuardIntegration) { + ConstructorEnabledModelManager manager; + std::string testPbtxt = buildOptGraphPbtxt(); + const std::string testModelsPath = getGenericFullPathForSrcTest("/ovms/src/test/llm_testing/facebook/opt-125m"); + if (!std::filesystem::exists(testModelsPath)) { + GTEST_SKIP() << "opt-125m model not present; skipping integration guard test"; + } + + ovms::MediapipeGraphConfig mgc{"mediaGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaGuard", mgc, testPbtxt, nullptr); + def.inputConfig = testPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + + auto counterPtr = def.getActiveInferenceCount(); + ASSERT_NE(counterPtr, nullptr); + EXPECT_EQ(counterPtr->load(), 0); + + { + std::unique_ptr executor; + ASSERT_EQ(def.create(executor), StatusCode::OK); + ASSERT_NE(executor, nullptr); + // Counter incremented: executor is alive. + EXPECT_EQ(counterPtr->load(), 1); + + // Backdate activity to look idle — should NOT unload because count > 0. + def.recordActivity(secondsAgo(60)); + EXPECT_FALSE(def.shouldUnloadDueToIdle()); + auto sleepStatus = def.putToSleep(); + EXPECT_EQ(sleepStatus, StatusCode::MEDIAPIPE_PUT_TO_SLEEP_ACTIVE_INFERENCES); + // putToSleep() should be rejected (counter > 0), state remains AVAILABLE. + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + } // executor destroyed here -> counter decremented back to 0 + + EXPECT_EQ(counterPtr->load(), 0); + // Completing the inference refreshed lastActivityTimeNs (the ActiveInferenceGuard + // destructor resets the idle timer), so the graph is NOT idle immediately after — + // this is the key behavior preventing an immediate re-unload right after a long + // generation finishes. + EXPECT_FALSE(def.shouldUnloadDueToIdle()); + // After the idle period elapses again (post-inference), it should unload. + def.recordActivity(secondsAgo(60)); + EXPECT_TRUE(def.shouldUnloadDueToIdle()); + EXPECT_EQ(def.putToSleep(), StatusCode::OK); + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Wake-failure recovery: a failed wake-up reload must leave the graph SLEEPING +// (retryable), NOT LOADING_PRECONDITION_FAILED (wedged). Then once the underlying +// problem is resolved, the next wake self-heals to AVAILABLE. +// ───────────────────────────────────────────────────────────────────────────── + +// Returns an LLM graph pbtxt whose models_path points at a nonexistent directory, +// so validate() fails (LLM_NODE_DIRECTORY_DOES_NOT_EXIST) — but it still contains +// HttpLLMCalculator, so the idle-unload scope check passes and we exercise the +// wake/reload/validate failure path. +static std::string buildBrokenOptGraphPbtxt() { + std::string testPbtxt = R"( + input_stream: "HTTP_REQUEST_PAYLOAD:input" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + + node: { + name: "llmNode" + calculator: "HttpLLMCalculator" + input_stream: "LOOPBACK:loopback" + input_stream: "HTTP_REQUEST_PAYLOAD:input" + input_side_packet: "LLM_NODE_RESOURCES:llm" + output_stream: "LOOPBACK:loopback" + output_stream: "HTTP_RESPONSE_PAYLOAD:output" + input_stream_info: { + tag_index: 'LOOPBACK:0', + back_edge: true + } + node_options: { + [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { + models_path: "/this/path/definitely/does/not/exist/opt-125m" + cache_size: 1 + } + } + input_stream_handler { + input_stream_handler: "SyncSetInputStreamHandler", + options { + [mediapipe.SyncSetInputStreamHandlerOptions.ext] { + sync_set { + tag_index: "LOOPBACK:0" + } + } + } + } + } + )"; + adjustConfigForTargetPlatform(testPbtxt); + return testPbtxt; +} + +TEST_F(LLMIdleUnloadTest, FailedWakeLeavesGraphSleepingAndRetryable) { + ConstructorEnabledModelManager manager; + std::string goodPbtxt = buildOptGraphPbtxt(); + std::string brokenPbtxt = buildBrokenOptGraphPbtxt(); + + ovms::MediapipeGraphConfig mgc{"mediaWakeFail", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("mediaWakeFail", mgc, goodPbtxt, nullptr); + def.inputConfig = goodPbtxt; + ASSERT_EQ(def.validate(manager), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + + // Idle-unload the healthy graph. + def.recordActivity(secondsAgo(60)); + ASSERT_EQ(def.putToSleep(), StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + + // Simulate the model becoming temporarily unavailable: swap in a broken config + // so the wake-up reload's validate() fails. + def.inputConfig = brokenPbtxt; + auto failStatus = def.wakeUpIfSleeping(manager); + EXPECT_FALSE(failStatus.ok()) << "expected wake-up to fail with broken model"; + // CRITICAL: the graph must be retryable, i.e. back in SLEEPING — not wedged in + // LOADING_PRECONDITION_FAILED. + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + + // A second attempt while still broken also fails but stays retryable. + auto failStatus2 = def.wakeUpIfSleeping(manager); + EXPECT_FALSE(failStatus2.ok()); + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + + // Restore the model: the next wake self-heals to AVAILABLE. + def.inputConfig = goodPbtxt; + auto okStatus = def.wakeUpIfSleeping(manager); + EXPECT_EQ(okStatus, StatusCode::OK) << okStatus.string(); + EXPECT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + EXPECT_NE(def.getGenAiServable("llmNode"), nullptr); +} diff --git a/src/test/mediapipe_framework_test.cpp b/src/test/mediapipe_framework_test.cpp index 4b1c644934..a085f02c27 100644 --- a/src/test/mediapipe_framework_test.cpp +++ b/src/test/mediapipe_framework_test.cpp @@ -41,7 +41,7 @@ #include "src/metrics/metric_config.hpp" #include "src/metrics/metric_module.hpp" #include "../precision.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" diff --git a/src/test/mediapipeflow_test.cpp b/src/test/mediapipeflow_test.cpp index c9fa5c22f2..94eeeba466 100644 --- a/src/test/mediapipeflow_test.cpp +++ b/src/test/mediapipeflow_test.cpp @@ -13,15 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. //***************************************************************************** +#include #include #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -54,7 +57,8 @@ #include "../model.hpp" #include "../ovms_exit_codes.hpp" #include "../precision.hpp" -#include "../servablemanagermodule.hpp" +#include "../servable_definition_unload_guard.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" @@ -1523,7 +1527,7 @@ TEST_F(MediapipeStreamFlowAddTest, Infer) { // Inference on unloaded mediapipe graph // Expect old stream to continue responding until closure // Expect new stream to be rejected -TEST_F(MediapipeStreamFlowAddTest, InferOnUnloadedGraph) { +TEST_F(MediapipeStreamFlowAddTest, InferOnSleepingGraph) { const ovms::Module* grpcModule = server.getModule(ovms::GRPC_SERVER_MODULE_NAME); KFSInferenceServiceImpl& impl = dynamic_cast(grpcModule)->getKFSGrpcImpl(); @@ -2509,7 +2513,8 @@ const std::string MediapipeConfigChanges::configFileWithGraphPathToReplace = R"( "model_config_list": [ {"config": { "name": "dummy", - "base_path": "/ovms/src/test/dummy" + "base_path": ")" + + getGenericFullPathForSrcTest("/ovms/src/test/dummy") + R"(" } } ], @@ -2527,7 +2532,8 @@ const std::string MediapipeConfigChanges::configFileWithEmptyBasePath = R"( "model_config_list": [ {"config": { "name": "dummy", - "base_path": "/ovms/src/test/dummy" + "base_path": ")" + + getGenericFullPathForSrcTest("/ovms/src/test/dummy") + R"(" } } ], @@ -2545,7 +2551,8 @@ const std::string MediapipeConfigChanges::configFileWithNoBasePath = R"( "model_config_list": [ {"config": { "name": "dummy", - "base_path": "/ovms/src/test/dummy" + "base_path": ")" + + getGenericFullPathForSrcTest("/ovms/src/test/dummy") + R"(" } } ], @@ -2587,7 +2594,8 @@ const std::string MediapipeConfigChanges::configFileWithoutGraph = R"( "model_config_list": [ {"config": { "name": "dummy", - "base_path": "/ovms/src/test/dummy" + "base_path": ")" + + getGenericFullPathForSrcTest("/ovms/src/test/dummy") + R"(" } } ] @@ -2922,6 +2930,115 @@ TEST_F(MediapipeConfigChanges, AddProperGraphThenRetireThenAddAgain) { checkStatus(modelManager, StatusCode::OK); } +TEST_F(MediapipeConfigChanges, RetireGraphWithIdleManagementEnabled) { + std::string configFileContent = configFileWithGraphPathToReplace; + std::string configFilePath = directoryPath + "/config.json"; + std::string graphFilePath = directoryPath + "/graph.pbtxt"; + const std::string modelPathToReplace{"XYZ"}; + configFileContent.replace(configFileContent.find(modelPathToReplace), modelPathToReplace.size(), graphFilePath); + createConfigFileWithContent(configFileContent, configFilePath); + createConfigFileWithContent(pbtxtContent, graphFilePath); + ConstructorEnabledModelManager modelManager(30'000'000); + modelManager.loadConfig(configFilePath); + const MediapipeFactory& factory = modelManager.getMediapipeFactory(); + auto definition = factory.findDefinitionByName(mgdName); + ASSERT_NE(nullptr, definition); + checkStatus(modelManager, StatusCode::OK); + // now we retire + configFileContent = configFileWithoutGraph; + createConfigFileWithContent(configFileContent, configFilePath); + modelManager.loadConfig(configFilePath); + definition = factory.findDefinitionByName(mgdName); + ASSERT_NE(nullptr, definition); + EXPECT_EQ(definition->getStatus().getStateCode(), PipelineDefinitionStateCode::RETIRED); + checkStatus(modelManager, StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE); + EXPECT_EQ(definition->getStatus().getStateCode(), PipelineDefinitionStateCode::RETIRED); +} + +TEST_F(MediapipeConfigChanges, RetiringGraphsGoesThroughLoadingQueue) { + std::string configFilePath = directoryPath + "/config.json"; + std::string graphFilePath = directoryPath + "/graph.pbtxt"; + createConfigFileWithContent(pbtxtContent, graphFilePath); + + const std::string dummyModelPath = getGenericFullPathForSrcTest("/ovms/src/test/dummy"); + auto configWithGraphs = [&graphFilePath, &dummyModelPath](const std::vector& graphNames) { + std::string entries; + for (const auto& name : graphNames) { + if (!entries.empty()) { + entries += ","; + } + entries += R"({"name":")" + name + R"(","graph_path":")" + graphFilePath + R"("})"; + } + return R"({"model_config_list":[{"config":{"name":"dummy","base_path":")" + dummyModelPath + R"("}}],)" + R"("mediapipe_config_list":[)" + + entries + "]}"; + }; + + createConfigFileWithContent(configWithGraphs({"graphA", "graphB", "graphC"}), configFilePath); + ConstructorEnabledModelManager modelManager; + ASSERT_EQ(modelManager.loadConfig(configFilePath), StatusCode::OK); + const MediapipeFactory& factory = modelManager.getMediapipeFactory(); + for (const auto& name : {"graphA", "graphB", "graphC"}) { + auto* definition = factory.findDefinitionByName(name); + ASSERT_NE(nullptr, definition) << name; + ASSERT_EQ(definition->getStatus().getStateCode(), PipelineDefinitionStateCode::AVAILABLE) << name; + } + + std::mutex retiredMtx; + std::vector retired; + // Installed after the initial load so that only the retirement reload is recorded. + modelManager.getLoadingQueue().setTaskObserver( + [&retiredMtx, &retired](TaskEvent event, const ServableLoadingTask& task) { + if (event != TaskEvent::Executed || task.type != ServableLoadingTaskType::RetireMediapipe) { + return; + } + std::lock_guard lock(retiredMtx); + retired.push_back(task.name); + }); + + createConfigFileWithContent(configWithGraphs({"graphB"}), configFilePath); + ASSERT_EQ(modelManager.loadConfig(configFilePath), StatusCode::OK); + + for (const auto& name : {"graphA", "graphC"}) { + auto* definition = factory.findDefinitionByName(name); + ASSERT_NE(nullptr, definition) << name; + EXPECT_EQ(definition->getStatus().getStateCode(), PipelineDefinitionStateCode::RETIRED) << name; + } + EXPECT_EQ(factory.findDefinitionByName("graphB")->getStatus().getStateCode(), PipelineDefinitionStateCode::AVAILABLE); + + std::lock_guard lock(retiredMtx); + std::sort(retired.begin(), retired.end()); + EXPECT_EQ(retired, (std::vector{"graphA", "graphC"})) + << "graphs dropped from the config must be retired through the loading queue like every other state change"; +} + +TEST_F(MediapipeConfigChanges, WakeUpDoesNotResurrectRetiredGraph) { + std::string configFileContent = configFileWithGraphPathToReplace; + std::string configFilePath = directoryPath + "/config.json"; + std::string graphFilePath = directoryPath + "/graph.pbtxt"; + const std::string modelPathToReplace{"XYZ"}; + configFileContent.replace(configFileContent.find(modelPathToReplace), modelPathToReplace.size(), graphFilePath); + createConfigFileWithContent(configFileContent, configFilePath); + createConfigFileWithContent(pbtxtContent, graphFilePath); + ConstructorEnabledModelManager modelManager; + ASSERT_EQ(modelManager.loadConfig(configFilePath), StatusCode::OK); + const MediapipeFactory& factory = modelManager.getMediapipeFactory(); + auto* definition = factory.findDefinitionByName(mgdName); + ASSERT_NE(nullptr, definition); + ASSERT_EQ(definition->getStatus().getStateCode(), PipelineDefinitionStateCode::AVAILABLE); + + createConfigFileWithContent(configFileWithoutGraph, configFilePath); + ASSERT_EQ(modelManager.loadConfig(configFilePath), StatusCode::OK); + ASSERT_EQ(definition->getStatus().getStateCode(), PipelineDefinitionStateCode::RETIRED); + + // A wake-up scheduled by a request thread working off a stale group snapshot must not + // bring back a graph the user removed from the config. + auto status = modelManager.requestServableWakeUp(mgdName, /*urgent=*/true).get(); + EXPECT_EQ(status, StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE) << status.string(); + EXPECT_EQ(definition->getStatus().getStateCode(), PipelineDefinitionStateCode::RETIRED); + checkStatus(modelManager, StatusCode::MEDIAPIPE_DEFINITION_NOT_LOADED_ANYMORE); +} + TEST_F(MediapipeConfigChanges, AddImproperGraphThenFixWithReloadThenBreakAgain) { std::string configFileContent = configFileWithGraphPathToReplace; std::string configFilePath = directoryPath + "/config.json"; @@ -4500,3 +4617,155 @@ TEST_F(UnaryQueueReinitTest, GraphIsReinitializedAfterCalculatorError) { ASSERT_TRUE(status.ok()); } } + +// --------------------------------------------------------------------------- +// Idle unload feature: putToSleep() guard correctness (issue #4141, model-free) +// Verifies FIX 1: putToSleep() must NOT tear down resources unless the state was +// actually AVAILABLE and the SleepEvent transition really happened. +// --------------------------------------------------------------------------- + +// A trivial pbtxt is enough; these tests never reach validate(), they drive the +// state machine directly to exercise putToSleep() preconditions. +static const std::string kIdleUnloadDummyPbtxt = R"( + input_stream: "in" + output_stream: "out" +)"; + +TEST(MediapipeIdleUnloadGuard, SleepIsNoOpWhenStateBegin) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + // Fresh definition is in BEGIN (validate never called). + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + ASSERT_EQ(def.putToSleep(), ovms::StatusCode::MEDIAPIPE_PUT_TO_SLEEP_STATE_NOT_AVAILABLE); + + // State unchanged and resources untouched. + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); + ASSERT_TRUE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); +} + +TEST(MediapipeIdleUnloadGuard, SleepIsNoOpWhenStateReloading) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + // Drive BEGIN -> AVAILABLE -> RELOADING. + def.forceValidationPassedEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + def.forceReloadEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + ASSERT_EQ(def.putToSleep(), ovms::StatusCode::MEDIAPIPE_PUT_TO_SLEEP_STATE_NOT_AVAILABLE); + + // Critical: unload() must NOT have cleared resources while RELOADING. + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + ASSERT_TRUE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); +} + +TEST(MediapipeIdleUnloadGuard, SleepTransitionsAndTearsDownWhenAvailable) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + def.forceValidationPassedEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + ASSERT_EQ(def.putToSleep(), ovms::StatusCode::OK); + + // Now it should have transitioned and released resources. + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), nullptr); + ASSERT_NE(def.sidePacketMapsPtrForTest(), mapsBefore); +} + +TEST(MediapipeIdleUnloadGuard, UnloadSkipsWhenRequestsInFlight) { + ovms::MediapipeGraphConfig mgc{"idleGuard", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("idleGuard", mgc, kIdleUnloadDummyPbtxt, nullptr); + def.forceValidationPassedEventForTest(); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + def.insertSidePacketMarkerForTest("marker"); + const void* mapsBefore = def.sidePacketMapsPtrForTest(); + + { + // Simulate an in-flight request by holding an unload guard (bumps the counter). + ovms::ServableDefinitionUnloadGuard guard(def); + ASSERT_EQ(def.requestsHandlesCounterForTest(), 1u); + + // putToSleep() must be rejected because counter > 0. + auto sleepStatus = def.putToSleep(); + ASSERT_EQ(sleepStatus, ovms::StatusCode::MEDIAPIPE_PUT_TO_SLEEP_REQUESTS_IN_FLIGHT); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + ASSERT_TRUE(def.hasSidePacketMarkerForTest("marker")); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), mapsBefore); + } + // After the guard releases, putToSleep() now proceeds. + ASSERT_EQ(def.requestsHandlesCounterForTest(), 0u); + ASSERT_EQ(def.putToSleep(), ovms::StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + ASSERT_EQ(def.sidePacketMapsPtrForTest(), nullptr); +} + +TEST(MediapipeIdleUnloadGuard, LazyLoadConstructorStartsSleeping) { + ovms::MediapipeGraphConfig mgc{"skipLoad", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("skipLoad", mgc, kIdleUnloadDummyPbtxt, nullptr, true); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +TEST(MediapipeIdleUnloadGuard, LazyLoadThenUnloadIsNoOp) { + ovms::MediapipeGraphConfig mgc{"skipLoad", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + DummyMediapipeGraphDefinition def("skipLoad", mgc, kIdleUnloadDummyPbtxt, nullptr, true); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + + ASSERT_EQ(def.putToSleep(), ovms::StatusCode::OK); + ASSERT_EQ(def.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +namespace { +class StubMetricProvider : public ovms::MetricProvider { +public: + ovms::MetricRegistry* getMetricRegistry() const override { return nullptr; } + const ovms::MetricConfig& getMetricConfig() const override { return config_; } + +private: + ovms::MetricConfig config_; +}; +} // namespace + +TEST(MediapipeIdleUnloadGuard, CreateDefinitionLazyLoad) { + ovms::MediapipeFactory factory(nullptr); + ovms::MediapipeGraphConfig mgc{"unloadedGraph", "", ""}; + mgc.setIdleUnloadTimeoutSeconds(10); + StubMetricProvider metrics; + ovms::ModelManager manager; + + auto status = factory.createDefinition("unloadedGraph", mgc, metrics, manager, true); + ASSERT_EQ(status, ovms::StatusCode::OK); + + auto* def = factory.findDefinitionByName("unloadedGraph"); + ASSERT_NE(def, nullptr); + ASSERT_EQ(def->getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +TEST(MediapipeIdleUnloadGuard, CreateDefinitionLazyLoadRejectsDuplicate) { + ovms::MediapipeFactory factory(nullptr); + ovms::MediapipeGraphConfig mgc{"dupGraph", "", ""}; + StubMetricProvider metrics; + ovms::ModelManager manager; + + auto status1 = factory.createDefinition("dupGraph", mgc, metrics, manager, true); + ASSERT_EQ(status1, ovms::StatusCode::OK); + + auto status2 = factory.createDefinition("dupGraph", mgc, metrics, manager, true); + ASSERT_EQ(status2, ovms::StatusCode::PIPELINE_DEFINITION_ALREADY_EXIST); +} diff --git a/src/test/metrics_flow_test.cpp b/src/test/metrics_flow_test.cpp index e50e72cf90..35a0fea375 100644 --- a/src/test/metrics_flow_test.cpp +++ b/src/test/metrics_flow_test.cpp @@ -32,7 +32,7 @@ #include "src/metrics/metric_config.hpp" #include "src/metrics/metric_module.hpp" #include "../precision.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" #include "constructor_enabled_model_manager.hpp" @@ -135,6 +135,8 @@ class ServerWithMockedManagerModule : public Server { module = this->createModule(GRPC_SERVER_MODULE_NAME); this->modules.emplace(GRPC_SERVER_MODULE_NAME, std::move(module)); } + // Modules hold a reference to manager; shut them down before manager is destroyed + ~ServerWithMockedManagerModule() override { shutdownModules(); } ConstructorEnabledModelManager& getManager() { return this->manager; diff --git a/src/test/mockmodelinstancechangingstates.hpp b/src/test/mockmodelinstancechangingstates.hpp index ebb0b4a019..2d92c0944a 100644 --- a/src/test/mockmodelinstancechangingstates.hpp +++ b/src/test/mockmodelinstancechangingstates.hpp @@ -32,8 +32,12 @@ class MockModelInstanceChangingStates : public ovms::ModelInstance { status = ovms::ModelVersionStatus(modelName, modelVersion, ovms::ModelVersionState::START); } virtual ~MockModelInstanceChangingStates() {} - ovms::Status loadModel(const ovms::ModelConfig& config) override { + ovms::Status loadModel(const ovms::ModelConfig& config, bool lazyLoad = false) override { this->status = ovms::ModelVersionStatus(config.getName(), config.getVersion()); + if (lazyLoad) { + this->status.setSleeping(); + return ovms::StatusCode::OK; + } this->status.setLoading(); status.setAvailable(); return ovms::StatusCode::OK; diff --git a/src/test/model_cache_test.cpp b/src/test/model_cache_test.cpp index 16b41a38e3..3c0f1e41b1 100644 --- a/src/test/model_cache_test.cpp +++ b/src/test/model_cache_test.cpp @@ -23,7 +23,7 @@ #include "../modelconfig.hpp" #include "../modelinstance.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "constructor_enabled_model_manager.hpp" #include "test_models_configs.hpp" #include "test_with_temp_dir.hpp" diff --git a/src/test/model_test.cpp b/src/test/model_test.cpp index e3ad4820cd..1749a498e4 100644 --- a/src/test/model_test.cpp +++ b/src/test/model_test.cpp @@ -22,7 +22,7 @@ #include "src/filesystem/filesystem.hpp" #include "../model.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "mockmodelinstancechangingstates.hpp" #include "test_models_configs.hpp" diff --git a/src/test/modelmanager_test.cpp b/src/test/modelmanager_test.cpp index 1d2419185b..9029872390 100644 --- a/src/test/modelmanager_test.cpp +++ b/src/test/modelmanager_test.cpp @@ -31,7 +31,7 @@ #include "../logging.hpp" #include "../model.hpp" #include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "../prediction_service_utils.hpp" #include "absl/synchronization/notification.h" #include "constructor_enabled_model_manager.hpp" @@ -151,7 +151,7 @@ class MockModel : public ovms::Model { public: MockModel() : Model("MOCK_NAME") {} - MOCK_METHOD(ovms::Status, addVersion, (const ovms::ModelConfig&, ov::Core&, ovms::MetricRegistry*, const ovms::MetricConfig*), (override)); + MOCK_METHOD(ovms::Status, addVersion, (const ovms::ModelConfig&, ov::Core&, ovms::MetricRegistry*, const ovms::MetricConfig*, bool), (override)); }; std::shared_ptr modelMock; @@ -484,6 +484,29 @@ TEST_F(ModelManager, ConfigParseNoModels) { EXPECT_EQ(status, ovms::StatusCode::OK); } +TEST_F(ModelManager, WakeUpDoesNotResurrectRetiredModel) { + std::string configFile = this->getFilePath("/ovms_config_file.json"); + createConfigFileWithContent( + R"({"model_config_list":[{"config":{"name":"dummy","base_path":")" + getGenericFullPathForSrcTest("/ovms/src/test/dummy") + R"("}}]})", + configFile); + ASSERT_EQ(fixtureManager.loadConfig(configFile), ovms::StatusCode::OK); + std::shared_ptr modelInstance; + std::unique_ptr modelInstanceUnloadGuardPtr; + ASSERT_EQ(fixtureManager.getModelInstance("dummy", 1, modelInstance, modelInstanceUnloadGuardPtr), ovms::StatusCode::OK); + modelInstance.reset(); + modelInstanceUnloadGuardPtr.reset(); + + createConfigFileWithContent("{ \"model_config_list\": [ ] }\n", configFile); + ASSERT_EQ(fixtureManager.loadConfig(configFile), ovms::StatusCode::OK); + ASSERT_EQ(fixtureManager.getModelInstance("dummy", 1, modelInstance, modelInstanceUnloadGuardPtr), ovms::StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE); + + // A wake-up scheduled by a request thread working off a stale group snapshot must not + // bring back a model the user removed from the config. + auto status = fixtureManager.requestServableWakeUp("dummy", /*urgent=*/true).get(); + EXPECT_EQ(status, ovms::StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE) << status.string(); + EXPECT_EQ(fixtureManager.getModelInstance("dummy", 1, modelInstance, modelInstanceUnloadGuardPtr), ovms::StatusCode::MODEL_VERSION_NOT_LOADED_ANYMORE); +} + #if (MEDIAPIPE_DISABLE == 1) TEST_F(ModelManager, ConfigParseDisableMediapipe) { auto status = fixtureManager.startFromFile("/ovms/src/test/mediapipe/config_mediapipe_add_adapter_full.json"); @@ -987,7 +1010,7 @@ TEST_F(ModelManagerWatcher, StartFromFile) { modelMock = std::make_shared(); MockModelManager manager; - EXPECT_CALL(*modelMock, addVersion(_, _, _, _)) + EXPECT_CALL(*modelMock, addVersion(_, _, _, _, _)) .Times(1) .WillRepeatedly(Return(ovms::Status(ovms::StatusCode::OK))); auto status = manager.startFromFile(fileToReload); @@ -1004,7 +1027,7 @@ TEST_F(ModelManagerWatcher, StartFromFileRelativePath) { modelMock = std::make_shared(); MockModelManager manager; - EXPECT_CALL(*modelMock, addVersion(_, _, _, _)) + EXPECT_CALL(*modelMock, addVersion(_, _, _, _, _)) .Times(1) .WillRepeatedly(Return(ovms::Status(ovms::StatusCode::OK))); auto status = manager.startFromFile(fileToReload); @@ -1066,7 +1089,7 @@ TEST_F(ModelManagerWatcher, ConfigReloadingShouldAddNewModel) { createConfigFileWithContent(getConfig1Model(this->getFilePath("/models/dummy1")), fileToReload); modelMock = std::make_shared(); MockModelManager manager; - EXPECT_CALL(*modelMock, addVersion(_, _, _, _)) + EXPECT_CALL(*modelMock, addVersion(_, _, _, _, _)) .WillRepeatedly(Return(ovms::Status(ovms::StatusCode::OK))); auto status = manager.startFromFile(fileToReload); @@ -1087,7 +1110,7 @@ TEST_F(ModelManagerWatcher, ConfigReloadingShouldAddNewModelRelativePath) { createConfigFileWithContent(relative_config_1_model, fileToReload); modelMock = std::make_shared(); MockModelManager manager; - EXPECT_CALL(*modelMock, addVersion(_, _, _, _)) + EXPECT_CALL(*modelMock, addVersion(_, _, _, _, _)) .WillRepeatedly(Return(ovms::Status(ovms::StatusCode::OK))); auto status = manager.startFromFile(fileToReload); @@ -1386,7 +1409,7 @@ TEST_F(ModelManager, ConfigReloadingWithTwoModelsWithTheSameName) { modelMock = std::make_shared(); MockModelManager manager; - EXPECT_CALL(*modelMock, addVersion(_, _, _, _)) + EXPECT_CALL(*modelMock, addVersion(_, _, _, _, _)) .Times(1) .WillRepeatedly(Return(ovms::Status(ovms::StatusCode::OK))); auto status = manager.startFromFile(fileToReload); @@ -1421,7 +1444,7 @@ TEST_F(ModelManager, ConfigReloadingWithTwoModelsWithTheSameNameRelativePath) { modelMock = std::make_shared(); MockModelManager manager; - EXPECT_CALL(*modelMock, addVersion(_, _, _, _)) + EXPECT_CALL(*modelMock, addVersion(_, _, _, _, _)) .Times(1) .WillRepeatedly(Return(ovms::Status(ovms::StatusCode::OK))); auto status = manager.startFromFile(fileToReload); @@ -2022,7 +2045,7 @@ class MockModelInstanceFakeLoad : public ovms::ModelInstance { ModelInstance("UNUSED_NAME", UNUSED_MODEL_VERSION, ieCore) {} protected: - ovms::Status loadModel(const ovms::ModelConfig& config) override { + ovms::Status loadModel(const ovms::ModelConfig& config, bool lazyLoad = false) override { status = ovms::ModelVersionStatus(name, version); status.setAvailable(); return ovms::StatusCode::OK; @@ -2076,7 +2099,7 @@ class ModelInstanceLoadedStuckInLoadingState : public ovms::ModelInstance { ModelInstance("UNUSED_NAME", UNUSED_MODEL_VERSION, ieCore) {} protected: - ovms::Status loadModel(const ovms::ModelConfig& config) override { + ovms::Status loadModel(const ovms::ModelConfig& config, bool lazyLoad = false) override { status = ovms::ModelVersionStatus(name, version); status.setLoading(); return ovms::StatusCode::OK; @@ -2113,7 +2136,7 @@ class ModelInstanceLoadedWaitInLoadingState : public ovms::ModelInstance { } protected: - ovms::Status loadModel(const ovms::ModelConfig& config) override { + ovms::Status loadModel(const ovms::ModelConfig& config, bool lazyLoad = false) override { this->status = ovms::ModelVersionStatus(name, version); this->status.setLoading(); this->thread = std::make_unique([this]() { diff --git a/src/test/multipart_calculator_test.cpp b/src/test/multipart_calculator_test.cpp index e3c870537a..c98d7abb1a 100644 --- a/src/test/multipart_calculator_test.cpp +++ b/src/test/multipart_calculator_test.cpp @@ -19,7 +19,7 @@ #include "../http_rest_api_handler.hpp" #include "../http_payload.hpp" #include "../module_names.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "test_http_utils.hpp" #include "test_utils.hpp" diff --git a/src/test/pipelinedefinitionstatus_test.cpp b/src/test/pipelinedefinitionstatus_test.cpp index 8c3d722f6c..d7f32e26e7 100644 --- a/src/test/pipelinedefinitionstatus_test.cpp +++ b/src/test/pipelinedefinitionstatus_test.cpp @@ -300,3 +300,173 @@ TEST(PipelineDefinitionStatus, ConvertToModelStatus) { ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); ASSERT_EQ((std::tuple(ModelVersionState::END, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); } + +// --------------------------------------------------------------------------- +// Idle unload feature: SLEEPING state transitions (issue #4141) +// --------------------------------------------------------------------------- + +TEST(PipelineDefinitionStatus, AvailableThenSleep) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +TEST(PipelineDefinitionStatus, SleepingThenReloadGoesToReloading) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); +} + +TEST(PipelineDefinitionStatus, SleepingThenReloadThenValidationPassGoesToAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +TEST(PipelineDefinitionStatus, SleepingThenValidationPassDefensiveGoesToAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +TEST(PipelineDefinitionStatus, SleepingThenRetireGoesToRetired) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + pds.handle(RetireEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); +} + +TEST(PipelineDefinitionStatus, SleepingIsNotAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + ASSERT_TRUE(pds.isAvailable()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + ASSERT_FALSE(pds.isAvailable()); + ASSERT_TRUE(pds.isSleeping()); + ASSERT_TRUE(pds.appearsAvailable()); +} + +TEST(PipelineDefinitionStatus, SleepingConvertsToModelStatusAvailable) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + // SLEEPING must report AVAILABLE so health checks / routing do not exclude the + // servable (it auto-reloads on the next inference request). + ASSERT_EQ((std::tuple(ModelVersionState::AVAILABLE, ModelVersionStatusErrorCode::OK)), pds.convertToModelStatus()); +} + +TEST(PipelineDefinitionStatus, SleepEventOnBeginTransitionsToSleeping) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::BEGIN); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +TEST(PipelineDefinitionStatus, SleepEventOnReloadingShouldThrow) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + ASSERT_THROW(pds.handle(SleepEvent()), std::logic_error); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); +} + +TEST(PipelineDefinitionStatus, SleepEventOnRetiredShouldThrow) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(RetireEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); + ASSERT_THROW(pds.handle(SleepEvent()), std::logic_error); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RETIRED); +} + +TEST(PipelineDefinitionStatus, SleepEventOnAvailableRequiredRevalidationTransitionsToSleeping) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(UsedModelChangedEvent(modelNotifyingDetails)); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE_REQUIRED_REVALIDATION); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +TEST(PipelineDefinitionStatus, SleepEventOnLoadingPreconditionFailedRevertsToSleeping) { + // A failed wake-up reload (validate -> LOADING_PRECONDITION_FAILED) is reverted + // to SLEEPING by wakeUpIfSleeping() via SleepEvent so the next request retries. + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationFailedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +TEST(PipelineDefinitionStatus, SleepingAfterFailedWakeIsRetryableViaReload) { + // Full retry path: AVAILABLE -> SLEEPING -> (wake) RELOADING -> (fail) FAILED + // -> (revert) SLEEPING -> (retry wake) RELOADING -> (pass) AVAILABLE. + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + pds.handle(ReloadEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::RELOADING); + pds.handle(ValidationFailedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED); + pds.handle(SleepEvent()); // wakeUpIfSleeping reverts on failure + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + pds.handle(ReloadEvent()); + pds.handle(ValidationPassedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::AVAILABLE); +} + +TEST(PipelineDefinitionStatus, SleepEventOnSleepingShouldThrow) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + ASSERT_THROW(pds.handle(SleepEvent()), std::logic_error); +} + +TEST(PipelineDefinitionStatus, SleepEventOnLoadingFailedRequiredRevalidationIsNoOp) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationFailedEvent()); + pds.handle(UsedModelChangedEvent(modelNotifyingDetails)); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::LOADING_PRECONDITION_FAILED_REQUIRED_REVALIDATION); +} + +TEST(PipelineDefinitionStatus, SleepingThenValidationFailedKeepsSleeping) { + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + pds.handle(ValidationFailedEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); +} + +TEST(PipelineDefinitionStatus, SleepingThenUsedModelChangedKeepsSleeping) { + // A subscribed model changing while the graph sleeps must not wake it up; + // the next wake-up reload revalidates anyway. + PipelineDefinitionStatus pds(unusedPipelineType, unusedPipelineName); + pds.handle(ValidationPassedEvent()); + pds.handle(SleepEvent()); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + // TODO @atobiszei idle - todo later - potentially to remove with DAGS. + pds.handle(UsedModelChangedEvent(modelNotifyingDetails)); + ASSERT_EQ(pds.getStateCode(), ovms::PipelineDefinitionStateCode::SLEEPING); + ASSERT_TRUE(pds.isSleeping()); +} diff --git a/src/test/pythonnode_test.cpp b/src/test/pythonnode_test.cpp index 955cc65499..2978c28482 100644 --- a/src/test/pythonnode_test.cpp +++ b/src/test/pythonnode_test.cpp @@ -41,7 +41,7 @@ #include "../precision.hpp" #include "../python/pythoninterpretermodule.hpp" #include "../python/pythonnoderesources.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../shape.hpp" #include "../stringutils.hpp" diff --git a/src/test/schema_test.cpp b/src/test/schema_test.cpp index 2b5aaf11fd..e4b0020bfb 100644 --- a/src/test/schema_test.cpp +++ b/src/test/schema_test.cpp @@ -2003,6 +2003,148 @@ TEST(SchemaTest, MediapipeConfigInModelConfigPositive) { auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); EXPECT_EQ(result, ovms::StatusCode::OK); } + +TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutPositive) { + const char* mediapipeConfigPositive = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_model", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path_base", + "idle_unload_timeout_seconds": 300 + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(mediapipeConfigPositive); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutNegativeValueRejected) { + const char* mediapipeConfigNegative = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_model", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path_base", + "idle_unload_timeout_seconds": -5 + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(mediapipeConfigNegative); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +TEST(SchemaTest, MediapipeConfigIdleUnloadTimeoutWrongTypeRejected) { + const char* mediapipeConfigNegative = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_model", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path_base", + "idle_unload_timeout_seconds": "notAnInteger" + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(mediapipeConfigNegative); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} +#endif + +TEST(SchemaTest, ModelConfigGroupNameValidString) { + const char* config = R"( + { + "model_config_list": [ + { + "config": { + "name": "dummy_model", + "base_path": "dummy_path", + "group_name": "rag" + } + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST(SchemaTest, ModelConfigGroupNameInvalidType) { + const char* config = R"( + { + "model_config_list": [ + { + "config": { + "name": "dummy_model", + "base_path": "dummy_path", + "group_name": 123 + } + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} + +#if (MEDIAPIPE_DISABLE == 0) +TEST(SchemaTest, MediapipeConfigGroupNameValidString) { + const char* config = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_graph", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path", + "group_name": "llm_group" + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::OK); +} + +TEST(SchemaTest, MediapipeConfigGroupNameInvalidType) { + const char* config = R"( + { + "model_config_list": [], + "mediapipe_config_list": [ + { + "name": "dummy_graph", + "graph_path": "graph.pbtxt", + "base_path": "dummy_path", + "group_name": 42 + } + ] + })"; + + rapidjson::Document configDoc; + configDoc.Parse(config); + auto result = ovms::validateJsonAgainstSchema(configDoc, ovms::MODELS_CONFIG_SCHEMA.c_str()); + EXPECT_EQ(result, ovms::StatusCode::JSON_INVALID); +} #endif TEST(SchemaTest, MediapipeConfigNegativeAdditionalMediapipeConfigField) { diff --git a/src/test/servable_group_manager_test.cpp b/src/test/servable_group_manager_test.cpp new file mode 100644 index 0000000000..1614bd17fa --- /dev/null +++ b/src/test/servable_group_manager_test.cpp @@ -0,0 +1,346 @@ +//***************************************************************************** +// Copyright 2024 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "src/servable_management/servable_group_manager.hpp" +#include "constructor_enabled_model_manager.hpp" +#include "src/model.hpp" +#include "src/servable_management/servable_loading_queue.hpp" +#include "src/servable_management/servable_loading_task.hpp" +#include "src/modelconfig.hpp" +#include "src/modelinstance.hpp" +#include "src/modelversionstatus.hpp" +#include "src/status.hpp" +#include "test_models.hpp" +#include "test_with_temp_dir.hpp" + +using namespace ovms; + +static std::unordered_map createModelConfigs( + const std::vector>& nameGroupPairs) { + std::unordered_map configs; + for (const auto& [name, group] : nameGroupPairs) { + ModelConfig config; + config.setName(name); + config.setGroupName(group); + configs.emplace(name, std::move(config)); + } + return configs; +} + +class ServableGroupManagerTest : public ::testing::Test { +protected: + ConstructorEnabledModelManager mm; +}; + +TEST_F(ServableGroupManagerTest, DisabledByDefault) { + ServableGroupManager mgr(0); + ASSERT_FALSE(mgr.isEnabled()); +} + +TEST_F(ServableGroupManagerTest, EnabledWithPositiveTimeout) { + ServableGroupManager mgr(30'000'000); + ASSERT_TRUE(mgr.isEnabled()); + ASSERT_EQ(mgr.getIdleTimeoutMicroseconds(), 30'000'000u); +} + +TEST_F(ServableGroupManagerTest, BuildGroups_DefaultGroupNames) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({ + {"model_a", "model_a"}, + {"model_b", "model_b"}, + {"model_c", "model_c"}, + }); + mgr.buildGroups(configs, mm); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 3u); + EXPECT_TRUE(groups.count("model_a")); + EXPECT_TRUE(groups.count("model_b")); + EXPECT_TRUE(groups.count("model_c")); +} + +TEST_F(ServableGroupManagerTest, BuildGroups_ExplicitGroupNames) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "rag"}, + {"model_c", "rag"}, + }); + mgr.buildGroups(configs, mm); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 1u); + EXPECT_TRUE(groups.count("rag")); + EXPECT_EQ(groups.at("rag").modelNames.size(), 3u); +} + +TEST_F(ServableGroupManagerTest, BuildGroups_PermanentGroup) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({ + {"model_a", "permanent"}, + {"model_b", "permanent"}, + {"model_c", "rag"}, + }); + mgr.buildGroups(configs, mm); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 2u); + EXPECT_TRUE(groups.at("permanent").isPermanent()); + EXPECT_FALSE(groups.at("rag").isPermanent()); + EXPECT_EQ(groups.at("permanent").modelNames.size(), 2u); +} + +TEST_F(ServableGroupManagerTest, BuildGroups_MixedGroups) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "rag"}, + {"model_c", "audio"}, + {"model_d", "audio"}, + {"model_e", "permanent"}, + }); + mgr.buildGroups(configs, mm); + const auto& groups = mgr.getGroups(); + ASSERT_EQ(groups.size(), 3u); + EXPECT_EQ(groups.at("rag").modelNames.size(), 2u); + EXPECT_EQ(groups.at("audio").modelNames.size(), 2u); + EXPECT_EQ(groups.at("permanent").modelNames.size(), 1u); +} + +TEST_F(ServableGroupManagerTest, GetGroupForServable) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "audio"}, + }); + mgr.buildGroups(configs, mm); + EXPECT_EQ(mgr.getGroupForServable("model_a"), "rag"); + EXPECT_EQ(mgr.getGroupForServable("model_b"), "audio"); + EXPECT_EQ(mgr.getGroupForServable("nonexistent"), ""); +} + +TEST_F(ServableGroupManagerTest, IsGroupLoaded_PermanentAlwaysTrue) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({ + {"model_a", "permanent"}, + {"model_b", "rag"}, + }); + mgr.buildGroups(configs, mm); + EXPECT_TRUE(mgr.isGroupLoaded("permanent")); + EXPECT_FALSE(mgr.isGroupLoaded("rag")); + EXPECT_FALSE(mgr.isGroupLoaded("nonexistent")); +} + +TEST_F(ServableGroupManagerTest, GetAllConfiguredServableNames) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({ + {"model_a", "rag"}, + {"model_b", "rag"}, + {"model_c", "permanent"}, + }); + mgr.buildGroups(configs, mm); + auto names = mgr.getAllConfiguredServableNames(); + ASSERT_EQ(names.size(), 3u); + std::set nameSet(names.begin(), names.end()); + EXPECT_TRUE(nameSet.count("model_a")); + EXPECT_TRUE(nameSet.count("model_b")); + EXPECT_TRUE(nameSet.count("model_c")); +} + +TEST_F(ServableGroupManagerTest, RecordActivityUpdatesTimestamp) { + ServableGroupManager mgr(30'000'000); + auto configs = createModelConfigs({{"model_a", "rag"}}); + mgr.buildGroups(configs, mm); + + // Record activity and verify no crash + mgr.recordActivity(); +} + +TEST_F(ServableGroupManagerTest, ActiveGroupNameInitiallyEmpty) { + ServableGroupManager mgr(30'000'000); + EXPECT_TRUE(mgr.getActiveGroupName().empty()); +} + +struct RecordedEvent { + TaskEvent event; + ServableLoadingTaskType type; + std::string name; + bool urgent; +}; + +class ServableGroupSwapTest : public TestWithTempDir { +protected: + std::unique_ptr mm; + std::mutex recordMtx; + std::vector events; + + static std::string swapConfig() { + return R"({"model_config_list": [ + {"config": {"name": "a1", "base_path": ")" + + dummy_model_location + R"(", "target_device": "CPU", "nireq": 1, "group_name": "groupA"}}, + {"config": {"name": "a2", "base_path": ")" + + dummy_model_location + R"(", "target_device": "CPU", "nireq": 1, "group_name": "groupA"}}, + {"config": {"name": "b1", "base_path": ")" + + dummy_model_location + R"(", "target_device": "CPU", "nireq": 1, "group_name": "groupB"}}, + {"config": {"name": "b2", "base_path": ")" + + dummy_model_location + R"(", "target_device": "CPU", "nireq": 1, "group_name": "groupB"}} + ]})"; + } + + void SetUp() override { + TestWithTempDir::SetUp(); + std::string configFilePath = directoryPath + "/config.json"; + std::ofstream(configFilePath) << swapConfig(); + + mm = std::make_unique(uint64_t{30'000'000}); + auto status = mm->loadConfig(configFilePath); + ASSERT_TRUE(status.ok()) << status.string(); + groupManager = mm->getGroupManager(); + ASSERT_NE(groupManager, nullptr); + ASSERT_TRUE(groupManager->getActiveGroupName().empty()); + + // Installed after loadConfig so that only swap traffic is recorded. + mm->getLoadingQueue().setTaskObserver( + [this](TaskEvent event, const ServableLoadingTask& task) { + std::lock_guard lock(recordMtx); + events.push_back({event, task.type, task.name, task.urgent}); + }); + } + + ServableGroupManager* groupManager = nullptr; + + void clearRecorded() { + std::lock_guard lock(recordMtx); + events.clear(); + } + + std::vector recorded(TaskEvent event) { + std::lock_guard lock(recordMtx); + std::vector filtered; + std::copy_if(events.begin(), events.end(), std::back_inserter(filtered), + [event](const RecordedEvent& e) { return e.event == event; }); + return filtered; + } + + static bool isUnload(ServableLoadingTaskType type) { + return type == ServableLoadingTaskType::PutToSleepModel || + type == ServableLoadingTaskType::PutToSleepMediapipe; + } + + static size_t countLoadsOf(const std::vector& tasks, const std::string& name) { + return std::count_if(tasks.begin(), tasks.end(), [&name](const RecordedEvent& t) { + return t.name == name && t.type == ServableLoadingTaskType::WakeUpModel; + }); + } + + void ensureLoaded(const std::string& servableName) { + auto status = groupManager->ensureServableLoaded(servableName, *mm); + ASSERT_TRUE(status.ok()) << servableName << ": " << status.string(); + auto instance = mm->findModelByName(servableName)->getDefaultModelInstance(); + ASSERT_NE(instance, nullptr); + EXPECT_EQ(instance->getStatus().getState(), ModelVersionState::AVAILABLE); + } + + void expectRequestedLoadsFirst(const std::string& requested) { + ASSERT_NO_FATAL_FAILURE(ensureLoaded(requested)); + + auto executed = recorded(TaskEvent::Executed); + ASSERT_FALSE(executed.empty()); + EXPECT_EQ(executed[0].name, requested) + << "the servable that triggered the wake-up should load first to shorten " + "time-to-first-response"; + EXPECT_TRUE(executed[0].urgent); + } +}; + +TEST_F(ServableGroupSwapTest, SwapUnloadsPreviousGroupBeforeLoadingNew) { + ASSERT_NO_FATAL_FAILURE(ensureLoaded("a1")); + ASSERT_EQ(groupManager->getActiveGroupName(), "groupA"); + clearRecorded(); + + ASSERT_NO_FATAL_FAILURE(ensureLoaded("b1")); + auto tasks = recorded(TaskEvent::Executed); + ASSERT_FALSE(tasks.empty()); + + std::set retired; + size_t firstLoadIdx = tasks.size(); + for (size_t i = 0; i < tasks.size(); ++i) { + if (isUnload(tasks[i].type)) { + retired.insert(tasks[i].name); + EXPECT_LT(i, firstLoadIdx) << "unload of " << tasks[i].name << " ran after a load"; + } else if (i < firstLoadIdx) { + firstLoadIdx = i; + } + } + EXPECT_EQ(retired, (std::set{"a1", "a2"})) + << "every member of the previously active group must be unloaded on swap"; + EXPECT_EQ(groupManager->getActiveGroupName(), "groupB"); + for (const char* name : {"a1", "a2"}) { + auto instance = mm->findModelByName(name)->getDefaultModelInstance(); + ASSERT_NE(instance, nullptr) << name << " must stay known so it can be woken up again"; + EXPECT_EQ(instance->getStatus().getState(), ModelVersionState::SLEEPING) + << name << " must not be servable after its group was swapped out"; + } +} + +TEST_F(ServableGroupSwapTest, RequestedServableIsScheduledOnlyOnce) { + ASSERT_NO_FATAL_FAILURE(ensureLoaded("b1")); + // Second request hits a group that is already active, so nothing should reload. + ASSERT_NO_FATAL_FAILURE(ensureLoaded("b2")); + + auto tasks = recorded(TaskEvent::Scheduled); + EXPECT_EQ(countLoadsOf(tasks, "b1"), 1u) + << "loadGroup() already loads every group member, so ensureServableLoaded() " + "must not schedule the requested servable a second time"; + EXPECT_EQ(countLoadsOf(tasks, "b2"), 1u) + << "b2 was already loaded as part of groupB - requesting it must not reload it"; +} + +TEST_F(ServableGroupSwapTest, AlreadyAvailableServableDoesNotTouchLoadingQueue) { + // we should try to load servable/push task to queue when its already loaded + ASSERT_NO_FATAL_FAILURE(ensureLoaded("b1")); + clearRecorded(); + + for (int i = 0; i < 5; ++i) { + ASSERT_NO_FATAL_FAILURE(ensureLoaded("b1")); + } + + EXPECT_TRUE(recorded(TaskEvent::Scheduled).empty()) + << "requesting an already available servable must be answered without a " + "loading queue round-trip"; + EXPECT_EQ(groupManager->getActiveGroupName(), "groupB"); +} + +// The requested servable must load first regardless of its position in the group's +// name ordering, so both directions are checked. +TEST_F(ServableGroupSwapTest, RequestedServableIsLoadedFirstWithinGroupLastAlphabetically) { + expectRequestedLoadsFirst("a2"); +} + +TEST_F(ServableGroupSwapTest, RequestedServableIsLoadedFirstWithinGroupFirstAlphabetically) { + expectRequestedLoadsFirst("a1"); +} diff --git a/src/test/servable_loading_queue_test.cpp b/src/test/servable_loading_queue_test.cpp new file mode 100644 index 0000000000..890560c645 --- /dev/null +++ b/src/test/servable_loading_queue_test.cpp @@ -0,0 +1,199 @@ +//***************************************************************************** +// Copyright 2026 Intel Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//***************************************************************************** +#include +#include +#include +#include + +#include + +#include "src/servable_management/servable_loading_queue.hpp" +#include "src/servable_management/servable_loading_task.hpp" + +using namespace ovms; + +// Gate that the test thread can use to block/unblock the processor +struct TaskGate { + std::mutex mtx; + std::condition_variable cv; + bool open = false; + + void wait() { + std::unique_lock lock(mtx); + cv.wait(lock, [this] { return open; }); + } + void release() { + std::lock_guard lock(mtx); + open = true; + cv.notify_all(); + } +}; + +class ServableLoadingQueueTest : public ::testing::Test { +protected: + std::mutex orderMtx; + std::vector executionOrder; + + void recordExecution(const std::string& name) { + std::lock_guard lock(orderMtx); + executionOrder.push_back(name); + } + + Status defaultProcessor(ServableLoadingTask& task) { + recordExecution(task.name); + if (task.name.find("bad") != std::string::npos) { + return StatusCode::MODEL_NAME_MISSING; + } + return StatusCode::OK; + } +}; + +TEST_F(ServableLoadingQueueTest, NonPriorityTaskCompletes) { + ServableLoadingQueue queue; + queue.start([this](ServableLoadingTask& task) { return defaultProcessor(task); }); + + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, "model_a"}; + auto future = queue.scheduleTask(std::move(task)); + auto status = future.get(); + + EXPECT_EQ(status, StatusCode::OK); + ASSERT_EQ(executionOrder.size(), 1); + EXPECT_EQ(executionOrder[0], "model_a"); +} + +TEST_F(ServableLoadingQueueTest, PriorityTaskCompletes) { + ServableLoadingQueue queue; + queue.start([this](ServableLoadingTask& task) { return defaultProcessor(task); }); + + bool isPriorityRequest{true}; + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, "urgent_model", isPriorityRequest}; + auto future = queue.scheduleTask(std::move(task)); + auto status = future.get(); + + EXPECT_EQ(status, StatusCode::OK); + ASSERT_EQ(executionOrder.size(), 1); + EXPECT_EQ(executionOrder[0], "urgent_model"); +} + +TEST_F(ServableLoadingQueueTest, ProcessorStatusPropagated) { + ServableLoadingQueue queue; + queue.start([this](ServableLoadingTask& task) { return defaultProcessor(task); }); + + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, "bad_model"}; + auto future = queue.scheduleTask(std::move(task)); + + EXPECT_EQ(future.get(), StatusCode::MODEL_NAME_MISSING); + ASSERT_EQ(executionOrder.size(), 1); + EXPECT_EQ(executionOrder[0], "bad_model"); +} + +TEST_F(ServableLoadingQueueTest, PriorityTaskRunsBeforeQueuedNonPriority) { + // Block the processor on the first task so we can queue up tasks behind it + TaskGate gate; + TaskGate processingStarted; + ServableLoadingQueue queue; + queue.start([this, &gate, &processingStarted](ServableLoadingTask& task) -> Status { + if (task.name == "blocker") { + processingStarted.release(); + gate.wait(); + } + return defaultProcessor(task); + }); + + // Task 1: non-priority, will block in processor + ServableLoadingTask blocker{ServableLoadingTaskType::LoadModel, "blocker"}; + auto blockerFuture = queue.scheduleTask(std::move(blocker)); + + // Wait until the worker is actually processing the blocker + processingStarted.wait(); + + // Task 2: non-priority, queued behind blocker + ServableLoadingTask normal{ServableLoadingTaskType::LoadModel, "normal"}; + auto normalFuture = queue.scheduleTask(std::move(normal)); + + // Task 3: priority, should jump ahead of "normal" + bool isPriorityRequest{true}; + ServableLoadingTask urgent{ServableLoadingTaskType::LoadModel, "urgent", isPriorityRequest}; + auto urgentFuture = queue.scheduleTask(std::move(urgent)); + + // Release the blocker — worker processes remaining tasks in queue order + gate.release(); + + EXPECT_EQ(blockerFuture.get(), StatusCode::OK); + EXPECT_EQ(urgentFuture.get(), StatusCode::OK); + EXPECT_EQ(normalFuture.get(), StatusCode::OK); + + ASSERT_EQ(executionOrder.size(), 3); + EXPECT_EQ(executionOrder[0], "blocker"); + EXPECT_EQ(executionOrder[1], "urgent"); + EXPECT_EQ(executionOrder[2], "normal"); +} + +TEST_F(ServableLoadingQueueTest, StopFinishesCurrentTaskNotPendingOnes) { + TaskGate gate; + TaskGate processingStarted; + ServableLoadingQueue queue; + queue.start([this, &gate, &processingStarted](ServableLoadingTask& task) -> Status { + if (task.name == "blocker") { + processingStarted.release(); + gate.wait(); + } + return defaultProcessor(task); + }); + + ServableLoadingTask blocker{ServableLoadingTaskType::LoadModel, "blocker"}; + auto f1 = queue.scheduleTask(std::move(blocker)); + + processingStarted.wait(); + + ServableLoadingTask pending{ServableLoadingTaskType::LoadModel, "pending"}; + auto f2 = queue.scheduleTask(std::move(pending)); + + // Signal stop while worker is still blocked — ensures !running before it loops + queue.requestStop(); + gate.release(); + queue.stop(); + + EXPECT_EQ(f1.get(), StatusCode::OK); + EXPECT_EQ(f2.get(), StatusCode::SERVER_SHUTTING_DOWN); + ASSERT_EQ(executionOrder.size(), 1); + EXPECT_EQ(executionOrder[0], "blocker"); +} + +TEST_F(ServableLoadingQueueTest, MultipleTasksProcessedSerially) { + std::atomic concurrency{0}; + std::atomic maxConcurrency{0}; + ServableLoadingQueue queue; + queue.start([&](ServableLoadingTask&) -> Status { + int cur = ++concurrency; + int prev = maxConcurrency.load(); + while (cur > prev && !maxConcurrency.compare_exchange_weak(prev, cur)) { + } + std::this_thread::yield(); + --concurrency; + return StatusCode::OK; + }); + + std::vector> futures; + for (int i = 0; i < 10; ++i) { + ServableLoadingTask task{ServableLoadingTaskType::LoadModel, "m" + std::to_string(i)}; + futures.push_back(queue.scheduleTask(std::move(task))); + } + for (auto& f : futures) { + EXPECT_EQ(f.get(), StatusCode::OK); + } + EXPECT_EQ(maxConcurrency.load(), 1); +} diff --git a/src/test/server_test.cpp b/src/test/server_test.cpp index 0665772689..7bef3347a9 100644 --- a/src/test/server_test.cpp +++ b/src/test/server_test.cpp @@ -29,11 +29,11 @@ #include "../logging.hpp" #include "../model.hpp" #include "../modelinstanceunloadguard.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "../module_names.hpp" #include "../ovms_exit_codes.hpp" #include "../prediction_service_utils.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../version.hpp" #include "c_api_test_utils.hpp" diff --git a/src/test/streaming_test.cpp b/src/test/streaming_test.cpp index edadeb15de..d5b6e3a21c 100644 --- a/src/test/streaming_test.cpp +++ b/src/test/streaming_test.cpp @@ -24,7 +24,7 @@ #include "../kfs_frontend/kfs_grpc_inference_service.hpp" #include "../mediapipe_internal/mediapipegraphdefinition.hpp" #include "../mediapipe_internal/mediapipegraphexecutor.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../status.hpp" #include "../stringutils.hpp" diff --git a/src/test/stress_test_utils.hpp b/src/test/stress_test_utils.hpp index 1d9be7ec52..9c0236831f 100644 --- a/src/test/stress_test_utils.hpp +++ b/src/test/stress_test_utils.hpp @@ -45,7 +45,7 @@ #include "../modelconfig.hpp" #include "../modelinstance.hpp" #include "../prediction_service_utils.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../status.hpp" #include "../stringutils.hpp" diff --git a/src/test/test_utils.cpp b/src/test/test_utils.cpp index 5a5028fe34..66bb005a49 100644 --- a/src/test/test_utils.cpp +++ b/src/test/test_utils.cpp @@ -30,7 +30,7 @@ #include "../capi_frontend/inferenceparameter.hpp" #include "../kfs_frontend/kfs_utils.hpp" #include "../network_utils.hpp" -#include "../servablemanagermodule.hpp" +#include "src/servable_management/servablemanagermodule.hpp" #include "../server.hpp" #include "../tensorinfo.hpp" diff --git a/src/test/test_utils.hpp b/src/test/test_utils.hpp index e75d821006..7ee955d61b 100644 --- a/src/test/test_utils.hpp +++ b/src/test/test_utils.hpp @@ -46,7 +46,7 @@ #endif #include "src/metrics/metric_registry.hpp" #include "../modelinstance.hpp" -#include "../modelmanager.hpp" +#include "src/servable_management/modelmanager.hpp" #include "../shape.hpp" #include "../status.hpp" #include "../tensorinfo.hpp" @@ -775,11 +775,29 @@ class DummyMediapipeGraphDefinition : public ovms::MediapipeGraphDefinition { ovms::GenAiServableMap& getGenAiServableMap() { return this->sidePacketMaps->genAiServableMap; } + // Test seams for idle-unload concurrency tests. + // Drive the underlying state machine directly. + void forceReloadEventForTest() { this->status.handle(ovms::ReloadEvent()); } + void forceValidationPassedEventForTest() { this->status.handle(ovms::ValidationPassedEvent()); } + // Identity of the sidePacketMaps shared_ptr, so a test can detect whether it + // was reset/swapped (unload uses clear(), not reset(), so the pointer must be stable). + const void* sidePacketMapsPtrForTest() const { return static_cast(this->sidePacketMaps.get()); } + bool sidePacketMapsEmptyForTest() { return this->sidePacketMaps->empty(); } + // Insert a harmless marker into a side-packet map so we can detect teardown. + void insertSidePacketMarkerForTest(const std::string& key) { + this->sidePacketMaps->genAiServableMap.insert({key, nullptr}); + } + bool hasSidePacketMarkerForTest(const std::string& key) { + return this->sidePacketMaps->genAiServableMap.count(key) > 0; + } + uint64_t requestsHandlesCounterForTest() const { return this->pendingCreateExecutorCount.load(); } + DummyMediapipeGraphDefinition(const std::string name, const ovms::MediapipeGraphConfig& config, std::string inputConfig, - ovms::PythonBackend* pythonBackend = nullptr) : - ovms::MediapipeGraphDefinition(name, config, nullptr, nullptr, pythonBackend) { + ovms::PythonBackend* pythonBackend = nullptr, + bool lazyLoad = false) : + ovms::MediapipeGraphDefinition(name, config, nullptr, nullptr, pythonBackend, lazyLoad) { this->inputConfig = inputConfig; }