From fcfb8236df0ff9ae28242015f76c7067b4816cdb Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 10 Jul 2026 09:28:25 +0200 Subject: [PATCH 1/5] initial version --- src/modelinstance.cpp | 1 + src/ov_utils.cpp | 86 +++++++++++++++++++++++++++++++++++++++++++ src/ov_utils.hpp | 10 +++++ 3 files changed, 97 insertions(+) diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 074cd0ec2e..8ecfcec251 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1076,6 +1076,7 @@ Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) { return this->compiledModel->get_property(key); }, std::string("compiled model: ") + getName(), std::string(" version: ") + std::to_string(getVersion()) + std::string("; target device: ") + targetDevice + ";"); + SPDLOG_LOGGER_INFO(modelmanager_logger, "Recommended target device: {}", recommendTargetDevice(ieCore)); return StatusCode::OK; } diff --git a/src/ov_utils.cpp b/src/ov_utils.cpp index aeb4ca1e11..5632dcf778 100644 --- a/src/ov_utils.cpp +++ b/src/ov_utils.cpp @@ -150,6 +150,92 @@ Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const st return StatusCode::OK; } +std::string recommendTargetDevice(ov::Core& core) { + auto availableDevices = core.get_available_devices(); + + struct GpuDeviceInfo { + std::string name; + bool isDiscrete = false; + int64_t freeMemBytes = 0; + }; + + std::vector gpuDevices; + + for (const auto& device : availableDevices) { + if (device.find("GPU") != 0) { + continue; + } + + GpuDeviceInfo info; + info.name = device; + + try { + auto deviceType = core.get_property(device, ov::device::type); + info.isDiscrete = (deviceType == ov::device::Type::DISCRETE); + } catch (const std::exception& e) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Failed to get DEVICE_TYPE for {}: {}", device, e.what()); + continue; + } + + try { + uint64_t totalMem = core.get_property(device, "GPU_DEVICE_TOTAL_MEM_SIZE").as(); + uint64_t usedMem = 0; + try { + auto memStats = core.get_property(device, "GPU_MEMORY_STATISTICS").as>(); + auto it = memStats.find("usm_device"); + if (it != memStats.end()) { + usedMem = it->second; + } + } catch (...) { + // Memory statistics may not be available before any model is loaded + } + info.freeMemBytes = static_cast(totalMem) - static_cast(usedMem); + } catch (const std::exception& e) { + SPDLOG_LOGGER_WARN(modelmanager_logger, "Failed to get memory info for {}: {}", device, e.what()); + info.freeMemBytes = 0; + } + + gpuDevices.push_back(info); + } + + if (gpuDevices.empty()) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "No GPU devices found, recommending CPU"); + return "CPU"; + } + + std::vector discreteGpus; + std::vector integratedGpus; + for (const auto& gpu : gpuDevices) { + if (gpu.isDiscrete) { + discreteGpus.push_back(gpu); + } else { + integratedGpus.push_back(gpu); + } + } + + if (discreteGpus.size() == 1) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Single discrete GPU found, recommending: {}", discreteGpus[0].name); + return discreteGpus[0].name; + } + + if (discreteGpus.size() > 1) { + auto best = std::max_element(discreteGpus.begin(), discreteGpus.end(), + [](const GpuDeviceInfo& a, const GpuDeviceInfo& b) { + return a.freeMemBytes < b.freeMemBytes; + }); + SPDLOG_LOGGER_INFO(modelmanager_logger, "Multiple discrete GPUs found, recommending {} with {} bytes free VRAM", best->name, best->freeMemBytes); + return best->name; + } + + if (!integratedGpus.empty()) { + SPDLOG_LOGGER_INFO(modelmanager_logger, "Integrated GPU found, recommending: {}", integratedGpus[0].name); + return integratedGpus[0].name; + } + + SPDLOG_LOGGER_INFO(modelmanager_logger, "Falling back to CPU"); + return "CPU"; +} + Status applyDefaultCpuProperties(ov::AnyMap& properties, uint16_t coreCount, uint16_t physicalCoresPerSocket, uint16_t socketsCount, uint16_t dockerCpuQuota) { if (properties.find(ov::hint::enable_cpu_pinning.name()) == properties.end()) { const bool cpuPinning = dockerCpuQuota <= 0; diff --git a/src/ov_utils.hpp b/src/ov_utils.hpp index cc62209d1d..cdefbc2141 100644 --- a/src/ov_utils.hpp +++ b/src/ov_utils.hpp @@ -52,6 +52,14 @@ std::optional getLayoutFromRTMap(const ov::RTMap& rtMap); Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const std::string& targetDevice, const ov::Core& ieCore); +// Determines the recommended target device based on available devices and their properties. +// Rules: +// - If only CPU is available (no GPU) - returns "CPU" +// - If a single discrete GPU is available - returns that device (e.g. "GPU.0") +// - If only integrated GPU(s) are available - returns the first integrated GPU +// - If multiple discrete GPUs are available - returns the one with the most free VRAM +std::string recommendTargetDevice(ov::Core& core); + // Applies resource-aware CPU defaults to an OpenVINO property map. // Sets inference_num_threads and (on Linux) enable_cpu_pinning only when not // already present in the map. When PERFORMANCE_HINT=THROUGHPUT is set, @@ -107,4 +115,6 @@ static void logOVPluginConfig(PropertyExtractor&& propertyExtractor, const std:: SPDLOG_LOGGER_DEBUG(modelmanager_logger, "{}; {}plugin configuration: {{ {} }}", loggingAuthor, loggingDetails, pluginConfigNameValuesString); } + + } // namespace ovms From f2da75a460154c2133259bf730f8923fecc57ede Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Fri, 10 Jul 2026 11:07:11 +0200 Subject: [PATCH 2/5] style --- src/ov_utils.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/ov_utils.hpp b/src/ov_utils.hpp index cdefbc2141..848f83a7f2 100644 --- a/src/ov_utils.hpp +++ b/src/ov_utils.hpp @@ -115,6 +115,4 @@ static void logOVPluginConfig(PropertyExtractor&& propertyExtractor, const std:: SPDLOG_LOGGER_DEBUG(modelmanager_logger, "{}; {}plugin configuration: {{ {} }}", loggingAuthor, loggingDetails, pluginConfigNameValuesString); } - - } // namespace ovms From 124390341a1a8aa337c4f9b37a545f95e758b4ff Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sat, 11 Jul 2026 01:27:54 +0200 Subject: [PATCH 3/5] initial version --- src/audio/speech_to_text/BUILD | 1 + src/audio/speech_to_text/s2t_servable.cpp | 10 ++++++++-- src/audio/text_to_speech/BUILD | 1 + src/audio/text_to_speech/t2s_servable.cpp | 8 +++++++- src/capi_frontend/server_settings.hpp | 2 +- src/cli_parser.cpp | 14 ++++++++------ src/config.cpp | 8 +++++--- src/embeddings/embeddings_calculator_ov.proto | 2 +- src/image_gen/BUILD | 1 + src/image_gen/pipelines.cpp | 4 +++- .../continuous_batching/servable_initializer.cpp | 4 ++++ .../language_model/legacy/servable_initializer.cpp | 5 +++++ src/llm/llm_calculator.proto | 2 +- .../legacy/servable_initializer.cpp | 5 +++++ src/modelconfig.hpp | 2 +- src/modelinstance.cpp | 8 ++++++-- src/modelmanager.cpp | 8 ++++++-- src/ov_utils.cpp | 3 ++- src/ov_utils.hpp | 2 +- src/rerank/rerank_calculator_ov.proto | 2 +- src/sidepacket_servable.cpp | 4 ++++ 21 files changed, 72 insertions(+), 24 deletions(-) diff --git a/src/audio/speech_to_text/BUILD b/src/audio/speech_to_text/BUILD index 770d9c5a39..d2a6eb9ef6 100644 --- a/src/audio/speech_to_text/BUILD +++ b/src/audio/speech_to_text/BUILD @@ -30,6 +30,7 @@ ovms_cc_library( "//src:libovmslogging", "//src:libmodelconfigjsonparser", "//src:libovmsstring_utils", + "//src:libovms_ov_utils", "s2t_calculator_cc_proto", "@com_google_absl//absl/status", "//third_party:genai", diff --git a/src/audio/speech_to_text/s2t_servable.cpp b/src/audio/speech_to_text/s2t_servable.cpp index 16051ea647..a989443acf 100644 --- a/src/audio/speech_to_text/s2t_servable.cpp +++ b/src/audio/speech_to_text/s2t_servable.cpp @@ -28,6 +28,7 @@ #include "src/http_payload.hpp" #include "src/json_parser.hpp" #include "src/logging.hpp" +#include "src/ov_utils.hpp" #include "src/stringutils.hpp" namespace ovms { @@ -42,6 +43,11 @@ SttServable::SttServable(const ::mediapipe::S2tCalculatorOptions& nodeOptions, c } else { parsedModelsPath = fsModelsPath; } + std::string device = nodeOptions.target_device(); + if (device.empty()) { + device = recommendTargetDevice(); + SPDLOG_INFO("No device specified for STT model, using recommended device: {}", device); + } ov::AnyMap config; auto status = JsonParser::parsePluginConfig(nodeOptions.plugin_config(), config); if (!status.ok()) { @@ -49,11 +55,11 @@ SttServable::SttServable(const ::mediapipe::S2tCalculatorOptions& nodeOptions, c throw std::runtime_error("Error during plugin_config option parsing"); } enableWordTimestamps = nodeOptions.enable_word_timestamps(); - if (enableWordTimestamps && nodeOptions.target_device() == "NPU") { + if (enableWordTimestamps && device == "NPU") { config["STATIC_PIPELINE"] = true; } config["word_timestamps"] = enableWordTimestamps; - sttPipeline = std::make_shared(parsedModelsPath.string(), nodeOptions.target_device(), config); + sttPipeline = std::make_shared(parsedModelsPath.string(), device, config); streamingExecutor = std::make_unique(sttPipeline, sttPipelineMutex); } diff --git a/src/audio/text_to_speech/BUILD b/src/audio/text_to_speech/BUILD index a3d10f9893..65d0f0dff6 100644 --- a/src/audio/text_to_speech/BUILD +++ b/src/audio/text_to_speech/BUILD @@ -26,6 +26,7 @@ ovms_cc_library( "@mediapipe//mediapipe/framework:calculator_framework", "//src:libovmslogging", "//src:libmodelconfigjsonparser", + "//src:libovms_ov_utils", "t2s_calculator_cc_proto", ], visibility = ["//visibility:public"], diff --git a/src/audio/text_to_speech/t2s_servable.cpp b/src/audio/text_to_speech/t2s_servable.cpp index 501c867e5e..a0af03b2cf 100644 --- a/src/audio/text_to_speech/t2s_servable.cpp +++ b/src/audio/text_to_speech/t2s_servable.cpp @@ -25,6 +25,7 @@ #include "src/status.hpp" #include "src/logging.hpp" #include "src/json_parser.hpp" +#include "src/ov_utils.hpp" #include "src/audio/text_to_speech/t2s_servable.hpp" @@ -94,13 +95,18 @@ TtsServable::TtsServable(const std::string& modelDir, const std::string& targetD } else { parsedModelsPath = fsModelsPath; } + std::string device = targetDevice; + if (device.empty()) { + device = recommendTargetDevice(); + SPDLOG_INFO("No device specified for TTS model, using recommended device: {}", device); + } ov::AnyMap config; Status status = JsonParser::parsePluginConfig(pluginConfig, config); if (!status.ok()) { SPDLOG_ERROR("Error during llm node plugin_config option parsing to JSON: {}", pluginConfig); throw std::runtime_error("Error during plugin_config option parsing"); } - ttsPipeline = std::make_shared(parsedModelsPath.string(), targetDevice, config); + ttsPipeline = std::make_shared(parsedModelsPath.string(), device, config); const ov::Shape speakerEmbeddingShape = ttsPipeline->get_speaker_embedding_shape(); for (const auto& voice : graphVoices) { std::filesystem::path voicePath(voice.path()); diff --git a/src/capi_frontend/server_settings.hpp b/src/capi_frontend/server_settings.hpp index 2dda86a6f4..a6968994dc 100644 --- a/src/capi_frontend/server_settings.hpp +++ b/src/capi_frontend/server_settings.hpp @@ -191,7 +191,7 @@ struct ImageGenerationGraphSettingsImpl { struct ExportSettings { std::string modelName = ""; std::string modelPath = "./"; - std::string targetDevice = "CPU"; + std::string targetDevice; std::optional restWorkers; std::optional extraQuantizationParams; std::optional vocoder; diff --git a/src/cli_parser.cpp b/src/cli_parser.cpp index e05109d905..74ed63f19a 100644 --- a/src/cli_parser.cpp +++ b/src/cli_parser.cpp @@ -289,8 +289,8 @@ std::variant> CLIParser::parse(int argc, char* cxxopts::value(), "NIREQ") ("target_device", - "Target device to run the inference", - cxxopts::value()->default_value("CPU"), + "Target device to run the inference. Default: auto-detected based on available devices.", + cxxopts::value()->default_value(""), "TARGET_DEVICE") ("plugin_config", "A dictionary of plugin configuration keys and their values, eg \"{\\\"NUM_STREAMS\\\": \\\"1\\\"}\". Default number of streams is optimized to optimal latency with low concurrency.", @@ -648,10 +648,12 @@ void CLIParser::prepareModel(ModelsSettingsImpl& modelsSettings, HFSettingsImpl& if (result->count("target_device")) { modelsSettings.targetDevice = result->operator[]("target_device").as(); - if (isHFPullOrPullAndStart(this->result)) { - hfSettings.exportSettings.targetDevice = modelsSettings.targetDevice; - } else { - modelsSettings.userSetSingleModelArguments.push_back("target_device"); + if (!modelsSettings.targetDevice.empty()) { + if (isHFPullOrPullAndStart(this->result)) { + hfSettings.exportSettings.targetDevice = modelsSettings.targetDevice; + } else { + modelsSettings.userSetSingleModelArguments.push_back("target_device"); + } } } diff --git a/src/config.cpp b/src/config.cpp index aa888aa4de..9d964557a9 100644 --- a/src/config.cpp +++ b/src/config.cpp @@ -197,7 +197,10 @@ bool Config::validate() { std::vector allowedTargetDevices = {"CPU", "GPU", "NPU", "AUTO"}; bool validDeviceSelected = false; - if (exportSettings.targetDevice.rfind("GPU.", 0) == 0) { + if (exportSettings.targetDevice.empty()) { + // Empty means auto-detect via recommendTargetDevice + validDeviceSelected = true; + } else if (exportSettings.targetDevice.rfind("GPU.", 0) == 0) { // Accept GPU.x where x is a number to select specific GPU card std::string indexPart = exportSettings.targetDevice.substr(4); validDeviceSelected = !indexPart.empty() && std::all_of(indexPart.begin(), indexPart.end(), ::isdigit); @@ -407,8 +410,7 @@ const std::string Config::precision() const { return this->modelsSettings.precis const std::string& Config::modelVersionPolicy() const { return this->modelsSettings.modelVersionPolicy; } uint32_t Config::nireq() const { return this->modelsSettings.nireq; } const std::string& Config::targetDevice() const { - static const std::string defaultTargetDevice = "CPU"; - return this->modelsSettings.targetDevice.empty() ? defaultTargetDevice : this->modelsSettings.targetDevice; + return this->modelsSettings.targetDevice; } const std::string& Config::Config::pluginConfig() const { return this->modelsSettings.pluginConfig; } bool Config::metricsEnabled() const { return this->serverSettings.metricsEnabled; } diff --git a/src/embeddings/embeddings_calculator_ov.proto b/src/embeddings/embeddings_calculator_ov.proto index 53f7921225..fa4db8e396 100644 --- a/src/embeddings/embeddings_calculator_ov.proto +++ b/src/embeddings/embeddings_calculator_ov.proto @@ -27,7 +27,7 @@ message EmbeddingsCalculatorOVOptions { } required string models_path = 1; optional bool normalize_embeddings = 2 [default = true]; - optional string target_device = 3 [default = "CPU"]; + optional string target_device = 3; optional string plugin_config = 4 [default = ""]; enum Pooling { CLS = 0; diff --git a/src/image_gen/BUILD b/src/image_gen/BUILD index ec99c26221..94a06bbaea 100644 --- a/src/image_gen/BUILD +++ b/src/image_gen/BUILD @@ -26,6 +26,7 @@ ovms_cc_library( "//src:libovmslogging", "//src:libovms_queue", "//src:libovmsstring_utils", + "//src:libovms_ov_utils", "//third_party:genai",], visibility = ["//visibility:public"], alwayslink = 1, diff --git a/src/image_gen/pipelines.cpp b/src/image_gen/pipelines.cpp index 3c6bba5e39..dcb3d223b1 100644 --- a/src/image_gen/pipelines.cpp +++ b/src/image_gen/pipelines.cpp @@ -23,6 +23,7 @@ #include #include "src/logging.hpp" +#include "src/ov_utils.hpp" #include "src/stringutils.hpp" namespace ovms { @@ -61,7 +62,8 @@ ImageGenerationPipelines::ImageGenerationPipelines(const ImageGenPipelineArgs& a args(args) { std::vector device; if (!args.device.size()) { - device.push_back("CPU"); + device.push_back(recommendTargetDevice()); + SPDLOG_INFO("No device specified for image generation model, using recommended device: {}", device[0]); } else { device = args.device; } diff --git a/src/llm/language_model/continuous_batching/servable_initializer.cpp b/src/llm/language_model/continuous_batching/servable_initializer.cpp index 1aaff99844..6621b2d414 100644 --- a/src/llm/language_model/continuous_batching/servable_initializer.cpp +++ b/src/llm/language_model/continuous_batching/servable_initializer.cpp @@ -190,6 +190,10 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptrdevice = nodeOptions.device(); + if (properties->device.empty()) { + properties->device = recommendTargetDevice(); + SPDLOG_INFO("No device specified for LLM CB model, using recommended device: {}", properties->device); + } properties->bestOfLimit = nodeOptions.best_of_limit(); properties->enableToolGuidedGeneration = nodeOptions.enable_tool_guided_generation(); diff --git a/src/llm/language_model/legacy/servable_initializer.cpp b/src/llm/language_model/legacy/servable_initializer.cpp index 52d041f74d..3e3029a27d 100644 --- a/src/llm/language_model/legacy/servable_initializer.cpp +++ b/src/llm/language_model/legacy/servable_initializer.cpp @@ -33,6 +33,7 @@ #include "../../../json_parser.hpp" #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" +#include "../../../ov_utils.hpp" #include "../../../status.hpp" #include "../../io_processing/parser_config_validation.hpp" #include "servable.hpp" @@ -85,6 +86,10 @@ Status LegacyServableInitializer::initialize(std::shared_ptr& ser properties->schedulerConfig.enable_prefix_caching = nodeOptions.enable_prefix_caching(); properties->device = nodeOptions.device(); + if (properties->device.empty()) { + properties->device = recommendTargetDevice(); + SPDLOG_INFO("No device specified for LLM legacy model, using recommended device: {}", properties->device); + } if (nodeOptions.has_draft_max_num_batched_tokens() || nodeOptions.has_draft_cache_size() || nodeOptions.has_draft_dynamic_split_fuse() || nodeOptions.has_draft_max_num_seqs() || nodeOptions.has_draft_block_size() || nodeOptions.has_draft_device()) { // Consider moving draft parameters to separate structure in node options, so it's validated on the proto level diff --git a/src/llm/llm_calculator.proto b/src/llm/llm_calculator.proto index c3e38f3408..751f8cf2a7 100644 --- a/src/llm/llm_calculator.proto +++ b/src/llm/llm_calculator.proto @@ -93,7 +93,7 @@ message LLMCalculatorOptions { optional bool dynamic_split_fuse = 5 [default = true]; - optional string device = 6 [default = "CPU"]; + optional string device = 6; optional string plugin_config = 7 [default = ""]; diff --git a/src/llm/visual_language_model/legacy/servable_initializer.cpp b/src/llm/visual_language_model/legacy/servable_initializer.cpp index 3b3cb61923..ec99b4950d 100644 --- a/src/llm/visual_language_model/legacy/servable_initializer.cpp +++ b/src/llm/visual_language_model/legacy/servable_initializer.cpp @@ -33,6 +33,7 @@ #include "../../../json_parser.hpp" #include "../../../logging.hpp" #include "../../../mediapipe_internal/mediapipe_utils.hpp" +#include "../../../ov_utils.hpp" #include "../../../status.hpp" #include "../../io_processing/parser_config_validation.hpp" #include "servable.hpp" @@ -84,6 +85,10 @@ Status VisualLanguageModelLegacyServableInitializer::initialize(std::shared_ptr< properties->schedulerConfig.enable_prefix_caching = nodeOptions.enable_prefix_caching(); properties->device = nodeOptions.device(); + if (properties->device.empty()) { + properties->device = recommendTargetDevice(); + SPDLOG_INFO("No device specified for VLM model, using recommended device: {}", properties->device); + } if (nodeOptions.has_draft_max_num_batched_tokens() || nodeOptions.has_draft_cache_size() || nodeOptions.has_draft_dynamic_split_fuse() || nodeOptions.has_draft_max_num_seqs() || nodeOptions.has_draft_block_size() || nodeOptions.has_draft_device()) { // Consider moving draft parameters to separate structure in node options, so it's validated on the proto level diff --git a/src/modelconfig.hpp b/src/modelconfig.hpp index 292e6917c0..353715f651 100644 --- a/src/modelconfig.hpp +++ b/src/modelconfig.hpp @@ -211,7 +211,7 @@ class ModelConfig { */ ModelConfig(const std::string& name = "", const std::string& basePath = "", - const std::string& targetDevice = "CPU", + const std::string& targetDevice = "", const std::string& configBatchSize = "", uint64_t nireq = 0, const std::string& cacheDir = "", diff --git a/src/modelinstance.cpp b/src/modelinstance.cpp index 8ecfcec251..e5ee64f4ff 100644 --- a/src/modelinstance.cpp +++ b/src/modelinstance.cpp @@ -1023,7 +1023,7 @@ plugin_config_t ModelInstance::prepareDefaultPluginConfig(const ModelConfig& con Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) { plugin_config_t pluginConfig = prepareDefaultPluginConfig(config); - if (config.getTargetDevice() == "CPU") { + if (this->targetDevice == "CPU") { Status status = applyDefaultCpuProperties(pluginConfig); if (!status.ok()) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to apply default CPU properties for model: {}; version: {}; error: {}", @@ -1076,7 +1076,6 @@ Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) { return this->compiledModel->get_property(key); }, std::string("compiled model: ") + getName(), std::string(" version: ") + std::to_string(getVersion()) + std::string("; target device: ") + targetDevice + ";"); - SPDLOG_LOGGER_INFO(modelmanager_logger, "Recommended target device: {}", recommendTargetDevice(ieCore)); return StatusCode::OK; } @@ -1239,6 +1238,11 @@ Status ModelInstance::loadModelImpl(const ModelConfig& config, const DynamicMode subscriptionManager.notifySubscribers(); this->path = config.getPath(); this->targetDevice = config.getTargetDevice(); + if (this->targetDevice.empty()) { + this->targetDevice = recommendTargetDevice(); + SPDLOG_LOGGER_INFO(modelmanager_logger, "No target device specified for model: {}; version: {}; using recommended device: {}", + config.getName(), config.getVersion(), this->targetDevice); + } this->config = config; auto status = fetchModelFilepaths(); diff --git a/src/modelmanager.cpp b/src/modelmanager.cpp index 544ad2a69b..c14c1efe4e 100644 --- a/src/modelmanager.cpp +++ b/src/modelmanager.cpp @@ -296,7 +296,9 @@ Status ModelManager::startFromConfig() { return status; } - status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *ieCore.get()); + status = validatePluginConfiguration(modelConfig.getPluginConfig(), + modelConfig.getTargetDevice().empty() ? recommendTargetDevice() : modelConfig.getTargetDevice(), + *ieCore.get()); if (!status.ok()) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys"); return status; @@ -728,7 +730,9 @@ Status ModelManager::ConfigLoader::loadModels(ModelManager& modelManager, const continue; } - status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *modelManager.ieCore.get()); + status = validatePluginConfiguration(modelConfig.getPluginConfig(), + modelConfig.getTargetDevice().empty() ? recommendTargetDevice() : modelConfig.getTargetDevice(), + *modelManager.ieCore.get()); if (!status.ok()) { SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys"); return status; diff --git a/src/ov_utils.cpp b/src/ov_utils.cpp index 5632dcf778..b0cb3fc61b 100644 --- a/src/ov_utils.cpp +++ b/src/ov_utils.cpp @@ -150,7 +150,8 @@ Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const st return StatusCode::OK; } -std::string recommendTargetDevice(ov::Core& core) { +std::string recommendTargetDevice() { + static ov::Core core; auto availableDevices = core.get_available_devices(); struct GpuDeviceInfo { diff --git a/src/ov_utils.hpp b/src/ov_utils.hpp index 848f83a7f2..53d0c4a73c 100644 --- a/src/ov_utils.hpp +++ b/src/ov_utils.hpp @@ -58,7 +58,7 @@ Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const st // - If a single discrete GPU is available - returns that device (e.g. "GPU.0") // - If only integrated GPU(s) are available - returns the first integrated GPU // - If multiple discrete GPUs are available - returns the one with the most free VRAM -std::string recommendTargetDevice(ov::Core& core); +std::string recommendTargetDevice(); // Applies resource-aware CPU defaults to an OpenVINO property map. // Sets inference_num_threads and (on Linux) enable_cpu_pinning only when not diff --git a/src/rerank/rerank_calculator_ov.proto b/src/rerank/rerank_calculator_ov.proto index a0bdd20fbf..fc985b02d7 100644 --- a/src/rerank/rerank_calculator_ov.proto +++ b/src/rerank/rerank_calculator_ov.proto @@ -32,7 +32,7 @@ message RerankCalculatorOVOptions { optional uint64 max_position_embeddings = 3; - optional string target_device = 4 [default = "CPU"]; + optional string target_device = 4; optional string plugin_config = 5 [default = ""]; } diff --git a/src/sidepacket_servable.cpp b/src/sidepacket_servable.cpp index 18778f77d1..7587f21789 100644 --- a/src/sidepacket_servable.cpp +++ b/src/sidepacket_servable.cpp @@ -62,6 +62,10 @@ SidepacketServable::SidepacketServable(const std::string& modelDir, const std::s void SidepacketServable::initialize(const std::string& modelDir, const std::string& inputTargetDevice, const std::string& pluginConfig, const std::string& graphPath) { this->targetDevice = inputTargetDevice; + if (this->targetDevice.empty()) { + this->targetDevice = recommendTargetDevice(); + SPDLOG_INFO("No target device specified for sidepacket servable, using recommended device: {}", this->targetDevice); + } auto fsModelsPath = std::filesystem::path(modelDir); if (fsModelsPath.is_relative()) { parsedModelsPath = (std::filesystem::path(graphPath) / fsModelsPath); From 3968888eb22cfff0b3b009a34e9a86664f18cceb Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sat, 11 Jul 2026 03:01:46 +0200 Subject: [PATCH 4/5] update tests --- src/test/c_api_tests.cpp | 2 +- src/test/graph_export_test.cpp | 12 ++++++------ src/test/lora_graph_export_test.cpp | 12 ++++++------ src/test/ovmsconfig_test.cpp | 18 +++++++++--------- src/test/pull_hf_model_test.cpp | 4 ++-- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/test/c_api_tests.cpp b/src/test/c_api_tests.cpp index e59f27cbf4..3a650d3bd2 100644 --- a/src/test/c_api_tests.cpp +++ b/src/test/c_api_tests.cpp @@ -233,7 +233,7 @@ TEST(CAPIConfigTest, MultiModelConfiguration) { EXPECT_EQ(cfg.precision(), ""); EXPECT_EQ(cfg.modelVersionPolicy(), ""); EXPECT_EQ(cfg.nireq(), 0); - EXPECT_EQ(cfg.targetDevice(), "CPU"); + EXPECT_EQ(cfg.targetDevice(), ""); EXPECT_EQ(cfg.pluginConfig(), ""); EXPECT_EQ(cfg.configPath(), getGenericFullPathForTmp("/tmp/config")); diff --git a/src/test/graph_export_test.cpp b/src/test/graph_export_test.cpp index ed1f4203f0..539975dbac 100644 --- a/src/test/graph_export_test.cpp +++ b/src/test/graph_export_test.cpp @@ -164,7 +164,7 @@ const std::string expectedDefaultGraphContents = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./", enable_prefix_caching: true, cache_size: 10, @@ -330,7 +330,7 @@ node { [type.googleapis.com / mediapipe.RerankCalculatorOVOptions]: { models_path: "./", max_allowed_chunks: 10000, - target_device: "CPU", + target_device: "", plugin_config: '{"NUM_STREAMS":"1"}', } } @@ -374,7 +374,7 @@ node { normalize_embeddings: true, truncate: false, pooling: CLS, - target_device: "CPU", + target_device: "", plugin_config: '{"NUM_STREAMS":"1"}', } } @@ -412,7 +412,7 @@ node { node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { models_path: "./" - target_device: "CPU" + target_device: "" } } } @@ -430,7 +430,7 @@ node { node_options: { [type.googleapis.com / mediapipe.T2sCalculatorOptions]: { models_path: "./" - target_device: "CPU" + target_device: "" voices: [ { name: "af_alloy", path: "./voices/af_alloy.bin" }, { name: "am_adam", path: "./voices/am_adam.bin" } @@ -493,7 +493,7 @@ node { node_options: { [type.googleapis.com / mediapipe.S2tCalculatorOptions]: { models_path: "./" - target_device: "CPU" + target_device: "" } } input_stream_handler { diff --git a/src/test/lora_graph_export_test.cpp b/src/test/lora_graph_export_test.cpp index 1ac03102a3..3d769a8a42 100644 --- a/src/test/lora_graph_export_test.cpp +++ b/src/test/lora_graph_export_test.cpp @@ -46,7 +46,7 @@ node: { node_options: { [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: { models_path: "./" - device: "CPU" + device: "" lora_adapters { alias: "pokemon" path: "loras/juliensimon/sd-pokemon-lora/pytorch_lora_weights.safetensors" mode: DYNAMIC } } } @@ -123,7 +123,7 @@ node: { node_options: { [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: { models_path: "./" - device: "CPU" + device: "" } } } @@ -374,7 +374,7 @@ node: { node_options: { [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: { models_path: "./" - device: "CPU" + device: "" lora_adapters { alias: "pokemon" path: "loras/pokemon/pytorch_lora_weights.safetensors" mode: DYNAMIC } } } @@ -410,7 +410,7 @@ node: { node_options: { [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: { models_path: "./" - device: "CPU" + device: "" lora_adapters { alias: "pokemon" path: "/path/to/weights.safetensors" mode: DYNAMIC } } } @@ -622,7 +622,7 @@ node: { node_options: { [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: { models_path: "./" - device: "CPU" + device: "" lora_adapters { alias: "pokemon" path: "loras/org/pokemon-lora/weights.safetensors" mode: DYNAMIC } lora_adapters { alias: "anime" path: "loras/org/anime-lora/weights.safetensors" mode: DYNAMIC } composite_lora_adapters { @@ -819,7 +819,7 @@ node: { node_options: { [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: { models_path: "./" - device: "CPU" + device: "" lora_adapters { alias: "landscape" path: "loras/civitai/landscapes-lora/Fantastic_Landscape.safetensors" mode: DYNAMIC } lora_adapters { alias: "portrait" path: "loras/org/portrait-lora/weights.safetensors" mode: DYNAMIC } composite_lora_adapters { diff --git a/src/test/ovmsconfig_test.cpp b/src/test/ovmsconfig_test.cpp index 1c5d8de75d..10674798af 100644 --- a/src/test/ovmsconfig_test.cpp +++ b/src/test/ovmsconfig_test.cpp @@ -1223,7 +1223,7 @@ TEST(OvmsGraphConfigTest, positiveTaskTextGen) { ASSERT_EQ(graphSettings.pipelineType.has_value(), false); ASSERT_EQ(exportSettings.modelPath, "./"); ASSERT_EQ(graphSettings.maxNumSeqs, 256); - ASSERT_EQ(exportSettings.targetDevice, "CPU"); + ASSERT_EQ(exportSettings.targetDevice, ""); ASSERT_EQ(exportSettings.pluginConfig.kvCachePrecision.has_value(), false); ASSERT_EQ(graphSettings.enablePrefixCaching, "true"); ASSERT_EQ(graphSettings.cacheSize, 0); @@ -1258,7 +1258,7 @@ TEST(OvmsExportHfSettingsTest, positiveDefault) { ASSERT_EQ(hfSettings.downloadPath, downloadPath); ASSERT_EQ(hfSettings.overwriteModels, true); ASSERT_EQ(hfSettings.exportSettings.precision, "int8"); - ASSERT_EQ(hfSettings.exportSettings.targetDevice, "CPU"); + ASSERT_EQ(hfSettings.exportSettings.targetDevice, ""); ASSERT_EQ(hfSettings.downloadType, ovms::GIT_CLONE_DOWNLOAD); ASSERT_EQ(hfSettings.exportSettings.extraQuantizationParams.has_value(), false); ASSERT_EQ(config.getServerSettings().serverMode, ovms::HF_PULL_MODE); @@ -1403,7 +1403,7 @@ TEST(OvmsGraphConfigTest, positiveDefault) { ASSERT_EQ(graphSettings.pipelineType.has_value(), false); ASSERT_EQ(exportSettings.modelPath, "./"); ASSERT_EQ(graphSettings.maxNumSeqs, 256); - ASSERT_EQ(exportSettings.targetDevice, "CPU"); + ASSERT_EQ(exportSettings.targetDevice, ""); ASSERT_EQ(exportSettings.pluginConfig.kvCachePrecision.has_value(), false); ASSERT_EQ(graphSettings.enablePrefixCaching, "true"); ASSERT_EQ(graphSettings.cacheSize, 0); @@ -1443,7 +1443,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultStart) { ASSERT_EQ(graphSettings.pipelineType.has_value(), false); ASSERT_EQ(exportSettings.modelPath, "./"); ASSERT_EQ(graphSettings.maxNumSeqs, 256); - ASSERT_EQ(exportSettings.targetDevice, "CPU"); + ASSERT_EQ(exportSettings.targetDevice, ""); ASSERT_EQ(exportSettings.pluginConfig.kvCachePrecision.has_value(), false); ASSERT_EQ(graphSettings.enablePrefixCaching, "true"); ASSERT_EQ(graphSettings.cacheSize, 0); @@ -1698,7 +1698,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultRerank) { ovms::RerankGraphSettingsImpl rerankGraphSettings = std::get(hfSettings.graphSettings); ASSERT_EQ(rerankGraphSettings.maxAllowedChunks, 10000); ASSERT_EQ(exportSettings.pluginConfig.numStreams, 1); - ASSERT_EQ(exportSettings.targetDevice, "CPU"); + ASSERT_EQ(exportSettings.targetDevice, ""); ASSERT_EQ(exportSettings.modelName, modelName); ASSERT_EQ(exportSettings.modelPath, "./"); } @@ -1867,7 +1867,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultImageGeneration) { ASSERT_EQ(config.getServerSettings().serverMode, ovms::HF_PULL_MODE); ASSERT_EQ(hfSettings.task, ovms::IMAGE_GENERATION_GRAPH); ovms::ImageGenerationGraphSettingsImpl imageGenerationGraphSettings = std::get(hfSettings.graphSettings); - ASSERT_EQ(exportSettings.targetDevice, "CPU"); + ASSERT_EQ(exportSettings.targetDevice, ""); ASSERT_TRUE(imageGenerationGraphSettings.maxResolution.empty()); ASSERT_TRUE(imageGenerationGraphSettings.defaultResolution.empty()); ASSERT_FALSE(imageGenerationGraphSettings.maxNumberImagesPerPrompt.has_value()); @@ -2005,7 +2005,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultEmbeddings) { ASSERT_EQ(embeddingsGraphSettings.truncate, "false"); ASSERT_EQ(embeddingsGraphSettings.pooling, "CLS"); ASSERT_EQ(exportSettings.pluginConfig.numStreams, 1); - ASSERT_EQ(exportSettings.targetDevice, "CPU"); + ASSERT_EQ(exportSettings.targetDevice, ""); ASSERT_EQ(exportSettings.modelName, modelName); } @@ -2148,7 +2148,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultTextToSpeech) { ASSERT_EQ(config.getServerSettings().serverMode, ovms::HF_PULL_MODE); ASSERT_EQ(hfSettings.task, ovms::TEXT_TO_SPEECH_GRAPH); ASSERT_EQ(hfSettings.exportSettings.pluginConfig.numStreams, 1); - ASSERT_EQ(hfSettings.exportSettings.targetDevice, "CPU"); + ASSERT_EQ(hfSettings.exportSettings.targetDevice, ""); ASSERT_EQ(hfSettings.exportSettings.modelName, modelName); } @@ -2283,7 +2283,7 @@ TEST(OvmsGraphConfigTest, positiveDefaultSpeechToText) { ASSERT_EQ(config.getServerSettings().serverMode, ovms::HF_PULL_MODE); ASSERT_EQ(hfSettings.task, ovms::SPEECH_TO_TEXT_GRAPH); ASSERT_EQ(hfSettings.exportSettings.pluginConfig.numStreams, 1); - ASSERT_EQ(hfSettings.exportSettings.targetDevice, "CPU"); + ASSERT_EQ(hfSettings.exportSettings.targetDevice, ""); ASSERT_EQ(hfSettings.exportSettings.modelName, modelName); } diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 4cee806be3..a424a2e5ed 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -777,7 +777,7 @@ const std::string expectedGraphContents = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./", enable_prefix_caching: true, cache_size: 0, @@ -814,7 +814,7 @@ const std::string expectedGraphContentsDraft = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./", enable_prefix_caching: true, cache_size: 0, From 93ef5d9ac10f530ef1b24fc0d500e9aea5895286 Mon Sep 17 00:00:00 2001 From: Dariusz Trawinski Date: Sun, 12 Jul 2026 00:41:47 +0200 Subject: [PATCH 5/5] unit tests --- src/test/graph_export_test.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/graph_export_test.cpp b/src/test/graph_export_test.cpp index 539975dbac..6e7e5ddcd8 100644 --- a/src/test/graph_export_test.cpp +++ b/src/test/graph_export_test.cpp @@ -48,7 +48,7 @@ const std::string expectedOneSettingPluginGraphContents = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./", plugin_config: '{"KV_CACHE_PRECISION":"u8"}', enable_prefix_caching: true, @@ -86,7 +86,7 @@ const std::string expectedFullPluginGraphContents = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./", plugin_config: '{"KV_CACHE_PRECISION":"u8","MAX_PROMPT_LEN":123,"MODEL_DISTRIBUTION_POLICY":"PIPELINE_PARALLEL"}', enable_prefix_caching: true, @@ -124,7 +124,7 @@ const std::string expectedGraphContentsWithResponseParser = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./", enable_prefix_caching: true, cache_size: 10, @@ -201,7 +201,7 @@ const std::string expectedDraftAndFuseGraphContents = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./", enable_prefix_caching: true, cache_size: 10, @@ -241,7 +241,7 @@ const std::string expectedGGUFGraphContents = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./PRETTY_GOOD_GGUF_MODEL.gguf", enable_prefix_caching: true, cache_size: 10, @@ -278,7 +278,7 @@ const std::string expectedGGUFGraphContents2 = R"( node_options: { [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { max_num_seqs:256, - device: "CPU", + device: "", models_path: "./PRETTY_GOOD_GGUF_MODEL_Q8-00001-of-20000.gguf", enable_prefix_caching: true, cache_size: 10, @@ -548,7 +548,7 @@ node: { node_options: { [type.googleapis.com / mediapipe.ImageGenCalculatorOptions]: { models_path: "./" - device: "CPU" + device: "" } } }