setting default recommended device

src/audio/speech_to_text/BUILD

@@ -30,6 +30,7 @@ ovms_cc_library(
         "//src:libovmslogging",
         "//src:libmodelconfigjsonparser",
         "//src:libovmsstring_utils",
+        "//src:libovms_ov_utils",
         "s2t_calculator_cc_proto",
         "@com_google_absl//absl/status",
         "//third_party:genai",

src/audio/speech_to_text/s2t_servable.cpp

@@ -28,6 +28,7 @@
 #include "src/http_payload.hpp"
 #include "src/json_parser.hpp"
 #include "src/logging.hpp"
+#include "src/ov_utils.hpp"
 #include "src/stringutils.hpp"
 
 namespace ovms {
@@ -42,18 +43,23 @@ SttServable::SttServable(const ::mediapipe::S2tCalculatorOptions& nodeOptions, c
     } else {
         parsedModelsPath = fsModelsPath;
     }
+    std::string device = nodeOptions.target_device();
+    if (device.empty()) {
+        device = recommendTargetDevice();
+        SPDLOG_INFO("No device specified for STT model, using recommended device: {}", device);
+    }
     ov::AnyMap config;
     auto status = JsonParser::parsePluginConfig(nodeOptions.plugin_config(), config);
     if (!status.ok()) {
         SPDLOG_ERROR("Error during llm node plugin_config option parsing to JSON: {}", nodeOptions.plugin_config());
         throw std::runtime_error("Error during plugin_config option parsing");
     }
     enableWordTimestamps = nodeOptions.enable_word_timestamps();
-    if (enableWordTimestamps && nodeOptions.target_device() == "NPU") {
+    if (enableWordTimestamps && device == "NPU") {
         config["STATIC_PIPELINE"] = true;
     }
     config["word_timestamps"] = enableWordTimestamps;
-    sttPipeline = std::make_shared<ov::genai::ASRPipeline>(parsedModelsPath.string(), nodeOptions.target_device(), config);
+    sttPipeline = std::make_shared<ov::genai::ASRPipeline>(parsedModelsPath.string(), device, config);
 
     streamingExecutor = std::make_unique<SttExecutorWrapper>(sttPipeline, sttPipelineMutex);
 }

src/audio/text_to_speech/BUILD

@@ -26,6 +26,7 @@ ovms_cc_library(
         "@mediapipe//mediapipe/framework:calculator_framework",
         "//src:libovmslogging",
         "//src:libmodelconfigjsonparser",
+        "//src:libovms_ov_utils",
         "t2s_calculator_cc_proto",
     ],
     visibility = ["//visibility:public"],

src/audio/text_to_speech/t2s_servable.cpp

@@ -25,6 +25,7 @@
 #include "src/status.hpp"
 #include "src/logging.hpp"
 #include "src/json_parser.hpp"
+#include "src/ov_utils.hpp"
 
 #include "src/audio/text_to_speech/t2s_servable.hpp"
 
@@ -94,13 +95,18 @@ TtsServable::TtsServable(const std::string& modelDir, const std::string& targetD
     } else {
         parsedModelsPath = fsModelsPath;
     }
+    std::string device = targetDevice;
+    if (device.empty()) {
+        device = recommendTargetDevice();
+        SPDLOG_INFO("No device specified for TTS model, using recommended device: {}", device);
+    }
     ov::AnyMap config;
     Status status = JsonParser::parsePluginConfig(pluginConfig, config);
     if (!status.ok()) {
         SPDLOG_ERROR("Error during llm node plugin_config option parsing to JSON: {}", pluginConfig);
         throw std::runtime_error("Error during plugin_config option parsing");
     }
-    ttsPipeline = std::make_shared<ov::genai::Text2SpeechPipeline>(parsedModelsPath.string(), targetDevice, config);
+    ttsPipeline = std::make_shared<ov::genai::Text2SpeechPipeline>(parsedModelsPath.string(), device, config);
     const ov::Shape speakerEmbeddingShape = ttsPipeline->get_speaker_embedding_shape();
     for (const auto& voice : graphVoices) {
         std::filesystem::path voicePath(voice.path());

src/capi_frontend/server_settings.hpp

@@ -191,7 +191,7 @@ struct ImageGenerationGraphSettingsImpl {
 struct ExportSettings {
     std::string modelName = "";
     std::string modelPath = "./";
-    std::string targetDevice = "CPU";
+    std::string targetDevice;
     std::optional<uint32_t> restWorkers;
     std::optional<std::string> extraQuantizationParams;
     std::optional<std::string> vocoder;

src/cli_parser.cpp

@@ -289,8 +289,8 @@ std::variant<bool, std::pair<int, std::string>> CLIParser::parse(int argc, char*
                 cxxopts::value<uint32_t>(),
                 "NIREQ")
             ("target_device",
-                "Target device to run the inference",
-                cxxopts::value<std::string>()->default_value("CPU"),
+                "Target device to run the inference. Default: auto-detected based on available devices.",
+                cxxopts::value<std::string>()->default_value(""),
                 "TARGET_DEVICE")
             ("plugin_config",
                 "A dictionary of plugin configuration keys and their values, eg \"{\\\"NUM_STREAMS\\\": \\\"1\\\"}\". Default number of streams is optimized to optimal latency with low concurrency.",
@@ -648,10 +648,12 @@ void CLIParser::prepareModel(ModelsSettingsImpl& modelsSettings, HFSettingsImpl&
 
     if (result->count("target_device")) {
         modelsSettings.targetDevice = result->operator[]("target_device").as<std::string>();
-        if (isHFPullOrPullAndStart(this->result)) {
-            hfSettings.exportSettings.targetDevice = modelsSettings.targetDevice;
-        } else {
-            modelsSettings.userSetSingleModelArguments.push_back("target_device");
+        if (!modelsSettings.targetDevice.empty()) {
+            if (isHFPullOrPullAndStart(this->result)) {
+                hfSettings.exportSettings.targetDevice = modelsSettings.targetDevice;
+            } else {
+                modelsSettings.userSetSingleModelArguments.push_back("target_device");
+            }
         }
     }
 

src/config.cpp

@@ -197,7 +197,10 @@ bool Config::validate() {
 
             std::vector allowedTargetDevices = {"CPU", "GPU", "NPU", "AUTO"};
             bool validDeviceSelected = false;
-            if (exportSettings.targetDevice.rfind("GPU.", 0) == 0) {
+            if (exportSettings.targetDevice.empty()) {
+                // Empty means auto-detect via recommendTargetDevice
+                validDeviceSelected = true;
+            } else if (exportSettings.targetDevice.rfind("GPU.", 0) == 0) {
                 // Accept GPU.x where x is a number to select specific GPU card
                 std::string indexPart = exportSettings.targetDevice.substr(4);
                 validDeviceSelected = !indexPart.empty() && std::all_of(indexPart.begin(), indexPart.end(), ::isdigit);
@@ -407,8 +410,7 @@ const std::string Config::precision() const { return this->modelsSettings.precis
 const std::string& Config::modelVersionPolicy() const { return this->modelsSettings.modelVersionPolicy; }
 uint32_t Config::nireq() const { return this->modelsSettings.nireq; }
 const std::string& Config::targetDevice() const {
-    static const std::string defaultTargetDevice = "CPU";
-    return this->modelsSettings.targetDevice.empty() ? defaultTargetDevice : this->modelsSettings.targetDevice;
+    return this->modelsSettings.targetDevice;
 }
 const std::string& Config::Config::pluginConfig() const { return this->modelsSettings.pluginConfig; }
 bool Config::metricsEnabled() const { return this->serverSettings.metricsEnabled; }

src/embeddings/embeddings_calculator_ov.proto

@@ -27,7 +27,7 @@ message EmbeddingsCalculatorOVOptions {
     }
     required string models_path = 1;
     optional bool normalize_embeddings = 2 [default = true];
-    optional string target_device = 3 [default = "CPU"];
+    optional string target_device = 3;
     optional string plugin_config = 4 [default = ""];
     enum Pooling {
       CLS = 0;

src/image_gen/BUILD

@@ -26,6 +26,7 @@ ovms_cc_library(
         "//src:libovmslogging",
         "//src:libovms_queue",
         "//src:libovmsstring_utils",
+        "//src:libovms_ov_utils",
         "//third_party:genai",],
     visibility = ["//visibility:public"],
     alwayslink = 1,

src/image_gen/pipelines.cpp

@@ -23,6 +23,7 @@
 #include <openvino/genai/image_generation/image2image_pipeline.hpp>
 
 #include "src/logging.hpp"
+#include "src/ov_utils.hpp"
 #include "src/stringutils.hpp"
 
 namespace ovms {
@@ -61,7 +62,8 @@ ImageGenerationPipelines::ImageGenerationPipelines(const ImageGenPipelineArgs& a
     args(args) {
     std::vector<std::string> device;
     if (!args.device.size()) {
-        device.push_back("CPU");
+        device.push_back(recommendTargetDevice());
+        SPDLOG_INFO("No device specified for image generation model, using recommended device: {}", device[0]);
     } else {
         device = args.device;
     }

src/llm/language_model/continuous_batching/servable_initializer.cpp

@@ -190,6 +190,10 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptr<GenAiSe
     }
 
     properties->device = nodeOptions.device();
+    if (properties->device.empty()) {
+        properties->device = recommendTargetDevice();
+        SPDLOG_INFO("No device specified for LLM CB model, using recommended device: {}", properties->device);
+    }
     properties->bestOfLimit = nodeOptions.best_of_limit();
     properties->enableToolGuidedGeneration = nodeOptions.enable_tool_guided_generation();
 

src/llm/language_model/legacy/servable_initializer.cpp

@@ -33,6 +33,7 @@
 #include "../../../json_parser.hpp"
 #include "../../../logging.hpp"
 #include "../../../mediapipe_internal/mediapipe_utils.hpp"
+#include "../../../ov_utils.hpp"
 #include "../../../status.hpp"
 #include "../../io_processing/parser_config_validation.hpp"
 #include "servable.hpp"
@@ -85,6 +86,10 @@ Status LegacyServableInitializer::initialize(std::shared_ptr<GenAiServable>& ser
     properties->schedulerConfig.enable_prefix_caching = nodeOptions.enable_prefix_caching();
 
     properties->device = nodeOptions.device();
+    if (properties->device.empty()) {
+        properties->device = recommendTargetDevice();
+        SPDLOG_INFO("No device specified for LLM legacy model, using recommended device: {}", properties->device);
+    }
 
     if (nodeOptions.has_draft_max_num_batched_tokens() || nodeOptions.has_draft_cache_size() || nodeOptions.has_draft_dynamic_split_fuse() || nodeOptions.has_draft_max_num_seqs() || nodeOptions.has_draft_block_size() || nodeOptions.has_draft_device()) {
         // Consider moving draft parameters to separate structure in node options, so it's validated on the proto level

src/llm/llm_calculator.proto

@@ -93,7 +93,7 @@ message LLMCalculatorOptions {
 
     optional bool dynamic_split_fuse = 5 [default = true];
 
-    optional string device = 6 [default = "CPU"];
+    optional string device = 6;
 
     optional string plugin_config = 7 [default = ""];
 

src/llm/visual_language_model/legacy/servable_initializer.cpp

@@ -33,6 +33,7 @@
 #include "../../../json_parser.hpp"
 #include "../../../logging.hpp"
 #include "../../../mediapipe_internal/mediapipe_utils.hpp"
+#include "../../../ov_utils.hpp"
 #include "../../../status.hpp"
 #include "../../io_processing/parser_config_validation.hpp"
 #include "servable.hpp"
@@ -84,6 +85,10 @@ Status VisualLanguageModelLegacyServableInitializer::initialize(std::shared_ptr<
     properties->schedulerConfig.enable_prefix_caching = nodeOptions.enable_prefix_caching();
 
     properties->device = nodeOptions.device();
+    if (properties->device.empty()) {
+        properties->device = recommendTargetDevice();
+        SPDLOG_INFO("No device specified for VLM model, using recommended device: {}", properties->device);
+    }
 
     if (nodeOptions.has_draft_max_num_batched_tokens() || nodeOptions.has_draft_cache_size() || nodeOptions.has_draft_dynamic_split_fuse() || nodeOptions.has_draft_max_num_seqs() || nodeOptions.has_draft_block_size() || nodeOptions.has_draft_device()) {
         // Consider moving draft parameters to separate structure in node options, so it's validated on the proto level

src/modelconfig.hpp

@@ -211,7 +211,7 @@ class ModelConfig {
          */
     ModelConfig(const std::string& name = "",
         const std::string& basePath = "",
-        const std::string& targetDevice = "CPU",
+        const std::string& targetDevice = "",
         const std::string& configBatchSize = "",
         uint64_t nireq = 0,
         const std::string& cacheDir = "",

src/modelinstance.cpp

@@ -1023,7 +1023,7 @@ plugin_config_t ModelInstance::prepareDefaultPluginConfig(const ModelConfig& con
 
 Status ModelInstance::loadOVCompiledModel(const ModelConfig& config) {
     plugin_config_t pluginConfig = prepareDefaultPluginConfig(config);
-    if (config.getTargetDevice() == "CPU") {
+    if (this->targetDevice == "CPU") {
         Status status = applyDefaultCpuProperties(pluginConfig);
         if (!status.ok()) {
             SPDLOG_LOGGER_ERROR(modelmanager_logger, "Failed to apply default CPU properties for model: {}; version: {}; error: {}",
@@ -1238,6 +1238,11 @@ Status ModelInstance::loadModelImpl(const ModelConfig& config, const DynamicMode
     subscriptionManager.notifySubscribers();
     this->path = config.getPath();
     this->targetDevice = config.getTargetDevice();
+    if (this->targetDevice.empty()) {
+        this->targetDevice = recommendTargetDevice();
+        SPDLOG_LOGGER_INFO(modelmanager_logger, "No target device specified for model: {}; version: {}; using recommended device: {}",
+            config.getName(), config.getVersion(), this->targetDevice);
+    }
     this->config = config;
     auto status = fetchModelFilepaths();
 

src/modelmanager.cpp

@@ -296,7 +296,9 @@ Status ModelManager::startFromConfig() {
         return status;
     }
 
-    status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *ieCore.get());
+    status = validatePluginConfiguration(modelConfig.getPluginConfig(),
+        modelConfig.getTargetDevice().empty() ? recommendTargetDevice() : modelConfig.getTargetDevice(),
+        *ieCore.get());
     if (!status.ok()) {
         SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys");
         return status;
@@ -728,7 +730,9 @@ Status ModelManager::ConfigLoader::loadModels(ModelManager& modelManager, const
             continue;
         }
 
-        status = validatePluginConfiguration(modelConfig.getPluginConfig(), modelConfig.getTargetDevice(), *modelManager.ieCore.get());
+        status = validatePluginConfiguration(modelConfig.getPluginConfig(),
+            modelConfig.getTargetDevice().empty() ? recommendTargetDevice() : modelConfig.getTargetDevice(),
+            *modelManager.ieCore.get());
         if (!status.ok()) {
             SPDLOG_LOGGER_ERROR(modelmanager_logger, "Plugin config contains unsupported keys");
             return status;

src/ov_utils.cpp

@@ -150,6 +150,93 @@ Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const st
     return StatusCode::OK;
 }
 
+std::string recommendTargetDevice() {
+    static ov::Core core;
+    auto availableDevices = core.get_available_devices();
+
+    struct GpuDeviceInfo {
+        std::string name;
+        bool isDiscrete = false;
+        int64_t freeMemBytes = 0;
+    };
+
+    std::vector<GpuDeviceInfo> gpuDevices;
+
+    for (const auto& device : availableDevices) {
+        if (device.find("GPU") != 0) {
+            continue;
+        }
+
+        GpuDeviceInfo info;
+        info.name = device;
+
+        try {
+            auto deviceType = core.get_property(device, ov::device::type);
+            info.isDiscrete = (deviceType == ov::device::Type::DISCRETE);
+        } catch (const std::exception& e) {
+            SPDLOG_LOGGER_WARN(modelmanager_logger, "Failed to get DEVICE_TYPE for {}: {}", device, e.what());
+            continue;
+        }
+
+        try {
+            uint64_t totalMem = core.get_property(device, "GPU_DEVICE_TOTAL_MEM_SIZE").as<uint64_t>();
+            uint64_t usedMem = 0;
+            try {
+                auto memStats = core.get_property(device, "GPU_MEMORY_STATISTICS").as<std::map<std::string, uint64_t>>();
+                auto it = memStats.find("usm_device");
+                if (it != memStats.end()) {
+                    usedMem = it->second;
+                }
+            } catch (...) {
+                // Memory statistics may not be available before any model is loaded
+            }
+            info.freeMemBytes = static_cast<int64_t>(totalMem) - static_cast<int64_t>(usedMem);
+        } catch (const std::exception& e) {
+            SPDLOG_LOGGER_WARN(modelmanager_logger, "Failed to get memory info for {}: {}", device, e.what());
+            info.freeMemBytes = 0;
+        }
+
+        gpuDevices.push_back(info);
+    }
+
+    if (gpuDevices.empty()) {
+        SPDLOG_LOGGER_INFO(modelmanager_logger, "No GPU devices found, recommending CPU");
+        return "CPU";
+    }
+
+    std::vector<GpuDeviceInfo> discreteGpus;
+    std::vector<GpuDeviceInfo> integratedGpus;
+    for (const auto& gpu : gpuDevices) {
+        if (gpu.isDiscrete) {
+            discreteGpus.push_back(gpu);
+        } else {
+            integratedGpus.push_back(gpu);
+        }
+    }
+
+    if (discreteGpus.size() == 1) {
+        SPDLOG_LOGGER_INFO(modelmanager_logger, "Single discrete GPU found, recommending: {}", discreteGpus[0].name);
+        return discreteGpus[0].name;
+    }
+
+    if (discreteGpus.size() > 1) {
+        auto best = std::max_element(discreteGpus.begin(), discreteGpus.end(),
+            [](const GpuDeviceInfo& a, const GpuDeviceInfo& b) {
+                return a.freeMemBytes < b.freeMemBytes;
+            });
+        SPDLOG_LOGGER_INFO(modelmanager_logger, "Multiple discrete GPUs found, recommending {} with {} bytes free VRAM", best->name, best->freeMemBytes);
+        return best->name;
+    }
+
+    if (!integratedGpus.empty()) {
+        SPDLOG_LOGGER_INFO(modelmanager_logger, "Integrated GPU found, recommending: {}", integratedGpus[0].name);
+        return integratedGpus[0].name;
+    }
+
+    SPDLOG_LOGGER_INFO(modelmanager_logger, "Falling back to CPU");
+    return "CPU";
+}
+
 Status applyDefaultCpuProperties(ov::AnyMap& properties, uint16_t coreCount, uint16_t physicalCoresPerSocket, uint16_t socketsCount, uint16_t dockerCpuQuota) {
     if (properties.find(ov::hint::enable_cpu_pinning.name()) == properties.end()) {
         const bool cpuPinning = dockerCpuQuota <= 0;

src/ov_utils.hpp

@@ -52,6 +52,14 @@ std::optional<ov::Layout> getLayoutFromRTMap(const ov::RTMap& rtMap);
 
 Status validatePluginConfiguration(const plugin_config_t& pluginConfig, const std::string& targetDevice, const ov::Core& ieCore);
 
+// Determines the recommended target device based on available devices and their properties.
+// Rules:
+// - If only CPU is available (no GPU) - returns "CPU"
+// - If a single discrete GPU is available - returns that device (e.g. "GPU.0")
+// - If only integrated GPU(s) are available - returns the first integrated GPU
+// - If multiple discrete GPUs are available - returns the one with the most free VRAM
+std::string recommendTargetDevice();
+
 // Applies resource-aware CPU defaults to an OpenVINO property map.
 // Sets inference_num_threads and (on Linux) enable_cpu_pinning only when not
 // already present in the map.  When PERFORMANCE_HINT=THROUGHPUT is set,

src/rerank/rerank_calculator_ov.proto

@@ -32,7 +32,7 @@ message RerankCalculatorOVOptions {
 
     optional uint64 max_position_embeddings = 3;
 
-    optional string target_device = 4 [default = "CPU"];
+    optional string target_device = 4;
 
     optional string plugin_config = 5 [default = ""];
 }