Propagate global --cache_dir to continuous batching pipeline (#4230)

docs/model_cache.md

@@ -23,6 +23,8 @@ Alternatively the location of the cache storage can be set using the parameter `
 
 The model server security context must have read-write access to the cache storage path.
 
+`--cache_dir` also applies to LLM text-generation servables using the continuous batching pipeline (GPU). With it set, the compiled-model/blob cache is persisted across restarts, so a model that has already been compiled (or idle-unloaded) reloads from the cache instead of recompiling. An explicit `CACHE_DIR` in a node's `plugin_config` takes precedence over the global `--cache_dir`.
+
 When using Model Server with configuration file, it is possible to serve more than one model. In such case, model cache is applied to all the models, with an exception to:
 - Models with custom loader (for security reasons explained earlier)
 - Models configured to shape `auto` or batch_size `auto`

src/llm/language_model/continuous_batching/servable_initializer.cpp

@@ -227,6 +227,8 @@ Status ContinuousBatchingServableInitializer::initialize(std::shared_ptr<GenAiSe
         return status;
     }
 
+    applyGlobalCacheDir(properties);
+
     if (properties->device == "CPU") {
         status = applyDefaultCpuProperties(properties->pluginConfig);
         if (!status.ok()) {

src/llm/language_model/legacy/servable_initializer.cpp

@@ -98,6 +98,8 @@ Status LegacyServableInitializer::initialize(std::shared_ptr<GenAiServable>& ser
         return status;
     }
 
+    applyGlobalCacheDir(properties);
+
     // Max prompt len is NPU specific property
     if (properties->device == "NPU") {
         auto it = properties->pluginConfig.find("MAX_PROMPT_LEN");

src/llm/servable_initializer.cpp

@@ -25,6 +25,7 @@
 
 #include <fstream>
 
+#include <openvino/openvino.hpp>
 #include <rapidjson/error/en.h>
 #include <rapidjson/istreamwrapper.h>
 
@@ -36,6 +37,7 @@
 #pragma GCC diagnostic pop
 #pragma warning(pop)
 
+#include "../config.hpp"
 #include "../logging.hpp"
 #include "../mediapipe_internal/mediapipe_utils.hpp"
 #include "../status.hpp"
@@ -169,6 +171,24 @@ void GenAiServableInitializer::loadChatTemplate(std::shared_ptr<GenAiServablePro
 #endif
 }
 
+void GenAiServableInitializer::applyGlobalCacheDir(std::shared_ptr<GenAiServableProperties> properties) {
+    // Propagate the global --cache_dir (ServerSettings) into the pipeline plugin config.
+    // Unlike the non-CB ModelInstance path (ModelInstance::setCacheOptions), these GenAI
+    // initializers construct the pipeline directly, so the server-level cache_dir is
+    // otherwise never applied and compiled-model cache artifacts are never persisted.
+    // An explicit CACHE_DIR in the node's plugin_config remains authoritative.
+    const std::string& globalCacheDir = Config::instance().cacheDir();
+    if (globalCacheDir.empty()) {
+        return;
+    }
+    if (properties->pluginConfig.find(ov::cache_dir.name()) == properties->pluginConfig.end()) {
+        properties->pluginConfig[ov::cache_dir.name()] = globalCacheDir;
+        SPDLOG_DEBUG("Applying global cache_dir to GenAI pipeline: {}", globalCacheDir);
+    } else {
+        SPDLOG_DEBUG("CACHE_DIR set explicitly in node plugin_config; keeping user value over global cache_dir");
+    }
+}
+
 #if (PYTHON_DISABLE == 0)
 // Helper function for case-insensitive comparison of file extensions
 static bool hasGGUFExtension(const std::filesystem::path& path) {

src/llm/servable_initializer.hpp

@@ -48,6 +48,11 @@ class GenAiServableInitializer {
 public:
     virtual ~GenAiServableInitializer() = default;
     static void loadChatTemplate(std::shared_ptr<GenAiServableProperties> properties, const std::string& chatTemplateDirectory);
+    // Propagates the global --cache_dir (ServerSettings) into the pipeline plugin config
+    // when the node did not set an explicit CACHE_DIR. Shared by every GenAI initializer
+    // (continuous batching and legacy, LM and VLM) since they all construct GenAI pipelines
+    // directly and would otherwise never apply the server-level cache_dir.
+    static void applyGlobalCacheDir(std::shared_ptr<GenAiServableProperties> properties);
 #if (PYTHON_DISABLE == 0)
     // Use Python Jinja module for template processing
     static void loadPyTemplateProcessor(std::shared_ptr<GenAiServableProperties> properties, const ExtraGenerationInfo& extraGenInfo);

src/llm/visual_language_model/legacy/servable_initializer.cpp

@@ -97,6 +97,8 @@ Status VisualLanguageModelLegacyServableInitializer::initialize(std::shared_ptr<
         return status;
     }
 
+    applyGlobalCacheDir(properties);
+
     try {
         properties->pipeline = std::make_shared<ov::genai::VLMPipeline>(parsedModelsPath, properties->device, properties->pluginConfig);
         properties->tokenizer = properties->pipeline->get_tokenizer();

src/test/llm/llmnode_test.cpp

@@ -37,6 +37,7 @@
 #endif
 
 #include "../../http_rest_api_handler.hpp"
+#include "../../config.hpp"
 #include "../../http_status_code.hpp"
 #include "../../json_parser.hpp"
 #include "../../llm/apis/openai_completions.hpp"
@@ -4499,6 +4500,217 @@ TEST_F(LLMVLMOptionsHttpTest, LLMVLMNodeOptionsCheckPluginConfig) {
     LLMNodeOptionsCheckPluginConfig(modelsPath);
 }
 
+// RAII guard that restores the global Config singleton (and optionally removes a temporary
+// cache directory) on scope exit. The cache_dir tests below mutate the process-wide Config
+// singleton; without this guard a failed ASSERT_* mid-test (which returns early) would leak
+// the modified --cache_dir into subsequent tests in this suite.
+struct GlobalCacheDirGuard {
+    std::string cacheDirToRemove;
+    explicit GlobalCacheDirGuard(std::string cacheDirToRemove = "") :
+        cacheDirToRemove(std::move(cacheDirToRemove)) {}
+    ~GlobalCacheDirGuard() {
+        char* reset_argv[] = {(char*)"ovms", (char*)"--model_path", (char*)"/path/to/model", (char*)"--model_name", (char*)"some_name", (char*)"--rest_port", (char*)"8080"};
+        ovms::Config::instance().parse(7, reset_argv);
+        if (!cacheDirToRemove.empty()) {
+            std::error_code ec;
+            std::filesystem::remove_all(cacheDirToRemove, ec);
+        }
+    }
+};
+
+// Verifies that the global --cache_dir (ServerSettings) is propagated into the
+// continuous batching pipeline plugin config, and that an explicit CACHE_DIR in
+// the node's plugin_config takes precedence over the global value.
+// Regression test for openvinotoolkit/model_server#4230.
+void LLMNodeOptionsCacheDirPropagation(std::string& modelsPath) {
+    // Restore the global cache_dir on scope exit even if an ASSERT below fails early.
+    GlobalCacheDirGuard cacheDirGuard;
+    // Seed the global cache_dir via the CLI parser (same path used in production).
+    char* n_argv[] = {(char*)"ovms", (char*)"--model_path", (char*)"/path/to/model", (char*)"--model_name", (char*)"some_name", (char*)"--rest_port", (char*)"8080", (char*)"--cache_dir", (char*)"/tmp/ovms_global_cache"};
+    int arg_count = 9;
+    ovms::Config::instance().parse(arg_count, n_argv);
+    ASSERT_EQ(ovms::Config::instance().cacheDir(), "/tmp/ovms_global_cache");
+
+    // Case 1: no CACHE_DIR in node plugin_config -> global value is applied.
+    {
+        std::string testPbtxt = R"(
+            input_stream: "HTTP_REQUEST_PAYLOAD:input"
+            output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+
+            node: {
+            name: "llmNode"
+            calculator: "HttpLLMCalculator"
+            input_stream: "LOOPBACK:loopback"
+            input_stream: "HTTP_REQUEST_PAYLOAD:input"
+            input_side_packet: "LLM_NODE_RESOURCES:llm"
+            output_stream: "LOOPBACK:loopback"
+            output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+            input_stream_info: {
+                tag_index: 'LOOPBACK:0',
+                back_edge: true
+            }
+            node_options: {
+                [type.googleapis.com / mediapipe.LLMCalculatorOptions]: {
+                    models_path: ")" +
+                                modelsPath + R"("
+                }
+            }
+            input_stream_handler {
+                input_stream_handler: "SyncSetInputStreamHandler",
+                options {
+                [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+                    sync_set {
+                    tag_index: "LOOPBACK:0"
+                    }
+                }
+                }
+            }
+            }
+        )";
+        adjustConfigForTargetPlatform(testPbtxt);
+        ::mediapipe::CalculatorGraphConfig config;
+        ASSERT_TRUE(::google::protobuf::TextFormat::ParseFromString(testPbtxt, &config));
+        std::shared_ptr<GenAiServable> servable;
+        ASSERT_EQ(initializeGenAiServable(servable, config.node(0), ""), StatusCode::OK);
+        auto properties = std::static_pointer_cast<ContinuousBatchingServableProperties>(servable->getProperties());
+        ASSERT_EQ(properties->pluginConfig.count("CACHE_DIR"), 1);
+        ASSERT_EQ(properties->pluginConfig["CACHE_DIR"].as<std::string>(), "/tmp/ovms_global_cache");
+    }
+
+    // Case 2: explicit CACHE_DIR in node plugin_config wins over the global value.
+    {
+        std::string testPbtxt = R"(
+            input_stream: "HTTP_REQUEST_PAYLOAD:input"
+            output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+
+            node: {
+            name: "llmNode"
+            calculator: "HttpLLMCalculator"
+            input_stream: "LOOPBACK:loopback"
+            input_stream: "HTTP_REQUEST_PAYLOAD:input"
+            input_side_packet: "LLM_NODE_RESOURCES:llm"
+            output_stream: "LOOPBACK:loopback"
+            output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+            input_stream_info: {
+                tag_index: 'LOOPBACK:0',
+                back_edge: true
+            }
+            node_options: {
+                [type.googleapis.com / mediapipe.LLMCalculatorOptions]: {
+                    models_path: ")" +
+                                modelsPath + R"("
+                    plugin_config: '{"CACHE_DIR": "/tmp/ovms_node_cache"}'
+                }
+            }
+            input_stream_handler {
+                input_stream_handler: "SyncSetInputStreamHandler",
+                options {
+                [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+                    sync_set {
+                    tag_index: "LOOPBACK:0"
+                    }
+                }
+                }
+            }
+            }
+        )";
+        adjustConfigForTargetPlatform(testPbtxt);
+        ::mediapipe::CalculatorGraphConfig config;
+        ASSERT_TRUE(::google::protobuf::TextFormat::ParseFromString(testPbtxt, &config));
+        std::shared_ptr<GenAiServable> servable;
+        ASSERT_EQ(initializeGenAiServable(servable, config.node(0), ""), StatusCode::OK);
+        auto properties = std::static_pointer_cast<ContinuousBatchingServableProperties>(servable->getProperties());
+        ASSERT_EQ(properties->pluginConfig.count("CACHE_DIR"), 1);
+        // The test harness may rewrite the path for the target platform, so match
+        // on substrings: the explicit node value must win over the global one.
+        std::string nodeCacheDir = properties->pluginConfig["CACHE_DIR"].as<std::string>();
+        ASSERT_NE(nodeCacheDir.find("ovms_node_cache"), std::string::npos) << "Explicit node CACHE_DIR should be used, got: " << nodeCacheDir;
+        ASSERT_EQ(nodeCacheDir.find("ovms_global_cache"), std::string::npos) << "Global cache_dir must not override explicit node CACHE_DIR, got: " << nodeCacheDir;
+    }
+    // GlobalCacheDirGuard restores the global cache_dir on scope exit.
+}
+TEST_F(LLMOptionsHttpTest, LLMNodeOptionsCacheDirPropagation) {
+    LLMNodeOptionsCacheDirPropagation(modelsPath);
+}
+TEST_F(LLMVLMOptionsHttpTest, LLMVLMNodeOptionsCacheDirPropagation) {
+    LLMNodeOptionsCacheDirPropagation(modelsPath);
+}
+
+// End-to-end regression test for #4230: LLMNodeOptionsCacheDirPropagation above only
+// verifies that --cache_dir lands in properties->pluginConfig; it does not prove that
+// OpenVINO Core actually persists compiled-model cache artifacts, which was the crux of
+// the original bug report (log said "cache enabled", nothing was ever written on disk).
+// This test constructs a real ContinuousBatchingPipeline against --cache_dir and asserts
+// that compiled-model cache artifacts actually land under it.
+TEST_F(LLMOptionsHttpTest, LLMNodeOptionsCacheDirWritesCacheArtifacts) {
+    std::string cacheDir = std::filesystem::temp_directory_path().string() +
+                            "/LLMNodeOptionsCacheDirWritesCacheArtifacts_" +
+                            ::testing::UnitTest::GetInstance()->current_test_info()->name();
+    std::filesystem::remove_all(cacheDir);
+    std::filesystem::create_directories(cacheDir);
+    // Restore the global cache_dir and remove the temp cache dir on scope exit, even if an
+    // ASSERT below fails early.
+    GlobalCacheDirGuard cacheDirGuard(cacheDir);
+
+    // Seed the global cache_dir via the CLI parser (same path used in production).
+    char* n_argv[] = {(char*)"ovms", (char*)"--model_path", (char*)"/path/to/model", (char*)"--model_name", (char*)"some_name", (char*)"--rest_port", (char*)"8080", (char*)"--cache_dir", (char*)cacheDir.c_str()};
+    int arg_count = 9;
+    ovms::Config::instance().parse(arg_count, n_argv);
+    ASSERT_EQ(ovms::Config::instance().cacheDir(), cacheDir);
+
+    std::string testPbtxt = R"(
+        input_stream: "HTTP_REQUEST_PAYLOAD:input"
+        output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+
+        node: {
+        name: "llmNode"
+        calculator: "HttpLLMCalculator"
+        input_stream: "LOOPBACK:loopback"
+        input_stream: "HTTP_REQUEST_PAYLOAD:input"
+        input_side_packet: "LLM_NODE_RESOURCES:llm"
+        output_stream: "LOOPBACK:loopback"
+        output_stream: "HTTP_RESPONSE_PAYLOAD:output"
+        input_stream_info: {
+            tag_index: 'LOOPBACK:0',
+            back_edge: true
+        }
+        node_options: {
+            [type.googleapis.com / mediapipe.LLMCalculatorOptions]: {
+                models_path: ")" +
+                             modelsPath + R"("
+            }
+        }
+        input_stream_handler {
+            input_stream_handler: "SyncSetInputStreamHandler",
+            options {
+            [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+                sync_set {
+                tag_index: "LOOPBACK:0"
+                }
+            }
+            }
+        }
+        }
+    )";
+    adjustConfigForTargetPlatform(testPbtxt);
+    ::mediapipe::CalculatorGraphConfig config;
+    ASSERT_TRUE(::google::protobuf::TextFormat::ParseFromString(testPbtxt, &config));
+    std::shared_ptr<GenAiServable> servable;
+    ASSERT_EQ(initializeGenAiServable(servable, config.node(0), ""), StatusCode::OK);
+
+    bool foundCacheArtifact = false;
+    for (const auto& entry : std::filesystem::recursive_directory_iterator(cacheDir)) {
+        if (entry.is_regular_file()) {
+            foundCacheArtifact = true;
+            break;
+        }
+    }
+    EXPECT_TRUE(foundCacheArtifact)
+        << "Expected compiled-model cache artifacts under --cache_dir after constructing the "
+        << "continuous batching pipeline, found none in: " << cacheDir;
+    // GlobalCacheDirGuard restores the global cache_dir and removes cacheDir on scope exit.
+}
+
 void LLMNodeOptionsCheckNonDefault(std::string& modelsPath) {
     std::string testPbtxt = R"(
         input_stream: "HTTP_REQUEST_PAYLOAD:input"