From f7a2f27918516d2cd3ec203a7d3c7baef2091b32 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 24 Jun 2026 09:06:39 +0000 Subject: [PATCH 01/11] feat: add standalone infinirt graph bridge Route graph capture and replay through a dlopen-based standalone InfiniRT bridge when enabled. Package the standalone InfiniRT runtime as libinfiniops_infinirt.so and ensure libinfiniops.so records that private dependency. --- src/infinicore/graph/graph.cc | 83 +++-- .../graph/standalone_infinirt_graph_bridge.cc | 287 ++++++++++++++++++ .../standalone_infinirt_graph_bridge.hpp | 28 ++ xmake.lua | 93 +++--- 4 files changed, 431 insertions(+), 60 deletions(-) create mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.cc create mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.hpp diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 3511b10a1..ee7cb5ca8 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -4,6 +4,7 @@ #include "infinicore/context/context.hpp" #ifdef USE_INFINIRT_GRAPH +#include "standalone_infinirt_graph_bridge.hpp" #include #endif @@ -36,9 +37,11 @@ DispatchableGraphOperator::~DispatchableGraphOperator() { #ifdef USE_INFINIRT_GRAPH struct Graph::DeviceGraph { - infinirtGraph_t graph; - infinirtGraphExec_t exec; - infinirtGraphNode_t node; + infinirtGraph_t graph = nullptr; + infinirtGraphExec_t exec = nullptr; + infinirtGraphNode_t node = nullptr; + infinirtStream_t stream = nullptr; + bool standalone = false; std::vector log_buffer; DeviceGraph() { @@ -47,15 +50,30 @@ struct Graph::DeviceGraph { ~DeviceGraph() { if (exec) { - infinirtGraphExecDestroy(exec); + if (standalone) { + standalone_infinirt::graph_exec_destroy(exec); + } else { + infinirtGraphExecDestroy(exec); + } } if (graph) { - infinirtGraphDestroy(graph); + if (standalone) { + standalone_infinirt::graph_destroy(graph); + } else { + infinirtGraphDestroy(graph); + } + } + if (standalone && stream) { + standalone_infinirt::destroy_wrapped_stream(stream); } } void launch() { - INFINICORE_CHECK_ERROR(infinirtGraphLuanch(exec, context::getStream())); + if (standalone) { + INFINICORE_CHECK_ERROR(standalone_infinirt::graph_launch(exec, stream)); + } else { + INFINICORE_CHECK_ERROR(infinirtGraphLuanch(exec, context::getStream())); + } } }; #else @@ -85,6 +103,22 @@ void Graph::instantiate() { #ifdef USE_INFINIRT_GRAPH // Reset device graph device_graph_ = std::make_unique(); + device_graph_->standalone = standalone_infinirt::available(); + device_graph_->stream = device_graph_->standalone + ? standalone_infinirt::wrap_stream(context::getDevice(), context::getStream()) + : context::getStream(); + if (device_graph_->standalone && device_graph_->stream == nullptr) { + spdlog::warn("Standalone InfiniRT graph bridge is enabled but failed to wrap the current stream. Falling back to eager execution."); + device_graph_.reset(); + return; + } + if (device_graph_->standalone) { + static bool logged_once = false; + if (!logged_once) { + logged_once = true; + spdlog::info("Using standalone InfiniRT graph bridge for graph capture and replay."); + } + } // warmup for (size_t iter = 0; iter < 5; ++iter) { @@ -92,35 +126,42 @@ void Graph::instantiate() { } infinicore::context::syncStream(); - if (infinirtStreamBeginCapture( - context::getStream(), - INFINIRT_STREAM_CAPTURE_MODE_RELAXED) - != INFINI_STATUS_SUCCESS) { + auto begin_status = device_graph_->standalone + ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) + : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); + if (begin_status != INFINI_STATUS_SUCCESS) { + spdlog::warn("Fail to begin device graph capture."); + device_graph_.reset(); return; } // Run and record this->run(); - if (infinirtStreamEndCapture( - context::getStream(), - &device_graph_.get()->graph) - != INFINI_STATUS_SUCCESS) { + auto end_status = device_graph_->standalone + ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) + : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); + if (end_status != INFINI_STATUS_SUCCESS) { + spdlog::warn("Fail to end device graph capture."); + device_graph_.reset(); return; } - if (infinirtGraphInstantiate( - &device_graph_.get()->exec, - device_graph_.get()->graph, - &device_graph_.get()->node, - device_graph_.get()->log_buffer.data(), - device_graph_.get()->log_buffer.size()) - != INFINI_STATUS_SUCCESS) { + auto instantiate_status = device_graph_->standalone + ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) + : infinirtGraphInstantiate( + &device_graph_.get()->exec, + device_graph_.get()->graph, + &device_graph_.get()->node, + device_graph_.get()->log_buffer.data(), + device_graph_.get()->log_buffer.size()); + if (instantiate_status != INFINI_STATUS_SUCCESS) { static bool warned_once = false; if (!warned_once) { warned_once = true; spdlog::warn("Fail to instantiate device graph: {}", std::string(device_graph_.get()->log_buffer.data())); } + device_graph_.reset(); } #endif } diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc new file mode 100644 index 000000000..db52a3975 --- /dev/null +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -0,0 +1,287 @@ +#include "standalone_infinirt_graph_bridge.hpp" + +#ifdef USE_STANDALONE_INFINIRT_GRAPH + +#include "../utils.hpp" + +#include +#include +#include + +#include + +namespace infinicore::graph::standalone_infinirt { +namespace { + +using StreamWrapFn = infiniRtStatus_t (*)(infiniRtDevice_t, void *, infiniRtStream_t *); +using StreamDestroyFn = infiniRtStatus_t (*)(infiniRtStream_t); +using StreamBeginCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtStreamCaptureMode_t); +using StreamEndCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtGraph_t *); +using GraphDestroyFn = infiniRtStatus_t (*)(infiniRtGraph_t); +using GraphInstantiateFn = infiniRtStatus_t (*)(infiniRtGraphExec_t *, infiniRtGraph_t); +using GraphExecDestroyFn = infiniRtStatus_t (*)(infiniRtGraphExec_t); +using GraphLaunchFn = infiniRtStatus_t (*)(infiniRtGraphExec_t, infiniRtStream_t); + +struct Api { + void *handle = nullptr; + StreamWrapFn stream_wrap = nullptr; + StreamDestroyFn stream_destroy = nullptr; + StreamBeginCaptureFn stream_begin_capture = nullptr; + StreamEndCaptureFn stream_end_capture = nullptr; + GraphDestroyFn graph_destroy = nullptr; + GraphInstantiateFn graph_instantiate = nullptr; + GraphExecDestroyFn graph_exec_destroy = nullptr; + GraphLaunchFn graph_launch = nullptr; + bool loaded = false; +}; + +bool truthy_env(const char *name) { + auto value = std::getenv(name); + if (value == nullptr) { + return false; + } + std::string text{value}; + return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; +} + +std::string standalone_library_path() { + if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { + return explicit_path; + } + if (auto root = std::getenv("INFINI_RT_ROOT")) { + auto lib = std::string(root) + "/lib/libinfinirt.so"; + void *handle = dlopen(lib.c_str(), RTLD_NOW | RTLD_LOCAL); + if (handle != nullptr) { + dlclose(handle); + return lib; + } + return std::string(root) + "/lib64/libinfinirt.so"; + } + return "libinfinirt.so"; +} + +template +bool load_symbol(void *handle, const char *name, T *symbol) { + *symbol = reinterpret_cast(dlsym(handle, name)); + return *symbol != nullptr; +} + +Api &api() { + static Api api_ = [] { + Api loaded{}; + const auto path = standalone_library_path(); + loaded.handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (loaded.handle == nullptr) { + spdlog::warn("Standalone InfiniRT graph bridge disabled: failed to load {}: {}", path, dlerror()); + return loaded; + } + + loaded.loaded = + load_symbol(loaded.handle, "infiniRtStreamWrap", &loaded.stream_wrap) + && load_symbol(loaded.handle, "infiniRtStreamDestroy", &loaded.stream_destroy) + && load_symbol(loaded.handle, "infiniRtStreamBeginCapture", &loaded.stream_begin_capture) + && load_symbol(loaded.handle, "infiniRtStreamEndCapture", &loaded.stream_end_capture) + && load_symbol(loaded.handle, "infiniRtGraphDestroy", &loaded.graph_destroy) + && load_symbol(loaded.handle, "infiniRtGraphInstantiate", &loaded.graph_instantiate) + && load_symbol(loaded.handle, "infiniRtGraphExecDestroy", &loaded.graph_exec_destroy) + && load_symbol(loaded.handle, "infiniRtGraphLaunch", &loaded.graph_launch); + + if (loaded.loaded) { + spdlog::info("Standalone InfiniRT graph bridge loaded: {}", path); + } else { + spdlog::warn("Standalone InfiniRT graph bridge disabled: required graph symbols missing in {}", path); + } + return loaded; + }(); + return api_; +} + +infiniRtDeviceType_t to_standalone_device_type(Device::Type type) { + switch (type) { + case Device::Type::CPU: + return INFINI_RT_DEVICE_CPU; + case Device::Type::NVIDIA: + return INFINI_RT_DEVICE_NVIDIA; + case Device::Type::CAMBRICON: + return INFINI_RT_DEVICE_CAMBRICON; + case Device::Type::ASCEND: + return INFINI_RT_DEVICE_ASCEND; + case Device::Type::METAX: + return INFINI_RT_DEVICE_METAX; + case Device::Type::MOORE: + return INFINI_RT_DEVICE_MOORE; + case Device::Type::ILUVATAR: + return INFINI_RT_DEVICE_ILUVATAR; + case Device::Type::KUNLUN: + return INFINI_RT_DEVICE_KUNLUN; + case Device::Type::HYGON: + return INFINI_RT_DEVICE_HYGON; + case Device::Type::QY: + return INFINI_RT_DEVICE_QY; + default: + return INFINI_RT_DEVICE_CPU; + } +} + +infiniStatus_t to_core_status(infiniRtStatus_t status) { + switch (status) { + case INFINI_RT_STATUS_SUCCESS: + return INFINI_STATUS_SUCCESS; + case INFINI_RT_STATUS_INVALID_ARGUMENT: + return INFINI_STATUS_BAD_PARAM; + case INFINI_RT_STATUS_UNSUPPORTED_DEVICE: + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + case INFINI_RT_STATUS_RUNTIME_ERROR: + default: + return INFINI_STATUS_INTERNAL_ERROR; + } +} + +infiniRtStreamCaptureMode_t to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { + switch (mode) { + case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: + return INFINI_RT_STREAM_CAPTURE_MODE_GLOBAL; + case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: + return INFINI_RT_STREAM_CAPTURE_MODE_THREAD_LOCAL; + case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: + return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; + } + return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; +} + +infiniRtDevice_t to_standalone_device(const Device &device) { + return infiniRtDevice_t{ + to_standalone_device_type(device.getType()), + static_cast(device.getIndex()), + }; +} + +} // namespace + +bool enabled() { + return truthy_env("INFINICORE_USE_STANDALONE_INFINIRT_GRAPH"); +} + +bool available() { + return enabled() && api().loaded; +} + +infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream) { + auto &rt = api(); + if (!rt.loaded || stream == nullptr) { + return nullptr; + } + + auto standalone_device = to_standalone_device(device); + infiniRtStream_t wrapped = nullptr; + if (rt.stream_wrap(standalone_device, stream, &wrapped) != INFINI_RT_STATUS_SUCCESS) { + return nullptr; + } + return reinterpret_cast(wrapped); +} + +void destroy_wrapped_stream(infinirtStream_t stream) { + if (stream == nullptr || !api().loaded) { + return; + } + api().stream_destroy(reinterpret_cast(stream)); +} + +infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { + if (!api().loaded || stream == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().stream_begin_capture( + reinterpret_cast(stream), + to_standalone_capture_mode(mode))); +} + +infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { + if (!api().loaded || stream == nullptr || graph == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().stream_end_capture( + reinterpret_cast(stream), + reinterpret_cast(graph))); +} + +infiniStatus_t graph_destroy(infinirtGraph_t graph) { + if (!api().loaded || graph == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_destroy(reinterpret_cast(graph))); +} + +infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { + if (!api().loaded || graph_exec == nullptr || graph == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_instantiate( + reinterpret_cast(graph_exec), + reinterpret_cast(graph))); +} + +infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { + if (!api().loaded || graph_exec == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_exec_destroy(reinterpret_cast(graph_exec))); +} + +infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { + if (!api().loaded || graph_exec == nullptr || stream == nullptr) { + return INFINI_STATUS_INTERNAL_ERROR; + } + return to_core_status(api().graph_launch( + reinterpret_cast(graph_exec), + reinterpret_cast(stream))); +} + +} // namespace infinicore::graph::standalone_infinirt + +#else + +namespace infinicore::graph::standalone_infinirt { + +bool enabled() { + return false; +} + +bool available() { + return false; +} + +infinirtStream_t wrap_stream(const Device &, infinirtStream_t) { + return nullptr; +} + +void destroy_wrapped_stream(infinirtStream_t) { +} + +infiniStatus_t stream_begin_capture(infinirtStream_t, infinirtStreamCaptureMode_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t stream_end_capture(infinirtStream_t, infinirtGraph_t *) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_destroy(infinirtGraph_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_instantiate(infinirtGraphExec_t *, infinirtGraph_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_exec_destroy(infinirtGraphExec_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +infiniStatus_t graph_launch(infinirtGraphExec_t, infinirtStream_t) { + return INFINI_STATUS_NOT_IMPLEMENTED; +} + +} // namespace infinicore::graph::standalone_infinirt + +#endif diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp new file mode 100644 index 000000000..0c58bff77 --- /dev/null +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "infinicore/device.hpp" +#include + +namespace infinicore::graph::standalone_infinirt { + +bool enabled(); + +bool available(); + +infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream); + +void destroy_wrapped_stream(infinirtStream_t stream); + +infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode); + +infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph); + +infiniStatus_t graph_destroy(infinirtGraph_t graph); + +infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph); + +infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec); + +infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream); + +} // namespace infinicore::graph::standalone_infinirt diff --git a/xmake.lua b/xmake.lua index d98fe6e01..016ab074a 100644 --- a/xmake.lua +++ b/xmake.lua @@ -260,10 +260,20 @@ option("graph") set_description("Whether to use device graph instantiating feature, such as cuda graph for nvidia") option_end() -if has_config("graph") then +option("standalone-infinirt-graph") + set_default(false) + set_showmenu(true) + set_description("Whether to route graph lifecycle calls through an installed standalone InfiniRT") +option_end() + +if has_config("graph") or has_config("standalone-infinirt-graph") then add_defines("USE_INFINIRT_GRAPH") end +if has_config("standalone-infinirt-graph") then + add_defines("USE_STANDALONE_INFINIRT_GRAPH") +end + -- InfiniCCL option("ccl") @@ -327,6 +337,30 @@ end local infiniops_external_built = false +local function find_standalone_infinirt(infinirt_root, xmake_os) + local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") + if not xmake_os.isfile(standalone_infinirt) then + standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") + end + if not xmake_os.isfile(standalone_infinirt) then + raise("Standalone InfiniRT library not found under: " .. infinirt_root) + end + return standalone_infinirt +end + +local function patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) + local private_soname = path.filename(private_infinirt) + xmake_os.cp(standalone_infinirt, private_infinirt) + xmake_os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) + xmake_os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) + xmake_os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + + local needed = xmake_os.iorunv("patchelf", {"--print-needed", infiniops_lib}) + if not needed:find(private_soname, 1, true) then + xmake_os.execv("patchelf", {"--add-needed", private_soname, infiniops_lib}) + end +end + local function filter_infiniops_ops_for_backend(infiniops_ops) if not infiniops_ops or #infiniops_ops == 0 then return infiniops_ops @@ -408,20 +442,10 @@ local function build_infiniops_external(xmake_os) xmake_os.execv("cmake", {"--build", infiniops_builddir, "--target", "infiniops"}) xmake_os.execv("cmake", {"--install", infiniops_builddir, "--prefix", INFINI_ROOT}) if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not xmake_os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") - end - if not xmake_os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) - end + local standalone_infinirt = find_standalone_infinirt(infinirt_root, xmake_os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") - local private_soname = "libinfiniops_infinirt.so" - local private_infinirt = path.join(INFINI_ROOT, "lib", private_soname) - xmake_os.cp(standalone_infinirt, private_infinirt) - xmake_os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - xmake_os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) - xmake_os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") + patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) end infiniops_external_built = true end @@ -624,6 +648,16 @@ target("infinicore_cpp_api") add_defines("CHAR_BIT=8", "INT_MIN=(-2147483647 - 1)", "INT_MAX=2147483647", "UINT_MAX=4294967295U") end add_includedirs(INFINI_ROOT.."/include", { public = true }) + local graph_infinirt_root = get_standalone_infinirt_root() + if has_config("standalone-infinirt-graph") then + if not graph_infinirt_root or graph_infinirt_root == "" then + raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") + end + add_includedirs(graph_infinirt_root .. "/include") + if is_plat("linux") then + add_links("dl") + end + end if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" add_includedirs(cuda_root .. "/include") @@ -651,19 +685,10 @@ target("infinicore_cpp_api") local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") local infinirt_root = get_standalone_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") - end - if not os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) - end + local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") - local private_soname = "libinfiniops_infinirt.so" - local private_infinirt = path.join(INFINI_ROOT, "lib", private_soname) - os.cp(standalone_infinirt, private_infinirt) - os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") + patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) end end) end @@ -923,19 +948,9 @@ target("_infinicore") end local infinirt_root = get_standalone_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") - end - if not os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) - end - local private_soname = "libinfiniops_infinirt.so" - local private_infinirt = path.join(INFINI_ROOT, "lib", private_soname) - os.cp(standalone_infinirt, private_infinirt) - os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) - os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) + local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) + local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") + patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) end os.mkdir(path.join(INFINI_ROOT, "lib")) if not infiniops_lib_installed then From 1493ff67b3cbd85b4bcac91826b57e782ba5df7c Mon Sep 17 00:00:00 2001 From: Jiacheng Huang Date: Fri, 3 Jul 2026 16:49:25 +0800 Subject: [PATCH 02/11] refactor: call standalone InfiniRT C++ graph API --- src/infinicore/graph/graph.cc | 43 ++-- .../graph/standalone_infinirt_graph_bridge.cc | 228 +++++------------- .../standalone_infinirt_graph_bridge.hpp | 6 +- xmake.lua | 3 - 4 files changed, 79 insertions(+), 201 deletions(-) diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index ee7cb5ca8..377e6dcc7 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -63,9 +63,6 @@ struct Graph::DeviceGraph { infinirtGraphDestroy(graph); } } - if (standalone && stream) { - standalone_infinirt::destroy_wrapped_stream(stream); - } } void launch() { @@ -103,16 +100,16 @@ void Graph::instantiate() { #ifdef USE_INFINIRT_GRAPH // Reset device graph device_graph_ = std::make_unique(); - device_graph_->standalone = standalone_infinirt::available(); - device_graph_->stream = device_graph_->standalone - ? standalone_infinirt::wrap_stream(context::getDevice(), context::getStream()) - : context::getStream(); - if (device_graph_->standalone && device_graph_->stream == nullptr) { - spdlog::warn("Standalone InfiniRT graph bridge is enabled but failed to wrap the current stream. Falling back to eager execution."); - device_graph_.reset(); - return; - } + device_graph_->standalone = standalone_infinirt::available(context::getDevice()); + device_graph_->stream = context::getStream(); if (device_graph_->standalone) { + auto set_device_status = standalone_infinirt::set_device(context::getDevice()); + if (set_device_status != INFINI_STATUS_SUCCESS) { + spdlog::warn("Standalone InfiniRT graph bridge failed to select the current device. Falling back to eager execution."); + device_graph_.reset(); + return; + } + static bool logged_once = false; if (!logged_once) { logged_once = true; @@ -127,8 +124,8 @@ void Graph::instantiate() { infinicore::context::syncStream(); auto begin_status = device_graph_->standalone - ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) - : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); + ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) + : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); if (begin_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to begin device graph capture."); device_graph_.reset(); @@ -139,8 +136,8 @@ void Graph::instantiate() { this->run(); auto end_status = device_graph_->standalone - ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) - : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); + ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) + : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); if (end_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to end device graph capture."); device_graph_.reset(); @@ -148,13 +145,13 @@ void Graph::instantiate() { } auto instantiate_status = device_graph_->standalone - ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) - : infinirtGraphInstantiate( - &device_graph_.get()->exec, - device_graph_.get()->graph, - &device_graph_.get()->node, - device_graph_.get()->log_buffer.data(), - device_graph_.get()->log_buffer.size()); + ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) + : infinirtGraphInstantiate( + &device_graph_.get()->exec, + device_graph_.get()->graph, + &device_graph_.get()->node, + device_graph_.get()->log_buffer.data(), + device_graph_.get()->log_buffer.size()); if (instantiate_status != INFINI_STATUS_SUCCESS) { static bool warned_once = false; if (!warned_once) { diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc index db52a3975..be4c90940 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -2,38 +2,16 @@ #ifdef USE_STANDALONE_INFINIRT_GRAPH -#include "../utils.hpp" - #include -#include #include -#include +#include namespace infinicore::graph::standalone_infinirt { namespace { -using StreamWrapFn = infiniRtStatus_t (*)(infiniRtDevice_t, void *, infiniRtStream_t *); -using StreamDestroyFn = infiniRtStatus_t (*)(infiniRtStream_t); -using StreamBeginCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtStreamCaptureMode_t); -using StreamEndCaptureFn = infiniRtStatus_t (*)(infiniRtStream_t, infiniRtGraph_t *); -using GraphDestroyFn = infiniRtStatus_t (*)(infiniRtGraph_t); -using GraphInstantiateFn = infiniRtStatus_t (*)(infiniRtGraphExec_t *, infiniRtGraph_t); -using GraphExecDestroyFn = infiniRtStatus_t (*)(infiniRtGraphExec_t); -using GraphLaunchFn = infiniRtStatus_t (*)(infiniRtGraphExec_t, infiniRtStream_t); - -struct Api { - void *handle = nullptr; - StreamWrapFn stream_wrap = nullptr; - StreamDestroyFn stream_destroy = nullptr; - StreamBeginCaptureFn stream_begin_capture = nullptr; - StreamEndCaptureFn stream_end_capture = nullptr; - GraphDestroyFn graph_destroy = nullptr; - GraphInstantiateFn graph_instantiate = nullptr; - GraphExecDestroyFn graph_exec_destroy = nullptr; - GraphLaunchFn graph_launch = nullptr; - bool loaded = false; -}; +using StandaloneDevice = infini::rt::Device; +using StandaloneRuntime = infini::rt::runtime::Runtime; bool truthy_env(const char *name) { auto value = std::getenv(name); @@ -44,116 +22,40 @@ bool truthy_env(const char *name) { return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; } -std::string standalone_library_path() { - if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { - return explicit_path; - } - if (auto root = std::getenv("INFINI_RT_ROOT")) { - auto lib = std::string(root) + "/lib/libinfinirt.so"; - void *handle = dlopen(lib.c_str(), RTLD_NOW | RTLD_LOCAL); - if (handle != nullptr) { - dlclose(handle); - return lib; - } - return std::string(root) + "/lib64/libinfinirt.so"; - } - return "libinfinirt.so"; +bool supports_device(Device::Type type) { + return type == Device::Type::NVIDIA; } -template -bool load_symbol(void *handle, const char *name, T *symbol) { - *symbol = reinterpret_cast(dlsym(handle, name)); - return *symbol != nullptr; +template +infiniStatus_t to_core_status(Status status) { + return status == StandaloneRuntime::kSuccess + ? INFINI_STATUS_SUCCESS + : INFINI_STATUS_INTERNAL_ERROR; } -Api &api() { - static Api api_ = [] { - Api loaded{}; - const auto path = standalone_library_path(); - loaded.handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); - if (loaded.handle == nullptr) { - spdlog::warn("Standalone InfiniRT graph bridge disabled: failed to load {}: {}", path, dlerror()); - return loaded; - } - - loaded.loaded = - load_symbol(loaded.handle, "infiniRtStreamWrap", &loaded.stream_wrap) - && load_symbol(loaded.handle, "infiniRtStreamDestroy", &loaded.stream_destroy) - && load_symbol(loaded.handle, "infiniRtStreamBeginCapture", &loaded.stream_begin_capture) - && load_symbol(loaded.handle, "infiniRtStreamEndCapture", &loaded.stream_end_capture) - && load_symbol(loaded.handle, "infiniRtGraphDestroy", &loaded.graph_destroy) - && load_symbol(loaded.handle, "infiniRtGraphInstantiate", &loaded.graph_instantiate) - && load_symbol(loaded.handle, "infiniRtGraphExecDestroy", &loaded.graph_exec_destroy) - && load_symbol(loaded.handle, "infiniRtGraphLaunch", &loaded.graph_launch); - - if (loaded.loaded) { - spdlog::info("Standalone InfiniRT graph bridge loaded: {}", path); - } else { - spdlog::warn("Standalone InfiniRT graph bridge disabled: required graph symbols missing in {}", path); - } - return loaded; - }(); - return api_; +StandaloneRuntime::Stream to_standalone_stream(infinirtStream_t stream) { + return reinterpret_cast(stream); } -infiniRtDeviceType_t to_standalone_device_type(Device::Type type) { - switch (type) { - case Device::Type::CPU: - return INFINI_RT_DEVICE_CPU; - case Device::Type::NVIDIA: - return INFINI_RT_DEVICE_NVIDIA; - case Device::Type::CAMBRICON: - return INFINI_RT_DEVICE_CAMBRICON; - case Device::Type::ASCEND: - return INFINI_RT_DEVICE_ASCEND; - case Device::Type::METAX: - return INFINI_RT_DEVICE_METAX; - case Device::Type::MOORE: - return INFINI_RT_DEVICE_MOORE; - case Device::Type::ILUVATAR: - return INFINI_RT_DEVICE_ILUVATAR; - case Device::Type::KUNLUN: - return INFINI_RT_DEVICE_KUNLUN; - case Device::Type::HYGON: - return INFINI_RT_DEVICE_HYGON; - case Device::Type::QY: - return INFINI_RT_DEVICE_QY; - default: - return INFINI_RT_DEVICE_CPU; - } +StandaloneRuntime::Graph to_standalone_graph(infinirtGraph_t graph) { + return reinterpret_cast(graph); } -infiniStatus_t to_core_status(infiniRtStatus_t status) { - switch (status) { - case INFINI_RT_STATUS_SUCCESS: - return INFINI_STATUS_SUCCESS; - case INFINI_RT_STATUS_INVALID_ARGUMENT: - return INFINI_STATUS_BAD_PARAM; - case INFINI_RT_STATUS_UNSUPPORTED_DEVICE: - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - case INFINI_RT_STATUS_RUNTIME_ERROR: - default: - return INFINI_STATUS_INTERNAL_ERROR; - } +StandaloneRuntime::GraphExec to_standalone_graph_exec(infinirtGraphExec_t graph_exec) { + return reinterpret_cast(graph_exec); } -infiniRtStreamCaptureMode_t to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { +decltype(StandaloneRuntime::kStreamCaptureModeRelaxed) +to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { switch (mode) { case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: - return INFINI_RT_STREAM_CAPTURE_MODE_GLOBAL; + return StandaloneRuntime::kStreamCaptureModeGlobal; case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: - return INFINI_RT_STREAM_CAPTURE_MODE_THREAD_LOCAL; + return StandaloneRuntime::kStreamCaptureModeThreadLocal; case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: - return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; + return StandaloneRuntime::kStreamCaptureModeRelaxed; } - return INFINI_RT_STREAM_CAPTURE_MODE_RELAXED; -} - -infiniRtDevice_t to_standalone_device(const Device &device) { - return infiniRtDevice_t{ - to_standalone_device_type(device.getType()), - static_cast(device.getIndex()), - }; + return StandaloneRuntime::kStreamCaptureModeRelaxed; } } // namespace @@ -162,79 +64,66 @@ bool enabled() { return truthy_env("INFINICORE_USE_STANDALONE_INFINIRT_GRAPH"); } -bool available() { - return enabled() && api().loaded; +bool available(const Device &device) { + return enabled() && supports_device(device.getType()); } -infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream) { - auto &rt = api(); - if (!rt.loaded || stream == nullptr) { - return nullptr; - } - - auto standalone_device = to_standalone_device(device); - infiniRtStream_t wrapped = nullptr; - if (rt.stream_wrap(standalone_device, stream, &wrapped) != INFINI_RT_STATUS_SUCCESS) { - return nullptr; - } - return reinterpret_cast(wrapped); -} - -void destroy_wrapped_stream(infinirtStream_t stream) { - if (stream == nullptr || !api().loaded) { - return; +infiniStatus_t set_device(const Device &device) { + if (!supports_device(device.getType())) { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; } - api().stream_destroy(reinterpret_cast(stream)); + return to_core_status(StandaloneRuntime::SetDevice(static_cast(device.getIndex()))); } infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { - if (!api().loaded || stream == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (stream == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().stream_begin_capture( - reinterpret_cast(stream), + return to_core_status(StandaloneRuntime::StreamBeginCapture( + to_standalone_stream(stream), to_standalone_capture_mode(mode))); } infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { - if (!api().loaded || stream == nullptr || graph == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (stream == nullptr || graph == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().stream_end_capture( - reinterpret_cast(stream), - reinterpret_cast(graph))); + return to_core_status(StandaloneRuntime::StreamEndCapture( + to_standalone_stream(stream), + reinterpret_cast(graph))); } infiniStatus_t graph_destroy(infinirtGraph_t graph) { - if (!api().loaded || graph == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_destroy(reinterpret_cast(graph))); + return to_core_status(StandaloneRuntime::GraphDestroy(to_standalone_graph(graph))); } infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { - if (!api().loaded || graph_exec == nullptr || graph == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph_exec == nullptr || graph == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_instantiate( - reinterpret_cast(graph_exec), - reinterpret_cast(graph))); + return to_core_status(StandaloneRuntime::GraphInstantiate( + reinterpret_cast(graph_exec), + to_standalone_graph(graph))); } infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { - if (!api().loaded || graph_exec == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph_exec == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_exec_destroy(reinterpret_cast(graph_exec))); + return to_core_status(StandaloneRuntime::GraphExecDestroy( + to_standalone_graph_exec(graph_exec))); } infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { - if (!api().loaded || graph_exec == nullptr || stream == nullptr) { - return INFINI_STATUS_INTERNAL_ERROR; + if (graph_exec == nullptr || stream == nullptr) { + return INFINI_STATUS_NULL_POINTER; } - return to_core_status(api().graph_launch( - reinterpret_cast(graph_exec), - reinterpret_cast(stream))); + return to_core_status(StandaloneRuntime::GraphLaunch( + to_standalone_graph_exec(graph_exec), + to_standalone_stream(stream))); } } // namespace infinicore::graph::standalone_infinirt @@ -247,15 +136,12 @@ bool enabled() { return false; } -bool available() { +bool available(const Device &) { return false; } -infinirtStream_t wrap_stream(const Device &, infinirtStream_t) { - return nullptr; -} - -void destroy_wrapped_stream(infinirtStream_t) { +infiniStatus_t set_device(const Device &) { + return INFINI_STATUS_NOT_IMPLEMENTED; } infiniStatus_t stream_begin_capture(infinirtStream_t, infinirtStreamCaptureMode_t) { diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp index 0c58bff77..0ebbb0a6b 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp @@ -7,11 +7,9 @@ namespace infinicore::graph::standalone_infinirt { bool enabled(); -bool available(); +bool available(const Device &device); -infinirtStream_t wrap_stream(const Device &device, infinirtStream_t stream); - -void destroy_wrapped_stream(infinirtStream_t stream); +infiniStatus_t set_device(const Device &device); infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode); diff --git a/xmake.lua b/xmake.lua index 016ab074a..39904c2b9 100644 --- a/xmake.lua +++ b/xmake.lua @@ -654,9 +654,6 @@ target("infinicore_cpp_api") raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") end add_includedirs(graph_infinirt_root .. "/include") - if is_plat("linux") then - add_links("dl") - end end if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" From 8a774ab7dfbbcc7cb1b4e8b141d2d7db6088f079 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Tue, 30 Jun 2026 17:06:34 +0800 Subject: [PATCH 03/11] fix: support ascend paged graph runtime Synchronize RoPE cache copies before the cache is consumed by Ascend kernels. Rename paged attention block_idx locals to page_block_idx to avoid AscendC kernel symbol conflicts while preserving the same block table address calculation. Detect common Ascend driver library layouts before linking libascend_hal.so instead of relying on one hard-coded driver path. --- src/infinicore/nn/rope.cc | 14 ++++++++++++++ .../ascend/paged_attention_ascend_kernel.cpp | 8 ++++---- .../paged_attention_prefill_ascend_kernel.cpp | 8 ++++---- xmake/ascend.lua | 9 ++++++++- 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/src/infinicore/nn/rope.cc b/src/infinicore/nn/rope.cc index 115e162e2..c8ab4f02d 100644 --- a/src/infinicore/nn/rope.cc +++ b/src/infinicore/nn/rope.cc @@ -1,6 +1,7 @@ #include "infinicore/nn/rope.hpp" #include "../../utils.h" #include "../utils.hpp" +#include "infinicore/context/context.hpp" #include "infinicore/ops/mrope.hpp" #include "infinicore/ops/rope.hpp" #include @@ -12,6 +13,16 @@ #include namespace infinicore::nn { +namespace { + +void sync_cache_copy(const Device &device) { + if (device.getType() == Device::Type::ASCEND) { + // Ascend H2D copies are async, and these host buffers leave scope after init. + infinicore::context::syncStream(); + } +} + +} // namespace RoPE::RoPE(size_t head_dim, size_t rotary_dim, @@ -95,6 +106,7 @@ void RoPE::initialize_cache() { auto cos_f32_cpu = Tensor::from_blob(cos_data.data(), {max_seq_len_, cache_dim}, DataType::F32, cpu_device); sin_cache_->copy_from(sin_f32_cpu); cos_cache_->copy_from(cos_f32_cpu); + sync_cache_copy(device_); } else if (dtype_ == DataType::BF16) { // Convert F32 to BF16 using the same conversion as Python's ml_dtypes.bfloat16 // This uses round-to-nearest-even (matching _f32_to_bf16 implementation) @@ -112,6 +124,7 @@ void RoPE::initialize_cache() { // copy_from handles cross-device copying to target device sin_cache_->copy_from(sin_bf16_cpu); cos_cache_->copy_from(cos_bf16_cpu); + sync_cache_copy(device_); } else if (dtype_ == DataType::F16) { // Convert F32 to F16 std::vector sin_f16_data(max_seq_len_ * cache_dim); @@ -127,6 +140,7 @@ void RoPE::initialize_cache() { sin_cache_->copy_from(sin_f16_cpu); cos_cache_->copy_from(cos_f16_cpu); + sync_cache_copy(device_); } else { throw std::runtime_error( "RoPE cache dtype conversion not yet supported for dtype: " diff --git a/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp b/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp index 0f47c2ff5..e6df4f50b 100644 --- a/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp +++ b/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp @@ -164,22 +164,22 @@ class PagedAttentionKernel { private: __aicore__ inline ptrdiff_t keyBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _k_batch_stride + static_cast(kv_head_idx) * _k_head_stride + static_cast(token_offset) * _k_row_stride; } __aicore__ inline ptrdiff_t valueBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _v_batch_stride + static_cast(kv_head_idx) * _v_head_stride + static_cast(token_offset) * _v_row_stride; diff --git a/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp b/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp index e1fce2785..db90b2056 100644 --- a/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp +++ b/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp @@ -197,22 +197,22 @@ class PagedAttentionPrefillKernel { } __aicore__ inline ptrdiff_t keyBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadPrefillIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _k_batch_stride + static_cast(kv_head_idx) * _k_head_stride + static_cast(token_offset) * _k_row_stride; } __aicore__ inline ptrdiff_t valueBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t block_idx = token_idx / _page_block_size; + const size_t page_block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadPrefillIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); return static_cast(physical_block) * _v_batch_stride + static_cast(kv_head_idx) * _v_head_stride + static_cast(token_offset) * _v_row_stride; diff --git a/xmake/ascend.lua b/xmake/ascend.lua index 6179cac13..07211b065 100644 --- a/xmake/ascend.lua +++ b/xmake/ascend.lua @@ -10,7 +10,14 @@ add_links("libascendcl.so") add_links("libnnopbase.so") add_links("libopapi.so") add_links("libruntime.so") -add_linkdirs(ASCEND_HOME .. "/../../driver/lib64/driver") +for _, driver_dir in ipairs({ + path.join(ASCEND_HOME, "../driver/lib64/driver"), + path.join(ASCEND_HOME, "../../driver/lib64/driver"), +}) do + if os.isdir(driver_dir) then + add_linkdirs(driver_dir) + end +end add_links("libascend_hal.so") local builddir = string.format( "%s/build/%s/%s/%s", From 9a8583e3cc2592de41d828468285b76b46a45702 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Mon, 6 Jul 2026 16:17:14 +0800 Subject: [PATCH 04/11] fix: support ascend standalone infinirt graph bridge --- .../graph/standalone_infinirt_graph_bridge.cc | 220 +++++++++++++++--- xmake.lua | 15 ++ xmake/ascend.lua | 23 +- 3 files changed, 217 insertions(+), 41 deletions(-) diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc index be4c90940..e74bcabde 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -2,6 +2,8 @@ #ifdef USE_STANDALONE_INFINIRT_GRAPH +#include "../utils.hpp" + #include #include @@ -11,7 +13,6 @@ namespace infinicore::graph::standalone_infinirt { namespace { using StandaloneDevice = infini::rt::Device; -using StandaloneRuntime = infini::rt::runtime::Runtime; bool truthy_env(const char *name) { auto value = std::getenv(name); @@ -22,40 +23,154 @@ bool truthy_env(const char *name) { return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; } +std::string standalone_library_path() { + if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { + return explicit_path; + } + if (auto root = std::getenv("INFINI_RT_ROOT")) { + return std::string(root) + "/lib/libinfinirt.so"; + } + return "libinfinirt.so"; +} + +void log_loaded_once() { + static bool logged_once = false; + if (!logged_once) { + logged_once = true; + spdlog::info("Standalone InfiniRT graph bridge loaded: {}", standalone_library_path()); + } +} + +template +constexpr bool standalone_device_enabled() { + return infini::rt::DeviceEnabled::value; +} + bool supports_device(Device::Type type) { - return type == Device::Type::NVIDIA; + switch (type) { + case Device::Type::NVIDIA: + return standalone_device_enabled(); + case Device::Type::ASCEND: + return standalone_device_enabled(); + default: + return false; + } } -template +thread_local Device::Type current_device_type = Device::Type::CPU; + +template infiniStatus_t to_core_status(Status status) { - return status == StandaloneRuntime::kSuccess + return status == Runtime::kSuccess ? INFINI_STATUS_SUCCESS : INFINI_STATUS_INTERNAL_ERROR; } -StandaloneRuntime::Stream to_standalone_stream(infinirtStream_t stream) { - return reinterpret_cast(stream); +template +infiniStatus_t set_device_impl(int index) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::SetDevice(index)); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t stream_begin_capture_impl(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + auto standalone_mode = Runtime::kStreamCaptureModeRelaxed; + switch (mode) { + case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: + standalone_mode = Runtime::kStreamCaptureModeGlobal; + break; + case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: + standalone_mode = Runtime::kStreamCaptureModeThreadLocal; + break; + case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: + standalone_mode = Runtime::kStreamCaptureModeRelaxed; + break; + } + return to_core_status(Runtime::StreamBeginCapture( + reinterpret_cast(stream), + standalone_mode)); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t stream_end_capture_impl(infinirtStream_t stream, infinirtGraph_t *graph) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::StreamEndCapture( + reinterpret_cast(stream), + reinterpret_cast(graph))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t graph_destroy_impl(infinirtGraph_t graph) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphDestroy( + reinterpret_cast(graph))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } +} + +template +infiniStatus_t graph_instantiate_impl(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphInstantiate( + reinterpret_cast(graph_exec), + reinterpret_cast(graph))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } } -StandaloneRuntime::Graph to_standalone_graph(infinirtGraph_t graph) { - return reinterpret_cast(graph); +template +infiniStatus_t graph_exec_destroy_impl(infinirtGraphExec_t graph_exec) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphExecDestroy( + reinterpret_cast(graph_exec))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } } -StandaloneRuntime::GraphExec to_standalone_graph_exec(infinirtGraphExec_t graph_exec) { - return reinterpret_cast(graph_exec); +template +infiniStatus_t graph_launch_impl(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { + if constexpr (standalone_device_enabled()) { + using Runtime = infini::rt::runtime::Runtime; + return to_core_status(Runtime::GraphLaunch( + reinterpret_cast(graph_exec), + reinterpret_cast(stream))); + } else { + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + } } -decltype(StandaloneRuntime::kStreamCaptureModeRelaxed) -to_standalone_capture_mode(infinirtStreamCaptureMode_t mode) { - switch (mode) { - case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: - return StandaloneRuntime::kStreamCaptureModeGlobal; - case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: - return StandaloneRuntime::kStreamCaptureModeThreadLocal; - case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: - return StandaloneRuntime::kStreamCaptureModeRelaxed; +template +infiniStatus_t dispatch_current( + infiniStatus_t (*nvidia_fn)(Args...), + infiniStatus_t (*ascend_fn)(Args...), + Args... args) { + switch (current_device_type) { + case Device::Type::NVIDIA: + return nvidia_fn(args...); + case Device::Type::ASCEND: + return ascend_fn(args...); + default: + return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; } - return StandaloneRuntime::kStreamCaptureModeRelaxed; } } // namespace @@ -65,65 +180,96 @@ bool enabled() { } bool available(const Device &device) { - return enabled() && supports_device(device.getType()); + if (!enabled() || !supports_device(device.getType())) { + return false; + } + log_loaded_once(); + return true; } infiniStatus_t set_device(const Device &device) { if (!supports_device(device.getType())) { return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; } - return to_core_status(StandaloneRuntime::SetDevice(static_cast(device.getIndex()))); + auto status = INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; + switch (device.getType()) { + case Device::Type::NVIDIA: + status = set_device_impl(static_cast(device.getIndex())); + break; + case Device::Type::ASCEND: + status = set_device_impl(static_cast(device.getIndex())); + break; + default: + break; + } + if (status == INFINI_STATUS_SUCCESS) { + current_device_type = device.getType(); + } + return status; } infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { if (stream == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::StreamBeginCapture( - to_standalone_stream(stream), - to_standalone_capture_mode(mode))); + return dispatch_current( + stream_begin_capture_impl, + stream_begin_capture_impl, + stream, + mode); } infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { if (stream == nullptr || graph == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::StreamEndCapture( - to_standalone_stream(stream), - reinterpret_cast(graph))); + return dispatch_current( + stream_end_capture_impl, + stream_end_capture_impl, + stream, + graph); } infiniStatus_t graph_destroy(infinirtGraph_t graph) { if (graph == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphDestroy(to_standalone_graph(graph))); + return dispatch_current( + graph_destroy_impl, + graph_destroy_impl, + graph); } infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { if (graph_exec == nullptr || graph == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphInstantiate( - reinterpret_cast(graph_exec), - to_standalone_graph(graph))); + return dispatch_current( + graph_instantiate_impl, + graph_instantiate_impl, + graph_exec, + graph); } infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { if (graph_exec == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphExecDestroy( - to_standalone_graph_exec(graph_exec))); + return dispatch_current( + graph_exec_destroy_impl, + graph_exec_destroy_impl, + graph_exec); } infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { if (graph_exec == nullptr || stream == nullptr) { return INFINI_STATUS_NULL_POINTER; } - return to_core_status(StandaloneRuntime::GraphLaunch( - to_standalone_graph_exec(graph_exec), - to_standalone_stream(stream))); + return dispatch_current( + graph_launch_impl, + graph_launch_impl, + graph_exec, + stream); } } // namespace infinicore::graph::standalone_infinirt diff --git a/xmake.lua b/xmake.lua index 39904c2b9..642e908f9 100644 --- a/xmake.lua +++ b/xmake.lua @@ -654,6 +654,21 @@ target("infinicore_cpp_api") raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") end add_includedirs(graph_infinirt_root .. "/include") + if has_config("ascend-npu") then + local ascend_home = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") + if ascend_home and ascend_home ~= "" then + for _, include_dir in ipairs({ + path.join(ascend_home, "include"), + path.join(ascend_home, "include/aclnn"), + path.join(ascend_home, "aarch64-linux/include"), + path.join(ascend_home, "aarch64-linux/include/aclnn"), + }) do + if os.isdir(include_dir) then + add_includedirs(include_dir) + end + end + end + end end if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" diff --git a/xmake/ascend.lua b/xmake/ascend.lua index 07211b065..af82ab162 100644 --- a/xmake/ascend.lua +++ b/xmake/ascend.lua @@ -1,11 +1,26 @@ add_defines("ENABLE_ASCEND_API") -local ASCEND_HOME = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") +local ASCEND_HOME = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") local SOC_VERSION = os.getenv("SOC_VERSION") -- Add include dirs -add_includedirs(ASCEND_HOME .. "/include") -add_includedirs(ASCEND_HOME .. "/include/aclnn") -add_linkdirs(ASCEND_HOME .. "/lib64") +for _, include_dir in ipairs({ + path.join(ASCEND_HOME, "include"), + path.join(ASCEND_HOME, "include/aclnn"), + path.join(ASCEND_HOME, "aarch64-linux/include"), + path.join(ASCEND_HOME, "aarch64-linux/include/aclnn"), +}) do + if os.isdir(include_dir) then + add_includedirs(include_dir) + end +end +for _, lib_dir in ipairs({ + path.join(ASCEND_HOME, "lib64"), + path.join(ASCEND_HOME, "aarch64-linux/lib64"), +}) do + if os.isdir(lib_dir) then + add_linkdirs(lib_dir) + end +end add_links("libascendcl.so") add_links("libnnopbase.so") add_links("libopapi.so") From e27cd0b01f36d0c35d72d12e2ea6c2b200812969 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Mon, 6 Jul 2026 16:58:07 +0800 Subject: [PATCH 05/11] fix: drop stale standalone InfiniRT C API bridge log The standalone bridge now calls the InfiniRT C++ runtime API directly, so remove the old loaded-library marker and keep the runtime log aligned with the actual path. --- src/infinicore/graph/graph.cc | 2 +- .../graph/standalone_infinirt_graph_bridge.cc | 26 +------------------ 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 377e6dcc7..0a09bf68e 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -113,7 +113,7 @@ void Graph::instantiate() { static bool logged_once = false; if (!logged_once) { logged_once = true; - spdlog::info("Using standalone InfiniRT graph bridge for graph capture and replay."); + spdlog::info("Using standalone InfiniRT C++ graph runtime API for graph capture and replay."); } } diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc index e74bcabde..7725d48c1 100644 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc @@ -2,8 +2,6 @@ #ifdef USE_STANDALONE_INFINIRT_GRAPH -#include "../utils.hpp" - #include #include @@ -23,24 +21,6 @@ bool truthy_env(const char *name) { return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; } -std::string standalone_library_path() { - if (auto explicit_path = std::getenv("INFINIRT_GRAPH_LIBRARY")) { - return explicit_path; - } - if (auto root = std::getenv("INFINI_RT_ROOT")) { - return std::string(root) + "/lib/libinfinirt.so"; - } - return "libinfinirt.so"; -} - -void log_loaded_once() { - static bool logged_once = false; - if (!logged_once) { - logged_once = true; - spdlog::info("Standalone InfiniRT graph bridge loaded: {}", standalone_library_path()); - } -} - template constexpr bool standalone_device_enabled() { return infini::rt::DeviceEnabled::value; @@ -180,11 +160,7 @@ bool enabled() { } bool available(const Device &device) { - if (!enabled() || !supports_device(device.getType())) { - return false; - } - log_loaded_once(); - return true; + return enabled() && supports_device(device.getType()); } infiniStatus_t set_device(const Device &device) { From f134f601c5e140bdb3e80f6d575930b2def5d8d1 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 8 Jul 2026 11:26:32 +0800 Subject: [PATCH 06/11] build: link Core against installed InfiniRT Use --infinirt-root or INFINI_RT_ROOT as the installed InfiniRT prefix for Core targets, install the runtime handle header with Core, and remove stale dependencies on the old in-tree infinirt target. The standalone graph build switch is removed because graph capture is now controlled by USE_INFINIRT_GRAPH and routed through the InfiniRT runtime API directly. --- xmake.lua | 124 ++++++++++++++++++++++----------------------- xmake/ali.lua | 1 - xmake/ascend.lua | 1 - xmake/bang.lua | 1 - xmake/hygon.lua | 1 - xmake/iluvatar.lua | 1 - xmake/kunlun.lua | 1 - xmake/metax.lua | 1 - xmake/moore.lua | 1 - xmake/qy.lua | 1 - 10 files changed, 60 insertions(+), 73 deletions(-) diff --git a/xmake.lua b/xmake.lua index 642e908f9..960ad5dcd 100644 --- a/xmake.lua +++ b/xmake.lua @@ -260,20 +260,10 @@ option("graph") set_description("Whether to use device graph instantiating feature, such as cuda graph for nvidia") option_end() -option("standalone-infinirt-graph") - set_default(false) - set_showmenu(true) - set_description("Whether to route graph lifecycle calls through an installed standalone InfiniRT") -option_end() - -if has_config("graph") or has_config("standalone-infinirt-graph") then +if has_config("graph") then add_defines("USE_INFINIRT_GRAPH") end -if has_config("standalone-infinirt-graph") then - add_defines("USE_STANDALONE_INFINIRT_GRAPH") -end - -- InfiniCCL option("ccl") @@ -302,10 +292,10 @@ option_end() option("infinirt-root") set_default("") set_showmenu(true) - set_description("Path to an installed standalone InfiniRT prefix used by --infiniops") + set_description("Path to an installed InfiniRT prefix") option_end() -local function get_standalone_infinirt_root() +local function get_infinirt_root() local infinirt_root = get_config("infinirt-root") if not infinirt_root or infinirt_root == "" then infinirt_root = os.getenv("INFINI_RT_ROOT") @@ -337,22 +327,52 @@ end local infiniops_external_built = false -local function find_standalone_infinirt(infinirt_root, xmake_os) - local standalone_infinirt = path.join(infinirt_root, "lib", "libinfinirt.so") - if not xmake_os.isfile(standalone_infinirt) then - standalone_infinirt = path.join(infinirt_root, "lib64", "libinfinirt.so") +local function find_infinirt_library(infinirt_root, xmake_os) + local infinirt_library = path.join(infinirt_root, "lib", "libinfinirt.so") + if not xmake_os.isfile(infinirt_library) then + infinirt_library = path.join(infinirt_root, "lib64", "libinfinirt.so") end - if not xmake_os.isfile(standalone_infinirt) then - raise("Standalone InfiniRT library not found under: " .. infinirt_root) + if not xmake_os.isfile(infinirt_library) then + raise("InfiniRT library not found under: " .. infinirt_root) end - return standalone_infinirt + return infinirt_library end -local function patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) +local function add_external_infinirt() + local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") + local infinirt_root = get_infinirt_root() + if infinirt_root and infinirt_root ~= "" then + add_includedirs(path.join(infinirt_root, "include"), { public = true }) + add_linkdirs(path.join(infinirt_root, "lib"), path.join(infinirt_root, "lib64")) + add_rpathdirs(path.join(infinirt_root, "lib"), path.join(infinirt_root, "lib64")) + else + add_includedirs(INFINI_ROOT .. "/include", { public = true }) + add_linkdirs(INFINI_ROOT .. "/lib") + add_rpathdirs(INFINI_ROOT .. "/lib") + end + if has_config("ascend-npu") then + local ascend_home = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") + if ascend_home and ascend_home ~= "" then + for _, include_dir in ipairs({ + path.join(ascend_home, "include"), + path.join(ascend_home, "include/aclnn"), + path.join(ascend_home, "aarch64-linux/include"), + path.join(ascend_home, "aarch64-linux/include/aclnn"), + }) do + if os.isdir(include_dir) then + add_includedirs(include_dir, { public = true }) + end + end + end + end + add_links("infinirt") +end + +local function patch_infiniops_private_infinirt(xmake_os, infiniops_lib, infinirt_library, private_infinirt) local private_soname = path.filename(private_infinirt) - xmake_os.cp(standalone_infinirt, private_infinirt) + xmake_os.cp(infinirt_library, private_infinirt) xmake_os.execv("patchelf", {"--set-soname", private_soname, private_infinirt}) - xmake_os.execv("patchelf", {"--replace-needed", standalone_infinirt, private_soname, infiniops_lib}) + xmake_os.execv("patchelf", {"--replace-needed", infinirt_library, private_soname, infiniops_lib}) xmake_os.execv("patchelf", {"--replace-needed", "libinfinirt.so", private_soname, infiniops_lib}) local needed = xmake_os.iorunv("patchelf", {"--print-needed", infiniops_lib}) @@ -410,7 +430,7 @@ local function build_infiniops_external(xmake_os) local infiniops_root = path.absolute(get_config("infiniops-root") or "submodules/InfiniOps", os.projectdir()) local infiniops_builddir = path.join(infiniops_root, "build") local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() local cmake_config_args = { "-S", infiniops_root, "-B", infiniops_builddir, @@ -442,10 +462,10 @@ local function build_infiniops_external(xmake_os) xmake_os.execv("cmake", {"--build", infiniops_builddir, "--target", "infiniops"}) xmake_os.execv("cmake", {"--install", infiniops_builddir, "--prefix", INFINI_ROOT}) if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = find_standalone_infinirt(infinirt_root, xmake_os) + local infinirt_library = find_infinirt_library(infinirt_root, xmake_os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") - patch_infiniops_private_infinirt(xmake_os, infiniops_lib, standalone_infinirt, private_infinirt) + patch_infiniops_private_infinirt(xmake_os, infiniops_lib, infinirt_library, private_infinirt) end infiniops_external_built = true end @@ -487,11 +507,7 @@ target_end() target("infiniop") set_kind("shared") - local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") - add_includedirs(INFINI_ROOT.."/include", { public = true }) - add_linkdirs(INFINI_ROOT.."/lib") - add_links("infinirt") - add_rpathdirs(INFINI_ROOT.."/lib") + add_external_infinirt() local public_cuda_root = get_config("cuda") or os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") if public_cuda_root and public_cuda_root ~= "" then @@ -565,6 +581,7 @@ target("infiniop") add_installfiles("include/infiniop/*.h", {prefixdir = "include/infiniop"}) add_installfiles("include/infiniop.h", {prefixdir = "include"}) add_installfiles("include/infinicore.h", {prefixdir = "include"}) + add_installfiles("include/infinirt.h", {prefixdir = "include"}) target_end() target("infiniccl") @@ -648,28 +665,7 @@ target("infinicore_cpp_api") add_defines("CHAR_BIT=8", "INT_MIN=(-2147483647 - 1)", "INT_MAX=2147483647", "UINT_MAX=4294967295U") end add_includedirs(INFINI_ROOT.."/include", { public = true }) - local graph_infinirt_root = get_standalone_infinirt_root() - if has_config("standalone-infinirt-graph") then - if not graph_infinirt_root or graph_infinirt_root == "" then - raise("--standalone-infinirt-graph requires --infinirt-root or INFINI_RT_ROOT") - end - add_includedirs(graph_infinirt_root .. "/include") - if has_config("ascend-npu") then - local ascend_home = os.getenv("ASCEND_HOME") or os.getenv("ASCEND_TOOLKIT_HOME") or os.getenv("ASCEND_HOME_PATH") - if ascend_home and ascend_home ~= "" then - for _, include_dir in ipairs({ - path.join(ascend_home, "include"), - path.join(ascend_home, "include/aclnn"), - path.join(ascend_home, "aarch64-linux/include"), - path.join(ascend_home, "aarch64-linux/include/aclnn"), - }) do - if os.isdir(include_dir) then - add_includedirs(include_dir) - end - end - end - end - end + add_external_infinirt() if has_config("nv-gpu") then local cuda_root = os.getenv("CUDA_HOME") or os.getenv("CUDA_PATH") or get_config("cuda") or "/usr/local/cuda" add_includedirs(cuda_root .. "/include") @@ -680,7 +676,7 @@ target("infinicore_cpp_api") raise("InfiniOps root not found: " .. infiniops_root) end get_infiniops_backend_cmake_arg() - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then add_includedirs(infinirt_root .. "/include") add_linkdirs(infinirt_root .. "/lib", infinirt_root .. "/lib64") @@ -695,19 +691,18 @@ target("infinicore_cpp_api") end) after_install(function (target) local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) + local infinirt_library = find_infinirt_library(infinirt_root, os) local infiniops_lib = path.join(INFINI_ROOT, "lib", "libinfiniops.so") local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") - patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) + patch_infiniops_private_infinirt(os, infiniops_lib, infinirt_library, private_infinirt) end end) end add_linkdirs(INFINI_ROOT.."/lib") - add_links("infiniop", "infinirt", "infiniccl") - add_rpathdirs(INFINI_ROOT.."/lib") + add_links("infiniop", "infiniccl") if get_config("flash-attn") and get_config("flash-attn") ~= "" then add_installfiles("(builddir)/$(plat)/$(arch)/$(mode)/flash-attn*.so", {prefixdir = "lib"}) @@ -936,14 +931,15 @@ target("_infinicore") set_kind("shared") local INFINI_ROOT = os.getenv("INFINI_ROOT") or (os.getenv(is_host("windows") and "HOMEPATH" or "HOME") .. "/.infini") add_includedirs(INFINI_ROOT.."/include", { public = true }) + add_external_infinirt() add_linkdirs(INFINI_ROOT.."/lib") - add_links("infiniop", "infinirt", "infiniccl") + add_links("infiniop", "infiniccl") add_files("src/infinicore/pybind11/**.cc") if has_config("infiniops") then - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then add_includedirs(infinirt_root .. "/include") add_linkdirs(infinirt_root .. "/lib", infinirt_root .. "/lib64") @@ -958,11 +954,11 @@ target("_infinicore") infiniops_lib = path.join(infiniops_root, "build", "src", "libinfiniops.so") infiniops_lib_installed = false end - local infinirt_root = get_standalone_infinirt_root() + local infinirt_root = get_infinirt_root() if infinirt_root and infinirt_root ~= "" then - local standalone_infinirt = find_standalone_infinirt(infinirt_root, os) + local infinirt_library = find_infinirt_library(infinirt_root, os) local private_infinirt = path.join(INFINI_ROOT, "lib", "libinfiniops_infinirt.so") - patch_infiniops_private_infinirt(os, infiniops_lib, standalone_infinirt, private_infinirt) + patch_infiniops_private_infinirt(os, infiniops_lib, infinirt_library, private_infinirt) end os.mkdir(path.join(INFINI_ROOT, "lib")) if not infiniops_lib_installed then diff --git a/xmake/ali.lua b/xmake/ali.lua index 940650d67..550f7e0fc 100644 --- a/xmake/ali.lua +++ b/xmake/ali.lua @@ -104,7 +104,6 @@ target_end() target("infiniccl-ali") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then set_policy("build.cuda.devlink", true) diff --git a/xmake/ascend.lua b/xmake/ascend.lua index af82ab162..f8b2fe9ac 100644 --- a/xmake/ascend.lua +++ b/xmake/ascend.lua @@ -91,7 +91,6 @@ target_end() target("infiniccl-ascend") set_kind("static") - add_deps("infinirt") add_deps("infini-utils") set_warnings("all", "error") set_languages("cxx17") diff --git a/xmake/bang.lua b/xmake/bang.lua index ffa85ef6d..0be267c94 100644 --- a/xmake/bang.lua +++ b/xmake/bang.lua @@ -65,7 +65,6 @@ target_end() target("infiniccl-cambricon") set_kind("static") - add_deps("infinirt") add_deps("infini-utils") set_warnings("all", "error") set_languages("cxx17") diff --git a/xmake/hygon.lua b/xmake/hygon.lua index ed7da7b85..c7a2142aa 100644 --- a/xmake/hygon.lua +++ b/xmake/hygon.lua @@ -193,7 +193,6 @@ target_end() target("infiniccl-hygon") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then diff --git a/xmake/iluvatar.lua b/xmake/iluvatar.lua index 046e6716e..fea75a5d8 100644 --- a/xmake/iluvatar.lua +++ b/xmake/iluvatar.lua @@ -102,7 +102,6 @@ target_end() target("infiniccl-iluvatar") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then diff --git a/xmake/kunlun.lua b/xmake/kunlun.lua index 84ba14082..4f3a40db2 100644 --- a/xmake/kunlun.lua +++ b/xmake/kunlun.lua @@ -108,7 +108,6 @@ target_end() target("infiniccl-kunlun") set_kind("static") - add_deps("infinirt") add_deps("infini-utils") set_warnings("all", "error") set_languages("cxx17") diff --git a/xmake/metax.lua b/xmake/metax.lua index 85407ed1b..3e700fff2 100644 --- a/xmake/metax.lua +++ b/xmake/metax.lua @@ -158,7 +158,6 @@ target_end() target("infiniccl-metax") set_kind("static") - add_deps("infinirt") on_install(function (target) end) set_warnings("all", "error") if not is_plat("windows") then diff --git a/xmake/moore.lua b/xmake/moore.lua index d34dd24f7..6e8800be8 100644 --- a/xmake/moore.lua +++ b/xmake/moore.lua @@ -81,7 +81,6 @@ target_end() target("infiniccl-moore") set_kind("static") - add_deps("infinirt") on_install(function (target) end) set_warnings("all", "error") if not is_plat("windows") then diff --git a/xmake/qy.lua b/xmake/qy.lua index 35d53d6ef..82cf92d6b 100644 --- a/xmake/qy.lua +++ b/xmake/qy.lua @@ -174,7 +174,6 @@ target_end() target("infiniccl-qy") set_kind("static") - add_deps("infinirt") on_install(function (target) end) if has_config("ccl") then add_rules("qy.cuda", {override = true}) From 63096d470ccf4c94c340911854170d43c6dd6a58 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 8 Jul 2026 11:26:45 +0800 Subject: [PATCH 07/11] fix: adapt InfiniRT runtime bridge for Ascend Extend the Core-to-InfiniRT device translation table for Ascend and the other runtime-backed devices. Add explicit stream conversion helpers because Core still stores streams as infinirtStream_t while the InfiniRT C++ runtime API takes infini::rt::runtime::Stream. Include infinirt.h here so the bridge sees the runtime handle typedefs. Use those conversions for stream creation, async allocation, memcpy, memset, event recording, and stream wait calls. --- src/bridge/infini/rt.hpp | 31 ++++++++++++++++--- .../allocators/stream_ordered_allocator.cc | 9 ++++-- src/infinicore/context/context_impl.cc | 10 ++++++ src/infinicore/context/runtime/runtime.cc | 18 ++++++----- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/bridge/infini/rt.hpp b/src/bridge/infini/rt.hpp index d53cc49ac..07366fac7 100644 --- a/src/bridge/infini/rt.hpp +++ b/src/bridge/infini/rt.hpp @@ -1,6 +1,5 @@ #pragma once -#include "infinicore.h" #include "infinirt.h" #include @@ -22,6 +21,18 @@ inline ::infini::rt::Device::Type translate(infiniDevice_t device) { return ::infini::rt::Device::Type::kCpu; case INFINI_DEVICE_NVIDIA: return ::infini::rt::Device::Type::kNvidia; + case INFINI_DEVICE_CAMBRICON: + return ::infini::rt::Device::Type::kCambricon; + case INFINI_DEVICE_ASCEND: + return ::infini::rt::Device::Type::kAscend; + case INFINI_DEVICE_METAX: + return ::infini::rt::Device::Type::kMetax; + case INFINI_DEVICE_MOORE: + return ::infini::rt::Device::Type::kMoore; + case INFINI_DEVICE_ILUVATAR: + return ::infini::rt::Device::Type::kIluvatar; + case INFINI_DEVICE_HYGON: + return ::infini::rt::Device::Type::kHygon; default: return ::infini::rt::Device::Type::kCount; } @@ -33,17 +44,29 @@ inline infiniDevice_t translate(::infini::rt::Device::Type device) { return INFINI_DEVICE_CPU; case ::infini::rt::Device::Type::kNvidia: return INFINI_DEVICE_NVIDIA; + case ::infini::rt::Device::Type::kCambricon: + return INFINI_DEVICE_CAMBRICON; + case ::infini::rt::Device::Type::kAscend: + return INFINI_DEVICE_ASCEND; + case ::infini::rt::Device::Type::kMetax: + return INFINI_DEVICE_METAX; + case ::infini::rt::Device::Type::kMoore: + return INFINI_DEVICE_MOORE; + case ::infini::rt::Device::Type::kIluvatar: + return INFINI_DEVICE_ILUVATAR; + case ::infini::rt::Device::Type::kHygon: + return INFINI_DEVICE_HYGON; default: return INFINI_DEVICE_TYPE_COUNT; } } -inline ::infini::rt::runtime::Stream translate(infinirtStream_t stream) { +inline ::infini::rt::runtime::Stream to_rt_stream(infinirtStream_t stream) { return reinterpret_cast<::infini::rt::runtime::Stream>(stream); } -inline ::infini::rt::runtime::Stream *translate(infinirtStream_t *stream) { - return reinterpret_cast<::infini::rt::runtime::Stream *>(stream); +inline infinirtStream_t to_core_stream(::infini::rt::runtime::Stream stream) { + return reinterpret_cast(stream); } } // namespace infinicore::bridge::infini::rt diff --git a/src/infinicore/context/allocators/stream_ordered_allocator.cc b/src/infinicore/context/allocators/stream_ordered_allocator.cc index 00df7fb47..06ce4a30d 100644 --- a/src/infinicore/context/allocators/stream_ordered_allocator.cc +++ b/src/infinicore/context/allocators/stream_ordered_allocator.cc @@ -13,7 +13,10 @@ std::byte *StreamOrderedAllocator::allocate(size_t size) { } void *ptr = nullptr; if (device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocAsync(&ptr, size, bridge::infini::rt::translate(context::getStream())))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocAsync( + &ptr, + size, + bridge::infini::rt::to_rt_stream(context::getStream())))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&ptr, size))); } @@ -25,7 +28,9 @@ void StreamOrderedAllocator::deallocate(std::byte *ptr) { return; } if (device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeAsync(ptr, bridge::infini::rt::translate(context::getStream())))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeAsync( + ptr, + bridge::infini::rt::to_rt_stream(context::getStream())))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free(ptr))); } diff --git a/src/infinicore/context/context_impl.cc b/src/infinicore/context/context_impl.cc index 06c85955c..64d10dcfc 100644 --- a/src/infinicore/context/context_impl.cc +++ b/src/infinicore/context/context_impl.cc @@ -82,6 +82,16 @@ ContextImpl::ContextImpl() { } } + if constexpr (infini::rt::DeviceEnabled::value) { + infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_ASCEND)); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::ASCEND)]))); + runtime_table_[static_cast(Device::Type::ASCEND)].resize(device_counter[static_cast(Device::Type::ASCEND)]); + if (device_counter[static_cast(Device::Type::ASCEND)] > 0) { + runtime_table_[static_cast(Device::Type::ASCEND)][0] = std::unique_ptr(new Runtime(Device(Device::Type::ASCEND, 0))); + current_runtime_ = runtime_table_[static_cast(Device::Type::ASCEND)][0].get(); + } + } + if (current_runtime_ == nullptr && !runtime_table_[static_cast(Device::Type::CPU)].empty()) { current_runtime_ = runtime_table_[static_cast(Device::Type::CPU)][0].get(); } diff --git a/src/infinicore/context/runtime/runtime.cc b/src/infinicore/context/runtime/runtime.cc index 783b018c2..18424d036 100644 --- a/src/infinicore/context/runtime/runtime.cc +++ b/src/infinicore/context/runtime/runtime.cc @@ -11,7 +11,9 @@ namespace infinicore { Runtime::Runtime(Device device) : device_(device), graph_manager_(std::make_unique()) { activate(); - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamCreate(bridge::infini::rt::translate(&stream_)))); + infini::rt::runtime::Stream stream = nullptr; + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamCreate(&stream))); + stream_ = bridge::infini::rt::to_core_stream(stream); INFINICORE_CHECK_ERROR(infiniopCreateHandle(&infiniop_handle_)); if (device_.getType() == Device::Type::CPU) { device_memory_allocator_ = std::make_unique(device); @@ -27,7 +29,7 @@ Runtime::~Runtime() { } device_memory_allocator_.reset(); infiniopDestroyHandle(infiniop_handle_); - (void)infini::rt::runtime::StreamDestroy(bridge::infini::rt::translate(stream_)); + (void)infini::rt::runtime::StreamDestroy(bridge::infini::rt::to_rt_stream(stream_)); } Runtime *Runtime::activate() { @@ -51,7 +53,7 @@ infiniopHandle_t Runtime::infiniopHandle() const { } void Runtime::syncStream() { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(bridge::infini::rt::translate(stream_)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(bridge::infini::rt::to_rt_stream(stream_)))); } void Runtime::syncDevice() { @@ -108,7 +110,7 @@ std::shared_ptr Runtime::reinstantiateBlob(std::shared_ptr blob) void Runtime::memcpyH2D(void *dst, const void *src, size_t size, bool async) { if (async && device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, bridge::infini::rt::translate(stream_)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, bridge::infini::rt::to_rt_stream(stream_)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice))); } @@ -120,7 +122,7 @@ void Runtime::memcpyD2H(void *dst, const void *src, size_t size) { void Runtime::memcpyD2D(void *dst, const void *src, size_t size, bool async) { if (async && device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, bridge::infini::rt::translate(stream_)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, bridge::infini::rt::to_rt_stream(stream_)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice))); } @@ -132,7 +134,7 @@ void Runtime::setDeviceMemory(void *ptr, int value, size_t count) { void Runtime::setDeviceMemoryAsync(void *ptr, int value, size_t count, infinirtStream_t stream) { if (device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, bridge::infini::rt::translate(stream)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, bridge::infini::rt::to_rt_stream(stream)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memset(ptr, value, count))); } @@ -155,7 +157,7 @@ void Runtime::recordEvent(infinirtEvent_t event, infinirtStream_t stream) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, bridge::infini::rt::translate(stream)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, bridge::infini::rt::to_rt_stream(stream)))); } bool Runtime::queryEvent(infinirtEvent_t event) { @@ -181,7 +183,7 @@ void Runtime::streamWaitEvent(infinirtStream_t stream, infinirtEvent_t event) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(bridge::infini::rt::translate(stream), event, 0))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(bridge::infini::rt::to_rt_stream(stream), event, 0))); } bool Runtime::isGraphRecording() const { From 5f399ee1a8d59388b00b624c64b16f903a735090 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Wed, 8 Jul 2026 11:27:00 +0800 Subject: [PATCH 08/11] feat: route graph capture through InfiniRT runtime Use the InfiniRT C++ runtime graph API directly for begin/end capture, instantiate, launch, and graph destruction. Drop the old standalone bridge and environment-variable-gated path so graph mode uses the installed InfiniRT runtime whenever USE_INFINIRT_GRAPH is enabled. --- src/infinicore/graph/graph.cc | 104 +++--- .../graph/standalone_infinirt_graph_bridge.cc | 295 ------------------ .../standalone_infinirt_graph_bridge.hpp | 26 -- 3 files changed, 47 insertions(+), 378 deletions(-) delete mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.cc delete mode 100644 src/infinicore/graph/standalone_infinirt_graph_bridge.hpp diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 0a09bf68e..2bfc15574 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -1,15 +1,19 @@ #include "graph_manager.hpp" +#include "../../bridge/infini/rt.hpp" #include "../utils.hpp" #include "infinicore/context/context.hpp" #ifdef USE_INFINIRT_GRAPH -#include "standalone_infinirt_graph_bridge.hpp" -#include +#include #endif namespace infinicore::graph { +#ifdef USE_INFINIRT_GRAPH +namespace rt_runtime = ::infini::rt::runtime; +#endif + /* ========================= * GraphTensor * ========================= */ @@ -37,40 +41,25 @@ DispatchableGraphOperator::~DispatchableGraphOperator() { #ifdef USE_INFINIRT_GRAPH struct Graph::DeviceGraph { - infinirtGraph_t graph = nullptr; - infinirtGraphExec_t exec = nullptr; - infinirtGraphNode_t node = nullptr; - infinirtStream_t stream = nullptr; - bool standalone = false; - std::vector log_buffer; - - DeviceGraph() { - log_buffer.resize(4 * 1024); - } + rt_runtime::Graph graph = nullptr; + rt_runtime::GraphExec exec = nullptr; + rt_runtime::Stream stream = nullptr; + ::infini::rt::Device::Type device_type = ::infini::rt::Device::Type::kCount; + int device_index = 0; ~DeviceGraph() { if (exec) { - if (standalone) { - standalone_infinirt::graph_exec_destroy(exec); - } else { - infinirtGraphExecDestroy(exec); - } + (void)rt_runtime::GraphExecDestroy(exec); } if (graph) { - if (standalone) { - standalone_infinirt::graph_destroy(graph); - } else { - infinirtGraphDestroy(graph); - } + (void)rt_runtime::GraphDestroy(graph); } } void launch() { - if (standalone) { - INFINICORE_CHECK_ERROR(standalone_infinirt::graph_launch(exec, stream)); - } else { - INFINICORE_CHECK_ERROR(infinirtGraphLuanch(exec, context::getStream())); - } + ::infini::rt::set_runtime_device_type(device_type); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(rt_runtime::SetDevice(device_index))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(rt_runtime::GraphLaunch(exec, stream))); } }; #else @@ -100,21 +89,21 @@ void Graph::instantiate() { #ifdef USE_INFINIRT_GRAPH // Reset device graph device_graph_ = std::make_unique(); - device_graph_->standalone = standalone_infinirt::available(context::getDevice()); - device_graph_->stream = context::getStream(); - if (device_graph_->standalone) { - auto set_device_status = standalone_infinirt::set_device(context::getDevice()); - if (set_device_status != INFINI_STATUS_SUCCESS) { - spdlog::warn("Standalone InfiniRT graph bridge failed to select the current device. Falling back to eager execution."); - device_graph_.reset(); - return; - } - - static bool logged_once = false; - if (!logged_once) { - logged_once = true; - spdlog::info("Using standalone InfiniRT C++ graph runtime API for graph capture and replay."); - } + auto current_device = context::getDevice(); + device_graph_->device_type = bridge::infini::rt::translate(static_cast(current_device.getType())); + device_graph_->device_index = static_cast(current_device.getIndex()); + device_graph_->stream = bridge::infini::rt::to_rt_stream(context::getStream()); + if (device_graph_->device_type == ::infini::rt::Device::Type::kCount) { + spdlog::warn("InfiniRT graph runtime does not support the current device. Falling back to eager execution."); + device_graph_.reset(); + return; + } + ::infini::rt::set_runtime_device_type(device_graph_->device_type); + auto set_device_status = bridge::infini::rt::translate(rt_runtime::SetDevice(device_graph_->device_index)); + if (set_device_status != INFINI_STATUS_SUCCESS) { + spdlog::warn("InfiniRT graph runtime failed to select the current device. Falling back to eager execution."); + device_graph_.reset(); + return; } // warmup @@ -123,9 +112,9 @@ void Graph::instantiate() { } infinicore::context::syncStream(); - auto begin_status = device_graph_->standalone - ? standalone_infinirt::stream_begin_capture(device_graph_->stream, INFINIRT_STREAM_CAPTURE_MODE_RELAXED) - : infinirtStreamBeginCapture(context::getStream(), INFINIRT_STREAM_CAPTURE_MODE_RELAXED); + auto begin_status = bridge::infini::rt::translate(rt_runtime::StreamBeginCapture( + device_graph_->stream, + rt_runtime::StreamCaptureMode::kStreamCaptureModeRelaxed)); if (begin_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to begin device graph capture."); device_graph_.reset(); @@ -135,30 +124,31 @@ void Graph::instantiate() { // Run and record this->run(); - auto end_status = device_graph_->standalone - ? standalone_infinirt::stream_end_capture(device_graph_->stream, &device_graph_.get()->graph) - : infinirtStreamEndCapture(context::getStream(), &device_graph_.get()->graph); + auto end_status = bridge::infini::rt::translate(rt_runtime::StreamEndCapture( + device_graph_->stream, + &device_graph_->graph)); if (end_status != INFINI_STATUS_SUCCESS) { spdlog::warn("Fail to end device graph capture."); device_graph_.reset(); return; } - auto instantiate_status = device_graph_->standalone - ? standalone_infinirt::graph_instantiate(&device_graph_.get()->exec, device_graph_.get()->graph) - : infinirtGraphInstantiate( - &device_graph_.get()->exec, - device_graph_.get()->graph, - &device_graph_.get()->node, - device_graph_.get()->log_buffer.data(), - device_graph_.get()->log_buffer.size()); + auto instantiate_status = bridge::infini::rt::translate(rt_runtime::GraphInstantiate( + &device_graph_->exec, + device_graph_->graph)); if (instantiate_status != INFINI_STATUS_SUCCESS) { static bool warned_once = false; if (!warned_once) { warned_once = true; - spdlog::warn("Fail to instantiate device graph: {}", std::string(device_graph_.get()->log_buffer.data())); + spdlog::warn("Fail to instantiate device graph."); } device_graph_.reset(); + return; + } + static bool logged_once = false; + if (!logged_once) { + logged_once = true; + spdlog::info("Using InfiniRT C++ graph runtime API for graph capture and replay."); } #endif } diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc b/src/infinicore/graph/standalone_infinirt_graph_bridge.cc deleted file mode 100644 index 7725d48c1..000000000 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.cc +++ /dev/null @@ -1,295 +0,0 @@ -#include "standalone_infinirt_graph_bridge.hpp" - -#ifdef USE_STANDALONE_INFINIRT_GRAPH - -#include -#include - -#include - -namespace infinicore::graph::standalone_infinirt { -namespace { - -using StandaloneDevice = infini::rt::Device; - -bool truthy_env(const char *name) { - auto value = std::getenv(name); - if (value == nullptr) { - return false; - } - std::string text{value}; - return text == "1" || text == "ON" || text == "on" || text == "true" || text == "TRUE"; -} - -template -constexpr bool standalone_device_enabled() { - return infini::rt::DeviceEnabled::value; -} - -bool supports_device(Device::Type type) { - switch (type) { - case Device::Type::NVIDIA: - return standalone_device_enabled(); - case Device::Type::ASCEND: - return standalone_device_enabled(); - default: - return false; - } -} - -thread_local Device::Type current_device_type = Device::Type::CPU; - -template -infiniStatus_t to_core_status(Status status) { - return status == Runtime::kSuccess - ? INFINI_STATUS_SUCCESS - : INFINI_STATUS_INTERNAL_ERROR; -} - -template -infiniStatus_t set_device_impl(int index) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::SetDevice(index)); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t stream_begin_capture_impl(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - auto standalone_mode = Runtime::kStreamCaptureModeRelaxed; - switch (mode) { - case INFINIRT_STREAM_CAPTURE_MODE_GLOBAL: - standalone_mode = Runtime::kStreamCaptureModeGlobal; - break; - case INFINIRT_STREAM_CAPTURE_MODE_THREAD_LOCAL: - standalone_mode = Runtime::kStreamCaptureModeThreadLocal; - break; - case INFINIRT_STREAM_CAPTURE_MODE_RELAXED: - standalone_mode = Runtime::kStreamCaptureModeRelaxed; - break; - } - return to_core_status(Runtime::StreamBeginCapture( - reinterpret_cast(stream), - standalone_mode)); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t stream_end_capture_impl(infinirtStream_t stream, infinirtGraph_t *graph) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::StreamEndCapture( - reinterpret_cast(stream), - reinterpret_cast(graph))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_destroy_impl(infinirtGraph_t graph) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphDestroy( - reinterpret_cast(graph))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_instantiate_impl(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphInstantiate( - reinterpret_cast(graph_exec), - reinterpret_cast(graph))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_exec_destroy_impl(infinirtGraphExec_t graph_exec) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphExecDestroy( - reinterpret_cast(graph_exec))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t graph_launch_impl(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { - if constexpr (standalone_device_enabled()) { - using Runtime = infini::rt::runtime::Runtime; - return to_core_status(Runtime::GraphLaunch( - reinterpret_cast(graph_exec), - reinterpret_cast(stream))); - } else { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -template -infiniStatus_t dispatch_current( - infiniStatus_t (*nvidia_fn)(Args...), - infiniStatus_t (*ascend_fn)(Args...), - Args... args) { - switch (current_device_type) { - case Device::Type::NVIDIA: - return nvidia_fn(args...); - case Device::Type::ASCEND: - return ascend_fn(args...); - default: - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } -} - -} // namespace - -bool enabled() { - return truthy_env("INFINICORE_USE_STANDALONE_INFINIRT_GRAPH"); -} - -bool available(const Device &device) { - return enabled() && supports_device(device.getType()); -} - -infiniStatus_t set_device(const Device &device) { - if (!supports_device(device.getType())) { - return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - } - auto status = INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; - switch (device.getType()) { - case Device::Type::NVIDIA: - status = set_device_impl(static_cast(device.getIndex())); - break; - case Device::Type::ASCEND: - status = set_device_impl(static_cast(device.getIndex())); - break; - default: - break; - } - if (status == INFINI_STATUS_SUCCESS) { - current_device_type = device.getType(); - } - return status; -} - -infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode) { - if (stream == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - stream_begin_capture_impl, - stream_begin_capture_impl, - stream, - mode); -} - -infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph) { - if (stream == nullptr || graph == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - stream_end_capture_impl, - stream_end_capture_impl, - stream, - graph); -} - -infiniStatus_t graph_destroy(infinirtGraph_t graph) { - if (graph == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_destroy_impl, - graph_destroy_impl, - graph); -} - -infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph) { - if (graph_exec == nullptr || graph == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_instantiate_impl, - graph_instantiate_impl, - graph_exec, - graph); -} - -infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec) { - if (graph_exec == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_exec_destroy_impl, - graph_exec_destroy_impl, - graph_exec); -} - -infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream) { - if (graph_exec == nullptr || stream == nullptr) { - return INFINI_STATUS_NULL_POINTER; - } - return dispatch_current( - graph_launch_impl, - graph_launch_impl, - graph_exec, - stream); -} - -} // namespace infinicore::graph::standalone_infinirt - -#else - -namespace infinicore::graph::standalone_infinirt { - -bool enabled() { - return false; -} - -bool available(const Device &) { - return false; -} - -infiniStatus_t set_device(const Device &) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t stream_begin_capture(infinirtStream_t, infinirtStreamCaptureMode_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t stream_end_capture(infinirtStream_t, infinirtGraph_t *) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_destroy(infinirtGraph_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_instantiate(infinirtGraphExec_t *, infinirtGraph_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_exec_destroy(infinirtGraphExec_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -infiniStatus_t graph_launch(infinirtGraphExec_t, infinirtStream_t) { - return INFINI_STATUS_NOT_IMPLEMENTED; -} - -} // namespace infinicore::graph::standalone_infinirt - -#endif diff --git a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp b/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp deleted file mode 100644 index 0ebbb0a6b..000000000 --- a/src/infinicore/graph/standalone_infinirt_graph_bridge.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include "infinicore/device.hpp" -#include - -namespace infinicore::graph::standalone_infinirt { - -bool enabled(); - -bool available(const Device &device); - -infiniStatus_t set_device(const Device &device); - -infiniStatus_t stream_begin_capture(infinirtStream_t stream, infinirtStreamCaptureMode_t mode); - -infiniStatus_t stream_end_capture(infinirtStream_t stream, infinirtGraph_t *graph); - -infiniStatus_t graph_destroy(infinirtGraph_t graph); - -infiniStatus_t graph_instantiate(infinirtGraphExec_t *graph_exec, infinirtGraph_t graph); - -infiniStatus_t graph_exec_destroy(infinirtGraphExec_t graph_exec); - -infiniStatus_t graph_launch(infinirtGraphExec_t graph_exec, infinirtStream_t stream); - -} // namespace infinicore::graph::standalone_infinirt From c82ac4c5b366d66dea4348e9d028edd8f8748316 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Mon, 13 Jul 2026 16:28:26 +0800 Subject: [PATCH 09/11] chore: drop unrelated Ascend paged attention renames --- .../ascend/paged_attention_ascend_kernel.cpp | 8 ++++---- .../ascend/paged_attention_prefill_ascend_kernel.cpp | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp b/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp index e6df4f50b..0f47c2ff5 100644 --- a/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp +++ b/src/infiniop/ops/paged_attention/ascend/paged_attention_ascend_kernel.cpp @@ -164,22 +164,22 @@ class PagedAttentionKernel { private: __aicore__ inline ptrdiff_t keyBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t page_block_idx = token_idx / _page_block_size; + const size_t block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); return static_cast(physical_block) * _k_batch_stride + static_cast(kv_head_idx) * _k_head_stride + static_cast(token_offset) * _k_row_stride; } __aicore__ inline ptrdiff_t valueBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t page_block_idx = token_idx / _page_block_size; + const size_t block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); return static_cast(physical_block) * _v_batch_stride + static_cast(kv_head_idx) * _v_head_stride + static_cast(token_offset) * _v_row_stride; diff --git a/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp b/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp index db90b2056..e1fce2785 100644 --- a/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp +++ b/src/infiniop/ops/paged_attention_prefill/ascend/paged_attention_prefill_ascend_kernel.cpp @@ -197,22 +197,22 @@ class PagedAttentionPrefillKernel { } __aicore__ inline ptrdiff_t keyBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t page_block_idx = token_idx / _page_block_size; + const size_t block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadPrefillIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); return static_cast(physical_block) * _k_batch_stride + static_cast(kv_head_idx) * _k_head_stride + static_cast(token_offset) * _k_row_stride; } __aicore__ inline ptrdiff_t valueBase(size_t seq_idx, size_t kv_head_idx, size_t token_idx) { - const size_t page_block_idx = token_idx / _page_block_size; + const size_t block_idx = token_idx / _page_block_size; const size_t token_offset = token_idx % _page_block_size; const int64_t physical_block = loadPrefillIndex( _block_tables_gm, - static_cast(seq_idx) * _block_table_batch_stride + static_cast(page_block_idx)); + static_cast(seq_idx) * _block_table_batch_stride + static_cast(block_idx)); return static_cast(physical_block) * _v_batch_stride + static_cast(kv_head_idx) * _v_head_stride + static_cast(token_offset) * _v_row_stride; From cafb121a1c6a7ae73261a21c80206ba0d2c89ab3 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Mon, 13 Jul 2026 16:28:29 +0800 Subject: [PATCH 10/11] fix: make Ascend H2D tensor copies synchronous --- src/infinicore/nn/rope.cc | 14 -------------- src/infinicore/tensor/copy.cc | 6 ++++-- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/infinicore/nn/rope.cc b/src/infinicore/nn/rope.cc index c8ab4f02d..115e162e2 100644 --- a/src/infinicore/nn/rope.cc +++ b/src/infinicore/nn/rope.cc @@ -1,7 +1,6 @@ #include "infinicore/nn/rope.hpp" #include "../../utils.h" #include "../utils.hpp" -#include "infinicore/context/context.hpp" #include "infinicore/ops/mrope.hpp" #include "infinicore/ops/rope.hpp" #include @@ -13,16 +12,6 @@ #include namespace infinicore::nn { -namespace { - -void sync_cache_copy(const Device &device) { - if (device.getType() == Device::Type::ASCEND) { - // Ascend H2D copies are async, and these host buffers leave scope after init. - infinicore::context::syncStream(); - } -} - -} // namespace RoPE::RoPE(size_t head_dim, size_t rotary_dim, @@ -106,7 +95,6 @@ void RoPE::initialize_cache() { auto cos_f32_cpu = Tensor::from_blob(cos_data.data(), {max_seq_len_, cache_dim}, DataType::F32, cpu_device); sin_cache_->copy_from(sin_f32_cpu); cos_cache_->copy_from(cos_f32_cpu); - sync_cache_copy(device_); } else if (dtype_ == DataType::BF16) { // Convert F32 to BF16 using the same conversion as Python's ml_dtypes.bfloat16 // This uses round-to-nearest-even (matching _f32_to_bf16 implementation) @@ -124,7 +112,6 @@ void RoPE::initialize_cache() { // copy_from handles cross-device copying to target device sin_cache_->copy_from(sin_bf16_cpu); cos_cache_->copy_from(cos_bf16_cpu); - sync_cache_copy(device_); } else if (dtype_ == DataType::F16) { // Convert F32 to F16 std::vector sin_f16_data(max_seq_len_ * cache_dim); @@ -140,7 +127,6 @@ void RoPE::initialize_cache() { sin_cache_->copy_from(sin_f16_cpu); cos_cache_->copy_from(cos_f16_cpu); - sync_cache_copy(device_); } else { throw std::runtime_error( "RoPE cache dtype conversion not yet supported for dtype: " diff --git a/src/infinicore/tensor/copy.cc b/src/infinicore/tensor/copy.cc index 1297d9f8c..9db8bebe8 100644 --- a/src/infinicore/tensor/copy.cc +++ b/src/infinicore/tensor/copy.cc @@ -43,11 +43,13 @@ void TensorImpl::copy_from(Tensor src) { } } else if (src->device().getType() == Device::Type::CPU) { context::setDevice(this->device()); + // Keep Ascend H2D synchronous because copy_from does not retain the host source. + const bool async = this->device().getType() != Device::Type::ASCEND; if (this->is_contiguous()) { - context::memcpyH2D(this->data(), src->data(), copy_size); + context::memcpyH2D(this->data(), src->data(), copy_size, async); } else { auto local_src = Tensor::empty(this->shape(), this->dtype(), this->device()); - context::memcpyH2D(local_src->data(), src->data(), copy_size); + context::memcpyH2D(local_src->data(), src->data(), copy_size, async); op::rearrange_(Tensor(const_cast(this)->shared_from_this()), local_src); } } From f65e359a1298b20f503580072fd290ff7cecc702 Mon Sep 17 00:00:00 2001 From: gongchensu Date: Mon, 13 Jul 2026 16:56:19 +0800 Subject: [PATCH 11/11] refactor: clarify InfiniRT bridge translation direction --- src/bridge/infini/rt.hpp | 8 ++++---- .../allocators/stream_ordered_allocator.cc | 4 ++-- src/infinicore/context/context_impl.cc | 6 +++--- src/infinicore/context/runtime/runtime.cc | 18 +++++++++--------- src/infinicore/graph/graph.cc | 4 ++-- src/infiniop/devices/handle.cc | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/bridge/infini/rt.hpp b/src/bridge/infini/rt.hpp index 07366fac7..b74b2250f 100644 --- a/src/bridge/infini/rt.hpp +++ b/src/bridge/infini/rt.hpp @@ -15,7 +15,7 @@ inline infiniStatus_t translate(::infini::rt::runtime::Error error) { } } -inline ::infini::rt::Device::Type translate(infiniDevice_t device) { +inline ::infini::rt::Device::Type translate_to(infiniDevice_t device) { switch (device) { case INFINI_DEVICE_CPU: return ::infini::rt::Device::Type::kCpu; @@ -38,7 +38,7 @@ inline ::infini::rt::Device::Type translate(infiniDevice_t device) { } } -inline infiniDevice_t translate(::infini::rt::Device::Type device) { +inline infiniDevice_t translate_from(::infini::rt::Device::Type device) { switch (device) { case ::infini::rt::Device::Type::kCpu: return INFINI_DEVICE_CPU; @@ -61,11 +61,11 @@ inline infiniDevice_t translate(::infini::rt::Device::Type device) { } } -inline ::infini::rt::runtime::Stream to_rt_stream(infinirtStream_t stream) { +inline ::infini::rt::runtime::Stream translate_to(infinirtStream_t stream) { return reinterpret_cast<::infini::rt::runtime::Stream>(stream); } -inline infinirtStream_t to_core_stream(::infini::rt::runtime::Stream stream) { +inline infinirtStream_t translate_from(::infini::rt::runtime::Stream stream) { return reinterpret_cast(stream); } diff --git a/src/infinicore/context/allocators/stream_ordered_allocator.cc b/src/infinicore/context/allocators/stream_ordered_allocator.cc index 06ce4a30d..c6c54e47d 100644 --- a/src/infinicore/context/allocators/stream_ordered_allocator.cc +++ b/src/infinicore/context/allocators/stream_ordered_allocator.cc @@ -16,7 +16,7 @@ std::byte *StreamOrderedAllocator::allocate(size_t size) { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MallocAsync( &ptr, size, - bridge::infini::rt::to_rt_stream(context::getStream())))); + bridge::infini::rt::translate_to(context::getStream())))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Malloc(&ptr, size))); } @@ -30,7 +30,7 @@ void StreamOrderedAllocator::deallocate(std::byte *ptr) { if (device_.getType() != Device::Type::CPU) { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::FreeAsync( ptr, - bridge::infini::rt::to_rt_stream(context::getStream())))); + bridge::infini::rt::translate_to(context::getStream())))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Free(ptr))); } diff --git a/src/infinicore/context/context_impl.cc b/src/infinicore/context/context_impl.cc index 64d10dcfc..4de56effb 100644 --- a/src/infinicore/context/context_impl.cc +++ b/src/infinicore/context/context_impl.cc @@ -64,7 +64,7 @@ ContextImpl &ContextImpl::singleton() { ContextImpl::ContextImpl() { std::vector device_counter(static_cast(Device::Type::COUNT), 0); - infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_CPU)); + infini::rt::set_runtime_device_type(bridge::infini::rt::translate_to(INFINI_DEVICE_CPU)); INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::CPU)]))); runtime_table_[static_cast(Device::Type::CPU)].resize(device_counter[static_cast(Device::Type::CPU)]); @@ -73,7 +73,7 @@ ContextImpl::ContextImpl() { } if constexpr (infini::rt::DeviceEnabled::value) { - infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_NVIDIA)); + infini::rt::set_runtime_device_type(bridge::infini::rt::translate_to(INFINI_DEVICE_NVIDIA)); INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::NVIDIA)]))); runtime_table_[static_cast(Device::Type::NVIDIA)].resize(device_counter[static_cast(Device::Type::NVIDIA)]); if (device_counter[static_cast(Device::Type::NVIDIA)] > 0) { @@ -83,7 +83,7 @@ ContextImpl::ContextImpl() { } if constexpr (infini::rt::DeviceEnabled::value) { - infini::rt::set_runtime_device_type(bridge::infini::rt::translate(INFINI_DEVICE_ASCEND)); + infini::rt::set_runtime_device_type(bridge::infini::rt::translate_to(INFINI_DEVICE_ASCEND)); INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::GetDeviceCount(&device_counter[static_cast(Device::Type::ASCEND)]))); runtime_table_[static_cast(Device::Type::ASCEND)].resize(device_counter[static_cast(Device::Type::ASCEND)]); if (device_counter[static_cast(Device::Type::ASCEND)] > 0) { diff --git a/src/infinicore/context/runtime/runtime.cc b/src/infinicore/context/runtime/runtime.cc index 18424d036..0d5cd6de8 100644 --- a/src/infinicore/context/runtime/runtime.cc +++ b/src/infinicore/context/runtime/runtime.cc @@ -13,7 +13,7 @@ Runtime::Runtime(Device device) : device_(device), graph_manager_(std::make_uniq activate(); infini::rt::runtime::Stream stream = nullptr; INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamCreate(&stream))); - stream_ = bridge::infini::rt::to_core_stream(stream); + stream_ = bridge::infini::rt::translate_from(stream); INFINICORE_CHECK_ERROR(infiniopCreateHandle(&infiniop_handle_)); if (device_.getType() == Device::Type::CPU) { device_memory_allocator_ = std::make_unique(device); @@ -29,11 +29,11 @@ Runtime::~Runtime() { } device_memory_allocator_.reset(); infiniopDestroyHandle(infiniop_handle_); - (void)infini::rt::runtime::StreamDestroy(bridge::infini::rt::to_rt_stream(stream_)); + (void)infini::rt::runtime::StreamDestroy(bridge::infini::rt::translate_to(stream_)); } Runtime *Runtime::activate() { - auto rt_device = bridge::infini::rt::translate(static_cast(device_.getType())); + auto rt_device = bridge::infini::rt::translate_to(static_cast(device_.getType())); INFINICORE_ASSERT(rt_device != infini::rt::Device::Type::kCount); infini::rt::set_runtime_device_type(rt_device); INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::SetDevice(static_cast(device_.getIndex())))); @@ -53,7 +53,7 @@ infiniopHandle_t Runtime::infiniopHandle() const { } void Runtime::syncStream() { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(bridge::infini::rt::to_rt_stream(stream_)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamSynchronize(bridge::infini::rt::translate_to(stream_)))); } void Runtime::syncDevice() { @@ -110,7 +110,7 @@ std::shared_ptr Runtime::reinstantiateBlob(std::shared_ptr blob) void Runtime::memcpyH2D(void *dst, const void *src, size_t size, bool async) { if (async && device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, bridge::infini::rt::to_rt_stream(stream_)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice, bridge::infini::rt::translate_to(stream_)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyHostToDevice))); } @@ -122,7 +122,7 @@ void Runtime::memcpyD2H(void *dst, const void *src, size_t size) { void Runtime::memcpyD2D(void *dst, const void *src, size_t size, bool async) { if (async && device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, bridge::infini::rt::to_rt_stream(stream_)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemcpyAsync(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice, bridge::infini::rt::translate_to(stream_)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memcpy(dst, src, size, infini::rt::runtime::kMemcpyDeviceToDevice))); } @@ -134,7 +134,7 @@ void Runtime::setDeviceMemory(void *ptr, int value, size_t count) { void Runtime::setDeviceMemoryAsync(void *ptr, int value, size_t count, infinirtStream_t stream) { if (device_.getType() != Device::Type::CPU) { - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, bridge::infini::rt::to_rt_stream(stream)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::MemsetAsync(ptr, value, count, bridge::infini::rt::translate_to(stream)))); } else { INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::Memset(ptr, value, count))); } @@ -157,7 +157,7 @@ void Runtime::recordEvent(infinirtEvent_t event, infinirtStream_t stream) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, bridge::infini::rt::to_rt_stream(stream)))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::EventRecord(event, bridge::infini::rt::translate_to(stream)))); } bool Runtime::queryEvent(infinirtEvent_t event) { @@ -183,7 +183,7 @@ void Runtime::streamWaitEvent(infinirtStream_t stream, infinirtEvent_t event) { if (stream == nullptr) { stream = stream_; } - INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(bridge::infini::rt::to_rt_stream(stream), event, 0))); + INFINICORE_CHECK_ERROR(bridge::infini::rt::translate(infini::rt::runtime::StreamWaitEvent(bridge::infini::rt::translate_to(stream), event, 0))); } bool Runtime::isGraphRecording() const { diff --git a/src/infinicore/graph/graph.cc b/src/infinicore/graph/graph.cc index 2bfc15574..9b28aae7f 100644 --- a/src/infinicore/graph/graph.cc +++ b/src/infinicore/graph/graph.cc @@ -90,9 +90,9 @@ void Graph::instantiate() { // Reset device graph device_graph_ = std::make_unique(); auto current_device = context::getDevice(); - device_graph_->device_type = bridge::infini::rt::translate(static_cast(current_device.getType())); + device_graph_->device_type = bridge::infini::rt::translate_to(static_cast(current_device.getType())); device_graph_->device_index = static_cast(current_device.getIndex()); - device_graph_->stream = bridge::infini::rt::to_rt_stream(context::getStream()); + device_graph_->stream = bridge::infini::rt::translate_to(context::getStream()); if (device_graph_->device_type == ::infini::rt::Device::Type::kCount) { spdlog::warn("InfiniRT graph runtime does not support the current device. Falling back to eager execution."); device_graph_.reset(); diff --git a/src/infiniop/devices/handle.cc b/src/infiniop/devices/handle.cc index 9ded0587b..9057f7203 100644 --- a/src/infiniop/devices/handle.cc +++ b/src/infiniop/devices/handle.cc @@ -25,7 +25,7 @@ #endif __INFINI_C infiniStatus_t infiniopSetRuntimeDevice(infiniDevice_t device, int device_id) { - auto rt_device = infinicore::bridge::infini::rt::translate(device); + auto rt_device = infinicore::bridge::infini::rt::translate_to(device); if (rt_device == infini::rt::Device::Type::kCount) { return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; } @@ -40,7 +40,7 @@ __INFINI_C infiniStatus_t infiniopCreateHandle(infiniopHandle_t *handle_ptr) { return INFINI_STATUS_NULL_POINTER; } - infiniDevice_t device = infinicore::bridge::infini::rt::translate(infini::rt::runtime_device_type()); + infiniDevice_t device = infinicore::bridge::infini::rt::translate_from(infini::rt::runtime_device_type()); if (device == INFINI_DEVICE_TYPE_COUNT) { return INFINI_STATUS_DEVICE_TYPE_NOT_SUPPORTED; }