diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54a3960ae28..88c478e440e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -460,20 +460,20 @@ jobs: sccache: s3 - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - uses: ./.github/actions/setup-prebuild + - name: Prepare FFI library + run: | + cargo build --profile ci -p vortex-ffi - name: Build and run C++ unit tests run: | mkdir -p vortex-cxx/build - cmake -S vortex-cxx -B vortex-cxx/build -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON + cmake -S vortex-cxx -B vortex-cxx/build -DBUILD_TESTS=ON -DSANITIZER=asan -DRUST_BUILD_PROFILE=ci cmake --build vortex-cxx/build --parallel $(nproc) - ctest --test-dir vortex-cxx/build -j $(nproc) -V + ./vortex-cxx/build/vortex_cxx_test - name: Build and run the example in release mode run: | - cmake -S vortex-cxx/examples -B vortex-cxx/examples/build -DCMAKE_BUILD_TYPE=Release + cmake -S vortex-cxx -B vortex-cxx/examples/build -DBUILD_EXAMPLES=1 -DCMAKE_BUILD_TYPE=Release -DRUST_BUILD_PROFILE=ci cmake --build vortex-cxx/examples/build --parallel $(nproc) - vortex-cxx/examples/build/hello-vortex vortex-cxx/examples/goldenfiles/example.vortex - - uses: ./.github/actions/check-rebuild - with: - command: "cargo build --profile ci --locked -p vortex-cxx --lib" + vortex-cxx/examples/build/examples/hello-vortex sqllogic-test: name: "SQL logic tests" diff --git a/Cargo.lock b/Cargo.lock index d99d2124e45..12ba041e5af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1435,17 +1435,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "codespan-reporting" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" -dependencies = [ - "serde", - "termcolor", - "unicode-width 0.2.2", -] - [[package]] name = "codspeed" version = "5.0.1" @@ -1997,68 +1986,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "cxx" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" -dependencies = [ - "cc", - "cxx-build", - "cxxbridge-cmd", - "cxxbridge-flags", - "cxxbridge-macro", - "foldhash 0.2.0", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" -dependencies = [ - "cc", - "codespan-reporting", - "indexmap 2.14.0", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.118", -] - -[[package]] -name = "cxxbridge-cmd" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" -dependencies = [ - "clap", - "codespan-reporting", - "indexmap 2.14.0", - "proc-macro2", - "quote", - "syn 2.0.118", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.194" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" -dependencies = [ - "indexmap 2.14.0", - "proc-macro2", - "quote", - "syn 2.0.118", -] - [[package]] name = "darling" version = "0.23.0" @@ -5905,15 +5832,6 @@ dependencies = [ "bitflags 2.13.0", ] -[[package]] -name = "link-cplusplus" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -8273,12 +8191,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scratch" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" - [[package]] name = "scroll" version = "0.11.0" @@ -9004,12 +8916,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" -[[package]] -name = "take_mut" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" - [[package]] name = "tap" version = "1.0.1" @@ -9055,15 +8961,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "terminal_size" version = "0.4.4" @@ -10127,21 +10024,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "vortex-cxx" -version = "0.1.0" -dependencies = [ - "anyhow", - "arrow-array 58.3.0", - "arrow-schema 58.3.0", - "async-fs", - "cxx", - "futures", - "paste", - "take_mut", - "vortex", -] - [[package]] name = "vortex-datafusion" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index ba54a50a2ab..68a025e6e29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ members = [ "vortex-cuda/gpu-scan-cli", "vortex-cuda/macros", "vortex-cuda/nvcomp", - "vortex-cxx", "vortex-ffi", "fuzz", "vortex-jni", diff --git a/vortex-cxx/.clang-format b/vortex-cxx/.clang-format new file mode 100644 index 00000000000..989f3d8a0e2 --- /dev/null +++ b/vortex-cxx/.clang-format @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors +NamespaceIndentation: None +BasedOnStyle: Google +IndentWidth: 4 +AccessModifierOffset: -4 diff --git a/vortex-cxx/.clang-tidy b/vortex-cxx/.clang-tidy index 80c94310cbc..8b08b258113 100644 --- a/vortex-cxx/.clang-tidy +++ b/vortex-cxx/.clang-tidy @@ -1,5 +1,5 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors Checks: | -*, @@ -43,7 +43,7 @@ Checks: | -bugprone-unused-local-non-trivial-variable, -bugprone-empty-catch, WarningsAsErrors: '*' -HeaderFilterRegex: '(cpp|examples)/.*\.(cpp|hpp)$' +HeaderFilterRegex: '(examples)/.*\.(cpp|hpp)$' FormatStyle: none CheckOptions: - key: readability-identifier-naming.ClassCase @@ -90,4 +90,3 @@ CheckOptions: value: lower_case - key: cppcoreguidelines-rvalue-reference-param-not-moved.IgnoreUnnamedParams value: true - diff --git a/vortex-cxx/CMakeLists.txt b/vortex-cxx/CMakeLists.txt index ab5474d0617..087f7230171 100644 --- a/vortex-cxx/CMakeLists.txt +++ b/vortex-cxx/CMakeLists.txt @@ -1,16 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -cmake_minimum_required(VERSION 3.22) +cmake_minimum_required(VERSION 3.10) +project(VortexCXX + VERSION 0.0.1 + LANGUAGES CXX) -# FetchContent from URL timestamp handling -cmake_policy(SET CMP0135 NEW) - -project(vortex) +include(FetchContent) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) find_program(SCCACHE_PROGRAM sccache) if (SCCACHE_PROGRAM) @@ -21,100 +21,121 @@ else () message(STATUS "Sccache not found") endif () -if (NOT CMAKE_BUILD_TYPE) - set(CMAKE_BUILD_TYPE Debug) -endif() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wpedantic") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wpedantic -Wno-dollar-in-identifier-extension") +option(BUILD_TESTS "Build tests" OFF) +option(BUILD_EXAMPLES "Build examples" OFF) -option(VORTEX_ENABLE_TESTING "Enable building test binary for vortex-cxx" OFF) -option(VORTEX_ENABLE_ASAN "Enable address sanitizer" OFF) - -include(FetchContent) -FetchContent_Declare( - Corrosion - GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git - GIT_TAG v0.5 -) -FetchContent_MakeAvailable(Corrosion) +set(SANITIZER "" CACHE STRING "Build with sanitizers") +set(TARGET_TRIPLE "" CACHE STRING "Rust target triple for FFI library") +set(RUST_BUILD_PROFILE "" CACHE STRING "Cargo profile name for Rust FFI library") -set(RUST_SOURCE_FILE lib.rs) +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE Debug) +endif() -# Import Rust crate using Corrosion -corrosion_import_crate( - MANIFEST_PATH ${CMAKE_CURRENT_SOURCE_DIR}/Cargo.toml - FEATURES ${CORROSION_FEATURES} - CRATES vortex-cxx -) +if (NOT SANITIZER STREQUAL "") + message(NOTICE "Sanitizer: ${SANITIZER}") + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(FATAL_ERROR "Only debug build is supported for sanitizer builds") + endif() -# Enable CXX bridge for the Rust crate -corrosion_add_cxxbridge(vortex_cxx_bridge CRATE vortex_cxx FILES ${RUST_SOURCE_FILE}) + if (SANITIZER STREQUAL "asan") + set(SANITIZER_FLAGS "-fsanitize=address,undefined,leak") + elseif (SANITIZER STREQUAL "tsan") + set(SANITIZER_FLAGS "-fsanitize=thread") + else() + message(FATAL_ERROR "Unknown sanitizer ${SANITIZER}") + endif() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SANITIZER_FLAGS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SANITIZER_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${SANITIZER_FLAGS}") +endif() -FetchContent_Declare( - nanoarrow - GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow.git - GIT_TAG a579fbf5d192e85b6249935e117de7d02a6dc4e9 # v0.8.0 -) -FetchContent_MakeAvailable(nanoarrow) +if (RUST_BUILD_PROFILE STREQUAL "") + if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + set(RUST_BUILD_PROFILE "release_debug") + else() + string(TOLOWER "${CMAKE_BUILD_TYPE}" RUST_BUILD_PROFILE) + endif() +endif() -file(GLOB_RECURSE CPP_SOURCE_FILE CONFIGURE_DEPENDS - "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/*.cpp" -) +set(VORTEX_FFI_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi") +set(VORTEX_FFI_LIB_DIR "${VORTEX_FFI_DIR}/../target/${TARGET_TRIPLE}/${RUST_BUILD_PROFILE}") +set(VORTEX_FFI_HEADERS "${VORTEX_FFI_DIR}/cinclude") + +if(APPLE) + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dylib") +elseif(WIN32) + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.lib") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.dll") +else() + set(VORTEX_FFI_STATIC "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.a") + set(VORTEX_FFI_SHARED "${VORTEX_FFI_LIB_DIR}/libvortex_ffi.so") +endif() -# Public headers -set(CPP_INCLUDE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/cpp/include -) +if(NOT EXISTS "${VORTEX_FFI_STATIC}") + message(FATAL_ERROR + "vortex-ffi static library not found at ${VORTEX_FFI_STATIC}. " + "Run: cargo build --profile -p vortex-ffi") +endif() -# Private headers -set(CPP_PRIVATE_INCLUDE_DIRS - ${CMAKE_CURRENT_SOURCE_DIR}/cpp/src -) +add_library(vortex_ffi STATIC IMPORTED) +set_target_properties(vortex_ffi PROPERTIES + IMPORTED_LOCATION "${VORTEX_FFI_STATIC}" + INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}") + +if(EXISTS "${VORTEX_FFI_SHARED}") + add_library(vortex_ffi_shared SHARED IMPORTED) + set_target_properties(vortex_ffi_shared PROPERTIES + IMPORTED_LOCATION "${VORTEX_FFI_SHARED}" + INTERFACE_INCLUDE_DIRECTORIES "${VORTEX_FFI_HEADERS}" + INTERFACE_LINK_OPTIONS "LINKER:-rpath,${VORTEX_FFI_LIB_DIR}") +endif() -# Create the main library combining C++ and Rust code -add_library(vortex STATIC ${CPP_SOURCE_FILE}) -target_include_directories(vortex PUBLIC ${CPP_INCLUDE_DIRS} - ${CMAKE_CURRENT_BINARY_DIR}/corrosion_generated/cxxbridge/vortex_cxx_bridge/include) -target_include_directories(vortex PRIVATE - ${CPP_PRIVATE_INCLUDE_DIRS} -) -target_link_libraries(vortex - PUBLIC nanoarrow_static - PRIVATE vortex_cxx_bridge +add_library(vortex_cxx STATIC ${CMAKE_CURRENT_SOURCE_DIR}/vortex.cpp) +target_include_directories(vortex_cxx PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude ) +target_link_libraries(vortex_cxx PUBLIC vortex_ffi) +add_library(vortex::cxx ALIAS vortex_cxx) + +if(TARGET vortex_ffi_shared) + add_library(vortex_cxx_shared SHARED ${CMAKE_CURRENT_SOURCE_DIR}/vortex.cpp) + set_target_properties(vortex_cxx_shared PROPERTIES POSITION_INDEPENDENT_CODE ON) + target_include_directories(vortex_cxx_shared PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/../vortex-ffi/cinclude + ) + target_link_libraries(vortex_cxx_shared PUBLIC vortex_ffi_shared) + add_library(vortex::cxx_shared ALIAS vortex_cxx_shared) +endif() -if (VORTEX_ENABLE_ASAN) - target_compile_options(vortex PRIVATE -fsanitize=leak,address,undefined -fno-omit-frame-pointer -fno-common -O1) - target_link_options(vortex PRIVATE -fsanitize=leak,address,undefined) +if (BUILD_TESTS OR BUILD_EXAMPLES) + FetchContent_Declare( + Nanoarrow + GIT_REPOSITORY https://github.com/apache/arrow-nanoarrow + GIT_TAG apache-arrow-nanoarrow-0.8.0 + ) + FetchContent_MakeAvailable(Nanoarrow) endif() -# Tests -if (VORTEX_ENABLE_TESTING) +if (BUILD_TESTS) FetchContent_Declare( - googletest - GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.17.0 - GIT_SHALLOW TRUE + Catch + GIT_REPOSITORY https://github.com/catchorg/Catch2.git + GIT_TAG v3.8.1 ) - FetchContent_MakeAvailable(googletest) + FetchContent_MakeAvailable(Catch) + include(Catch) + target_compile_definitions(Catch2 PRIVATE CATCH_CONFIG_NO_POSIX_SIGNALS) enable_testing() - add_executable(vortex_cxx_test cpp/tests/basic_test.cpp cpp/tests/test_data_generator.cpp) - target_include_directories(vortex_cxx_test PUBLIC ${CPP_INCLUDE_DIRS}) - target_include_directories(vortex_cxx_test PRIVATE cpp/tests) - target_link_libraries(vortex_cxx_test PRIVATE gtest_main vortex nanoarrow_static) - target_include_directories(vortex_cxx_test PRIVATE - ${CMAKE_CURRENT_BINARY_DIR}/corrosion_generated/cxxbridge/vortex_cxx_bridge/include - ) - # Platform-specific configuration - if(APPLE) - set(APPLE_LINK_FLAGS "-framework CoreFoundation -framework Security") - endif() - target_link_libraries(vortex_cxx_test PRIVATE vortex_cxx_bridge ${APPLE_LINK_FLAGS}) - if (VORTEX_ENABLE_ASAN) - target_compile_options(vortex_cxx_test PRIVATE -fsanitize=leak,address,undefined -fno-omit-frame-pointer -fno-common -O1) - target_link_options(vortex_cxx_test PRIVATE -fsanitize=leak,address,undefined) - endif() - include(GoogleTest) - gtest_discover_tests(vortex_cxx_test) + add_subdirectory(tests) +endif() + +if (BUILD_EXAMPLES) + add_subdirectory(examples) endif() diff --git a/vortex-cxx/Cargo.toml b/vortex-cxx/Cargo.toml deleted file mode 100644 index d75f7b74cee..00000000000 --- a/vortex-cxx/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "vortex-cxx" -authors = { workspace = true } -categories = { workspace = true } -description = "C++ bindings for Vortex generated from Rust using cxx" -edition = { workspace = true } -homepage = { workspace = true } -include = { workspace = true } -keywords = { workspace = true } -license = { workspace = true } -publish = false -readme = { workspace = true } -repository = { workspace = true } -rust-version = { workspace = true } -version = { workspace = true } - -[lints] -workspace = true - -[lib] -crate-type = ["staticlib"] - -[dependencies] -anyhow = { workspace = true } -arrow-array = { workspace = true, features = ["ffi"] } -arrow-schema = { workspace = true } -async-fs = { workspace = true } -cxx = "1.0" -futures = { workspace = true } -paste = { workspace = true } -take_mut = { workspace = true } -vortex = { workspace = true } diff --git a/vortex-cxx/README.md b/vortex-cxx/README.md index aa761f98b1a..c83e04cdf19 100644 --- a/vortex-cxx/README.md +++ b/vortex-cxx/README.md @@ -1,35 +1,34 @@ -# Vortex C++ Bindings +# Vortex C++ bindings +## Requirements -This directory contains C++ bindings for Vortex using the [cxx](https://cxx.rs/) crate. The bindings provide a C++ interface to Vortex file operations, including roundtripping with Arrow Array stream with advanced pushdown support. +- CMake 3.10+ +- C++20 compiler +- Rust toolchain for building `vortex-ffi`. -## Building +## Build -### Requirements +```sh +cargo build --release -p vortex-ffi +cmake -Bbuild -DCMAKE_BUILD_TYPE=Release +cmake --build build -j +``` -- CMake 3.22 or higher -- C++20 compatible compiler -- Rust toolchain (for building the Rust components) -- (optional) Ninja (`ninja-build`) +This will generate `libvortex_cxx` shared and static libraries. +You can use `vortex_cxx` and `vortex_cxx_shared` CMake targets. -### Build Steps +## Test -```bash -mkdir build -cmake -Bbuild -GNinja +```sh +cargo build --release -p vortex-ffi +cmake -Bbuild -DBUILD_TESTS=ON cmake --build build -j +ctest --test-dir build -j "$(nproc)" ``` -### Running Tests +## Run examples -```bash -# Enable tests in CMake -cmake -Bbuild -DVORTEX_ENABLE_TESTING=ON -GNinja +```sh +cmake -Bbuild -DBUILD_EXAMPLES=ON cmake --build build -j -./vortex_cxx_test +./build/examples/hello-vortex ``` - -## C++ Coding Convention - -We use `.clang-tidy` and `.clang-format` to setup the coding convention. Both are borrowed from DuckDB. - -`cppcoreguidelines-avoid-non-const-global-variables` is removed from `.clang-tidy` because GTest violates it. diff --git a/vortex-cxx/cpp/include/vortex.hpp b/vortex-cxx/cpp/include/vortex.hpp deleted file mode 100644 index 1f2dd1625c1..00000000000 --- a/vortex-cxx/cpp/include/vortex.hpp +++ /dev/null @@ -1,11 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/write_options.hpp" -#include "vortex/exception.hpp" -#include "vortex/expr.hpp" -#include "vortex/scalar.hpp" \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/dtype.hpp b/vortex-cxx/cpp/include/vortex/dtype.hpp deleted file mode 100644 index 9bf4a16d9bc..00000000000 --- a/vortex-cxx/cpp/include/vortex/dtype.hpp +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include -#include "vortex_cxx_bridge/lib.h" - -namespace vortex { - -enum class PType : uint8_t { - U8 = 0, - U16, - U32, - U64, - I8, - I16, - I32, - I64, - F16, - F32, - F64, -}; - -namespace dtype { - class DType { - public: - DType() = delete; - explicit DType(rust::Box impl) : impl_(std::move(impl)) { - } - DType(DType &&other) noexcept = default; - DType &operator=(DType &&other) = default; - ~DType() = default; - - DType(const DType &) = delete; - DType &operator=(const DType &) = delete; - - std::string ToString() const; - - const rust::Box &GetImpl() { - return impl_; - } - - private: - rust::Box impl_; - }; - - // Factory functions - DType null(); - DType bool_(bool nullable = false); - DType primitive(PType ptype, bool nullable = false); - DType int8(bool nullable = false); - DType int16(bool nullable = false); - DType int32(bool nullable = false); - DType int64(bool nullable = false); - DType uint8(bool nullable = false); - DType uint16(bool nullable = false); - DType uint32(bool nullable = false); - DType uint64(bool nullable = false); - DType float16(bool nullable = false); - DType float32(bool nullable = false); - DType float64(bool nullable = false); - DType decimal(uint8_t precision = 10, int8_t scale = 0, bool nullable = false); - DType utf8(bool nullable = false); - DType binary(bool nullable = false); - /// TODO: Other DTypes are only supported by creating from Arrow for now. - DType from_arrow(struct ArrowSchema &schema, bool non_nullable = false); -} // namespace dtype - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/exception.hpp b/vortex-cxx/cpp/include/vortex/exception.hpp deleted file mode 100644 index a7fe44fb91c..00000000000 --- a/vortex-cxx/cpp/include/vortex/exception.hpp +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include - -namespace vortex { - -/// TODO(xinyu): better error handling -class VortexException : public std::runtime_error { -public: - explicit VortexException(const std::string &message) : std::runtime_error(message) { - } -}; - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/expr.hpp b/vortex-cxx/cpp/include/vortex/expr.hpp deleted file mode 100644 index 3060a8e3536..00000000000 --- a/vortex-cxx/cpp/include/vortex/expr.hpp +++ /dev/null @@ -1,51 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "vortex/scalar.hpp" -#include "vortex_cxx_bridge/lib.h" -#include - -namespace vortex::expr { -class Expr { -public: - Expr() = delete; - explicit Expr(rust::Box impl) : impl_(std::move(impl)) { - } - Expr(Expr &&other) noexcept = default; - Expr &operator=(Expr &&other) noexcept = default; - ~Expr() = default; - - Expr(const Expr &) = delete; - Expr &operator=(const Expr &) = delete; - - rust::Box IntoImpl() && { - return std::move(impl_); - } - - const ffi::Expr &Impl() const & { - return *impl_; - } - -private: - rust::Box impl_; -}; - -Expr literal(scalar::Scalar scalar); -Expr root(); -Expr column(std::string_view name); -Expr get_item(std::string_view field, Expr expr); -Expr not_(Expr expr); -Expr is_null(Expr expr); -Expr eq(Expr lhs, Expr rhs); -Expr not_eq_(Expr lhs, Expr rhs); -Expr gt(Expr lhs, Expr rhs); -Expr gt_eq(Expr lhs, Expr rhs); -Expr lt(Expr lhs, Expr rhs); -Expr lt_eq(Expr lhs, Expr rhs); -Expr and_(Expr lhs, Expr rhs); -Expr or_(Expr lhs, Expr rhs); -Expr checked_add(Expr lhs, Expr rhs); -Expr select(const std::vector &fields, Expr child); -} // namespace vortex::expr \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/file.hpp b/vortex-cxx/cpp/include/vortex/file.hpp deleted file mode 100644 index c7b93e326d8..00000000000 --- a/vortex-cxx/cpp/include/vortex/file.hpp +++ /dev/null @@ -1,38 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include "vortex_cxx_bridge/lib.h" - -namespace vortex { -class ScanBuilder; - -class VortexFile { -public: - static VortexFile Open(const std::string &path); - static VortexFile Open(const uint8_t *data, size_t length); - - VortexFile(VortexFile &&other) noexcept = default; - VortexFile &operator=(VortexFile &&other) noexcept = default; - ~VortexFile() = default; - - VortexFile(const VortexFile &) = delete; - VortexFile &operator=(const VortexFile &) = delete; - - /// Get the number of rows in the file. - uint64_t RowCount() const; - - /// Create a scan builder for the file. - /// The scan builder can be used to scan the file. - ScanBuilder CreateScanBuilder() const; - -private: - explicit VortexFile(rust::Box impl) : impl_(std::move(impl)) { - } - - rust::Box impl_; -}; - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/scalar.hpp b/vortex-cxx/cpp/include/vortex/scalar.hpp deleted file mode 100644 index 77706aef98a..00000000000 --- a/vortex-cxx/cpp/include/vortex/scalar.hpp +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include "dtype.hpp" -#include "vortex_cxx_bridge/lib.h" - -namespace vortex::scalar { -class Scalar { -public: - Scalar() = delete; - explicit Scalar(rust::Box impl) : impl_(std::move(impl)) { - } - Scalar(Scalar &&other) noexcept = default; - Scalar &operator=(Scalar &&other) noexcept = default; - ~Scalar() = default; - - Scalar(const Scalar &) = delete; - Scalar &operator=(const Scalar &) = delete; - - rust::Box IntoImpl() && { - return std::move(impl_); - } - -private: - rust::Box impl_; -}; - -// Factory functions for creating scalar values -Scalar bool_(bool value); -Scalar int8(int8_t value); -Scalar int16(int16_t value); -Scalar int32(int32_t value); -Scalar int64(int64_t value); -Scalar uint8(uint8_t value); -Scalar uint16(uint16_t value); -Scalar uint32(uint32_t value); -Scalar uint64(uint64_t value); -Scalar float32(float value); -Scalar float64(double value); -Scalar string(std::string_view value); -Scalar binary(const uint8_t *data, size_t length); -/// TODO: Other Scalars are only supported by casting for now. -Scalar cast(Scalar scalar, dtype::DType dtype); -} // namespace vortex::scalar diff --git a/vortex-cxx/cpp/include/vortex/scan.hpp b/vortex-cxx/cpp/include/vortex/scan.hpp deleted file mode 100644 index 7debb9ef6aa..00000000000 --- a/vortex-cxx/cpp/include/vortex/scan.hpp +++ /dev/null @@ -1,108 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "vortex/expr.hpp" -#include -#include "vortex_cxx_bridge/lib.h" - -#include -#include - -namespace vortex { -/// The StreamDriver internally holds a `RecordBatchIteratorAdapter` from the Rust side, which is thread-safe -/// and cloneable. The `RecordBatchIteratorAdapter` internally holds a `WorkStealingArrayIterator`. -class StreamDriver { -public: - StreamDriver(StreamDriver &&other) noexcept = default; - StreamDriver &operator=(StreamDriver &&other) noexcept = default; - ~StreamDriver() = default; - - StreamDriver(const StreamDriver &) = delete; - StreamDriver &operator=(const StreamDriver &) = delete; - - /// Create a stream of record batches. - /// - /// This function is thread-safe and can be called from multiple threads to create one stream per - /// thread to make progress on the same StreamDriver that is built from a ScanBuilder concurrently. - /// - /// Within each thread, the record batches will be emitted in the original order they are within - /// the scan. Between threads, the order is not guaranteed. - /// - /// Example: If the scan contains batches [b0, b1, b2, b3, b4, b5] and two threads call this - /// function respectively to make progress on their own stream, Thread 1 might receive [b0, - /// b2, b4] and Thread 2 might receive [b1, b3, b5]. Each thread maintains order within its - /// subset, but overall ordering between threads is not guaranteed (e.g., Thread 2 could emit b1 - /// before Thread 1 emits b0). - ArrowArrayStream CreateArrayStream() const; - -private: - friend class ScanBuilder; - - explicit StreamDriver(rust::Box impl) : impl_(std::move(impl)) { - } - - rust::Box impl_; -}; - -class ScanBuilder { -public: - ScanBuilder(ScanBuilder &&other) noexcept = default; - ScanBuilder &operator=(ScanBuilder &&other) noexcept { - if (this != &other) { - impl_ = std::move(other.impl_); - } - return *this; - } - ~ScanBuilder() = default; - - ScanBuilder(const ScanBuilder &) = delete; - ScanBuilder &operator=(const ScanBuilder &) = delete; - - /// Only include rows that match the filter expressions. - ScanBuilder &WithFilter(expr::Expr &&expr) &; - ScanBuilder &WithFilter(const expr::Expr &expr) &; - ScanBuilder &&WithFilter(expr::Expr &&expr) &&; - ScanBuilder &&WithFilter(const expr::Expr &expr) &&; - - /// Only include columns that match the projection expressions. - ScanBuilder &WithProjection(expr::Expr &&expr) &; - ScanBuilder &WithProjection(const expr::Expr &expr) &; - ScanBuilder &&WithProjection(expr::Expr &&expr) &&; - ScanBuilder &&WithProjection(const expr::Expr &expr) &&; - - /// Only include rows in the range [row_range_start, row_range_end). - ScanBuilder &WithRowRange(uint64_t row_range_start, uint64_t row_range_end) &; - ScanBuilder &&WithRowRange(uint64_t row_range_start, uint64_t row_range_end) &&; - - /// Only include rows with the given indices. - ScanBuilder &WithIncludeByIndex(const uint64_t *indices, std::size_t size) &; - ScanBuilder &&WithIncludeByIndex(const uint64_t *indices, std::size_t size) &&; - - /// Set the limit on the number of rows to scan out. - ScanBuilder &WithLimit(uint64_t limit) &; - ScanBuilder &&WithLimit(uint64_t limit) &&; - - /// Set the output schema on the scan builder. - /// TODO: currently if pass in this option, the schema needs to be the schema after adding projection. - ScanBuilder &WithOutputSchema(ArrowSchema &output_schema) &; - ScanBuilder &&WithOutputSchema(ArrowSchema &output_schema) &&; - - /// Take ownership and consume the scan builder to a stream of record batches. - ArrowArrayStream IntoStream() &&; - - /// Take ownership and consume the scan builder to a stream driver. - /// Under the hood, this function calls `ScanBuilder::into_record_batch_reader` and holds a - /// `WorkStealingArrayIterator` in StreamDriver. - StreamDriver IntoStreamDriver() &&; - -private: - friend class VortexFile; - - explicit ScanBuilder(rust::Box impl) : impl_(std::move(impl)) { - } - - rust::Box impl_; -}; -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/include/vortex/write_options.hpp b/vortex-cxx/cpp/include/vortex/write_options.hpp deleted file mode 100644 index 08b903bf2fb..00000000000 --- a/vortex-cxx/cpp/include/vortex/write_options.hpp +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include -#include "vortex_cxx_bridge/lib.h" - -namespace vortex { - -class VortexWriteOptions { -public: - VortexWriteOptions() : impl_(ffi::write_options_new()) { - } - VortexWriteOptions(VortexWriteOptions &&other) noexcept = default; - VortexWriteOptions &operator=(VortexWriteOptions &&other) noexcept = default; - ~VortexWriteOptions() = default; - - VortexWriteOptions(const VortexWriteOptions &) = delete; - VortexWriteOptions &operator=(const VortexWriteOptions &) = delete; - - /// Write an ArrowArrayStream to a Vortex file - void WriteArrayStream(ArrowArrayStream &stream, const std::string &path); - -private: - rust::Box impl_; -}; - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/src/dtype.cpp b/vortex-cxx/cpp/src/dtype.cpp deleted file mode 100644 index 5b85f36835e..00000000000 --- a/vortex-cxx/cpp/src/dtype.cpp +++ /dev/null @@ -1,90 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/dtype.hpp" -#include "vortex/exception.hpp" - -#include "rust/cxx.h" - -namespace vortex::dtype { -DType null() { - return DType(ffi::dtype_null()); -} - -DType bool_(bool nullable) { - return DType(ffi::dtype_bool(nullable)); -} - -DType primitive(PType ptype, bool nullable) { - return DType(ffi::dtype_primitive(static_cast(ptype), nullable)); -} - -DType int8(bool nullable) { - return primitive(PType::I8, nullable); -} - -DType int16(bool nullable) { - return primitive(PType::I16, nullable); -} - -DType int32(bool nullable) { - return primitive(PType::I32, nullable); -} - -DType int64(bool nullable) { - return primitive(PType::I64, nullable); -} - -DType uint8(bool nullable) { - return primitive(PType::U8, nullable); -} - -DType uint16(bool nullable) { - return primitive(PType::U16, nullable); -} - -DType uint32(bool nullable) { - return primitive(PType::U32, nullable); -} - -DType uint64(bool nullable) { - return primitive(PType::U64, nullable); -} - -DType float16(bool nullable) { - return primitive(PType::F16, nullable); -} - -DType float32(bool nullable) { - return primitive(PType::F32, nullable); -} - -DType float64(bool nullable) { - return primitive(PType::F64, nullable); -} - -DType decimal(uint8_t precision, int8_t scale, bool nullable) { - return DType(ffi::dtype_decimal(precision, scale, nullable)); -} - -DType utf8(bool nullable) { - return DType(ffi::dtype_utf8(nullable)); -} - -DType binary(bool nullable) { - return DType(ffi::dtype_binary(nullable)); -} - -DType from_arrow(struct ArrowSchema &schema, bool non_nullable) { - try { - return DType(ffi::from_arrow(reinterpret_cast(&schema), non_nullable)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} -// Methods -std::string DType::ToString() const { - auto rust_str = impl_->to_string(); - return std::string(rust_str.data(), rust_str.length()); -} -} // namespace vortex::dtype diff --git a/vortex-cxx/cpp/src/expr.cpp b/vortex-cxx/cpp/src/expr.cpp deleted file mode 100644 index 0e2d67395c1..00000000000 --- a/vortex-cxx/cpp/src/expr.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/expr.hpp" -#include "vortex_cxx_bridge/lib.h" - -namespace vortex::expr { - -Expr literal(scalar::Scalar scalar) { - return Expr(ffi::literal(std::move(scalar).IntoImpl())); -} - -Expr root() { - return Expr(ffi::root()); -} - -Expr column(std::string_view name) { - return Expr(ffi::column(rust::String(name.data(), name.length()))); -} - -Expr get_item(std::string_view field, Expr child) { - return Expr(ffi::get_item(rust::String(field.data(), field.length()), std::move(child).IntoImpl())); -} - -Expr not_(Expr child) { - return Expr(ffi::not_(std::move(child).IntoImpl())); -} - -Expr is_null(Expr child) { - return Expr(ffi::is_null(std::move(child).IntoImpl())); -} - -// Macro to define binary operator functions -#define DEFINE_BINARY_OP(name) \ - Expr name(Expr lhs, Expr rhs) { \ - return Expr(ffi::name(std::move(lhs).IntoImpl(), std::move(rhs).IntoImpl())); \ - } - -DEFINE_BINARY_OP(eq) -DEFINE_BINARY_OP(not_eq_) -DEFINE_BINARY_OP(gt) -DEFINE_BINARY_OP(gt_eq) -DEFINE_BINARY_OP(lt) -DEFINE_BINARY_OP(lt_eq) -DEFINE_BINARY_OP(and_) -DEFINE_BINARY_OP(or_) -DEFINE_BINARY_OP(checked_add) - -#undef DEFINE_BINARY_OP - -Expr select(const std::vector &fields, Expr child) { - ::rust::Vec<::rust::String> rs_fields; - rs_fields.reserve(fields.size()); - for (std::string_view f : fields) { - rs_fields.emplace_back(f.data(), f.length()); - } - return Expr(ffi::select(rs_fields, std::move(child).IntoImpl())); -} - -} // namespace vortex::expr \ No newline at end of file diff --git a/vortex-cxx/cpp/src/file.cpp b/vortex-cxx/cpp/src/file.cpp deleted file mode 100644 index 078c305639d..00000000000 --- a/vortex-cxx/cpp/src/file.cpp +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/exception.hpp" -#include "rust/cxx.h" - -namespace vortex { - -VortexFile VortexFile::Open(const uint8_t *data, size_t length) { - try { - rust::Slice slice(data, length); - return VortexFile(ffi::open_file_from_buffer(slice)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -VortexFile VortexFile::Open(const std::string &path) { - try { - return VortexFile(ffi::open_file(path)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -uint64_t VortexFile::RowCount() const { - return impl_->row_count(); -} - -ScanBuilder VortexFile::CreateScanBuilder() const { - return ScanBuilder(impl_->scan_builder()); -} - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/src/scalar.cpp b/vortex-cxx/cpp/src/scalar.cpp deleted file mode 100644 index 87502ee2f00..00000000000 --- a/vortex-cxx/cpp/src/scalar.cpp +++ /dev/null @@ -1,66 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/scalar.hpp" - -#include "vortex_cxx_bridge/lib.h" - -namespace vortex::scalar { - -Scalar bool_(bool value) { - return Scalar(ffi::bool_scalar_new(value)); -} - -Scalar int8(int8_t value) { - return Scalar(ffi::i8_scalar_new(value)); -} - -Scalar int16(int16_t value) { - return Scalar(ffi::i16_scalar_new(value)); -} - -Scalar int32(int32_t value) { - return Scalar(ffi::i32_scalar_new(value)); -} - -Scalar int64(int64_t value) { - return Scalar(ffi::i64_scalar_new(value)); -} - -Scalar uint8(uint8_t value) { - return Scalar(ffi::u8_scalar_new(value)); -} - -Scalar uint16(uint16_t value) { - return Scalar(ffi::u16_scalar_new(value)); -} - -Scalar uint32(uint32_t value) { - return Scalar(ffi::u32_scalar_new(value)); -} - -Scalar uint64(uint64_t value) { - return Scalar(ffi::u64_scalar_new(value)); -} - -Scalar float32(float value) { - return Scalar(ffi::f32_scalar_new(value)); -} - -Scalar float64(double value) { - return Scalar(ffi::f64_scalar_new(value)); -} - -Scalar string(std::string_view value) { - return Scalar(ffi::string_scalar_new(rust::Str(value.data(), value.length()))); -} - -Scalar binary(const uint8_t *data, size_t length) { - return Scalar(ffi::binary_scalar_new(rust::Slice(data, length))); -} - -Scalar cast(Scalar scalar, dtype::DType dtype) { - return Scalar(std::move(scalar).IntoImpl()->cast_scalar(*std::move(dtype).GetImpl())); -} - -} // namespace vortex::scalar \ No newline at end of file diff --git a/vortex-cxx/cpp/src/scan.cpp b/vortex-cxx/cpp/src/scan.cpp deleted file mode 100644 index 9b40ec00e0d..00000000000 --- a/vortex-cxx/cpp/src/scan.cpp +++ /dev/null @@ -1,116 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/scan.hpp" -#include "vortex/exception.hpp" -#include "rust/cxx.h" -#include "vortex/expr.hpp" - -namespace vortex { -ScanBuilder &ScanBuilder::WithFilter(expr::Expr &&expr) & { - impl_->with_filter(std::move(expr).IntoImpl()); - return *this; -} -ScanBuilder &ScanBuilder::WithFilter(const expr::Expr &expr) & { - impl_->with_filter_ref(expr.Impl()); - return *this; -} -ScanBuilder &&ScanBuilder::WithFilter(expr::Expr &&expr) && { - impl_->with_filter(std::move(expr).IntoImpl()); - return std::move(*this); -} -ScanBuilder &&ScanBuilder::WithFilter(const expr::Expr &expr) && { - impl_->with_filter_ref(expr.Impl()); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithProjection(expr::Expr &&expr) & { - impl_->with_projection(std::move(expr).IntoImpl()); - return *this; -} -ScanBuilder &ScanBuilder::WithProjection(const expr::Expr &expr) & { - impl_->with_projection_ref(expr.Impl()); - return *this; -} -ScanBuilder &&ScanBuilder::WithProjection(expr::Expr &&expr) && { - impl_->with_projection(std::move(expr).IntoImpl()); - return std::move(*this); -} -ScanBuilder &&ScanBuilder::WithProjection(const expr::Expr &expr) && { - impl_->with_projection_ref(expr.Impl()); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithRowRange(uint64_t row_range_start, uint64_t row_range_end) & { - impl_->with_row_range(row_range_start, row_range_end); - return *this; -} -ScanBuilder &&ScanBuilder::WithRowRange(uint64_t row_range_start, uint64_t row_range_end) && { - impl_->with_row_range(row_range_start, row_range_end); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithLimit(uint64_t limit) & { - impl_->with_limit(limit); - return *this; -} - -ScanBuilder &&ScanBuilder::WithLimit(uint64_t limit) && { - impl_->with_limit(limit); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithIncludeByIndex(const uint64_t *indices, std::size_t size) & { - impl_->with_include_by_index(rust::Slice(indices, size)); - return *this; -} - -ScanBuilder &&ScanBuilder::WithIncludeByIndex(const uint64_t *indices, std::size_t size) && { - impl_->with_include_by_index(rust::Slice(indices, size)); - return std::move(*this); -} - -ScanBuilder &ScanBuilder::WithOutputSchema(ArrowSchema &output_schema) & { - try { - impl_->with_output_schema(reinterpret_cast(&output_schema)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } - return *this; -} - -ScanBuilder &&ScanBuilder::WithOutputSchema(ArrowSchema &output_schema) && { - try { - impl_->with_output_schema(reinterpret_cast(&output_schema)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } - return std::move(*this); -} - -ArrowArrayStream ScanBuilder::IntoStream() && { - try { - ArrowArrayStream stream; - ffi::scan_builder_into_stream(std::move(impl_), reinterpret_cast(&stream)); - return stream; - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -StreamDriver ScanBuilder::IntoStreamDriver() && { - try { - rust::Box reader = - ffi::scan_builder_into_threadsafe_cloneable_reader(std::move(impl_)); - return StreamDriver(std::move(reader)); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -ArrowArrayStream StreamDriver::CreateArrayStream() const { - ArrowArrayStream stream; - impl_->clone_a_stream(reinterpret_cast(&stream)); - return stream; -} -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/src/write_options.cpp b/vortex-cxx/cpp/src/write_options.cpp deleted file mode 100644 index 712a6933208..00000000000 --- a/vortex-cxx/cpp/src/write_options.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "vortex/write_options.hpp" -#include "vortex/exception.hpp" - -#include "rust/cxx.h" - -namespace vortex { -void VortexWriteOptions::WriteArrayStream(ArrowArrayStream &stream, const std::string &path) { - try { - ffi::write_array_stream(std::move(impl_), reinterpret_cast(&stream), path); - } catch (const rust::cxxbridge1::Error &e) { - throw VortexException(e.what()); - } -} - -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/tests/basic_test.cpp b/vortex-cxx/cpp/tests/basic_test.cpp deleted file mode 100644 index eb44715285a..00000000000 --- a/vortex-cxx/cpp/tests/basic_test.cpp +++ /dev/null @@ -1,510 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/write_options.hpp" -#include "vortex/scalar.hpp" -#include "test_data_generator.hpp" -#include "vortex_cxx_bridge/lib.h" - -#include -#include - -class VortexTest : public ::testing::Test { -protected: - // Helper function to create unique temporary files for each test - static std::string GetUniqueTempFile(const std::string &suffix = "vortex") { - std::filesystem::path temp_dir = std::filesystem::temp_directory_path(); - std::filesystem::path vortex_test_dir = temp_dir / "vortex_test"; - - if (!std::filesystem::exists(vortex_test_dir)) { - std::filesystem::create_directories(vortex_test_dir); - } - - // Use a unique random filename to prevent races between parallel test runs - std::string unique_name = "test_" + std::to_string(std::random_device {}()) + "_" + suffix; - return (vortex_test_dir / unique_name).string(); - } - - // Write test data to a unique temporary file and return the path - static std::string WriteTestData(const std::string &suffix = "test_data.vortex") { - std::string path = GetUniqueTempFile(suffix); - auto stream = vortex::testing::CreateTestDataStream(); - auto write_options = vortex::ffi::write_options_new(); - vortex::ffi::write_array_stream(std::move(write_options), - reinterpret_cast(&stream), - path.c_str()); - return path; - } - - // Helper function to create and initialize array view - nanoarrow::UniqueArrayView CreateArrayView(const nanoarrow::UniqueArray &array, - const nanoarrow::UniqueSchema &schema) { - nanoarrow::UniqueArrayView array_view; - ArrowError error; - ArrowErrorCode init_result = ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), &error); - if (init_result != NANOARROW_OK) { - std::cerr << "Error: " << error.message << '\n'; - std::abort(); - } - ArrowErrorCode set_result = ArrowArrayViewSetArray(array_view.get(), array.get(), nullptr); - if (set_result != NANOARROW_OK) { - std::cerr << "Error: " << error.message << '\n'; - std::abort(); - } - return array_view; - } - - std::pair - StreamToUniqueStreamSchema(ArrowArrayStream &stream) { - nanoarrow::UniqueArrayStream array_stream; - ArrowArrayStreamMove(&stream, array_stream.get()); - ArrowError error; - nanoarrow::UniqueSchema schema; - ArrowErrorCode set_result = ArrowArrayStreamGetSchema(array_stream.get(), schema.get(), &error); - if (set_result != NANOARROW_OK) { - std::cerr << "Error: " << error.message << '\n'; - std::abort(); - } - return {std::move(array_stream), std::move(schema)}; - } - - nanoarrow::UniqueArray ReadFirstArrayFromUniqueStream(nanoarrow::UniqueArrayStream &array_stream) { - nanoarrow::UniqueArray array; - int get_next_result = array_stream->get_next(array_stream.get(), array.get()); - EXPECT_EQ(get_next_result, 0); - return array; - } - - std::pair - ReadFirstArrayFromStream(ArrowArrayStream reference_stream) { - auto [ref_array_stream, ref_schema] = StreamToUniqueStreamSchema(reference_stream); - auto ref_array = ReadFirstArrayFromUniqueStream(ref_array_stream); - return {std::move(ref_array), std::move(ref_schema)}; - } - - /// Both array are struct of int64 - void ValidateArray(const nanoarrow::UniqueArray &actual_array, - const nanoarrow::UniqueSchema &actual_schema, - const nanoarrow::UniqueArray &ref_array, - const nanoarrow::UniqueSchema &ref_schema) { - // Basic properties validation - ASSERT_EQ(actual_schema->n_children, ref_schema->n_children); - - auto actual_view = CreateArrayView(actual_array, actual_schema); - auto ref_view = CreateArrayView(ref_array, ref_schema); - - ASSERT_EQ(actual_array->length, ref_array->length); - - // Compare all fields - for (int64_t field_idx = 0; field_idx < actual_schema->n_children; ++field_idx) { - auto actual_field = actual_view->children[field_idx]; - auto expected_field = ref_view->children[field_idx]; - - ASSERT_EQ(actual_field->array->length, expected_field->array->length); - - for (int64_t i = 0; i < actual_field->array->length; ++i) { - int64_t actual_value = ArrowArrayViewGetIntUnsafe(actual_field, i); - int64_t expected_value = ArrowArrayViewGetIntUnsafe(expected_field, i); - - ASSERT_EQ(actual_value, expected_value); - } - } - } - - /// Both array are struct of int64 - void ValidateArrayWithSelection(const nanoarrow::UniqueArray &actual_array, - const nanoarrow::UniqueSchema &actual_schema, - const nanoarrow::UniqueArray &ref_array, - const nanoarrow::UniqueSchema &ref_schema, - const std::vector &row_indices) { - // Basic properties validation - ASSERT_EQ(actual_schema->n_children, ref_schema->n_children); - if (row_indices.empty()) { - ASSERT_EQ(actual_array->length, 0); - return; - } - auto actual_view = CreateArrayView(actual_array, actual_schema); - auto ref_view = CreateArrayView(ref_array, ref_schema); - - ASSERT_EQ(actual_array->length, row_indices.size()); - - // Selective row comparison using indices - ASSERT_EQ(actual_array->length, static_cast(row_indices.size())); - - for (int64_t i = 0; i < static_cast(row_indices.size()); ++i) { - int64_t ref_idx = row_indices[i]; - - for (int64_t field_idx = 0; field_idx < actual_schema->n_children; ++field_idx) { - int64_t actual_val = ArrowArrayViewGetIntUnsafe(actual_view->children[field_idx], i); - int64_t expected_val = ArrowArrayViewGetIntUnsafe(ref_view->children[field_idx], ref_idx); - - ASSERT_EQ(actual_val, expected_val); - } - } - } - - // Helper to execute scan builder and get array+schema - std::pair ScanFirstArrayFromTestData( - const std::function &configureScanBuilder) { - auto test_data_path = WriteTestData(); - auto file = vortex::VortexFile::Open(test_data_path); - auto scan_builder = file.CreateScanBuilder(); - auto stream = configureScanBuilder(scan_builder); - - auto [array_stream, schema] = StreamToUniqueStreamSchema(stream); - auto array = ReadFirstArrayFromUniqueStream(array_stream); - - return {std::move(array), std::move(schema)}; - } - - /// Validate array with projection - only checks specified field indices - void ValidateArrayWithProjection(const nanoarrow::UniqueArray &actual_array, - const nanoarrow::UniqueSchema &actual_schema, - const nanoarrow::UniqueArray &ref_array, - const nanoarrow::UniqueSchema &ref_schema, - const std::vector &field_idxs) { - ASSERT_EQ(actual_schema->n_children, field_idxs.size()); - - auto actual_view = CreateArrayView(actual_array, actual_schema); - auto ref_view = CreateArrayView(ref_array, ref_schema); - - ASSERT_EQ(actual_array->length, ref_array->length); - - // Compare only the specified fields - for (int64_t i = 0; i < actual_array->length; ++i) { - for (size_t proj_idx = 0; proj_idx < field_idxs.size(); ++proj_idx) { - int64_t ref_field_idx = field_idxs[proj_idx]; - int64_t actual_val = ArrowArrayViewGetIntUnsafe(actual_view->children[proj_idx], i); - int64_t expected_val = ArrowArrayViewGetIntUnsafe(ref_view->children[ref_field_idx], i); - ASSERT_EQ(actual_val, expected_val); - } - } - } - - // Top-level test helper that all tests can use - void - RunScanBuilderTest(const std::function &configureScanBuilder, - ArrowArrayStream expected_stream, - const std::vector &expected_row_indices = {}, - bool selection = false) { - - auto [array, schema] = ScanFirstArrayFromTestData(configureScanBuilder); - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(expected_stream); - selection == false - ? ValidateArray(array, schema, ref_array, ref_schema) - : ValidateArrayWithSelection(array, schema, ref_array, ref_schema, expected_row_indices); - } - - // New helper for projection tests - void RunScanBuilderProjectionTest( - const std::function &configureScanBuilder, - ArrowArrayStream expected_stream, - const std::vector &field_idxs) { - - auto [array, schema] = ScanFirstArrayFromTestData(configureScanBuilder); - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(expected_stream); - - ValidateArrayWithProjection(array, schema, ref_array, ref_schema, field_idxs); - } -}; - -TEST_F(VortexTest, ScanToStream) { - RunScanBuilderTest([](vortex::ScanBuilder &builder) { return std::move(builder).IntoStream(); }, - vortex::testing::CreateTestDataStream()); -} - -TEST_F(VortexTest, ScanBuilderWithLimitWithRowRange) { - // Test field "a" and "b" - should contain values from rows 1-2 from original data (indices 1 and - // 2) - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - return std::move(scan_builder.WithLimit(2).WithRowRange(1, 4)).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {1, 2}, - true); -} - -TEST_F(VortexTest, ScanBuilderWithIncludeByIndex) { - std::vector include_by_index = {1, 3}; - - RunScanBuilderTest( - [&include_by_index](vortex::ScanBuilder &scan_builder) { - return std::move( - scan_builder.WithIncludeByIndex(include_by_index.data(), include_by_index.size())) - .IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {1, 3}, - true); -} - -TEST_F(VortexTest, ScanBuilderWithRowRangeWithIncludeByIndex) { - std::vector include_by_index = {1, 3, 4}; - - RunScanBuilderTest( - [&include_by_index](vortex::ScanBuilder &scan_builder) { - return std::move(scan_builder.WithRowRange(2, 5).WithIncludeByIndex(include_by_index.data(), - include_by_index.size())) - .IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {3, 4}, - true); -} - -TEST_F(VortexTest, WriteArrayStream) { - auto test_data_path = WriteTestData(); - auto file = vortex::VortexFile::Open(test_data_path); - auto stream = file.CreateScanBuilder().IntoStream(); - - // Write the stream to a new Vortex file - std::string test_output_path = GetUniqueTempFile("write_output.vortex"); - vortex::VortexWriteOptions write_options; - ASSERT_NO_THROW(write_options.WriteArrayStream(stream, test_output_path)); - - // Verify the written file - auto written_file = vortex::VortexFile::Open(test_output_path); - ASSERT_EQ(written_file.RowCount(), 5); - - // Verify data integrity by reading from the written file - auto out_stream = written_file.CreateScanBuilder().IntoStream(); - auto [array_stream, schema] = StreamToUniqueStreamSchema(out_stream); - auto array = ReadFirstArrayFromUniqueStream(array_stream); - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(vortex::testing::CreateTestDataStream()); - ValidateArray(array, schema, ref_array, ref_schema); -} - -TEST_F(VortexTest, ConcurrentMultiStreamRead) { - std::string test_data_path_1m = GetUniqueTempFile("concurrent_1m.vortex"); - auto stream_1m = vortex::testing::CreateTestData1MStream(); - auto write_options = vortex::ffi::write_options_new(); - vortex::ffi::write_array_stream(std::move(write_options), - reinterpret_cast(&stream_1m), - test_data_path_1m.c_str()); - - auto file = vortex::VortexFile::Open(test_data_path_1m); - auto stream_driver = file.CreateScanBuilder().IntoStreamDriver(); - - // Structure to hold batch data with first ID and nanoarrow array - struct BatchData { - int64_t first_id; - nanoarrow::UniqueArray array; - - BatchData(int64_t first_id, nanoarrow::UniqueArray array) - : first_id(first_id), array(std::move(array)) { - } - }; - - std::vector thread1_batches; - std::vector thread2_batches; - - // Helper function to read from a stream and collect batches - auto read_stream = [&](std::vector &batches) { - // Each thread creates its own stream - auto stream = stream_driver.CreateArrayStream(); - auto [array_stream, schema] = StreamToUniqueStreamSchema(stream); - - std::vector local_batches; - - while (true) { - nanoarrow::UniqueArray array; - int get_next_result = array_stream->get_next(array_stream.get(), array.get()); - - if (get_next_result != 0) { - std::cerr << "Error: " << array_stream->get_last_error(array_stream.get()) << '\n'; - std::abort(); - } - - if (array->length == 0) { - break; // Empty array indicates end - } - - auto array_view = CreateArrayView(array, schema); - - int64_t first_id = ArrowArrayViewGetIntUnsafe(array_view->children[0], 0); - - local_batches.emplace_back(first_id, std::move(array)); - } - batches = std::move(local_batches); - }; - - // Launch two threads - std::thread thread1(read_stream, std::ref(thread1_batches)); - std::thread thread2(read_stream, std::ref(thread2_batches)); - - // Wait for both threads to complete - thread1.join(); - thread2.join(); - - // Combine all batches from both threads - std::vector all_batches; - all_batches.insert(all_batches.end(), - std::make_move_iterator(thread1_batches.begin()), - std::make_move_iterator(thread1_batches.end())); - all_batches.insert(all_batches.end(), - std::make_move_iterator(thread2_batches.begin()), - std::make_move_iterator(thread2_batches.end())); - - // Sort batches by first ID to ensure proper validation order - std::sort(all_batches.begin(), all_batches.end(), [](const BatchData &a, const BatchData &b) { - return a.first_id < b.first_id; - }); - - // Create reference data for validation - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(vortex::testing::CreateTestData1MStream()); - auto ref_array_view = CreateArrayView(ref_array, ref_schema); - - // Validate all data against reference - constexpr size_t EXPECTED_ROWS = static_cast(1024) * 1024; - size_t total_rows_read = 0; - int64_t reference_offset = 0; - - auto stream_for_schema = stream_driver.CreateArrayStream(); - auto [_, schema] = StreamToUniqueStreamSchema(stream_for_schema); - for (const auto &batch : all_batches) { - auto array_view = CreateArrayView(batch.array, schema); - - for (int64_t i = 0; i < batch.array->length; ++i) { - - int64_t actual_id = ArrowArrayViewGetIntUnsafe(array_view->children[0], i); - int32_t actual_value = - static_cast(ArrowArrayViewGetIntUnsafe(array_view->children[1], i)); - - int64_t expected_id = ArrowArrayViewGetIntUnsafe(ref_array_view->children[0], reference_offset); - int32_t expected_value = static_cast( - ArrowArrayViewGetIntUnsafe(ref_array_view->children[1], reference_offset)); - - ASSERT_EQ(actual_id, expected_id); - ASSERT_EQ(actual_value, expected_value); - reference_offset++; - } - total_rows_read += batch.array->length; - } - - // Verify we read all expected data - ASSERT_EQ(total_rows_read, EXPECTED_ROWS) - << "Expected to read " << EXPECTED_ROWS << " rows, but read " << total_rows_read << " rows"; - ASSERT_EQ(reference_offset, EXPECTED_ROWS) << "Reference validation didn't cover all rows"; - - ASSERT_GT(all_batches.size(), 1) << "Expected multiple batches, but got " << all_batches.size(); -} - -namespace ve = vortex::expr; -namespace vs = vortex::scalar; - -TEST_F(VortexTest, ScanBuilderWithFilter) { - // Test filtering with eq(column("a"), val) - should return only rows where column "a" equals 30 - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - auto filter = ve::eq(ve::column("a"), ve::literal(vs::int32(30))); - return std::move(scan_builder.WithFilter(std::move(filter))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {2}, - true); // Row index 2 corresponds to value 30 -} - -TEST_F(VortexTest, ScanBuilderWithFilterLvalueref) { - // Test filtering with eq(column("a"), val) - should return only rows where column "a" equals 30 - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - const auto filter = ve::eq(ve::column("a"), ve::literal(vs::int32(30))); - return std::move(scan_builder.WithFilter(filter)).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {2}, - true); // Row index 2 corresponds to value 30 -} - -TEST_F(VortexTest, ScanBuilderWithFilterNoMatches) { - // Test filtering with eq(column("a"), val) where no rows match - should return empty result - RunScanBuilderTest( - [](vortex::ScanBuilder &scan_builder) { - auto filter = ve::eq(ve::column("a"), - ve::literal(vs::int32(999)) // Value that doesn't exist in test data - ); - return std::move(scan_builder.WithFilter(std::move(filter))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {}, - true); // No matching rows -} - -TEST_F(VortexTest, ScanBuilderWithFilterUsingDTypeFromArrowAndScalarCast) { - // Test filtering using DType::from_arrow and Scalar::cast functionality - // This test creates a filter expression by casting a scalar to match the column type - - // Test DType::from_arrow with int32 schema - nanoarrow::UniqueSchema int32_schema; - ArrowSchemaInit(int32_schema.get()); - ArrowErrorCode result = ArrowSchemaSetType(int32_schema.get(), NANOARROW_TYPE_INT32); - EXPECT_EQ(result, NANOARROW_OK); - result = ArrowSchemaSetName(int32_schema.get(), "test_field"); - EXPECT_EQ(result, NANOARROW_OK); - - auto dtype = vortex::dtype::from_arrow(*int32_schema.get()); - - // Use the casted scalar in filter expression - create a new scalar for lambda - RunScanBuilderTest( - [&](vortex::ScanBuilder &scan_builder) { - auto test_scalar = vs::cast(vs::int64(30), std::move(dtype)); - auto filter = ve::eq(ve::column("a"), ve::literal(std::move(test_scalar))); - return std::move(scan_builder.WithFilter(std::move(filter))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {2}, - true); // Row index 2 corresponds to value 30 -} - -TEST_F(VortexTest, ScanBuilderWithProjectionSingleColumn) { - // Test projection selecting only column "a" (field index 0) - RunScanBuilderProjectionTest( - [](vortex::ScanBuilder &scan_builder) { - auto projection = ve::select({"a"}, ve::root()); - return std::move(scan_builder.WithProjection(std::move(projection))).IntoStream(); - }, - vortex::testing::CreateTestDataStream(), - {0}); -} - -TEST_F(VortexTest, OpenFromBuffer) { - std::string test_file_path = GetUniqueTempFile("buffer.vortex"); - auto stream = vortex::testing::CreateTestDataStream(); - auto write_options = vortex::ffi::write_options_new(); - vortex::ffi::write_array_stream(std::move(write_options), - reinterpret_cast(&stream), - test_file_path.c_str()); - - std::ifstream file(test_file_path, std::ios::binary | std::ios::ate); - ASSERT_TRUE(file.is_open()) << "Failed to open file: " << test_file_path; - - std::streamsize file_size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(file_size); - ASSERT_TRUE(file.read(reinterpret_cast(buffer.data()), file_size)) - << "Failed to read file into buffer"; - file.close(); - - auto vortex_file = vortex::VortexFile::Open(buffer.data(), buffer.size()); - ASSERT_EQ(vortex_file.RowCount(), 5); - - auto scan_stream = vortex_file.CreateScanBuilder().IntoStream(); - auto [array_stream, schema] = StreamToUniqueStreamSchema(scan_stream); - auto array = ReadFirstArrayFromUniqueStream(array_stream); - - auto [ref_array, ref_schema] = ReadFirstArrayFromStream(vortex::testing::CreateTestDataStream()); - ValidateArray(array, schema, ref_array, ref_schema); -} diff --git a/vortex-cxx/cpp/tests/test_data_generator.cpp b/vortex-cxx/cpp/tests/test_data_generator.cpp deleted file mode 100644 index 67dcb290509..00000000000 --- a/vortex-cxx/cpp/tests/test_data_generator.cpp +++ /dev/null @@ -1,125 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "test_data_generator.hpp" -#include -#include -#include -#include -#include - -namespace vortex { -namespace testing { - - ArrowArrayStream CreateTestDataStream() { - // Create a simple two-column struct with int32 data - // Schema: struct{a: int32, b: int32} - nanoarrow::UniqueSchema schema; - ArrowSchemaInit(schema.get()); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateChildren(schema.get(), 2)); - ArrowSchemaInit(schema->children[0]); - ArrowSchemaInit(schema->children[1]); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "a")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "b")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32)); - - // Create arrays for each field - nanoarrow::UniqueArray field_a, field_b; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_a.get(), NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_b.get(), NANOARROW_TYPE_INT32)); - - // Reserve for 5 elements - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_a.get())); - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_b.get())); - - // Add data: [10, 20, 30, 40, 50] - std::vector data = {10, 20, 30, 40, 50}; - for (int32_t value : data) { - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_a.get(), value)); - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_b.get(), value)); - } - - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(field_a.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(field_b.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - - // Create struct array - nanoarrow::UniqueArray struct_array; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(struct_array.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowArrayAllocateChildren(struct_array.get(), 2)); - struct_array->length = 5; - ArrowArrayMove(field_a.get(), struct_array->children[0]); - ArrowArrayMove(field_b.get(), struct_array->children[1]); - - // Create vector and move array into it - std::vector arrays; - arrays.push_back(std::move(struct_array)); - - // Create stream - ArrowArrayStream stream; - nanoarrow::VectorArrayStream vector_stream(schema.get(), std::move(arrays)); - vector_stream.ToArrayStream(&stream); - - return stream; - } - - ArrowArrayStream CreateTestData1MStream() { - constexpr size_t NUM_ROWS = 1024UL * 1024; - - // Create schema: struct{id: int64, value: int32} - nanoarrow::UniqueSchema schema; - ArrowSchemaInit(schema.get()); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateChildren(schema.get(), 2)); - ArrowSchemaInit(schema->children[0]); - ArrowSchemaInit(schema->children[1]); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "id")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "value")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32)); - - // Create arrays for each field - nanoarrow::UniqueArray id_field, value_field; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(id_field.get(), NANOARROW_TYPE_INT64)); - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(value_field.get(), NANOARROW_TYPE_INT32)); - - // Reserve space - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(id_field.get())); - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(value_field.get())); - - // Add data - for (size_t i = 0; i < NUM_ROWS; ++i) { - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(id_field.get(), static_cast(i))); - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(value_field.get(), static_cast(i * 2))); - } - - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(id_field.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - NANOARROW_THROW_NOT_OK( - ArrowArrayFinishBuilding(value_field.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - - // Create struct array - nanoarrow::UniqueArray struct_array; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(struct_array.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowArrayAllocateChildren(struct_array.get(), 2)); - struct_array->length = NUM_ROWS; - ArrowArrayMove(id_field.get(), struct_array->children[0]); - ArrowArrayMove(value_field.get(), struct_array->children[1]); - - // Create vector and move array into it - std::vector arrays; - arrays.push_back(std::move(struct_array)); - - // Create stream - ArrowArrayStream stream; - nanoarrow::VectorArrayStream vector_stream(schema.get(), std::move(arrays)); - vector_stream.ToArrayStream(&stream); - - return stream; - } - -} // namespace testing -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/cpp/tests/test_data_generator.hpp b/vortex-cxx/cpp/tests/test_data_generator.hpp deleted file mode 100644 index ec5dba55d50..00000000000 --- a/vortex-cxx/cpp/tests/test_data_generator.hpp +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include - -namespace vortex { -namespace testing { - - /// Create test data with structure {a: [10, 20, 30, 40, 50], b: [10, 20, 30, 40, 50]} - /// This stream only has one Array - ArrowArrayStream CreateTestDataStream(); - - /// Create 1M rows of test data with structure {id: [0..1M], value: [0, 2, 4, ..., 2M]} - /// This stream only has one Array - ArrowArrayStream CreateTestData1MStream(); - -} // namespace testing -} // namespace vortex \ No newline at end of file diff --git a/vortex-cxx/examples/.clang-tidy b/vortex-cxx/examples/.clang-tidy new file mode 100644 index 00000000000..e302d65b598 --- /dev/null +++ b/vortex-cxx/examples/.clang-tidy @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors +InheritParentConfig: true +Checks: | + -google-build-using-namespace, + -bugprone-unchecked-optional-access, diff --git a/vortex-cxx/examples/CMakeLists.txt b/vortex-cxx/examples/CMakeLists.txt index 348d80e1547..1eb3650c4e4 100644 --- a/vortex-cxx/examples/CMakeLists.txt +++ b/vortex-cxx/examples/CMakeLists.txt @@ -1,34 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright the Vortex contributors -cmake_minimum_required(VERSION 3.22) - -project(vortex-examples) - -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - -find_program(SCCACHE_PROGRAM sccache) -if (SCCACHE_PROGRAM) - set(CMAKE_C_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") - set(CMAKE_CXX_COMPILER_LAUNCHER "${SCCACHE_PROGRAM}") - message(STATUS "Sccache found: ${SCCACHE_PROGRAM}") -else () - message(STATUS "Sccache not found") -endif () - -include(FetchContent) -FetchContent_Declare( - vortex - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../ - SOURCE_SUBDIR vortex-cxx -) -FetchContent_MakeAvailable(vortex) - -add_executable(hello-vortex hello-vortex.cpp) - -if(APPLE) - set(APPLE_LINK_FLAGS "-framework CoreFoundation -framework Security") -endif() - -target_link_libraries(hello-vortex PRIVATE vortex ${APPLE_LINK_FLAGS}) +foreach(name hello-vortex dtype write_sample scan scan_to_arrow) + add_executable(${name} ${name}.cpp) + target_link_libraries(${name} PRIVATE vortex_cxx_shared nanoarrow_shared) + if(APPLE) + target_link_libraries(${name} PRIVATE + "-framework CoreFoundation" "-framework Security") + endif() +endforeach() diff --git a/vortex-cxx/examples/README.md b/vortex-cxx/examples/README.md deleted file mode 100644 index 40e7708fa33..00000000000 --- a/vortex-cxx/examples/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# C++ examples - -This example shows how to interface with the C++ API using CMake. - -```bash -mkdir -p build -cd build -cmake .. -make -j$(nproc) -./hello-vortex -``` \ No newline at end of file diff --git a/vortex-cxx/examples/dtype.cpp b/vortex-cxx/examples/dtype.cpp new file mode 100644 index 00000000000..4e899c89319 --- /dev/null +++ b/vortex-cxx/examples/dtype.cpp @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include +#include +#include + +using vortex::DataSource; +using vortex::DataType; +using vortex::DataTypeVariant; +using vortex::Session; +using vortex::VortexException; + +static void print_dtype(const DataType& dtype); + +static std::string_view ptype_name(vortex::PType p) { + switch (p) { + case vortex::PType::U8: + return "uint8_t"; + case vortex::PType::U16: + return "uint16_t"; + case vortex::PType::U32: + return "uint32_t"; + case vortex::PType::U64: + return "uint64_t"; + case vortex::PType::I8: + return "int8_t"; + case vortex::PType::I16: + return "int16_t"; + case vortex::PType::I32: + return "int32_t"; + case vortex::PType::I64: + return "int64_t"; + case vortex::PType::F16: + return "float16"; + case vortex::PType::F32: + return "float"; + case vortex::PType::F64: + return "double"; + } + return "?"; +} + +static void print_struct(const DataType& dtype) { + std::cout << "struct(\n"; + const size_t n = dtype.num_fields(); + for (size_t i = 0; i < n; ++i) { + std::cout << " " << dtype.field_name(i) << " = "; + print_dtype(dtype.field_dtype(i)); + } + std::cout << ")"; +} + +static void print_dtype(const DataType& dtype) { + switch (dtype.variant()) { + case DataTypeVariant::Null: + std::cout << "null"; + break; + case DataTypeVariant::Bool: + std::cout << "bool"; + break; + case DataTypeVariant::Utf8: + std::cout << "utf8"; + break; + case DataTypeVariant::Binary: + std::cout << "binary"; + break; + case DataTypeVariant::Extension: + std::cout << "extension"; + break; + case DataTypeVariant::Primitive: + std::cout << "primitive(" << ptype_name(dtype.primitive_ptype()) + << ")"; + break; + case DataTypeVariant::Struct: + print_struct(dtype); + break; + case DataTypeVariant::List: + std::cout << "list("; + print_dtype(dtype.list_element()); + std::cout << ")"; + break; + case DataTypeVariant::FixedSizeList: + std::cout << "fixed_list("; + print_dtype(dtype.list_element()); + std::cout << ")"; + break; + case DataTypeVariant::Decimal: + std::cout << "decimal(precision=" + << static_cast(dtype.decimal_precision()) + << ", scale=" << static_cast(dtype.decimal_scale()) + << ")"; + break; + } + std::cout << (dtype.nullable() ? '?' : ' ') << '\n'; +} + +int main(int argc, char** argv) { + if (argc != 2) { + std::cerr << "Usage: dtype \n"; + return 1; + } + + Session session; + DataSource ds(session, argv[1]); + DataType dt = ds.dtype(); + std::cout << "dtype: "; + print_dtype(dt); + return 0; +} diff --git a/vortex-cxx/examples/hello-vortex.cpp b/vortex-cxx/examples/hello-vortex.cpp index c6a7fec3b9a..abc3c2a4009 100644 --- a/vortex-cxx/examples/hello-vortex.cpp +++ b/vortex-cxx/examples/hello-vortex.cpp @@ -1,129 +1,69 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "nanoarrow/common/inline_types.h" -#include "nanoarrow/hpp/unique.hpp" -#include "nanoarrow/nanoarrow.hpp" -#include "vortex/file.hpp" -#include "vortex/scan.hpp" -#include "vortex/write_options.hpp" -#include -#include #include #include #include +#include + +#include "../tests/sink.hpp" + +using vortex::Array; +using vortex::DataSource; +using vortex::DataType; +using vortex::Partition; +using vortex::Scan; +using vortex::ScanOptions; +using vortex::Session; +using vortex::StructColumnBuilder; +using vortex::StructFieldsBuilder; +using vortex::Validity; +using vortex::ValidityType; +namespace dtype = vortex::dtype; +namespace fs = std::filesystem; -/// Create test data with structure {a: [10, 20, 30, 40, 50], b: [100, 200, 300, 400, 500]} -ArrowArrayStream CreateTestDataStream() { - // Create a simple two-column struct with int32 data - // Schema: struct{a: int32, b: int32} - nanoarrow::UniqueSchema schema; - ArrowSchemaInit(schema.get()); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowSchemaAllocateChildren(schema.get(), 2)); - ArrowSchemaInit(schema->children[0]); - ArrowSchemaInit(schema->children[1]); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[0], "a")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[0], NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetName(schema->children[1], "b")); - NANOARROW_THROW_NOT_OK(ArrowSchemaSetType(schema->children[1], NANOARROW_TYPE_INT32)); - - // Create arrays for each field - nanoarrow::UniqueArray field_a, field_b; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_a.get(), NANOARROW_TYPE_INT32)); - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(field_b.get(), NANOARROW_TYPE_INT32)); - - // Reserve for 5 elements - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_a.get())); - NANOARROW_THROW_NOT_OK(ArrowArrayStartAppending(field_b.get())); - - // Add data: a=[10, 20, 30, 40, 50], b=[100, 200, 300, 400, 500] - std::vector data_a = {10, 20, 30, 40, 50}; - std::vector data_b = {100, 200, 300, 400, 500}; - for (size_t i = 0; i < data_a.size(); ++i) { - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_a.get(), data_a[i])); - NANOARROW_THROW_NOT_OK(ArrowArrayAppendInt(field_b.get(), data_b[i])); - } - - NANOARROW_THROW_NOT_OK(ArrowArrayFinishBuilding(field_a.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); - NANOARROW_THROW_NOT_OK(ArrowArrayFinishBuilding(field_b.get(), NANOARROW_VALIDATION_LEVEL_NONE, nullptr)); +int main() { + const fs::path out = + fs::temp_directory_path() / "hello_vortex_example.vortex"; + std::cout << "Writing to: " << out.string() << '\n'; - // Create struct array - nanoarrow::UniqueArray struct_array; - NANOARROW_THROW_NOT_OK(ArrowArrayInitFromType(struct_array.get(), NANOARROW_TYPE_STRUCT)); - NANOARROW_THROW_NOT_OK(ArrowArrayAllocateChildren(struct_array.get(), 2)); - struct_array->length = 5; - ArrowArrayMove(field_a.get(), struct_array->children[0]); - ArrowArrayMove(field_b.get(), struct_array->children[1]); + Session session; - // Create vector and move array into it - std::vector arrays; - arrays.push_back(std::move(struct_array)); + DataType type = + dtype::structure(StructFieldsBuilder() + .AddField("age", dtype::uint8()) + .AddField("height", dtype::uint16(true)) + .Finalize()); - // Create stream - ArrowArrayStream stream; - nanoarrow::VectorArrayStream vector_stream(schema.get(), std::move(arrays)); - vector_stream.ToArrayStream(&stream); + Validity non_nullable{ValidityType::NonNullable}; + Validity all_valid{ValidityType::AllValid}; - return stream; -} + const std::vector ages{10, 20, 30, 40, 50}; + const std::vector heights{100, 200, 300, 400, 500}; -int main() { - // Create a temporary file path - std::filesystem::path temp_dir = std::filesystem::temp_directory_path(); - std::string vortex_file = (temp_dir / "hello_vortex_example.vortex").string(); + Array age_arr(vortex::PType::U8, ages.data(), ages.size(), non_nullable); + Array height_arr(vortex::PType::U16, heights.data(), heights.size(), + all_valid); - std::cout << "=== Vortex C++ Example ===" << '\n'; - std::cout << "Writing to: " << vortex_file << '\n'; + Array struct_arr = StructColumnBuilder(non_nullable, ages.size()) + .AddField("age", age_arr) + .AddField("height", height_arr) + .Finalize(); - // Write test data to a Vortex file { - auto stream = CreateTestDataStream(); - vortex::VortexWriteOptions write_options; - write_options.WriteArrayStream(stream, vortex_file); - std::cout << "Wrote test data to file" << '\n'; + vortex_test::ArraySink sink(session, out.string(), type); + sink.push(struct_arr); + std::move(sink).close(); } - auto check_stream = [](ArrowArrayStream &stream) { - nanoarrow::UniqueArray array; - int get_next_result = stream.get_next(&stream, array.get()); - assert(get_next_result == 0); - std::cout << "Number of rows: " << array->length << '\n'; - std::cout << "Number of columns in schema: " << array->n_children << '\n'; - }; + DataSource ds(session, out.string()); + std::cout << "Row count: " << ds.row_count().value << '\n'; - // 1. Classic C++ builder pattern - std::cout << "\n1. Classic C++ builder pattern:" << '\n'; - { - auto builder = vortex::VortexFile::Open(vortex_file).CreateScanBuilder(); - builder.WithLimit(3); - auto stream = std::move(builder).IntoStream(); - check_stream(stream); - } - // 2. One-line Rusty function chain - std::cout << "\n2. One-line Rusty function chain:" << '\n'; - { - auto stream = vortex::VortexFile::Open(vortex_file).CreateScanBuilder().WithLimit(3).IntoStream(); - check_stream(stream); - } - // 3. Conditionally set the builder - std::cout << "\n3. Conditionally set the builder:" << '\n'; - { - auto limit = 1; - auto builder = vortex::VortexFile::Open(vortex_file).CreateScanBuilder(); - if (limit > 0) { - // prefer C++ way - builder.WithLimit(1); - // Rusty way is Ok, but you have to move the builder to an rvalue. - // builder = std::move(builder).WithLimit(3); - } - auto stream = std::move(builder).IntoStream(); - check_stream(stream); - } - - // Clean up - std::filesystem::remove(vortex_file); - std::cout << "\nCleaned up temporary file" << '\n'; + Scan scan = ds.scan(ScanOptions{}.limit(3)); + auto partition = scan.next_partition(); + auto array = partition.value().next(); + std::cout << "Read " << array.value().length() << " rows\n"; + fs::remove(out); return 0; -} \ No newline at end of file +} diff --git a/vortex-cxx/examples/scan.cpp b/vortex-cxx/examples/scan.cpp new file mode 100644 index 00000000000..c8866cc57ce --- /dev/null +++ b/vortex-cxx/examples/scan.cpp @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +#include +#include +#include +#include +#include +#include + +using vortex::DataSource; +using vortex::Estimate; +using vortex::EstimateType; +using vortex::Partition; +using vortex::Scan; +using vortex::Session; +using vortex::VortexException; + +static void print_estimate(const char* what, const Estimate& est) { + switch (est.type) { + case EstimateType::Unknown: + std::cout << what << ": unknown\n"; + break; + case EstimateType::Exact: + std::cout << what << ": " << est.value << '\n'; + break; + case EstimateType::Inexact: + std::cout << what << ": approximately " << est.value << '\n'; + break; + } +} + +struct ScanStats { + size_t partitions = 0; + size_t arrays = 0; + size_t rows = 0; +}; + +static ScanStats worker(Scan& scan, std::mutex& mu, size_t worker_id) { + ScanStats stats; + while (true) { + std::optional partition; + { + const std::lock_guard g{mu}; + partition = scan.next_partition(); + } + if (!partition.has_value()) { + break; + } + ++stats.partitions; + + const Estimate rows = partition->row_count(); + std::cout << "Thread " << (worker_id + 1) << " processing partition " + << stats.partitions << ", "; + print_estimate("row count", rows); + + while (auto array = partition->next()) { + ++stats.arrays; + stats.rows += array->length(); + } + } + std::cout << "Thread " << (worker_id + 1) << " finished, processed " + << stats.partitions << " partitions, " << stats.arrays + << " arrays, " << stats.rows << " rows\n"; + return stats; +} + +int main(int argc, char** argv) { + size_t num_threads = 0; + int opt = 0; + while ((opt = getopt(argc, argv, "j:")) != -1) { + switch (opt) { + case 'j': + num_threads = static_cast(std::atoi(optarg)); + break; + default: + std::cerr << "Multi-threaded file scan\nUsage: scan [-j " + "threads] \n"; + return 1; + } + } + if (optind + 1 != argc) { + std::cerr << "Multi-threaded file scan\nUsage: scan [-j threads] \n"; + return 1; + } + const char* paths = argv[optind]; + + const Session session; + std::cout << "Opening files: " << paths << '\n'; + const DataSource ds = DataSource(session, paths); + + print_estimate("Data source row count", ds.row_count()); + + Scan scan = ds.scan(); + print_estimate("Partition count", scan.partition_estimate()); + + if (num_threads == 0) { + const auto est = scan.partition_estimate(); + num_threads = est.type == EstimateType::Unknown ? 1 : est.value; + } + + std::cout << "Starting scan, using " << num_threads << " threads\n"; + std::mutex mu; + std::vector threads; + threads.reserve(num_threads); + std::vector results(num_threads); + + for (size_t i = 0; i < num_threads; ++i) { + threads.emplace_back( + [i, &scan, &mu, &results]() { results[i] = worker(scan, mu, i); }); + } + for (auto& t : threads) { + t.join(); + } + + ScanStats total; + for (const auto& r : results) { + total.partitions += r.partitions; + total.arrays += r.arrays; + total.rows += r.rows; + } + std::cout << "Finished scan, processed " << total.partitions + << " partitions, " << total.arrays << " arrays, " << total.rows + << " rows\n"; + return 0; +} diff --git a/vortex-cxx/examples/scan_to_arrow.cpp b/vortex-cxx/examples/scan_to_arrow.cpp new file mode 100644 index 00000000000..66724212855 --- /dev/null +++ b/vortex-cxx/examples/scan_to_arrow.cpp @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include +#include +#include +#include + +using vortex::ArrowStream; +using vortex::DataSource; +using vortex::DataType; +using vortex::Scan; +using vortex::Session; +using vortex::VortexException; + +int main(int argc, char** argv) { + if (argc != 2) { + std::cerr + << "Scan vortex files to Arrow\nUsage: scan_to_arrow \n"; + return 1; + } + const char* paths = argv[1]; + + Session session; + DataSource ds = DataSource(session, paths); + Scan scan = ds.scan(); + + DataType out_dtype = ds.dtype(); + ArrowSchema schema = out_dtype.ToArrowSchema(); + char schema_buf[10 * 1024]; + const int64_t schema_len = + ArrowSchemaToString(&schema, schema_buf, sizeof schema_buf, 1); + std::cout << "arrow schema: " + << std::string_view{schema_buf, static_cast(schema_len)} + << '\n'; + if (schema.release != nullptr) { + schema.release(&schema); + } + + ArrowError arrow_error{}; + ArrowErrorInit(&arrow_error); + + size_t partition_idx = 0; + while (auto partition = scan.next_partition()) { + ArrowStream stream = std::move(*partition).into_arrow_stream(session); + + size_t rows = 0; + size_t arrays = 0; + ArrowArray array = {}; + while (ArrowArrayStreamGetNext(stream.raw(), &array, &arrow_error) == + NANOARROW_OK && + array.release != nullptr) { + rows += array.length; + ++arrays; + array.release(&array); + std::memset(&array, 0, sizeof(array)); + } + std::cout << "Read partition " << partition_idx << " to Arrow, " + << arrays << " arrays, " << rows << " rows\n"; + ++partition_idx; + } + return 0; +} diff --git a/vortex-cxx/examples/write_sample.cpp b/vortex-cxx/examples/write_sample.cpp new file mode 100644 index 00000000000..725ae7ec5af --- /dev/null +++ b/vortex-cxx/examples/write_sample.cpp @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: CC-BY-4.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include +#include +#include +#include + +#include "../tests/sink.hpp" + +using vortex::Array; +using vortex::DataType; +using vortex::Session; +using vortex::StructColumnBuilder; +using vortex::StructFieldsBuilder; +using vortex::Validity; +using vortex::ValidityType; +using vortex::VortexException; +namespace dtype = vortex::dtype; + +constexpr size_t SAMPLE_ROWS = 200; + +static DataType sample_dtype() { + return dtype::structure(StructFieldsBuilder() + .AddField("age", dtype::uint8()) + .AddField("height", dtype::uint16(true)) + .Finalize()); +} + +static Array sample_array() { + Validity nn{ValidityType::NonNullable}; + Validity av{ValidityType::AllValid}; + + std::vector ages(SAMPLE_ROWS); + std::vector heights(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + ages[i] = static_cast(i); + heights[i] = static_cast(std::rand() % (i + 1)); + } + + Array age(vortex::PType::U8, ages.data(), ages.size(), nn); + Array height(vortex::PType::U16, heights.data(), heights.size(), av); + + return StructColumnBuilder(nn, SAMPLE_ROWS) + .AddField("age", age) + .AddField("height", height) + .Finalize(); +} + +int main(int argc, char** argv) { + if (argc != 2) { + std::cerr << "Usage: write_sample \n"; + return 1; + } + + Session session; + vortex_test::ArraySink sink(session, argv[1], sample_dtype()); + sink.push(sample_array()); + std::move(sink).close(); + return 0; +} diff --git a/vortex-cxx/src/dtype.rs b/vortex-cxx/src/dtype.rs deleted file mode 100644 index 0c125bcf822..00000000000 --- a/vortex-cxx/src/dtype.rs +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::fmt::Display; -use std::fmt::Formatter; - -use anyhow::Result; -use arrow_array::ffi::FFI_ArrowSchema; -use arrow_schema::Field; -use vortex::dtype::DType as RustDType; -use vortex::dtype::DecimalDType; -use vortex::dtype::Nullability; -use vortex::dtype::PType as RustPType; -use vortex::dtype::arrow::FromArrowType; - -use crate::ffi; -pub(crate) struct DType { - pub(crate) inner: RustDType, -} - -pub(crate) fn dtype_null() -> Box { - Box::new(DType { - inner: RustDType::Null, - }) -} - -pub(crate) fn dtype_bool(nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Bool(nullability_from_bool(nullable)), - }) -} - -pub(crate) fn dtype_primitive(ptype: ffi::PType, nullable: bool) -> Box { - let vortex_ptype = match ptype { - ffi::PType::U8 => RustPType::U8, - ffi::PType::U16 => RustPType::U16, - ffi::PType::U32 => RustPType::U32, - ffi::PType::U64 => RustPType::U64, - ffi::PType::I8 => RustPType::I8, - ffi::PType::I16 => RustPType::I16, - ffi::PType::I32 => RustPType::I32, - ffi::PType::I64 => RustPType::I64, - ffi::PType::F16 => RustPType::F16, - ffi::PType::F32 => RustPType::F32, - ffi::PType::F64 => RustPType::F64, - _ => unreachable!(), - }; - Box::new(DType { - inner: RustDType::Primitive(vortex_ptype, nullability_from_bool(nullable)), - }) -} - -pub(crate) fn dtype_decimal(precision: u8, scale: i8, nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Decimal( - DecimalDType::new(precision, scale), - nullability_from_bool(nullable), - ), - }) -} - -pub(crate) fn dtype_utf8(nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Utf8(nullability_from_bool(nullable)), - }) -} - -pub(crate) fn dtype_binary(nullable: bool) -> Box { - Box::new(DType { - inner: RustDType::Binary(nullability_from_bool(nullable)), - }) -} - -impl Display for DType { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{0}", self.inner) - } -} - -pub(crate) fn nullability_from_bool(nullable: bool) -> Nullability { - if nullable { - Nullability::Nullable - } else { - Nullability::NonNullable - } -} - -pub(crate) unsafe fn from_arrow(ffi_schema: *mut u8, non_nullable: bool) -> Result> { - let arrow_schema = unsafe { FFI_ArrowSchema::from_raw(ffi_schema as *mut FFI_ArrowSchema) }; - let arrow_dtype = arrow_schema::DataType::try_from(&arrow_schema)?; - Ok(Box::new(DType { - inner: RustDType::from_arrow(&Field::new("_", arrow_dtype, !non_nullable)), - })) -} diff --git a/vortex-cxx/src/expr.rs b/vortex-cxx/src/expr.rs deleted file mode 100644 index b980db7111b..00000000000 --- a/vortex-cxx/src/expr.rs +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use vortex::dtype::FieldName; -use vortex::expr::Expression; - -use crate::scalar::Scalar; - -pub(crate) struct Expr { - pub(crate) inner: Expression, -} - -pub(crate) fn literal(scalar: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::lit(scalar.inner), - }) -} - -pub(crate) fn root() -> Box { - Box::new(Expr { - inner: vortex::expr::root(), - }) -} - -pub(crate) fn column(name: String) -> Box { - Box::new(Expr { - inner: vortex::expr::get_item(name, vortex::expr::root()), - }) -} - -pub(crate) fn get_item(field: String, child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::get_item(field, child.inner), - }) -} - -pub(crate) fn not_(child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::not(child.inner), - }) -} - -pub(crate) fn is_null(child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::is_null(child.inner), - }) -} - -macro_rules! binary_op { - ($fn_name:ident $(, $suffix:tt)?) => { - paste::paste! { - pub(crate) fn [<$fn_name $($suffix)?>]( - lhs: Box, - rhs: Box, - ) -> Box { - Box::new(Expr { - inner: vortex::expr::$fn_name(lhs.inner, rhs.inner), - }) - } - } - }; -} - -binary_op!(eq); -binary_op!(not_eq, _); -binary_op!(gt); -binary_op!(gt_eq); -binary_op!(lt); -binary_op!(lt_eq); -binary_op!(and, _); -binary_op!(or, _); -binary_op!(checked_add); - -pub(crate) fn select(fields: Vec, child: Box) -> Box { - Box::new(Expr { - inner: vortex::expr::select( - fields.into_iter().map(FieldName::from).collect::>(), - child.inner, - ), - }) -} diff --git a/vortex-cxx/src/lib.rs b/vortex-cxx/src/lib.rs deleted file mode 100644 index b6d002513f6..00000000000 --- a/vortex-cxx/src/lib.rs +++ /dev/null @@ -1,139 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#![allow(clippy::boxed_local)] - -mod dtype; -mod expr; -mod read; -mod scalar; -mod write; - -use std::sync::LazyLock; - -use dtype::*; -use expr::*; -use read::*; -use scalar::*; -use vortex::VortexSessionDefault; -use vortex::io::runtime::BlockingRuntime; -use vortex::io::runtime::current::CurrentThreadRuntime; -use vortex::io::session::RuntimeSessionExt; -use vortex::session::VortexSession; -use write::*; - -/// By default, the C++ API uses a current-thread runtime, providing control of the threading -/// model to the C++ side. -// TODO(ngates): in the future, we could expose an API for C++ to spawn threads that can drive -// this runtime. -pub(crate) static RUNTIME: LazyLock = - LazyLock::new(CurrentThreadRuntime::new); -pub(crate) static SESSION: LazyLock = - LazyLock::new(|| VortexSession::default().with_handle(RUNTIME.handle())); - -#[cxx::bridge(namespace = "vortex::ffi")] -#[allow(let_underscore_drop)] -#[allow(clippy::absolute_paths)] -mod ffi { - extern "Rust" { - type DType; - // Factory functions for creating DType - fn dtype_null() -> Box; - fn dtype_bool(nullable: bool) -> Box; - fn dtype_primitive(ptype: PType, nullable: bool) -> Box; - fn dtype_decimal(precision: u8, scale: i8, nullable: bool) -> Box; - fn dtype_utf8(nullable: bool) -> Box; - fn dtype_binary(nullable: bool) -> Box; - unsafe fn from_arrow(ffi_schema: *mut u8, non_nullable: bool) -> Result>; - // Methods for DType - fn to_string(self: &DType) -> String; - - type Scalar; - fn bool_scalar_new(value: bool) -> Box; - fn i8_scalar_new(value: i8) -> Box; - fn i16_scalar_new(value: i16) -> Box; - fn i32_scalar_new(value: i32) -> Box; - fn i64_scalar_new(value: i64) -> Box; - fn u8_scalar_new(value: u8) -> Box; - fn u16_scalar_new(value: u16) -> Box; - fn u32_scalar_new(value: u32) -> Box; - fn u64_scalar_new(value: u64) -> Box; - fn f32_scalar_new(value: f32) -> Box; - fn f64_scalar_new(value: f64) -> Box; - fn string_scalar_new(value: &str) -> Box; - fn binary_scalar_new(value: &[u8]) -> Box; - fn cast_scalar(self: &Scalar, dtype: &DType) -> Result>; - - type Expr; - fn literal(scalar: Box) -> Box; - fn root() -> Box; - fn column(name: String) -> Box; - fn get_item(field: String, child: Box) -> Box; - fn not_(child: Box) -> Box; - fn is_null(child: Box) -> Box; - // binary op - fn eq(lhs: Box, rhs: Box) -> Box; - fn not_eq_(lhs: Box, rhs: Box) -> Box; - fn gt(lhs: Box, rhs: Box) -> Box; - fn gt_eq(lhs: Box, rhs: Box) -> Box; - fn lt(lhs: Box, rhs: Box) -> Box; - fn lt_eq(lhs: Box, rhs: Box) -> Box; - fn and_(lhs: Box, rhs: Box) -> Box; - fn or_(lhs: Box, rhs: Box) -> Box; - fn checked_add(lhs: Box, rhs: Box) -> Box; - fn select(fields: Vec, child: Box) -> Box; - - type VortexFile; - fn row_count(self: &VortexFile) -> u64; - fn scan_builder(self: &VortexFile) -> Result>; - fn open_file(path: &str) -> Result>; - fn open_file_from_buffer(data: &[u8]) -> Result>; - - type VortexScanBuilder; - fn with_filter(self: &mut VortexScanBuilder, filter: Box); - fn with_filter_ref(self: &mut VortexScanBuilder, filter: &Expr); - fn with_projection(self: &mut VortexScanBuilder, projection: Box); - fn with_projection_ref(self: &mut VortexScanBuilder, projection: &Expr); - fn with_row_range(self: &mut VortexScanBuilder, row_range_start: u64, row_range_end: u64); - fn with_include_by_index(self: &mut VortexScanBuilder, include_by_index: &[u64]); - fn with_limit(self: &mut VortexScanBuilder, limit: usize); - unsafe fn with_output_schema( - self: &mut VortexScanBuilder, - output_schema: *mut u8, - ) -> Result<()>; - unsafe fn scan_builder_into_stream( - builder: Box, - out_stream: *mut u8, - ) -> Result<()>; - fn scan_builder_into_threadsafe_cloneable_reader( - builder: Box, - ) -> Result>; - - type ThreadsafeCloneableReader; - unsafe fn clone_a_stream(self: &ThreadsafeCloneableReader, out_stream: *mut u8); - - type VortexWriteOptions; - fn write_options_new() -> Box; - unsafe fn write_array_stream( - options: Box, - input_stream: *mut u8, - path: &str, - ) -> Result<()>; - } - - #[repr(u8)] - #[derive(Debug, Clone, Copy)] - enum PType { - U8, - U16, - U32, - U64, - I8, - I16, - I32, - I64, - F16, - F32, - F64, - } -} diff --git a/vortex-cxx/src/read.rs b/vortex-cxx/src/read.rs deleted file mode 100644 index 480023084ef..00000000000 --- a/vortex-cxx/src/read.rs +++ /dev/null @@ -1,198 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::sync::Arc; - -use anyhow::Result; -use arrow_array::RecordBatch; -use arrow_array::RecordBatchReader; -use arrow_array::cast::AsArray; -use arrow_array::ffi::FFI_ArrowSchema; -use arrow_array::ffi_stream::FFI_ArrowArrayStream; -use arrow_schema::ArrowError; -use arrow_schema::Field; -use arrow_schema::Schema; -use arrow_schema::SchemaRef; -use futures::stream::TryStreamExt; -use vortex::array::ArrayRef; -use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; -use vortex::array::legacy_session; -use vortex::buffer::Buffer; -use vortex::file::OpenOptionsSessionExt; -use vortex::io::runtime::BlockingRuntime; -use vortex::layout::scan::arrow::RecordBatchIteratorAdapter; -use vortex::layout::scan::scan_builder::ScanBuilder; -use vortex::scan::selection::Selection; - -use crate::RUNTIME; -use crate::SESSION; -use crate::expr::Expr; - -pub(crate) struct VortexFile { - inner: vortex::file::VortexFile, -} - -impl VortexFile { - pub(crate) fn row_count(&self) -> u64 { - self.inner.row_count() - } - - pub(crate) fn scan_builder(&self) -> Result> { - Ok(Box::new(VortexScanBuilder { - inner: self.inner.scan()?, - output_schema: None, - })) - } -} - -/// File operations - using blocking operations for simplicity -/// TODO(xinyu): object store (see vortex-ffi) -pub(crate) fn open_file(path: &str) -> Result> { - let file = RUNTIME.block_on(SESSION.open_options().open_path(std::path::Path::new(path)))?; - Ok(Box::new(VortexFile { inner: file })) -} - -pub(crate) fn open_file_from_buffer(data: &[u8]) -> Result> { - let buffer = Buffer::from(data.to_vec()); - let file = SESSION.open_options().open_buffer(buffer)?; - Ok(Box::new(VortexFile { inner: file })) -} - -pub(crate) struct VortexScanBuilder { - inner: ScanBuilder, - output_schema: Option, -} - -impl VortexScanBuilder { - pub(crate) fn with_filter(&mut self, filter: Box) { - take_mut::take(&mut self.inner, |inner| inner.with_filter(filter.inner)); - } - - pub(crate) fn with_filter_ref(&mut self, filter: &Expr) { - take_mut::take(&mut self.inner, |inner| { - inner.with_filter(filter.inner.clone()) - }); - } - - pub(crate) fn with_projection(&mut self, filter: Box) { - take_mut::take(&mut self.inner, |inner| inner.with_projection(filter.inner)); - } - - pub(crate) fn with_projection_ref(&mut self, filter: &Expr) { - take_mut::take(&mut self.inner, |inner| { - inner.with_projection(filter.inner.clone()) - }); - } - - pub(crate) fn with_row_range(&mut self, row_range_start: u64, row_range_end: u64) { - take_mut::take(&mut self.inner, |inner| { - inner.with_row_range(row_range_start..row_range_end) - }); - } - - pub(crate) fn with_include_by_index(&mut self, include_by_index: &[u64]) { - let selection = Selection::IncludeByIndex(Buffer::copy_from(include_by_index)); - take_mut::take(&mut self.inner, |inner| inner.with_selection(selection)); - } - - pub(crate) fn with_limit(&mut self, limit: usize) { - take_mut::take(&mut self.inner, |inner| inner.with_limit(limit as u64)); - } - - pub(crate) unsafe fn with_output_schema(&mut self, output_schema: *mut u8) -> Result<()> { - let ffi_schema = - unsafe { FFI_ArrowSchema::from_raw(output_schema as *mut FFI_ArrowSchema) }; - self.output_schema = Some(Arc::new(Schema::try_from(&ffi_schema)?)); - Ok(()) - } -} - -/// # Safety -/// -/// out_stream should be properly aligned according to the Arrow C stream interface and valid for write. -pub(crate) unsafe fn scan_builder_into_stream( - builder: Box, - out_stream: *mut u8, -) -> Result<()> { - let schema = match builder.output_schema { - Some(schema) => schema, - None => { - let dtype = builder.inner.dtype()?; - let arrow_schema = dtype.to_arrow_schema()?; - Arc::new(arrow_schema) - } - }; - let reader = builder.inner.into_record_batch_reader(schema, &*RUNTIME)?; - let stream = FFI_ArrowArrayStream::new(Box::new(reader)); - let out_stream = out_stream as *mut FFI_ArrowArrayStream; - // # Safety - // Arrow C stream interface - unsafe { std::ptr::write(out_stream, stream) }; - Ok(()) -} - -trait ThreadsafeCloneableReaderTrait: RecordBatchReader + Send + 'static { - fn clone_boxed(&self) -> Box; -} - -impl ThreadsafeCloneableReaderTrait for T -where - T: RecordBatchReader + Send + Clone + 'static, -{ - fn clone_boxed(&self) -> Box { - Box::new(self.clone()) - } -} - -pub(crate) struct ThreadsafeCloneableReader { - inner: Box, -} - -#[allow(clippy::disallowed_methods)] -pub(crate) fn scan_builder_into_threadsafe_cloneable_reader( - builder: Box, -) -> Result, Box> { - let schema = match builder.output_schema { - Some(schema) => schema, - None => { - let dtype = builder.inner.dtype()?; - let arrow_schema = dtype.to_arrow_schema()?; - Arc::new(arrow_schema) - } - }; - let target = Field::new_struct("", schema.fields.clone(), false); - - let stream = builder - .inner - .map(move |b| { - SESSION - .arrow() - .execute_arrow( - b, - Some(&target), - &mut legacy_session().create_execution_ctx(), - ) - .map(|struct_array| RecordBatch::from(struct_array.as_struct())) - }) - .into_stream()? - .map_err(|e| ArrowError::ExternalError(Box::new(e))); - - let iter = RUNTIME.block_on_stream_thread_safe(|_h| stream); - let rbr = RecordBatchIteratorAdapter::new(iter, schema); - - Ok(Box::new(ThreadsafeCloneableReader { - inner: Box::new(rbr), - })) -} - -impl ThreadsafeCloneableReader { - pub(crate) fn clone_a_stream(&self, out_stream: *mut u8) { - let cloned_reader = self.inner.clone_boxed(); - let stream = FFI_ArrowArrayStream::new(cloned_reader); - let out_stream = out_stream as *mut FFI_ArrowArrayStream; - // # Safety - // Arrow C stream interface - unsafe { std::ptr::write(out_stream, stream) }; - } -} diff --git a/vortex-cxx/src/scalar.rs b/vortex-cxx/src/scalar.rs deleted file mode 100644 index 24cd38be0f1..00000000000 --- a/vortex-cxx/src/scalar.rs +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use anyhow::Result; - -use crate::dtype::DType; - -pub(crate) struct Scalar { - pub(crate) inner: vortex::scalar::Scalar, -} - -macro_rules! primitive_scalar_new { - ($name:ident, $type:ty) => { - pub(crate) fn $name(value: $type) -> Box { - Box::new(Scalar { - inner: vortex::scalar::Scalar::from(value), - }) - } - }; -} - -primitive_scalar_new!(bool_scalar_new, bool); // bool is not primitive but reuse the macro here -primitive_scalar_new!(i8_scalar_new, i8); -primitive_scalar_new!(i16_scalar_new, i16); -primitive_scalar_new!(i32_scalar_new, i32); -primitive_scalar_new!(i64_scalar_new, i64); -primitive_scalar_new!(u8_scalar_new, u8); -primitive_scalar_new!(u16_scalar_new, u16); -primitive_scalar_new!(u32_scalar_new, u32); -primitive_scalar_new!(u64_scalar_new, u64); -primitive_scalar_new!(f32_scalar_new, f32); -primitive_scalar_new!(f64_scalar_new, f64); - -pub(crate) fn string_scalar_new(value: &str) -> Box { - Box::new(Scalar { - inner: vortex::scalar::Scalar::from(value), - }) -} - -pub(crate) fn binary_scalar_new(value: &[u8]) -> Box { - Box::new(Scalar { - inner: vortex::scalar::Scalar::from(value), - }) -} - -impl Scalar { - pub(crate) fn cast_scalar(&self, dtype: &DType) -> Result> { - Ok(Box::new(Scalar { - inner: self.inner.cast(&dtype.inner)?, - })) - } -} diff --git a/vortex-cxx/src/write.rs b/vortex-cxx/src/write.rs deleted file mode 100644 index ff77c5a3dc5..00000000000 --- a/vortex-cxx/src/write.rs +++ /dev/null @@ -1,70 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use anyhow::Result; -use arrow_array::RecordBatchReader; -use arrow_array::ffi_stream::ArrowArrayStreamReader; -use arrow_array::ffi_stream::FFI_ArrowArrayStream; -use vortex::array::ArrayRef; -use vortex::array::arrow::FromArrowArray; -use vortex::array::iter::ArrayIteratorAdapter; -use vortex::array::iter::ArrayIteratorExt; -use vortex::array::stream::ArrayStream; -use vortex::dtype::DType; -use vortex::dtype::arrow::FromArrowType; -use vortex::error::VortexError; -use vortex::file::VortexWriteOptions as WriteOptions; -use vortex::file::WriteOptionsSessionExt; -use vortex::io::VortexWrite; -use vortex::io::runtime::BlockingRuntime; - -use crate::RUNTIME; -use crate::SESSION; - -pub(crate) struct VortexWriteOptions { - inner: WriteOptions, -} - -pub(crate) fn write_options_new() -> Box { - Box::new(VortexWriteOptions { - inner: SESSION.write_options(), - }) -} - -/// Convert an ArrowArrayStreamReader to a Vortex ArrayStream -fn arrow_stream_to_vortex_stream(reader: ArrowArrayStreamReader) -> Result { - let array_iter = ArrayIteratorAdapter::new( - DType::from_arrow(reader.schema()), - reader.map(|result| { - result - .map_err(VortexError::from) - .and_then(|record_batch| ArrayRef::from_arrow(record_batch, false)) - }), - ); - - Ok(array_iter.into_array_stream()) -} - -/// # Safety -/// -/// input_stream should be valid FFI_ArrowArrayStream. -/// See [`FFI_ArrowArrayStream::from_raw`] -pub(crate) unsafe fn write_array_stream( - options: Box, - input_stream: *mut u8, - path: &str, -) -> Result<()> { - let path = path.to_string(); - - let stream_reader = - unsafe { ArrowArrayStreamReader::from_raw(input_stream as *mut FFI_ArrowArrayStream) }?; - - let vortex_stream = arrow_stream_to_vortex_stream(stream_reader)?; - - RUNTIME.block_on(async { - let mut file = async_fs::File::create(path).await?; - options.inner.write(&mut file, vortex_stream).await?; - file.shutdown().await?; - Ok(()) - }) -} diff --git a/vortex-cxx/tests/.clang-tidy b/vortex-cxx/tests/.clang-tidy new file mode 100644 index 00000000000..ffb9e2daf91 --- /dev/null +++ b/vortex-cxx/tests/.clang-tidy @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +InheritParentConfig: true +Checks: | + -google-build-using-namespace, + -performance-unnecessary-copy-initialization, + -bugprone-unchecked-optional-access, diff --git a/vortex-cxx/tests/CMakeLists.txt b/vortex-cxx/tests/CMakeLists.txt new file mode 100644 index 00000000000..c9465609677 --- /dev/null +++ b/vortex-cxx/tests/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors +FetchContent_Declare( + magic_enum + GIT_REPOSITORY https://github.com/Neargye/magic_enum.git + GIT_TAG v0.9.7 +) +FetchContent_MakeAvailable(magic_enum) + +file(GLOB TEST_FILES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp") + +add_executable(vortex_cxx_test ${TEST_FILES}) +target_include_directories(vortex_cxx_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(vortex_cxx_test PRIVATE + vortex_cxx_shared + Catch2::Catch2WithMain + magic_enum::magic_enum + nanoarrow_shared) + +if(APPLE) + target_link_libraries(vortex_cxx_test PRIVATE + "-framework CoreFoundation" "-framework Security") +endif() + +catch_discover_tests(vortex_cxx_test) diff --git a/vortex-cxx/tests/common.hpp b/vortex-cxx/tests/common.hpp new file mode 100644 index 00000000000..11d8fe85ffe --- /dev/null +++ b/vortex-cxx/tests/common.hpp @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include +#include +#include +#include +#include + +#include "sink.hpp" + +namespace vortex_test { + +namespace fs = std::filesystem; +using namespace vortex; + +class TempPath { +public: + TempPath() = default; + explicit TempPath(fs::path p) : path_(std::move(p)) {} + + TempPath(const TempPath&) = delete; + TempPath& operator=(const TempPath&) = delete; + + TempPath(TempPath&& other) noexcept : path_(std::move(other.path_)) { + other.path_.clear(); + } + TempPath& operator=(TempPath&& other) noexcept { + if (this != &other) { + reset(); + path_ = std::move(other.path_); + other.path_.clear(); + } + return *this; + } + + ~TempPath() { reset(); } + + const fs::path& path() const noexcept { return path_; } + const char* c_str() const noexcept { return path_.c_str(); } + + static TempPath Unique(const std::string& suffix = ".vortex") { + auto dir = fs::temp_directory_path() / "vortex_cxx_test"; + fs::create_directories(dir); + std::string name = + "t_" + std::to_string(std::random_device{}()) + suffix; + return TempPath{dir / name}; + } + +private: + void reset() noexcept { + if (!path_.empty()) { + std::error_code ec; + fs::remove(path_, ec); + } + } + + fs::path path_; +}; + +inline DataType sample_dtype() { + return dtype::structure(StructFieldsBuilder() + .AddField("age", dtype::uint8()) + .AddField("height", dtype::uint16(true)) + .Finalize()); +} + +constexpr size_t SAMPLE_ROWS = 100; + +inline std::vector sample_ages() { + std::vector buf(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + buf[i] = static_cast(i); + } + return buf; +} + +inline std::vector sample_heights() { + std::vector buf(SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + buf[i] = static_cast((i + 1) % 200); + } + return buf; +} + +inline Array sample_array() { + Validity nn{ValidityType::NonNullable}; + Validity av{ValidityType::AllValid}; + + auto ages = sample_ages(); + auto heights = sample_heights(); + + Array age_arr(vortex::PType::U8, ages.data(), ages.size(), nn); + Array height_arr(vortex::PType::U16, heights.data(), heights.size(), av); + + return StructColumnBuilder(nn, SAMPLE_ROWS) + .AddField("age", age_arr) + .AddField("height", height_arr) + .Finalize(); +} + +inline TempPath write_sample(const Session& session) { + TempPath path = TempPath::Unique(); + ArraySink sink(session, path.path().string(), sample_dtype()); + sink.push(sample_array()); + std::move(sink).close(); + return path; +} +} // namespace vortex_test diff --git a/vortex-cxx/tests/sink.hpp b/vortex-cxx/tests/sink.hpp new file mode 100644 index 00000000000..34c117d43f7 --- /dev/null +++ b/vortex-cxx/tests/sink.hpp @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include +#include +#include + +namespace vortex_test { + +class ArraySink { +public: + ArraySink(const vortex::Session& session, const std::string& path, + const vortex::DataType& dtype) { + vx_error* err = nullptr; + vx_array_sink* sink = vx_array_sink_open_file( + vortex::Access::c_ptr(session), path.c_str(), + vortex::Access::c_ptr(dtype), &err); + vortex::CheckError(err); + impl_.reset(sink); + } + + ArraySink(const ArraySink&) = delete; + ArraySink& operator=(const ArraySink&) = delete; + ArraySink(ArraySink&&) noexcept = default; + ArraySink& operator=(ArraySink&&) noexcept = default; + + void push(const vortex::Array& array) { + vx_error* err = nullptr; + vx_array_sink_push(impl_.get(), vortex::Access::c_ptr(array), &err); + vortex::CheckError(err); + } + + void close() && { + vx_error* err = nullptr; + vx_array_sink* sink = impl_.release(); + vx_array_sink_close(sink, &err); + vortex::CheckError(err); + } + +private: + struct Deleter { + void operator()(vx_array_sink* ptr) const noexcept { + if (ptr == nullptr) { + return; + } + vx_error* err = nullptr; + vx_array_sink_close(ptr, &err); + if (err != nullptr) { + vx_error_free(err); + } + } + }; + std::unique_ptr impl_; +}; + +} // namespace vortex_test diff --git a/vortex-cxx/tests/test_array.cpp b/vortex-cxx/tests/test_array.cpp new file mode 100644 index 00000000000..31b42305e6a --- /dev/null +++ b/vortex-cxx/tests/test_array.cpp @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include + +using namespace vortex; + +TEST_CASE("Null array creation", "[array]") { + Array array(1999UL); + REQUIRE(array.nullable()); + REQUIRE(array.has_dtype(DataTypeVariant::Null)); + REQUIRE(array.length() == 1999); +} + +TEST_CASE("Primitive array creation", "[array]") { + std::vector buffer(20, 1); + buffer[3] = 8; + + Validity validity{ValidityType::AllValid}; + + Array array(vortex::PType::U8, buffer.data(), buffer.size(), validity); + REQUIRE(array.has_dtype(DataTypeVariant::Primitive)); + REQUIRE(array.is_primitive(vortex::PType::U8)); + REQUIRE(array.length() == buffer.size()); + for (size_t i = 0; i < buffer.size(); ++i) { + REQUIRE(array.get_u8(i) == buffer[i]); + } + buffer.assign(buffer.size(), 0); + REQUIRE(array.get_u8(3) == 8); +} + +TEST_CASE("Struct column builder", "[array]") { + Validity nn{ValidityType::NonNullable}; + + Array field(size_t{5}); + StructColumnBuilder builder(nn, 2); + builder.AddField("age", field); + + SECTION("dropped without finalize") {} + + SECTION("finalized") { + Array struct_array = std::move(builder).Finalize(); + REQUIRE(struct_array.length() == 5); + REQUIRE(struct_array.has_dtype(DataTypeVariant::Struct)); + } +} + +TEST_CASE("Array is arc-cloneable", "[array]") { + Array a(size_t{4}); + Array b = a; + REQUIRE(a.length() == 4); + REQUIRE(b.length() == 4); +} diff --git a/vortex-cxx/tests/test_arrow.cpp b/vortex-cxx/tests/test_arrow.cpp new file mode 100644 index 00000000000..794683cecd8 --- /dev/null +++ b/vortex-cxx/tests/test_arrow.cpp @@ -0,0 +1,98 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include + +#include +#include + +typedef struct ArrowSchema FFI_ArrowSchema; +typedef struct ArrowArray FFI_ArrowArray; +typedef struct ArrowArrayStream FFI_ArrowArrayStream; +#define USE_OWN_ARROW 1 + +#include + +#include "common.hpp" + +using namespace vortex; +using vortex_test::sample_dtype; +using vortex_test::TempPath; +using vortex_test::write_sample; + +TEST_CASE("dtype to ArrowSchema", "[arrow]") { + DataType d = sample_dtype(); + ArrowSchema schema = d.ToArrowSchema(); + + nanoarrow::UniqueSchema unique_schema; + ArrowSchemaMove(&schema, unique_schema.get()); + REQUIRE(unique_schema->format != nullptr); + REQUIRE(unique_schema->n_children == 2); +} + +TEST_CASE("Import Arrow array as Vortex array", "[arrow]") { + nanoarrow::UniqueSchema schema; + REQUIRE(ArrowSchemaInitFromType(schema.get(), NANOARROW_TYPE_STRUCT) == + NANOARROW_OK); + REQUIRE(ArrowSchemaAllocateChildren(schema.get(), 1) == NANOARROW_OK); + REQUIRE(ArrowSchemaInitFromType(schema->children[0], + NANOARROW_TYPE_INT32) == NANOARROW_OK); + REQUIRE(ArrowSchemaSetName(schema->children[0], "a") == NANOARROW_OK); + + nanoarrow::UniqueArray arr; + REQUIRE(ArrowArrayInitFromSchema(arr.get(), schema.get(), nullptr) == + NANOARROW_OK); + REQUIRE(ArrowArrayStartAppending(arr.get()) == NANOARROW_OK); + for (int i : {10, 20, 30}) { + REQUIRE(ArrowArrayAppendInt(arr->children[0], i) == NANOARROW_OK); + REQUIRE(ArrowArrayFinishElement(arr.get()) == NANOARROW_OK); + } + REQUIRE(ArrowArrayFinishBuildingDefault(arr.get(), nullptr) == + NANOARROW_OK); + + ArrowArray raw_arr = {}; + ArrowSchema raw_schema = {}; + ArrowArrayMove(arr.get(), &raw_arr); + ArrowSchemaMove(schema.get(), &raw_schema); + + Array vx(&raw_arr, &raw_schema, false); + REQUIRE(vx.length() == 3); + REQUIRE(vx.has_dtype(DataTypeVariant::Struct)); + + Array a = vx.field(0); + REQUIRE(a.is_primitive(vortex::PType::I32)); + REQUIRE(a.get_i32(0) == 10); + REQUIRE(a.get_i32(2) == 30); +} + +TEST_CASE("Scan partition to ArrowArrayStream", "[arrow]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource(session, path.path().string()); + + Scan scan = ds.scan(); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + + ArrowStream vx_stream = + std::move(partition.value()).into_arrow_stream(session); + nanoarrow::UniqueArrayStream owned; + ArrowArrayStreamMove(vx_stream.raw(), owned.get()); + + nanoarrow::UniqueSchema schema; + ArrowError err{}; + REQUIRE(ArrowArrayStreamGetSchema(owned.get(), schema.get(), &err) == + NANOARROW_OK); + REQUIRE(schema->n_children == 2); + + size_t rows = 0; + while (true) { + nanoarrow::UniqueArray chunk; + int rc = owned->get_next(owned.get(), chunk.get()); + REQUIRE(rc == NANOARROW_OK); + if (chunk->release == nullptr) { + break; + } + rows += chunk->length; + } + REQUIRE(rows == vortex_test::SAMPLE_ROWS); +} diff --git a/vortex-cxx/tests/test_dtype.cpp b/vortex-cxx/tests/test_dtype.cpp new file mode 100644 index 00000000000..f5b6815bada --- /dev/null +++ b/vortex-cxx/tests/test_dtype.cpp @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include + +using namespace vortex; + +TEST_CASE("DataType Null", "[dtype]") { + auto d = dtype::null(); + REQUIRE(d.variant() == DataTypeVariant::Null); + REQUIRE(d.nullable()); +} + +TEST_CASE("DataType Decimal", "[dtype]") { + auto d = dtype::decimal(5, 2, false); + REQUIRE(d.variant() == DataTypeVariant::Decimal); + REQUIRE(d.decimal_precision() == 5); + REQUIRE(d.decimal_scale() == 2); + REQUIRE_FALSE(d.nullable()); + + REQUIRE_THROWS_AS(d.num_fields(), VortexException); + REQUIRE_THROWS_AS(d.list_element(), VortexException); +} + +TEST_CASE("DataType copy", "[dtype]") { + auto d = dtype::int32(true); + DataType d2 = d; + REQUIRE(d2.variant() == DataTypeVariant::Primitive); + REQUIRE(d2.primitive_ptype() == vortex::PType::I32); + REQUIRE(d2.nullable()); + REQUIRE(d.variant() == DataTypeVariant::Primitive); + REQUIRE(d2.variant() == DataTypeVariant::Primitive); +} + +TEST_CASE("Struct fields builder", "[struct]") { + StructFieldsBuilder b; + b.AddField("col1", dtype::uint8()); + b.AddField("col2", dtype::binary(true)); +} + +TEST_CASE("Struct fields builder finalize", "[struct]") { + StructFields fields = StructFieldsBuilder() + .AddField("col1", dtype::uint8()) + .AddField("col2", dtype::binary(true)) + .Finalize(); + + REQUIRE(fields.num_fields() == 2); + REQUIRE(fields.field_name(0) == "col1"); + REQUIRE(fields.field_name(1) == "col2"); + + auto d0 = fields.field_dtype(0); + REQUIRE_FALSE(d0.nullable()); + REQUIRE(d0.variant() == DataTypeVariant::Primitive); + REQUIRE(d0.primitive_ptype() == vortex::PType::U8); + + auto d1 = fields.field_dtype(1); + REQUIRE(d1.nullable()); + REQUIRE(d1.variant() == DataTypeVariant::Binary); +} + +TEST_CASE("Struct dtype construction", "[struct]") { + DataType d = dtype::structure( + StructFieldsBuilder().AddField("a", dtype::int32()).Finalize()); + REQUIRE(d.variant() == DataTypeVariant::Struct); +} + +TEST_CASE("Every dtype factory produces the right variant", "[dtype]") { + REQUIRE(dtype::boolean().variant() == DataTypeVariant::Bool); + REQUIRE(dtype::utf8().variant() == DataTypeVariant::Utf8); + REQUIRE(dtype::binary().variant() == DataTypeVariant::Binary); + REQUIRE(dtype::int8().variant() == DataTypeVariant::Primitive); + REQUIRE(dtype::int8().primitive_ptype() == vortex::PType::I8); + REQUIRE(dtype::int16().primitive_ptype() == vortex::PType::I16); + REQUIRE(dtype::int32().primitive_ptype() == vortex::PType::I32); + REQUIRE(dtype::int64().primitive_ptype() == vortex::PType::I64); + REQUIRE(dtype::uint8().primitive_ptype() == vortex::PType::U8); + REQUIRE(dtype::uint16().primitive_ptype() == vortex::PType::U16); + REQUIRE(dtype::uint32().primitive_ptype() == vortex::PType::U32); + REQUIRE(dtype::uint64().primitive_ptype() == vortex::PType::U64); + REQUIRE(dtype::float16().primitive_ptype() == vortex::PType::F16); + REQUIRE(dtype::float32().primitive_ptype() == vortex::PType::F32); + REQUIRE(dtype::float64().primitive_ptype() == vortex::PType::F64); +} + +TEST_CASE("Nullable primitives", "[dtype]") { + REQUIRE_FALSE(dtype::int32(false).nullable()); + REQUIRE(dtype::int32(true).nullable()); + REQUIRE_FALSE(dtype::utf8().nullable()); + REQUIRE(dtype::utf8(true).nullable()); +} + +TEST_CASE("List dtype", "[dtype]") { + DataType inner = dtype::int32(); + DataType lst = dtype::list(std::move(inner), true); + REQUIRE(lst.variant() == DataTypeVariant::List); + REQUIRE(lst.nullable()); + DataType elem = lst.list_element(); + REQUIRE(elem.variant() == DataTypeVariant::Primitive); + REQUIRE(elem.primitive_ptype() == vortex::PType::I32); +} + +TEST_CASE("Fixed-size list dtype", "[dtype]") { + DataType lst = dtype::fixed_size_list(dtype::float64(), 4); + REQUIRE(lst.variant() == DataTypeVariant::FixedSizeList); + REQUIRE(lst.list_element().primitive_ptype() == vortex::PType::F64); +} + +TEST_CASE("Struct introspection", "[dtype]") { + DataType d = dtype::structure(StructFieldsBuilder() + .AddField("a", dtype::int64()) + .AddField("b", dtype::utf8(true)) + .Finalize()); + REQUIRE(d.num_fields() == 2); + REQUIRE(d.field_name(0) == "a"); + REQUIRE(d.field_name(1) == "b"); + REQUIRE(d.field_dtype(0).primitive_ptype() == vortex::PType::I64); + REQUIRE(d.field_dtype(1).variant() == DataTypeVariant::Utf8); + REQUIRE(d.field_dtype(1).nullable()); +} diff --git a/vortex-cxx/tests/test_enum_sizes.cpp b/vortex-cxx/tests/test_enum_sizes.cpp new file mode 100644 index 00000000000..2afe8332fdf --- /dev/null +++ b/vortex-cxx/tests/test_enum_sizes.cpp @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include + +using magic_enum::enum_count; + +static_assert(enum_count() == + enum_count()); +static_assert(enum_count() == + enum_count()); +static_assert(enum_count() == enum_count()); +static_assert(enum_count() == + enum_count()); +static_assert(enum_count() == + enum_count()); diff --git a/vortex-cxx/tests/test_expression.cpp b/vortex-cxx/tests/test_expression.cpp new file mode 100644 index 00000000000..97b9a8e8a94 --- /dev/null +++ b/vortex-cxx/tests/test_expression.cpp @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include + +using namespace vortex; + +TEST_CASE("Expression clone", "[expr]") { + Expression clone = expr::root(); + { + Expression orig = expr::column("age"); + clone = orig; + } + Expression another = clone; + (void)another; + SUCCEED(); +} + +TEST_CASE("Apply root()", "[expr]") { + std::vector data = {10, 20, 30, 40, 50}; + Validity nn{ValidityType::NonNullable}; + Array array(vortex::PType::I32, data.data(), data.size(), nn); + + Array applied = array.apply(expr::root()); + + REQUIRE(applied.length() == data.size()); + REQUIRE(applied.has_dtype(DataTypeVariant::Primitive)); + REQUIRE(applied.is_primitive(vortex::PType::I32)); + for (size_t i = 0; i < data.size(); ++i) { + REQUIRE(applied.get_i32(i) == data[i]); + } +} + +TEST_CASE("Apply projection", "[expr]") { + Validity nn{ValidityType::NonNullable}; + + std::vector ages = {10, 20, 30}; + std::vector heights = {150, 160, 170}; + + Array age_arr(vortex::PType::U8, ages.data(), ages.size(), nn); + Array height_arr(vortex::PType::U16, heights.data(), heights.size(), nn); + + Array struct_arr = StructColumnBuilder(nn, 2) + .AddField("age", age_arr) + .AddField("height", height_arr) + .Finalize(); + + Array projected = struct_arr.apply(expr::column("age")); + REQUIRE(projected.length() == ages.size()); + REQUIRE(projected.is_primitive(vortex::PType::U8)); + for (size_t i = 0; i < ages.size(); ++i) { + REQUIRE(projected.get_u8(i) == ages[i]); + } +} + +TEST_CASE("Apply arithmetic", "[expr]") { + std::vector data = {1, 2, 3, 4, 5}; + Validity nn{ValidityType::NonNullable}; + Array array(vortex::PType::I32, data.data(), data.size(), nn); + + Expression e = expr::add(expr::root(), expr::literal(scalar::int32(10))); + Array applied = array.apply(e); + + REQUIRE(applied.length() == data.size()); + REQUIRE(applied.is_primitive(vortex::PType::I32)); + for (size_t i = 0; i < data.size(); ++i) { + REQUIRE(applied.get_i32(i) == data[i] + 10); + } +} + +TEST_CASE("Apply error", "[expr]") { + std::vector data = {1, 2, 3}; + Validity nn{ValidityType::NonNullable}; + Array array(vortex::PType::U8, data.data(), data.size(), nn); + + Expression bad = expr::add(expr::root(), expr::literal(scalar::int32(1))); + REQUIRE_THROWS_AS(array.apply(bad), VortexException); +} diff --git a/vortex-cxx/tests/test_scalar.cpp b/vortex-cxx/tests/test_scalar.cpp new file mode 100644 index 00000000000..5023a1e7e23 --- /dev/null +++ b/vortex-cxx/tests/test_scalar.cpp @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include + +using namespace vortex; + +TEST_CASE("Boolean scalar", "[scalar]") { + Scalar s = scalar::boolean(true); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Bool); +} + +TEST_CASE("Integer scalars", "[scalar]") { + REQUIRE(scalar::uint8(42).dtype().primitive_ptype() == vortex::PType::U8); + REQUIRE(scalar::uint16(42).dtype().primitive_ptype() == vortex::PType::U16); + REQUIRE(scalar::uint32(42).dtype().primitive_ptype() == vortex::PType::U32); + REQUIRE(scalar::uint64(42).dtype().primitive_ptype() == vortex::PType::U64); + REQUIRE(scalar::int8(-1).dtype().primitive_ptype() == vortex::PType::I8); + REQUIRE(scalar::int16(-1).dtype().primitive_ptype() == vortex::PType::I16); + REQUIRE(scalar::int32(-1).dtype().primitive_ptype() == vortex::PType::I32); + REQUIRE(scalar::int64(-1).dtype().primitive_ptype() == vortex::PType::I64); +} + +TEST_CASE("Float scalars", "[scalar]") { + REQUIRE(scalar::float32(1.5F).dtype().primitive_ptype() == + vortex::PType::F32); + REQUIRE(scalar::float64(1.5).dtype().primitive_ptype() == + vortex::PType::F64); + REQUIRE(scalar::float16_bits(0x3C00).dtype().primitive_ptype() == + vortex::PType::F16); +} + +TEST_CASE("Nullable scalars", "[scalar]") { + Scalar s = scalar::int32(0, true); + REQUIRE(s.dtype().nullable()); + REQUIRE_FALSE(s.is_null()); +} + +TEST_CASE("Null scalar", "[scalar]") { + Scalar s = scalar::null(dtype::int32(true)); + REQUIRE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Primitive); +} + +TEST_CASE("UTF-8 scalar", "[scalar]") { + Scalar s = scalar::utf8("hello"); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Utf8); + + REQUIRE_FALSE(scalar::utf8("").is_null()); + s = scalar::utf8("Широкая строка"); + REQUIRE(s.dtype().variant() == DataTypeVariant::Utf8); +} + +TEST_CASE("Binary scalar", "[scalar]") { + const uint8_t bytes[] = {1, 2, 3, 0, 4}; + Scalar s = scalar::binary(bytes, sizeof bytes); + REQUIRE_FALSE(s.is_null()); + REQUIRE(s.dtype().variant() == DataTypeVariant::Binary); +} + +TEST_CASE("Decimal scalars", "[scalar]") { + Scalar d32 = scalar::decimal_i32(1234, 5, 2); + REQUIRE(d32.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d32.dtype().decimal_precision() == 5); + REQUIRE(d32.dtype().decimal_scale() == 2); + + Scalar d64 = scalar::decimal_i64(99999, 12, 3); + REQUIRE(d64.dtype().variant() == DataTypeVariant::Decimal); + REQUIRE(d64.dtype().decimal_precision() == 12); + REQUIRE(d64.dtype().decimal_scale() == 3); +} + +TEST_CASE("Scalar clone", "[scalar]") { + Scalar a = scalar::int64(42); + Scalar b = a; + REQUIRE_FALSE(a.is_null()); + REQUIRE_FALSE(b.is_null()); + REQUIRE(a.dtype().primitive_ptype() == vortex::PType::I64); + REQUIRE(b.dtype().primitive_ptype() == vortex::PType::I64); +} + +TEST_CASE("Scalar assign", "[scalar]") { + Scalar a = scalar::int32(1); + Scalar b = scalar::int32(2); + b = a; + REQUIRE(b.dtype().primitive_ptype() == vortex::PType::I32); +} diff --git a/vortex-cxx/tests/test_scan.cpp b/vortex-cxx/tests/test_scan.cpp new file mode 100644 index 00000000000..00c32673297 --- /dev/null +++ b/vortex-cxx/tests/test_scan.cpp @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.hpp" + +using namespace vortex; +using Catch::Matchers::ContainsSubstring; +using vortex_test::SAMPLE_ROWS; +using vortex_test::TempPath; +using vortex_test::write_sample; + +TEST_CASE("Non-existent data source", "[datasource]") { + Session session; + + REQUIRE_THROWS_AS(DataSource(session, "nonexistent"), VortexException); + REQUIRE_THROWS_AS(DataSource(session, nullptr, 0), VortexException); +} + +TEST_CASE("Bad glob", "[datasource]") { + Session session; + try { + DataSource ds(session, "nonexistent"); + (void)ds; + FAIL("expected exception"); + } catch (const VortexException& e) { + REQUIRE_THAT(std::string(e.what()), + ContainsSubstring("No files matched")); + } +} + +TEST_CASE("Read dtype", "[datasource]") { + Session session; + TempPath path = write_sample(session); + + DataSource ds = DataSource(session, path.path().string()); + + Estimate row_count = ds.row_count(); + REQUIRE(row_count.type == EstimateType::Exact); + REQUIRE(row_count.value == SAMPLE_ROWS); + + DataType dt = ds.dtype(); + REQUIRE(dt.variant() == DataTypeVariant::Struct); + REQUIRE(dt.num_fields() == 2); + REQUIRE(dt.field_name(0) == "age"); + REQUIRE(dt.field_name(1) == "height"); +} + +static void verify_age_field(const Array& age) { + REQUIRE(age.has_dtype(DataTypeVariant::Primitive)); + REQUIRE(age.is_primitive(vortex::PType::U8)); + REQUIRE(age.length() == SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + REQUIRE(age.get_u8(i) == static_cast(i)); + } +} + +static void verify_height_field(const Array& height) { + REQUIRE(height.has_dtype(DataTypeVariant::Primitive)); + REQUIRE(height.is_primitive(vortex::PType::U16)); + REQUIRE(height.length() == SAMPLE_ROWS); + for (size_t i = 0; i < SAMPLE_ROWS; ++i) { + REQUIRE(height.get_u16(i) > 0); + } +} + +static void verify_sample_array(const Array& array) { + REQUIRE(array.length() == SAMPLE_ROWS); + REQUIRE(array.has_dtype(DataTypeVariant::Struct)); + + verify_age_field(array.field(0)); + verify_height_field(array.field(1)); + + REQUIRE_THROWS_AS(array.field(2), VortexException); +} + +TEST_CASE("Basic scan", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource(session, path.path().string()); + + Scan scan = ds.scan(); + REQUIRE(scan.partition_estimate().type == EstimateType::Exact); + REQUIRE(scan.partition_estimate().value == 1); + + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + + auto rows = partition.value().row_count(); + REQUIRE(rows.type == EstimateType::Exact); + REQUIRE(rows.value == SAMPLE_ROWS); + + REQUIRE_FALSE(scan.next_partition().has_value()); + + auto array = partition.value().next(); + REQUIRE(array.has_value()); + REQUIRE_FALSE(partition.value().next().has_value()); + + verify_sample_array(*array); +} + +TEST_CASE("Basic scan from buffer", "[scan]") { + Session session; + TempPath path = write_sample(session); + + std::ifstream file(path.path(), std::ios::binary | std::ios::ate); + const auto size = file.tellg(); + file.seekg(0, std::ios::beg); + std::vector buffer(static_cast(size)); + REQUIRE(file.read(buffer.data(), size)); + + DataSource ds = DataSource(session, buffer.data(), buffer.size()); + Scan scan = ds.scan(); + auto p = scan.next_partition(); + REQUIRE(p.has_value()); + auto array = p.value().next(); + REQUIRE(array.has_value()); + verify_sample_array(*array); +} + +TEST_CASE("Multithreaded scan", "[scan]") { + Session session; + + constexpr size_t NUM_FILES = 10; + std::vector paths; + paths.reserve(NUM_FILES); + std::string joined; + for (size_t i = 0; i < NUM_FILES; ++i) { + paths.push_back(write_sample(session)); + if (i > 0) { + joined += ","; + } + joined += paths.back().path().string(); + } + + DataSource ds = DataSource(session, joined); + + Scan scan = ds.scan(); + REQUIRE(scan.partition_estimate().type == EstimateType::Inexact); + REQUIRE(scan.partition_estimate().value == NUM_FILES); + + std::mutex mu; + std::vector threads; + threads.reserve(NUM_FILES); + std::vector> arrays(NUM_FILES); + + for (size_t i = 0; i < NUM_FILES; ++i) { + threads.emplace_back([&, i] { + std::optional partition; + { + std::lock_guard g{mu}; + partition = scan.next_partition(); + } + REQUIRE(partition.has_value()); + + auto rc = partition.value().row_count(); + REQUIRE(rc.type == EstimateType::Exact); + REQUIRE(rc.value == SAMPLE_ROWS); + + arrays[i] = partition.value().next(); + }); + } + + for (auto& t : threads) { + t.join(); + } + + REQUIRE_FALSE(scan.next_partition().has_value()); + + for (const auto& a : arrays) { + REQUIRE(a.has_value()); + verify_sample_array(*a); + } +} + +TEST_CASE("Project field", "[projection]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource(session, path.path().string()); + + ScanOptions opts; + opts.projection(expr::column("age")); + + Scan scan = ds.scan(opts); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition.value().next(); + REQUIRE(array.has_value()); + verify_age_field(*array); +} + +TEST_CASE("Filter age", "[filter]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource(session, path.path().string()); + + constexpr uint8_t threshold = 50; + ScanOptions opts; + opts.filter(expr::gt_eq(expr::column("age"), + expr::literal(scalar::uint8(threshold)))); + + Scan scan = ds.scan(opts); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition.value().next(); + REQUIRE(array.has_value()); + + REQUIRE(array.value().length() == SAMPLE_ROWS - threshold); + Array age = array.value().field(0); + for (size_t i = 0; i < age.length(); ++i) { + REQUIRE(age.get_u8(i) == threshold + i); + } +} + +TEST_CASE("Broken filter", "[filter]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource(session, path.path().string()); + + ScanOptions opts; + opts.filter( + expr::eq(expr::column("age"), expr::literal(scalar::int32(67)))); + + Scan scan = ds.scan(opts); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + + REQUIRE_THROWS_AS(partition.value().next(), VortexException); +} + +TEST_CASE("Row range and limit", "[scan]") { + Session session; + TempPath path = write_sample(session); + DataSource ds = DataSource(session, path.path().string()); + + ScanOptions opts; + opts.row_range(10, 25).limit(5); + + Scan scan = ds.scan(opts); + auto partition = scan.next_partition(); + REQUIRE(partition.has_value()); + auto array = partition.value().next(); + REQUIRE(array.has_value()); + + REQUIRE(array.value().length() == 5); + Array age = array.value().field(0); + for (size_t i = 0; i < age.length(); ++i) { + REQUIRE(age.get_u8(i) == 10 + i); + } +} diff --git a/vortex-cxx/tests/test_string_binary.cpp b/vortex-cxx/tests/test_string_binary.cpp new file mode 100644 index 00000000000..c448fa63106 --- /dev/null +++ b/vortex-cxx/tests/test_string_binary.cpp @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include + +using namespace vortex; +using namespace std::string_view_literals; + +TEST_CASE("String", "[string]") { + for (std::string_view input : + {"ololo"sv, "Широкая строка"sv, "مرحبا بالعالم"sv}) { + String s{input}; + REQUIRE(s.view() == input); + REQUIRE(s.size() == input.size()); + + String copy = s; + REQUIRE(copy.view() == input); + } +} + +TEST_CASE("Binary", "[binary]") { + // Include a NUL byte to test we never use strlen + constexpr std::string_view data = std::string_view{"abc\0def", 7}; + Binary b{data}; + REQUIRE(b.size() == data.size()); + REQUIRE(std::string_view{b.data(), b.size()} == data); + + Binary c = b; + REQUIRE(c.view() == data); +} diff --git a/vortex-cxx/vortex.cpp b/vortex-cxx/vortex.cpp new file mode 100644 index 00000000000..443681a5124 --- /dev/null +++ b/vortex-cxx/vortex.cpp @@ -0,0 +1,917 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "vortex.hpp" + +#include + +#include +#include +#include + +namespace vortex { + +// Non-owning view over borrowed vx_string +static inline std::string_view as_view(const vx_string* s) noexcept { + return {vx_string_ptr(s), vx_string_len(s)}; +} + +void CheckError(vx_error* error) { + if (error == nullptr) { + return; + } + const vx_string* msg = vx_error_get_message(error); + const std::string message{vx_string_ptr(msg), vx_string_len(msg)}; + vx_error_free(error); + throw VortexException(message); +} + +void String::Deleter::operator()(const vx_string* ptr) const noexcept { + vx_string_free(ptr); +} + +String::String(std::string_view s) : impl_(vx_string_new(s.data(), s.size())) {} + +String::String(const String& other) + : impl_(vx_string_clone(other.impl_.get())) {} + +String& String::operator=(const String& other) { + if (this != &other) { + impl_.reset(vx_string_clone(other.impl_.get())); + } + return *this; +} + +std::string_view String::view() const { + const vx_string* p = impl_.get(); + return {vx_string_ptr(p), vx_string_len(p)}; +} + +size_t String::size() const { return vx_string_len(impl_.get()); } + +void Binary::Deleter::operator()(const vx_binary* ptr) const noexcept { + vx_binary_free(ptr); +} + +Binary::Binary(const char* data, size_t len) + : impl_(vx_binary_new(data, len)) {} + +Binary::Binary(const Binary& other) + : impl_(vx_binary_clone(other.impl_.get())) {} + +Binary& Binary::operator=(const Binary& other) { + if (this != &other) { + impl_.reset(vx_binary_clone(other.impl_.get())); + } + return *this; +} + +size_t Binary::size() const { return vx_binary_len(impl_.get()); } + +const char* Binary::data() const { return vx_binary_ptr(impl_.get()); } + +std::string_view Binary::view() const { + const vx_binary* p = impl_.get(); + return {vx_binary_ptr(p), vx_binary_len(p)}; +} + +void Session::Deleter::operator()(vx_session* ptr) const noexcept { + vx_session_free(ptr); +} + +Session::Session() : impl_(vx_session_new()) { + if (impl_ == nullptr) { + throw VortexException("failed to create vortex session"); + } +} + +Session::Session(const Session& other) + : impl_(vx_session_clone(other.impl_.get())) {} + +Session& Session::operator=(const Session& other) { + if (this != &other) { + impl_.reset(vx_session_clone(other.impl_.get())); + } + return *this; +} + +void StructFieldsBuilder::Deleter::operator()( + vx_struct_fields_builder* ptr) const noexcept { + vx_struct_fields_builder_free(ptr); +} + +StructFieldsBuilder::StructFieldsBuilder() + : impl_(vx_struct_fields_builder_new()) {} + +StructFieldsBuilder& StructFieldsBuilder::AddField(String name, + DataType dtype) & { + vx_struct_fields_builder_add_field(impl_.get(), + Access::release(std::move(name)), + Access::release(std::move(dtype))); + return *this; +} + +StructFieldsBuilder&& StructFieldsBuilder::AddField(String name, + DataType dtype) && { + AddField(std::move(name), std::move(dtype)); + return std::move(*this); +} + +StructFieldsBuilder& StructFieldsBuilder::AddField(std::string_view name, + DataType dtype) & { + return AddField(String(name), std::move(dtype)); +} + +StructFieldsBuilder&& StructFieldsBuilder::AddField(std::string_view name, + DataType dtype) && { + AddField(String(name), std::move(dtype)); + return std::move(*this); +} + +StructFields StructFieldsBuilder::Finalize() && { + vx_struct_fields_builder* b = impl_.release(); + return Access::adopt(vx_struct_fields_builder_finalize(b)); +} + +void StructFields::Deleter::operator()(vx_struct_fields* ptr) const noexcept { + vx_struct_fields_free(ptr); +} + +size_t StructFields::num_fields() const { + return vx_struct_fields_nfields(impl_.get()); +} + +std::string_view StructFields::field_name(size_t idx) const { + return as_view(vx_struct_fields_field_name(impl_.get(), idx)); +} + +DataType StructFields::field_dtype(size_t idx) const { + return Access::adopt( + vx_struct_fields_field_dtype(impl_.get(), idx)); +} + +void DataType::Deleter::operator()(const vx_dtype* ptr) const noexcept { + vx_dtype_free(ptr); +} + +DataType::DataType(const vx_dtype* owned) : impl_(owned) {} + +DataType::DataType(const DataType& other) + : impl_(vx_dtype_clone(other.impl_.get())) {} + +DataType& DataType::operator=(const DataType& other) { + if (this != &other) { + impl_.reset(vx_dtype_clone(other.impl_.get())); + } + return *this; +} + +DataType DataType::FromArrowSchema(ArrowSchema* schema) { + vx_error* err = nullptr; + const vx_dtype* dtype = vx_dtype_from_arrow_schema(schema, &err); + CheckError(err); + return DataType(dtype); +} + +ArrowSchema DataType::ToArrowSchema() const { + ArrowSchema schema{}; + vx_error* err = nullptr; + if (vx_dtype_to_arrow_schema(impl_.get(), &schema, &err) != 0) { + CheckError(err); + } + return schema; +} + +DataTypeVariant DataType::variant() const { + return static_cast(vx_dtype_get_variant(impl_.get())); +} + +bool DataType::nullable() const { return vx_dtype_is_nullable(impl_.get()); } + +PType DataType::primitive_ptype() const { + return static_cast(vx_dtype_primitive_ptype(impl_.get())); +} + +uint8_t DataType::decimal_precision() const { + return vx_dtype_decimal_precision(impl_.get()); +} + +int8_t DataType::decimal_scale() const { + return vx_dtype_decimal_scale(impl_.get()); +} + +size_t DataType::num_fields() const { + const vx_struct_fields* fields = vx_dtype_struct_dtype(impl_.get()); + if (fields == nullptr) { + throw VortexException("DataType is not a struct"); + } + return vx_struct_fields_nfields(fields); +} + +std::string_view DataType::field_name(size_t idx) const { + const vx_struct_fields* fields = vx_dtype_struct_dtype(impl_.get()); + if (fields == nullptr) { + throw VortexException("DataType is not a struct"); + } + return as_view(vx_struct_fields_field_name(fields, idx)); +} + +DataType DataType::field_dtype(size_t idx) const { + const vx_struct_fields* fields = vx_dtype_struct_dtype(impl_.get()); + if (fields == nullptr) { + throw VortexException("DataType is not a struct"); + } + return DataType(vx_struct_fields_field_dtype(fields, idx)); +} + +DataType DataType::list_element() const { + const vx_dtype* self = impl_.get(); + const vx_dtype_variant v = vx_dtype_get_variant(self); + const vx_dtype* elem = nullptr; + if (v == DTYPE_LIST) { + elem = vx_dtype_list_element(self); + } else if (v == DTYPE_FIXED_SIZE_LIST) { + elem = vx_dtype_fixed_size_list_element(self); + } else { + throw VortexException("DataType is not a list"); + } + return DataType(vx_dtype_clone(elem)); +} + +namespace dtype { + +DataType null() { return Access::adopt(vx_dtype_new_null()); } +DataType boolean(bool nullable) { + return Access::adopt(vx_dtype_new_bool(nullable)); +} +DataType primitive(vx_ptype ptype, bool nullable) { + return Access::adopt(vx_dtype_new_primitive(ptype, nullable)); +} +DataType int8(bool n) { return primitive(PTYPE_I8, n); } +DataType int16(bool n) { return primitive(PTYPE_I16, n); } +DataType int32(bool n) { return primitive(PTYPE_I32, n); } +DataType int64(bool n) { return primitive(PTYPE_I64, n); } +DataType uint8(bool n) { return primitive(PTYPE_U8, n); } +DataType uint16(bool n) { return primitive(PTYPE_U16, n); } +DataType uint32(bool n) { return primitive(PTYPE_U32, n); } +DataType uint64(bool n) { return primitive(PTYPE_U64, n); } +DataType float16(bool n) { return primitive(PTYPE_F16, n); } +DataType float32(bool n) { return primitive(PTYPE_F32, n); } +DataType float64(bool n) { return primitive(PTYPE_F64, n); } +DataType utf8(bool n) { return Access::adopt(vx_dtype_new_utf8(n)); } +DataType binary(bool n) { + return Access::adopt(vx_dtype_new_binary(n)); +} +DataType decimal(uint8_t p, int8_t s, bool n) { + return Access::adopt(vx_dtype_new_decimal(p, s, n)); +} +DataType list(DataType element, bool n) { + return Access::adopt( + vx_dtype_new_list(Access::release(std::move(element)), n)); +} +DataType fixed_size_list(DataType element, uint32_t size, bool n) { + return Access::adopt(vx_dtype_new_fixed_size_list( + Access::release(std::move(element)), size, n)); +} +DataType structure(StructFields fields, bool n) { + return Access::adopt( + vx_dtype_new_struct(Access::release(std::move(fields)), n)); +} + +} // namespace dtype + +void Scalar::Deleter::operator()(vx_scalar* ptr) const noexcept { + vx_scalar_free(ptr); +} + +Scalar::Scalar(const Scalar& other) + : impl_(vx_scalar_clone(other.impl_.get())) {} + +Scalar& Scalar::operator=(const Scalar& other) { + if (this != &other) { + impl_.reset(vx_scalar_clone(other.impl_.get())); + } + return *this; +} + +bool Scalar::is_null() const { return vx_scalar_is_null(impl_.get()); } + +DataType Scalar::dtype() const { + return Access::adopt( + vx_dtype_clone(vx_scalar_dtype(impl_.get()))); +} + +namespace scalar { + +Scalar boolean(bool v, bool n) { + return Access::adopt(vx_scalar_new_bool(v, n)); +} +Scalar uint8(uint8_t v, bool n) { + return Access::adopt(vx_scalar_new_u8(v, n)); +} +Scalar uint16(uint16_t v, bool n) { + return Access::adopt(vx_scalar_new_u16(v, n)); +} +Scalar uint32(uint32_t v, bool n) { + return Access::adopt(vx_scalar_new_u32(v, n)); +} +Scalar uint64(uint64_t v, bool n) { + return Access::adopt(vx_scalar_new_u64(v, n)); +} +Scalar int8(int8_t v, bool n) { + return Access::adopt(vx_scalar_new_i8(v, n)); +} +Scalar int16(int16_t v, bool n) { + return Access::adopt(vx_scalar_new_i16(v, n)); +} +Scalar int32(int32_t v, bool n) { + return Access::adopt(vx_scalar_new_i32(v, n)); +} +Scalar int64(int64_t v, bool n) { + return Access::adopt(vx_scalar_new_i64(v, n)); +} +Scalar float32(float v, bool n) { + return Access::adopt(vx_scalar_new_f32(v, n)); +} +Scalar float64(double v, bool n) { + return Access::adopt(vx_scalar_new_f64(v, n)); +} +Scalar float16_bits(uint16_t bits, bool n) { + return Access::adopt(vx_scalar_new_f16_bits(bits, n)); +} + +Scalar utf8(std::string_view value, bool nullable) { + vx_error* err = nullptr; + vx_scalar* s = + vx_scalar_new_utf8(value.data(), value.size(), nullable, &err); + CheckError(err); + return Access::adopt(s); +} + +Scalar binary(const uint8_t* ptr, size_t len, bool nullable) { + vx_error* err = nullptr; + vx_scalar* s = vx_scalar_new_binary(ptr, len, nullable, &err); + CheckError(err); + return Access::adopt(s); +} + +Scalar null(const DataType& dtype) { + vx_error* err = nullptr; + vx_scalar* s = vx_scalar_new_null(Access::c_ptr(dtype), &err); + CheckError(err); + return Access::adopt(s); +} + +Scalar decimal_i32(int32_t v, uint8_t p, int8_t s, bool n) { + vx_error* err = nullptr; + vx_scalar* sc = vx_scalar_new_decimal_i32(v, p, s, n, &err); + CheckError(err); + return Access::adopt(sc); +} + +Scalar decimal_i64(int64_t v, uint8_t p, int8_t s, bool n) { + vx_error* err = nullptr; + vx_scalar* sc = vx_scalar_new_decimal_i64(v, p, s, n, &err); + CheckError(err); + return Access::adopt(sc); +} + +} // namespace scalar + +Expression::Expression(const Expression& other) + : impl_(vx_expression_clone(other.impl_.get())) {} + +Expression& Expression::operator=(const Expression& other) { + if (this != &other) { + impl_.reset(vx_expression_clone(other.impl_.get())); + } + return *this; +} + +void Expression::Deleter::operator()(const vx_expression* ptr) const noexcept { + // vx_expression_free takes vx_expression*; both variants describe the + // same handle, so the cast just aligns the type. + vx_expression_free(const_cast(ptr)); // NOLINT +} + +namespace expr { +Expression root() { return Access::adopt(vx_expression_root()); } + +Expression literal(const Scalar& scalar) { + vx_error* err = nullptr; + vx_expression* e = vx_expression_literal(Access::c_ptr(scalar), &err); + CheckError(err); + return Access::adopt(e); +} + +Expression get_item(std::string_view name, const Expression& child) { + const std::string zero_terminated{name}; + return Access::adopt( + vx_expression_get_item(zero_terminated.c_str(), Access::c_ptr(child))); +} + +Expression column(std::string_view name) { return get_item(name, root()); } + +Expression select(const std::vector& names, + const Expression& child) { + std::vector ptrs; + ptrs.reserve(names.size()); + for (const auto& s : names) { + ptrs.push_back(s.c_str()); + } + return Access::adopt( + vx_expression_select(ptrs.data(), ptrs.size(), Access::c_ptr(child))); +} + +Expression not_(const Expression& child) { + return Access::adopt(vx_expression_not(Access::c_ptr(child))); +} + +Expression is_null(const Expression& child) { + return Access::adopt( + vx_expression_is_null(Access::c_ptr(child))); +} + +Expression binary_op(BinaryOperator op, const Expression& lhs, + const Expression& rhs) { + return Access::adopt( + vx_expression_binary(static_cast(op), + Access::c_ptr(lhs), Access::c_ptr(rhs))); +} + +Expression and_(const std::vector& children) { + std::vector ptrs; + ptrs.reserve(children.size()); + for (const Expression& e : children) { + ptrs.push_back(Access::c_ptr(e)); + } + return Access::adopt( + vx_expression_and(ptrs.data(), ptrs.size())); +} + +Expression or_(const std::vector& children) { + std::vector ptrs; + ptrs.reserve(children.size()); + for (const Expression& e : children) { + ptrs.push_back(Access::c_ptr(e)); + } + return Access::adopt( + vx_expression_or(ptrs.data(), ptrs.size())); +} + +Expression list_contains(const Expression& list, const Expression& value) { + return Access::adopt( + vx_expression_list_contains(Access::c_ptr(list), Access::c_ptr(value))); +} + +Expression eq(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Eq, l, r); +} +Expression not_eq_(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::NotEq, l, r); +} +Expression gt(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Gt, l, r); +} +Expression gt_eq(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Gte, l, r); +} +Expression lt(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Lt, l, r); +} +Expression lt_eq(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Lte, l, r); +} +Expression add(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Add, l, r); +} +Expression sub(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Sub, l, r); +} +Expression mul(const Expression& l, const Expression& r) { + return binary_op(BinaryOperator::Mul, l, r); +} + +} // namespace expr + +static void CheckPtype(const vx_array* array, vx_ptype expected) { + if (!vx_array_is_primitive(array, expected)) { + throw VortexException("Array ptype does not match accessor"); + } +} + +void Array::Deleter::operator()(const vx_array* ptr) const noexcept { + vx_array_free(ptr); +} + +Array::Array(const Array& other) : impl_(vx_array_clone(other.impl_.get())) {} + +Array& Array::operator=(const Array& other) { + if (this != &other) { + impl_.reset(vx_array_clone(other.impl_.get())); + } + return *this; +} + +Array::Array(size_t len) : impl_(vx_array_new_null(len)) {} + +Array::Array(PType ptype, const void* data, size_t len, + const Validity& validity) { + vx_error* err = nullptr; + const vx_validity c_validity{ + static_cast(validity.type), + validity.type == ValidityType::Array && validity.validity.has_value() + ? Access::c_ptr(*validity.validity) + : nullptr}; + impl_.reset(vx_array_new_primitive(static_cast(ptype), data, len, + &c_validity, &err)); + CheckError(err); +} + +Array::Array(ArrowArray* array, ArrowSchema* schema, bool nullable) { + vx_error* err = nullptr; + impl_.reset(vx_array_from_arrow(array, schema, nullable, &err)); + CheckError(err); +} + +size_t Array::length() const { return vx_array_len(impl_.get()); } + +bool Array::nullable() const { return vx_array_is_nullable(impl_.get()); } + +bool Array::has_dtype(DataTypeVariant v) const { + return vx_array_has_dtype(impl_.get(), static_cast(v)); +} + +bool Array::is_primitive(PType p) const { + return vx_array_is_primitive(impl_.get(), static_cast(p)); +} + +DataType Array::dtype() const { + return Access::adopt(vx_dtype_clone(vx_array_dtype(impl_.get()))); +} + +Array Array::field(size_t index) const { + vx_error* err = nullptr; + const vx_array* field = vx_array_get_field(impl_.get(), index, &err); + CheckError(err); + return Access::adopt(field); +} + +Array Array::slice(size_t start, size_t stop) const { + vx_error* err = nullptr; + const vx_array* slice = vx_array_slice(impl_.get(), start, stop, &err); + CheckError(err); + return Access::adopt(slice); +} + +Array Array::apply(const Expression& expr) const { + vx_error* err = nullptr; + const vx_array* applied = + vx_array_apply(impl_.get(), Access::c_ptr(expr), &err); + CheckError(err); + return Access::adopt(applied); +} + +uint8_t Array::get_u8(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_U8); + return vx_array_get_u8(p, i); +} + +uint16_t Array::get_u16(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_U16); + return vx_array_get_u16(p, i); +} + +uint32_t Array::get_u32(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_U32); + return vx_array_get_u32(p, i); +} + +uint64_t Array::get_u64(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_U64); + return vx_array_get_u64(p, i); +} + +int8_t Array::get_i8(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_I8); + return vx_array_get_i8(p, i); +} + +int16_t Array::get_i16(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_I16); + return vx_array_get_i16(p, i); +} + +int32_t Array::get_i32(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_I32); + return vx_array_get_i32(p, i); +} + +int64_t Array::get_i64(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_I64); + return vx_array_get_i64(p, i); +} + +float Array::get_f32(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_F32); + return vx_array_get_f32(p, i); +} + +double Array::get_f64(size_t i) const { + const vx_array* p = impl_.get(); + CheckPtype(p, PTYPE_F64); + return vx_array_get_f64(p, i); +} + +String Array::get_utf8(uint32_t i) const { + const vx_array* p = impl_.get(); + if (!vx_array_has_dtype(p, DTYPE_UTF8)) { + throw VortexException("Array is not utf8"); + } + return Access::adopt(vx_array_get_utf8(p, i)); +} + +Validity Array::validity() const { + vx_validity v{}; + vx_error* err = nullptr; + vx_array_get_validity(impl_.get(), &v, &err); + CheckError(err); + const ValidityType type = static_cast(v.type); + Array arr = v.array == nullptr ? Access::adopt(vx_array_new_null(0)) + : Access::adopt(v.array); + return {type, arr}; +} + +bool Array::element_is_invalid(size_t index) const { + vx_error* err = nullptr; + const bool res = vx_array_element_is_invalid(impl_.get(), index, &err); + CheckError(err); + return res; +} + +size_t Array::invalid_count() const { + vx_error* err = nullptr; + const size_t n = vx_array_invalid_count(impl_.get(), &err); + CheckError(err); + return n; +} + +void StructColumnBuilder::Deleter::operator()( + vx_struct_column_builder* ptr) const noexcept { + vx_struct_column_builder_free(ptr); +} + +StructColumnBuilder::StructColumnBuilder(const Validity& validity, + size_t capacity) { + vx_validity raw{}; + raw.type = static_cast(validity.type); + raw.array = + validity.type == ValidityType::Array && validity.validity.has_value() + ? Access::c_ptr(*validity.validity) + : nullptr; + impl_.reset(vx_struct_column_builder_new(&raw, capacity)); +} + +StructColumnBuilder& StructColumnBuilder::AddField(std::string_view name, + const Array& field) & { + vx_error* err = nullptr; + const char* const data = name.data(); + vx_struct_column_builder_add_field(impl_.get(), data, Access::c_ptr(field), + &err); + CheckError(err); + return *this; +} + +StructColumnBuilder&& StructColumnBuilder::AddField(std::string_view name, + const Array& field) && { + AddField(name, field); + return std::move(*this); +} + +Array StructColumnBuilder::Finalize() && { + vx_error* err = nullptr; + vx_struct_column_builder* builder = impl_.release(); + const vx_array* a = vx_struct_column_builder_finalize(builder, &err); + CheckError(err); + return Access::adopt(a); +} + +void Partition::Deleter::operator()(vx_partition* ptr) const noexcept { + vx_partition_free(ptr); +} + +Estimate Partition::row_count() const { + vx_estimate est{}; + vx_error* err = nullptr; + if (vx_partition_row_count(impl_.get(), &est, &err) != 0) { + CheckError(err); + } + return Estimate{static_cast(est.type), est.estimate}; +} + +std::optional Partition::next() { + vx_error* err = nullptr; + const vx_array* array = vx_partition_next(impl_.get(), &err); + CheckError(err); + if (array == nullptr) { + return std::nullopt; + } + return Access::adopt(array); +} + +ArrowStream Partition::into_arrow_stream(const Session& session) && { + ArrowArrayStream stream{}; + vx_error* err = nullptr; + vx_partition* partition = impl_.release(); + if (vx_partition_scan_arrow(Access::c_ptr(session), partition, &stream, + &err) != 0) { + CheckError(err); + } + return Access::adopt(Session(session), stream); +} + +ArrowStream::ArrowStream(ArrowStream&& other) noexcept + : session_(std::move(other.session_)), stream_(other.stream_) { + other.stream_ = ArrowArrayStream{}; +} + +ArrowStream& ArrowStream::operator=(ArrowStream&& other) noexcept { + if (this != &other) { + if (stream_.release != nullptr) { + stream_.release(&stream_); + } + session_ = std::move(other.session_); + stream_ = other.stream_; + other.stream_ = ArrowArrayStream{}; + } + return *this; +} + +ArrowStream::~ArrowStream() { + if (stream_.release != nullptr) { + stream_.release(&stream_); + } +} + +void Scan::Deleter::operator()(vx_scan* ptr) const noexcept { + vx_scan_free(ptr); +} + +DataType Scan::dtype() const { + vx_error* err = nullptr; + const vx_dtype* dtype = vx_scan_dtype(impl_.get(), &err); + CheckError(err); + return Access::adopt(vx_dtype_clone(dtype)); +} + +std::optional Scan::next_partition() { + vx_error* err = nullptr; + vx_partition* partition = vx_scan_next_partition(impl_.get(), &err); + CheckError(err); + if (partition == nullptr) { + return std::nullopt; + } + return Access::adopt(partition); +} + +ScanOptions& ScanOptions::projection(Expression p) & { + projection_ = std::move(p); + return *this; +} +ScanOptions&& ScanOptions::projection(Expression p) && { + projection_ = std::move(p); + return std::move(*this); +} + +ScanOptions& ScanOptions::filter(Expression f) & { + filter_ = std::move(f); + return *this; +} +ScanOptions&& ScanOptions::filter(Expression f) && { + filter_ = std::move(f); + return std::move(*this); +} + +ScanOptions& ScanOptions::row_range(uint64_t begin, uint64_t end) & { + row_range_ = {begin, end}; + return *this; +} +ScanOptions&& ScanOptions::row_range(uint64_t begin, uint64_t end) && { + row_range_ = {begin, end}; + return std::move(*this); +} + +ScanOptions& ScanOptions::limit(uint64_t l) & { + limit_ = l; + return *this; +} +ScanOptions&& ScanOptions::limit(uint64_t l) && { + limit_ = l; + return std::move(*this); +} + +ScanOptions& ScanOptions::ordered(bool o) & { + ordered_ = o; + return *this; +} + +ScanOptions& ScanOptions::include_indices( + const std::vector& indices) & { + selection_ = indices; + selection_include_ = VX_SELECTION_INCLUDE_RANGE; + return *this; +} + +ScanOptions&& ScanOptions::include_indices( + const std::vector& indices) && { + selection_ = indices; + selection_include_ = VX_SELECTION_INCLUDE_RANGE; + return std::move(*this); +} + +ScanOptions& ScanOptions::exclude_indices( + const std::vector& indices) & { + selection_ = indices; + selection_include_ = VX_SELECTION_EXCLUDE_RANGE; + return *this; +} + +vx_scan_options ScanOptions::to_raw() const noexcept { + vx_scan_options raw{}; + raw.projection = + projection_.has_value() ? Access::c_ptr(*projection_) : nullptr; + raw.filter = filter_.has_value() ? Access::c_ptr(*filter_) : nullptr; + raw.row_range_begin = row_range_.first; + raw.row_range_end = row_range_.second; + raw.selection.idx = selection_.empty() ? nullptr : selection_.data(); + raw.selection.idx_len = selection_.size(); + raw.selection.include = selection_include_; + raw.limit = limit_; + raw.ordered = ordered_; + return raw; +} + +void DataSource::Deleter::operator()(const vx_data_source* ptr) const noexcept { + vx_data_source_free(ptr); +} + +DataSource::DataSource(const Session& session, const std::string& paths) { + vx_data_source_options opts{}; + opts.paths = paths.c_str(); + vx_error* err = nullptr; + const vx_data_source* ds = + vx_data_source_new(Access::c_ptr(session), &opts, &err); + CheckError(err); + impl_.reset(ds); +} + +DataSource::DataSource(const Session& session, const void* data, size_t len) { + vx_error* err = nullptr; + const vx_data_source* ds = + vx_data_source_new_buffer(Access::c_ptr(session), data, len, &err); + CheckError(err); + impl_.reset(ds); +} + +DataSource::DataSource(const DataSource& other) + : impl_(vx_data_source_clone(other.impl_.get())) {} + +DataSource& DataSource::operator=(const DataSource& other) { + if (this != &other) { + impl_.reset(vx_data_source_clone(other.impl_.get())); + } + return *this; +} + +Estimate DataSource::row_count() const { + vx_estimate est{}; + vx_data_source_get_row_count(impl_.get(), &est); + return Estimate{static_cast(est.type), est.estimate}; +} + +DataType DataSource::dtype() const { + return Access::adopt( + vx_dtype_clone(vx_data_source_dtype(impl_.get()))); +} + +Scan DataSource::scan(const ScanOptions& options) const { + const vx_scan_options raw = options.to_raw(); + vx_estimate estimate{}; + vx_error* err = nullptr; + vx_scan* scan = vx_data_source_scan(impl_.get(), &raw, &estimate, &err); + CheckError(err); + return Access::adopt( + scan, + Estimate{static_cast(estimate.type), estimate.estimate}); +} + +} // namespace vortex diff --git a/vortex-cxx/vortex.hpp b/vortex-cxx/vortex.hpp new file mode 100644 index 00000000000..be6b973db6c --- /dev/null +++ b/vortex-cxx/vortex.hpp @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * C++ bindings for Vortex. + * When invoked on a moved-out object, methods throw VortexException. + */ +namespace vortex { + +class Array; +class ArrowStream; +class DataSource; +class DataType; +class Partition; +class Scalar; +class Scan; +class Session; +class String; +class StructFields; +struct Validity; + +// Implementation detail, don't use directly +struct Access { + template + static T adopt(Args&&... args) { + return T(std::forward(args)...); + } + template + static auto release(T&& t) noexcept { + return std::forward(t).release(); + } + template + static auto c_ptr(const T& t) noexcept { + return t.impl_.get(); + } +}; + +// Implementation detail, don't use directly +void CheckError(vx_error* error); + +class VortexException : public std::runtime_error { +public: + explicit VortexException(const std::string& message) + : std::runtime_error(message) {} +}; + +enum class ValidityType { + // Items can't be null + NonNullable = VX_VALIDITY_NON_NULLABLE, + // All items are valid + AllValid = VX_VALIDITY_ALL_VALID, + // All items are invalid + AllInvalid = VX_VALIDITY_ALL_INVALID, + /** + * Items validity is determined by a boolean array. True values in boolean + * array are valid, false values are invalid (null) + */ + Array = VX_VALIDITY_ARRAY +}; + +enum class DataTypeVariant { + Null = DTYPE_NULL, + Bool = DTYPE_BOOL, + // Primitives (e.g., u8, i16, f32, etc.) + Primitive = DTYPE_PRIMITIVE, + // Variable-length UTF-8 string + Utf8 = DTYPE_UTF8, + // Variable-length binary + Binary = DTYPE_BINARY, + // Nested struct + Struct = DTYPE_STRUCT, + // Nested list + List = DTYPE_LIST, + // User-defined extension + Extension = DTYPE_EXTENSION, + // Decimal with fixed precision and scale + Decimal = DTYPE_DECIMAL, + // Nested fixed-size list + FixedSizeList = DTYPE_FIXED_SIZE_LIST, +}; + +// Primitive type +enum class PType { + U8 = PTYPE_U8, + U16 = PTYPE_U16, + U32 = PTYPE_U32, + U64 = PTYPE_U64, + I8 = PTYPE_I8, + I16 = PTYPE_I16, + I32 = PTYPE_I32, + I64 = PTYPE_I64, + F16 = PTYPE_F16, + F32 = PTYPE_F32, + F64 = PTYPE_F64, +}; + +enum class EstimateType { + Unknown = VX_ESTIMATE_UNKNOWN, + Exact = VX_ESTIMATE_EXACT, + Inexact = VX_ESTIMATE_INEXACT, +}; + +struct Estimate { + EstimateType type; + uint64_t value; +}; + +enum class BinaryOperator { + // x = y + Eq = VX_OPERATOR_EQ, + // x != y + NotEq = VX_OPERATOR_NOT_EQ, + // x > y + Gt = VX_OPERATOR_GT, + // x >= y + Gte = VX_OPERATOR_GTE, + // x < y + Lt = VX_OPERATOR_LT, + // x <= y + Lte = VX_OPERATOR_LTE, + // Boolean AND + KleeneAnd = VX_OPERATOR_KLEENE_AND, + // Boolean OR + KleeneOr = VX_OPERATOR_KLEENE_OR, + /** + * The sum of the arguments. + * Errors at runtime if the sum would overflow or underflow. + */ + Add = VX_OPERATOR_ADD, + /** + * The difference between the arguments. + * Errors at runtime if the sum would overflow or underflow. + * The result is null at any index where either input is null. + */ + Sub = VX_OPERATOR_SUB, + // Multiply two numbers + Mul = VX_OPERATOR_MUL, + // Divide the left side by the right side + Div = VX_OPERATOR_DIV, +}; + +// A variable length Utf8 string +class String { +public: + explicit String(std::string_view s); + + String(const String& other); + String(String&&) noexcept = default; + String& operator=(const String& other); + String& operator=(String&&) noexcept = default; + + std::string_view view() const; + size_t size() const; + +private: + friend struct Access; + explicit String(const vx_string* owned) : impl_(owned) {} + const vx_string* release() && noexcept { return impl_.release(); } + + struct Deleter { + void operator()(const vx_string* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +/* + * A variable length binary data string. Unlinke String, data in Binary doesn't + * need to be Utf8 + */ +class Binary { +public: + Binary(const char* data, size_t len); + explicit Binary(std::string_view s) : Binary(s.data(), s.size()) {} + + Binary(const Binary& other); + Binary(Binary&&) noexcept = default; + Binary& operator=(const Binary& other); + Binary& operator=(Binary&&) noexcept = default; + + size_t size() const; + const char* data() const; + std::string_view view() const; + +private: + friend struct Access; + explicit Binary(const vx_binary* owned) : impl_(owned) {} + + struct Deleter { + void operator()(const vx_binary* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +class Session { +public: + Session(); + + Session(const Session& other); + Session(Session&&) noexcept = default; + Session& operator=(const Session& other); + Session& operator=(Session&&) noexcept = default; + +private: + friend struct Access; + + struct Deleter { + void operator()(vx_session* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +class StructFields { +public: + StructFields(const StructFields&) = delete; + StructFields& operator=(const StructFields&) = delete; + StructFields(StructFields&&) noexcept = default; + StructFields& operator=(StructFields&&) noexcept = default; + + size_t num_fields() const; + std::string_view field_name(size_t idx) const; + /// Owned DataType for field at "idx" + DataType field_dtype(size_t idx) const; + +private: + friend struct Access; + explicit StructFields(vx_struct_fields* owned) : impl_(owned) {} + vx_struct_fields* release() && noexcept { return impl_.release(); } + + struct Deleter { + void operator()(vx_struct_fields* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +class StructFieldsBuilder { +public: + StructFieldsBuilder(); + + StructFieldsBuilder& AddField(String name, DataType dtype) &; + StructFieldsBuilder&& AddField(String name, DataType dtype) &&; + StructFieldsBuilder& AddField(std::string_view name, DataType dtype) &; + StructFieldsBuilder&& AddField(std::string_view name, DataType dtype) &&; + + StructFields Finalize() &&; + +private: + struct Deleter { + void operator()(vx_struct_fields_builder* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +/** + * A Vortex data type. + * + * Data types in Vortex are purely logical, meaning they confer no information + * about how the data is physically stored. + */ +class DataType { +public: + DataType(const DataType& other); + DataType(DataType&&) noexcept = default; + DataType& operator=(const DataType& other); + DataType& operator=(DataType&&) noexcept = default; + + /* + * Consume ArrowSchema and convert it into DataType. Don't use schema + * after this call + */ + static DataType FromArrowSchema(ArrowSchema* schema); + /* + * Convert this dtype to an owned Arrow C schema. Caller is responsible for + * calling schema's release() callback + */ + ArrowSchema ToArrowSchema() const; + + DataTypeVariant variant() const; + bool nullable() const; + + PType primitive_ptype() const; + uint8_t decimal_precision() const; + int8_t decimal_scale() const; + + size_t num_fields() const; + std::string_view field_name(size_t idx) const; + // Owned DataType for field at "idx" + DataType field_dtype(size_t idx) const; + + // Owned DataType for list element + DataType list_element() const; + +private: + friend struct Access; + explicit DataType(const vx_dtype* owned); + const vx_dtype* release() && noexcept { return impl_.release(); } + + struct Deleter { + void operator()(const vx_dtype* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +namespace dtype { + +DataType null(); +DataType boolean(bool nullable = false); +DataType primitive(PType ptype, bool nullable = false); +DataType int8(bool nullable = false); +DataType int16(bool nullable = false); +DataType int32(bool nullable = false); +DataType int64(bool nullable = false); +DataType uint8(bool nullable = false); +DataType uint16(bool nullable = false); +DataType uint32(bool nullable = false); +DataType uint64(bool nullable = false); +DataType float16(bool nullable = false); +DataType float32(bool nullable = false); +DataType float64(bool nullable = false); +DataType utf8(bool nullable = false); +DataType binary(bool nullable = false); +DataType decimal(uint8_t precision, int8_t scale, bool nullable = false); +DataType list(DataType element, bool nullable = false); +DataType fixed_size_list(DataType element, uint32_t size, + bool nullable = false); +DataType structure(StructFields fields, bool nullable = false); + +// Consume ArrowSchema and return a DataType from it +inline DataType from_arrow(ArrowSchema* schema) { + return DataType::FromArrowSchema(schema); +} +} // namespace dtype + +/* + * A single value with associated DataType. + */ +class Scalar { +public: + Scalar(const Scalar& other); + Scalar(Scalar&&) noexcept = default; + Scalar& operator=(const Scalar& other); + Scalar& operator=(Scalar&&) noexcept = default; + + bool is_null() const; + DataType dtype() const; + +private: + friend struct Access; + explicit Scalar(vx_scalar* owned) : impl_(owned) {} + vx_scalar* release() && noexcept { return impl_.release(); } + + struct Deleter { + void operator()(vx_scalar* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +namespace scalar { + +Scalar boolean(bool value, bool nullable = false); +Scalar uint8(uint8_t value, bool nullable = false); +Scalar uint16(uint16_t value, bool nullable = false); +Scalar uint32(uint32_t value, bool nullable = false); +Scalar uint64(uint64_t value, bool nullable = false); +Scalar int8(int8_t value, bool nullable = false); +Scalar int16(int16_t value, bool nullable = false); +Scalar int32(int32_t value, bool nullable = false); +Scalar int64(int64_t value, bool nullable = false); +Scalar float32(float value, bool nullable = false); +Scalar float64(double value, bool nullable = false); +Scalar float16_bits(uint16_t bits, bool nullable = false); +Scalar utf8(std::string_view value, bool nullable = false); +Scalar binary(const uint8_t* ptr, size_t len, bool nullable = false); +Scalar null(const DataType& dtype); +Scalar decimal_i32(int32_t value, uint8_t precision, int8_t scale, + bool nullable = false); +Scalar decimal_i64(int64_t value, uint8_t precision, int8_t scale, + bool nullable = false); +} // namespace scalar + +class Expression { +public: + Expression(const Expression&); + Expression& operator=(const Expression&); + Expression(Expression&&) noexcept = default; + Expression& operator=(Expression&&) noexcept = default; + +private: + friend struct Access; + explicit Expression(const vx_expression* owned) : impl_(owned) {} + const vx_expression* release() && noexcept { return impl_.release(); } + + struct Deleter { + void operator()(const vx_expression* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +namespace expr { + +Expression root(); +Expression literal(const Scalar& scalar); +Expression get_item(std::string_view name, const Expression& child); +Expression column(std::string_view name); +Expression select(const std::vector& names, + const Expression& child); +Expression not_(const Expression& child); +Expression is_null(const Expression& child); +Expression binary_op(BinaryOperator op, const Expression& lhs, + const Expression& rhs); +Expression and_(const std::vector& children); +Expression or_(const std::vector& children); +Expression list_contains(const Expression& list, const Expression& value); + +Expression eq(const Expression& l, const Expression& r); +Expression not_eq_(const Expression& l, const Expression& r); +Expression gt(const Expression& l, const Expression& r); +Expression gt_eq(const Expression& l, const Expression& r); +Expression lt(const Expression& l, const Expression& r); +Expression lt_eq(const Expression& l, const Expression& r); +Expression add(const Expression& l, const Expression& r); +Expression sub(const Expression& l, const Expression& r); +Expression mul(const Expression& l, const Expression& r); +} // namespace expr + +/** + * Arrays are reference-counted handles to owned memory buffers that hold + * scalars. These buffers can be held in a number of physical encodings to + * perform lightweight compression that exploits the particular data + * distribution of the array's values. + * + * Every data type recognized by Vortex also has a canonical physical + * encoding format, which arrays can be canonicalized into for ease of + * access in compute functions. + * + * As an implementation detail, Arrays are Arc'ed inside, so cloning an + * array is a cheap operation. + */ +class Array { +public: + Array(const Array& other); + Array(Array&&) noexcept = default; + Array& operator=(const Array& other); + Array& operator=(Array&&) noexcept = default; + + // Create a Null array + explicit Array(size_t len); + + // Create a Primitive array + Array(PType ptype, const void* data, size_t len, const Validity& validity); + + /* + * Create Array from Arrow. Consumes array and schema. + * Don't use array or schema after this call + */ + Array(ArrowArray* array, ArrowSchema* schema, bool nullable); + + size_t length() const; + bool nullable() const; + bool has_dtype(DataTypeVariant v) const; + bool is_primitive(PType p) const; + DataType dtype() const; + + Array field(size_t index) const; + Array slice(size_t start, size_t stop) const; + Array apply(const Expression& expr) const; + + // These methods throw VortexException if array's ptype does not match + + uint8_t get_u8(size_t i) const; + uint16_t get_u16(size_t i) const; + uint32_t get_u32(size_t i) const; + uint64_t get_u64(size_t i) const; + int8_t get_i8(size_t i) const; + int16_t get_i16(size_t i) const; + int32_t get_i32(size_t i) const; + int64_t get_i64(size_t i) const; + float get_f32(size_t i) const; + double get_f64(size_t i) const; + String get_utf8(uint32_t i) const; + + Validity validity() const; + bool element_is_invalid(size_t index) const; + size_t invalid_count() const; + +private: + friend struct Access; + explicit Array(const vx_array* owned) : impl_(owned) {} + const vx_array* release() && noexcept { return impl_.release(); } + + struct Deleter { + void operator()(const vx_array* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +struct Validity { + ValidityType type; + std::optional validity; + + explicit Validity(ValidityType type) : type(type) {} + Validity(ValidityType type, const Array& array) + : type(type), validity(array) {} +}; + +class StructColumnBuilder { +public: + /* + * Create a struct builder with a capacity hint. If you don't know the + * capacity, set it to 0. + */ + StructColumnBuilder(const Validity& validity, size_t capacity); + + StructColumnBuilder& AddField(std::string_view name, const Array& field) &; + StructColumnBuilder&& AddField(std::string_view name, + const Array& field) &&; + Array Finalize() &&; + +private: + struct Deleter { + void operator()(vx_struct_column_builder* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +/** + * A partition is an independent unit of work. Call next() repeatedly to + * retrieve Arrays. + */ +class Partition { +public: + Partition(const Partition&) = delete; + Partition& operator=(const Partition&) = delete; + Partition(Partition&&) noexcept = default; + Partition& operator=(Partition&&) noexcept = default; + + Estimate row_count() const; + + std::optional next(); + + // Consume partition into an Arrow stream. + ArrowStream into_arrow_stream(const Session& session) &&; + +private: + friend struct Access; + explicit Partition(vx_partition* owned) : impl_(owned) {} + vx_partition* release() && noexcept { return impl_.release(); } + + struct Deleter { + void operator()(vx_partition* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; + +/* + * Wrapper around ArrowArrayStream which releases the array in destructor. + */ +class ArrowStream { +public: + ArrowStream(const ArrowStream&) = delete; + ArrowStream& operator=(const ArrowStream&) = delete; + ArrowStream(ArrowStream&& other) noexcept; + ArrowStream& operator=(ArrowStream&& other) noexcept; + + ~ArrowStream(); + + ArrowArrayStream* raw() noexcept { return &stream_; } + +private: + friend struct Access; + ArrowStream(Session session, ArrowArrayStream stream) noexcept + : session_(std::move(session)), stream_(stream) {} + + Session session_; + ArrowArrayStream stream_{}; +}; + +/** + * A scan is a single traversal of a DataSource with projections and filters. + * A scan can be consumed only once. + */ +class Scan { +public: + Scan(const Scan&) = delete; + Scan& operator=(const Scan&) = delete; + Scan(Scan&&) noexcept = default; + Scan& operator=(Scan&&) noexcept = default; + + Estimate partition_estimate() const noexcept { return estimate_; } + DataType dtype() const; + std::optional next_partition(); + +private: + friend struct Access; + Scan(vx_scan* owned, Estimate estimate) noexcept + : impl_(owned), estimate_(estimate) {} + + struct Deleter { + void operator()(vx_scan* ptr) const noexcept; + }; + std::unique_ptr impl_; + Estimate estimate_{EstimateType::Unknown, 0}; +}; + +class ScanOptions { +public: + ScanOptions() = default; + + ScanOptions& projection(Expression p) &; + ScanOptions&& projection(Expression p) &&; + + ScanOptions& filter(Expression f) &; + ScanOptions&& filter(Expression f) &&; + + ScanOptions& row_range(uint64_t begin, uint64_t end) &; + ScanOptions&& row_range(uint64_t begin, uint64_t end) &&; + + ScanOptions& limit(uint64_t l) &; + ScanOptions&& limit(uint64_t l) &&; + + ScanOptions& ordered(bool o) &; + + ScanOptions& include_indices(const std::vector& indices) &; + ScanOptions&& include_indices(const std::vector& indices) &&; + ScanOptions& exclude_indices(const std::vector& indices) &; + +private: + friend class DataSource; + + vx_scan_options to_raw() const noexcept; + + std::optional projection_; + std::optional filter_; + std::pair row_range_{0, 0}; + uint64_t limit_ = 0; + bool ordered_ = false; + std::vector selection_; + vx_scan_selection_include selection_include_ = VX_SELECTION_INCLUDE_ALL; +}; + +/** + * A reference to one or more (possibly remote) paths. + * Creating DataSource opens the first matched path to read the schema. + * All other I/O is deferred until a scan is requested. Multiple scans may + * be requested from a single data source. + */ +class DataSource { +public: + // Open file paths + DataSource(const Session& session, const std::string& paths); + // Open buffer + DataSource(const Session& session, const void* data, size_t len); + + DataSource(const DataSource& other); + DataSource(DataSource&&) noexcept = default; + DataSource& operator=(const DataSource& other); + DataSource& operator=(DataSource&&) noexcept = default; + + Estimate row_count() const; + DataType dtype() const; + Scan scan(const ScanOptions& options = {}) const; + +private: + friend struct Access; + + struct Deleter { + void operator()(const vx_data_source* ptr) const noexcept; + }; + std::unique_ptr impl_; +}; +} // namespace vortex diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index 0203202e437..b4711905816 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -1102,6 +1102,8 @@ void vx_expression_free(vx_expression *ptr); */ vx_expression *vx_expression_root(void); +vx_expression *vx_expression_clone(const vx_expression *ptr); + /** * Create a literal expression from a scalar. * @@ -1194,7 +1196,7 @@ vx_expression_binary(vx_binary_operator operator_, const vx_expression *lhs, con * * Returns the logical negation of the input boolean expression. */ -const vx_expression *vx_expression_not(const vx_expression *child); +vx_expression *vx_expression_not(const vx_expression *child); /** * Create an expression that checks for null values. diff --git a/vortex-ffi/src/expression.rs b/vortex-ffi/src/expression.rs index 269e1e5d941..0cd79ca399f 100644 --- a/vortex-ffi/src/expression.rs +++ b/vortex-ffi/src/expression.rs @@ -66,6 +66,16 @@ pub unsafe extern "C" fn vx_expression_root() -> *mut vx_expression { vx_expression::new(root()) } +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_expression_clone( + ptr: *const vx_expression, +) -> *mut vx_expression { + if ptr.is_null() { + return ptr::null_mut(); + } + vx_expression::new(vx_expression::as_ref(ptr).clone()) +} + /// Create a literal expression from a scalar. /// /// Literal expressions are useful for constants in expression trees, especially scan @@ -263,9 +273,9 @@ pub unsafe extern "C" fn vx_expression_binary( /// /// Returns the logical negation of the input boolean expression. #[unsafe(no_mangle)] -pub unsafe extern "C" fn vx_expression_not(child: *const vx_expression) -> *const vx_expression { +pub unsafe extern "C" fn vx_expression_not(child: *const vx_expression) -> *mut vx_expression { if child.is_null() { - return child; + return ptr::null_mut(); } vx_expression::new(not(vx_expression::as_ref(child).clone())) }