From db4a1664fadd58f12f4a668c33d02da9da7ca317 Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 19:40:24 -0500 Subject: [PATCH 01/10] gpu: account for overflow capacity and fixed partition scratch Linear k scaling underestimated Tiny at k=22: the fixed partition tile and full key capacity peaked at 41.96 MiB against a 16.63 MiB estimate. Scale the table portion by the shared capacity formula and retain the partition allowance so admission and the regression guard use the same estimate. --- src/host/GpuPipeline.cu | 2 +- src/host/VramBudget.hpp | 12 ++++++++++-- tools/parity/vram_budget_test.cpp | 6 ++++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/host/GpuPipeline.cu b/src/host/GpuPipeline.cu index 5ef4cac..8542e5f 100644 --- a/src/host/GpuPipeline.cu +++ b/src/host/GpuPipeline.cu @@ -4828,7 +4828,7 @@ GpuPipelineResult run_gpu_pipeline_streaming_impl( : "") + ". The tier floor is derived from the declared peak, so it is now " "too low. Re-measure and update the per-tier peak constant in " - "BatchPlotter.cpp."); + "VramBudget.hpp."); } if (stats.verbose) { diff --git a/src/host/VramBudget.hpp b/src/host/VramBudget.hpp index 8527923..a8a3a47 100644 --- a/src/host/VramBudget.hpp +++ b/src/host/VramBudget.hpp @@ -1,5 +1,7 @@ #pragma once +#include "PoolSizing.hpp" + #include #include #include @@ -36,8 +38,14 @@ inline std::uint64_t streaming_base_peak_bytes(int k, StreamingTier tier) case StreamingTier::Tiny: mib = 1064; break; case StreamingTier::Pinned: mib = 1150; break; } - auto const bytes = mib << 20; - return k < 28 ? bytes >> (28 - k) : bytes << (k - 28); + // Capacity includes the section overflow allowance, which does not scale + // as 2^k. Tiny also keeps a fixed 24 MiB partition tile at smaller k. + std::uint64_t const fixed_mib = tier == StreamingTier::Tiny ? 24 : 0; + int const section_bits = k < 28 ? 2 : k - 26; + auto const cap = max_pairs_per_section(k, section_bits) << section_bits; + auto const reference_mib = (max_pairs_per_section(28, 2) << 2) >> 20; + return ((mib - fixed_mib) * cap + reference_mib - 1) / reference_mib + + (fixed_mib << 20); } inline bool vram_fits(std::uint64_t free, std::uint64_t peak, diff --git a/tools/parity/vram_budget_test.cpp b/tools/parity/vram_budget_test.cpp index d259f5d..02d88a8 100644 --- a/tools/parity/vram_budget_test.cpp +++ b/tools/parity/vram_budget_test.cpp @@ -23,6 +23,12 @@ int main() assert(vram_scratch_budget(peak + buffer + 812, peak, buffer) == 812); } } + // Small plots retain the full partition tile and proportionally more + // overflow capacity. Scaling the k=28 byte count alone underestimates both. + assert(streaming_base_peak_bytes(22, StreamingTier::Tiny) == 42 * MiB); + assert(streaming_base_peak_bytes(26, StreamingTier::Tiny) == 288 * MiB); + assert(streaming_base_peak_bytes(26, StreamingTier::Plain) >= 1848 * MiB); + assert(streaming_base_peak_bytes(28, StreamingTier::Tiny) == 1064 * MiB); // Target card capacities after a representative 390 MiB context. The // picker must include the buffer exactly once and stop at Tiny. for (auto [gib, expected] : std::array{ From 213e397c5dcb628e54a9b7efc588505fcd2d0a2c Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 19:40:24 -0500 Subject: [PATCH 02/10] keygen: keep the CLVM runtime out of production binaries Only BLS derivation and plot-ID hashing are needed by the FFI. Depend on chia-bls directly and retain chia-protocol as a test oracle for the exact streamable encoding. This removes the unused VM and its static LGPL dependencies from the release dependency graph. --- keygen-rs/Cargo.lock | 794 +------------------------------------------ keygen-rs/Cargo.toml | 4 +- keygen-rs/src/lib.rs | 113 +++--- 3 files changed, 80 insertions(+), 831 deletions(-) diff --git a/keygen-rs/Cargo.lock b/keygen-rs/Cargo.lock index 80fdf9f..26a3432 100644 --- a/keygen-rs/Cargo.lock +++ b/keygen-rs/Cargo.lock @@ -8,45 +8,6 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" -[[package]] -name = "asn1-rs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" -dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", - "displaydoc", - "nom", - "num-traits", - "rusticata-macros", - "thiserror 1.0.69", - "time", -] - -[[package]] -name = "asn1-rs-derive" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "asn1-rs-impl" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "autocfg" version = "1.5.0" @@ -59,12 +20,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - [[package]] name = "base64ct" version = "1.8.3" @@ -137,12 +92,6 @@ version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - [[package]] name = "cc" version = "1.2.60" @@ -161,28 +110,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" -[[package]] -name = "chia" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "676e3b6f57c1b028bcaedb215109e3c96678d4b41079930aa88f4e130a5f0e1b" -dependencies = [ - "chia-bls 0.48.0", - "chia-client", - "chia-consensus", - "chia-datalayer", - "chia-protocol", - "chia-puzzle-types", - "chia-secp", - "chia-serde", - "chia-sha2 0.48.0", - "chia-ssl", - "chia-traits 0.48.0", - "clvm-traits", - "clvm-utils", - "clvmr", -] - [[package]] name = "chia-bls" version = "0.38.2" @@ -206,84 +133,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e698e6c8cb9ce00f626d810fcbae6ded7fed8806c4b550a5f58e9b7bbee565a9" dependencies = [ "blst", - "chia-serde", "chia-sha2 0.48.0", "chia-traits 0.48.0", "hex", "hkdf", "linked-hash-map", - "serde", "sha2 0.10.9", "thiserror 2.0.18", ] -[[package]] -name = "chia-client" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "001273eef8bc9c4b1c4603d9ae9b231de6acabeb955ba23feadeca0a211a0cb2" -dependencies = [ - "chia-protocol", - "chia-traits 0.48.0", - "futures-util", - "thiserror 2.0.18", - "tokio", - "tokio-tungstenite", - "tungstenite", -] - -[[package]] -name = "chia-consensus" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbdfa6fe38f2ffaa6457f6c17a643fe7682126b6b6ac04326c4f45889369604a" -dependencies = [ - "bitflags", - "chia-bls 0.48.0", - "chia-protocol", - "chia-puzzle-types", - "chia-puzzles", - "chia-sha2 0.48.0", - "chia-traits 0.48.0", - "chia_streamable_macro 0.48.0", - "clvm-traits", - "clvm-utils", - "clvmr", - "hex", - "hex-literal", - "thiserror 2.0.18", -] - -[[package]] -name = "chia-datalayer" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4ff5d73fdbd7d69023c0d12c658a3486d128065a844df970013242dad7130a6" -dependencies = [ - "bitvec", - "chia-datalayer-macro", - "chia-protocol", - "chia-sha2 0.48.0", - "chia-traits 0.48.0", - "chia_streamable_macro 0.48.0", - "indexmap", - "num-traits", - "rayon", - "thiserror 2.0.18", - "zstd", -] - -[[package]] -name = "chia-datalayer-macro" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6bc4f3706422ca3f84c76107b9262c5d2cad590e9c96810529fea8d6fa6e03" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "chia-pos2" version = "0.6.0" @@ -303,7 +161,6 @@ checksum = "23b6b29c18eda3a4b4f882611e2227dde34345778c3e3bd3d3d8d84354b30208" dependencies = [ "chia-bls 0.48.0", "chia-pos2", - "chia-serde", "chia-sha2 0.48.0", "chia-traits 0.48.0", "chia_streamable_macro 0.48.0", @@ -311,57 +168,6 @@ dependencies = [ "clvm-utils", "clvmr", "hex", - "serde", - "serde_arrays", -] - -[[package]] -name = "chia-puzzle-types" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8247ccae54f944c37adad740fb4a552fda0bab7f3f1f35e32c7ecf7b5dddb1b3" -dependencies = [ - "chia-bls 0.48.0", - "chia-protocol", - "chia-puzzles", - "chia-sha2 0.48.0", - "clvm-traits", - "clvm-utils", - "clvmr", - "hex-literal", - "num-bigint", -] - -[[package]] -name = "chia-puzzles" -version = "0.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "553363ce9550cfde0ae1306a0a79bf9afd36ea09e0222b874b6287b44ad9d178" -dependencies = [ - "hex", - "hex-literal", -] - -[[package]] -name = "chia-secp" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5edfac9a0e1aa3121f5e9e53638eee650de4c0db2820f81e77e2c2f19a4ab087" -dependencies = [ - "chia-sha2 0.48.0", - "hex", - "k256", - "p256", -] - -[[package]] -name = "chia-serde" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0199bf6ebf466516f033aa5fb89e414a0121a8c93c4a8225c1e4b78abe18f7" -dependencies = [ - "hex", - "serde", ] [[package]] @@ -382,19 +188,6 @@ dependencies = [ "sha2 0.10.9", ] -[[package]] -name = "chia-ssl" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4f21993e0859eb5098fc59cfcd7a42ccad6ab31c983ae68964eaeb164ed9ca" -dependencies = [ - "getrandom 0.4.2", - "rcgen", - "rsa", - "thiserror 2.0.18", - "time", -] - [[package]] name = "chia-traits" version = "0.38.2" @@ -458,8 +251,6 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06d7b1b6e7a83e1d4a4864ac34ef3f7933ef87fcd3f2fa2ff09f9d38a19f61af" dependencies = [ - "chia-bls 0.48.0", - "chia-secp", "clvm-derive", "clvmr", "num-bigint", @@ -541,31 +332,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - [[package]] name = "crypto-bigint" version = "0.7.5" @@ -578,7 +344,6 @@ dependencies = [ "hybrid-array", "num-traits", "rand_core 0.10.1", - "serdect", "subtle", "zeroize", ] @@ -604,17 +369,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "crypto-primes" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a" -dependencies = [ - "crypto-bigint", - "libm", - "rand_core 0.10.1", -] - [[package]] name = "ctutils" version = "0.4.2" @@ -625,12 +379,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "data-encoding" -version = "2.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" - [[package]] name = "der" version = "0.8.0" @@ -642,29 +390,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "der-parser" -version = "9.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" -dependencies = [ - "asn1-rs", - "displaydoc", - "nom", - "num-bigint", - "num-traits", - "rusticata-macros", -] - -[[package]] -name = "deranged" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" -dependencies = [ - "powerfmt", -] - [[package]] name = "digest" version = "0.10.7" @@ -688,17 +413,6 @@ dependencies = [ "ctutils", ] -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "ecdsa" version = "0.17.0" @@ -781,49 +495,6 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" -[[package]] -name = "futures-core" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" - -[[package]] -name = "futures-macro" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" - -[[package]] -name = "futures-task" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" - -[[package]] -name = "futures-util" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" -dependencies = [ - "futures-core", - "futures-macro", - "futures-sink", - "futures-task", - "pin-project-lite", - "slab", -] - [[package]] name = "generic-array" version = "0.14.9" @@ -834,17 +505,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - [[package]] name = "getrandom" version = "0.3.4" @@ -963,22 +623,6 @@ dependencies = [ "digest 0.11.2", ] -[[package]] -name = "http" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - [[package]] name = "hybrid-array" version = "0.4.14" @@ -1136,33 +780,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "mio" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -1173,12 +790,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-conv" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967" - [[package]] name = "num-integer" version = "0.1.46" @@ -1207,15 +818,6 @@ dependencies = [ "libc", ] -[[package]] -name = "oid-registry" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" -dependencies = [ - "asn1-rs", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -1241,16 +843,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "pem" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" -dependencies = [ - "base64", - "serde_core", -] - [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -1260,22 +852,6 @@ dependencies = [ "base64ct", ] -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkcs1" -version = "0.8.0-rc.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "986d2e952779af96ea048f160fd9194e1751b4faea78bcf3ceb456efe008088e" -dependencies = [ - "der", - "spki", -] - [[package]] name = "pkcs8" version = "0.11.0" @@ -1286,28 +862,17 @@ dependencies = [ "spki", ] -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - [[package]] name = "pos2_keygen" version = "0.1.0" dependencies = [ "bech32", - "chia", + "chia-bls 0.48.0", + "chia-protocol", "hex", "sha2 0.11.0", ] -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1435,40 +1000,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rcgen" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" -dependencies = [ - "pem", - "ring", - "rustls-pki-types", - "time", - "x509-parser", - "yasna", -] - [[package]] name = "rfc6979" version = "0.6.0" @@ -1479,56 +1010,6 @@ dependencies = [ "hmac 0.13.0", ] -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rsa" -version = "0.10.0-rc.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b2aa4ba0d89f73d1e332df05be0eeab8840351c36ca5654341dfdb57bb3caf" -dependencies = [ - "const-oid", - "crypto-bigint", - "crypto-primes", - "digest 0.11.2", - "pkcs1", - "pkcs8", - "rand_core 0.10.1", - "signature", - "spki", - "zeroize", -] - -[[package]] -name = "rusticata-macros" -version = "4.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" -dependencies = [ - "nom", -] - -[[package]] -name = "rustls-pki-types" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" -dependencies = [ - "zeroize", -] - [[package]] name = "ryu" version = "1.0.23" @@ -1583,15 +1064,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_arrays" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a16b99c5ea4fe3daccd14853ad260ec00ea043b2708d1fd1da3106dcd8d9df" -dependencies = [ - "serde", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -1694,22 +1166,6 @@ dependencies = [ "rand_core 0.10.1", ] -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "socket2" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - [[package]] name = "spki" version = "0.8.0" @@ -1737,17 +1193,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "tap" version = "1.0.1" @@ -1803,63 +1248,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "time" -version = "0.3.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" -dependencies = [ - "deranged", - "itoa", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" - -[[package]] -name = "time-macros" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" -dependencies = [ - "num-conv", - "time-core", -] - -[[package]] -name = "tokio" -version = "1.52.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" -dependencies = [ - "bytes", - "libc", - "mio", - "pin-project-lite", - "socket2", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-tungstenite" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" -dependencies = [ - "futures-util", - "log", - "tokio", - "tungstenite", -] - [[package]] name = "toml_datetime" version = "0.6.11" @@ -1877,23 +1265,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tungstenite" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" -dependencies = [ - "bytes", - "data-encoding", - "http", - "httparse", - "log", - "rand", - "sha1", - "thiserror 2.0.18", - "utf-8", -] - [[package]] name = "typenum" version = "1.20.0" @@ -1912,30 +1283,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -1998,94 +1351,6 @@ dependencies = [ "safe_arch", ] -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - [[package]] name = "winnow" version = "0.5.40" @@ -2203,33 +1468,6 @@ dependencies = [ "tap", ] -[[package]] -name = "x509-parser" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" -dependencies = [ - "asn1-rs", - "data-encoding", - "der-parser", - "lazy_static", - "nom", - "oid-registry", - "ring", - "rusticata-macros", - "thiserror 1.0.69", - "time", -] - -[[package]] -name = "yasna" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" -dependencies = [ - "time", -] - [[package]] name = "zerocopy" version = "0.8.48" @@ -2275,31 +1513,3 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] diff --git a/keygen-rs/Cargo.toml b/keygen-rs/Cargo.toml index 44327f5..6d9ff7e 100644 --- a/keygen-rs/Cargo.toml +++ b/keygen-rs/Cargo.toml @@ -2,17 +2,19 @@ name = "pos2_keygen" version = "0.1.0" edition = "2021" +license = "MIT" publish = false [lib] crate-type = ["staticlib"] [dependencies] -chia = "0.48" +chia-bls = "0.48" bech32 = "0.12" sha2 = "0.11" [dev-dependencies] +chia-protocol = "0.48" hex = "0.4" [profile.release] diff --git a/keygen-rs/src/lib.rs b/keygen-rs/src/lib.rs index 550702c..26c7854 100644 --- a/keygen-rs/src/lib.rs +++ b/keygen-rs/src/lib.rs @@ -1,17 +1,16 @@ -// pos2_keygen — C-callable shim around chia (chia-bls + chia-protocol) +// pos2_keygen — C-callable shim around chia-bls // that derives a v2 plot's plot_id and memo from caller-supplied farmer + // pool keys plus a 32-byte master-SK seed. The GPU plotter uses the returned // plot_id / memo to drive the existing batch path. // // The heavy lifting (BLS12-381 arithmetic, EIP-2333 HD derivation, Chia's -// taproot construction, compute_plot_id_v2 hashing) lives in chia-rs; this +// taproot construction) uses chia-rs; this // crate just sequences the calls and lays out the memo bytes the same way // chia-blockchain's create_v2_plots does, so the resulting plots are // byte-identical to `chia plots create --v2`. -use chia::bls::{PublicKey, SecretKey}; -use chia::protocol::{compute_plot_id_v2, Bytes32}; -use chia::sha2::Sha256; +use chia_bls::{PublicKey, SecretKey}; +use sha2::{Digest, Sha256}; // --------------------------------------------------------------------------- // Result codes returned across the FFI boundary. @@ -70,6 +69,27 @@ fn generate_plot_public_key( } } +// Chia's protocol crate also requires the CLVM runtime. Keep only the plot-ID +// encoding here and compare it with that crate in tests. Integers in Chia's +// streamable format are big-endian, unlike the subseed index below. +fn compute_plot_id_v2( + strength: u8, + plot_pk: &PublicKey, + pool_key: &[u8], + plot_index: u16, + meta_group: u8, +) -> [u8; 32] { + let mut group = Sha256::new(); + group.update([strength]); + group.update(plot_pk.to_bytes()); + group.update(pool_key); + let mut id = Sha256::new(); + id.update(group.finalize()); + id.update(plot_index.to_be_bytes()); + id.update([meta_group]); + id.finalize().into() +} + /// Derives a v2 plot's plot_id and memo from caller-supplied keys. /// /// Inputs: @@ -126,45 +146,34 @@ pub unsafe extern "C" fn pos2_keygen_derive_plot( Err(_) => return POS2_BAD_FARMER_PK, }; - let (pool_pk_opt, pool_ph_opt, pool_key_slice): (Option, Option, &[u8]) = - match pool_kind { - x if x == POS2_POOL_PK => { - let bytes: &[u8; 48] = match unsafe { (pool_key_ptr as *const [u8; 48]).as_ref() } { - Some(b) => b, - None => return POS2_BAD_POOL_KEY, - }; - let pk = match PublicKey::from_bytes(bytes) { - Ok(pk) => pk, - Err(_) => return POS2_BAD_POOL_KEY, - }; - (Some(pk), None, &bytes[..]) - } - x if x == POS2_POOL_PH => { - let bytes: &[u8; 32] = match unsafe { (pool_key_ptr as *const [u8; 32]).as_ref() } { - Some(b) => b, - None => return POS2_BAD_POOL_KEY, - }; - let ph: Bytes32 = (*bytes).into(); - (None, Some(ph), &bytes[..]) + let (include_taproot, pool_key_slice): (bool, &[u8]) = match pool_kind { + x if x == POS2_POOL_PK => { + let bytes: &[u8; 48] = match unsafe { (pool_key_ptr as *const [u8; 48]).as_ref() } { + Some(b) => b, + None => return POS2_BAD_POOL_KEY, + }; + if PublicKey::from_bytes(bytes).is_err() { + return POS2_BAD_POOL_KEY; } - _ => return POS2_BAD_POOL_KIND, - }; + (false, &bytes[..]) + } + x if x == POS2_POOL_PH => { + let bytes: &[u8; 32] = match unsafe { (pool_key_ptr as *const [u8; 32]).as_ref() } { + Some(b) => b, + None => return POS2_BAD_POOL_KEY, + }; + (true, &bytes[..]) + } + _ => return POS2_BAD_POOL_KIND, + }; let master_sk = SecretKey::from_seed(seed); let local_sk = master_sk_to_local_sk(&master_sk); let local_pk = local_sk.public_key(); - let include_taproot = pool_ph_opt.is_some(); let plot_pk = generate_plot_public_key(&local_pk, &farmer_pk, include_taproot); - let plot_id: Bytes32 = compute_plot_id_v2( - strength, - &plot_pk, - pool_pk_opt.as_ref(), - pool_ph_opt.as_ref(), - plot_index, - meta_group, - ); + let plot_id = compute_plot_id_v2(strength, &plot_pk, pool_key_slice, plot_index, meta_group); let master_sk_bytes = master_sk.to_bytes(); let memo_len = pool_key_slice.len() + 48 /* farmer_pk */ + master_sk_bytes.len(); @@ -176,7 +185,7 @@ pub unsafe extern "C" fn pos2_keygen_derive_plot( } unsafe { - std::ptr::copy_nonoverlapping(plot_id.as_ref().as_ptr(), out_plot_id, 32); + std::ptr::copy_nonoverlapping(plot_id.as_ptr(), out_plot_id, 32); let dst = out_memo_buf; std::ptr::copy_nonoverlapping(pool_key_slice.as_ptr(), dst, pool_key_slice.len()); std::ptr::copy_nonoverlapping(farmer_pk_bytes.as_ptr(), dst.add(pool_key_slice.len()), 48); @@ -244,7 +253,6 @@ pub unsafe extern "C" fn pos2_keygen_derive_subseed( idx: u64, out_seed: *mut u8, // 32 bytes ) -> i32 { - use sha2::{Digest, Sha256}; if base_seed.is_null() || out_seed.is_null() { return POS2_BAD_SEED; } @@ -263,6 +271,35 @@ pub unsafe extern "C" fn pos2_keygen_derive_subseed( mod tests { use super::*; + #[test] + fn plot_ids_match_chia_protocol() { + let plot_pk = SecretKey::from_seed(&[0x11; 32]).public_key(); + let pool_pk = SecretKey::from_seed(&[0x22; 32]).public_key(); + let contract = chia_protocol::Bytes32::new([0x33; 32]); + for strength in [1, 2, 32, 63] { + for index in [0, 1, 255, 256, u16::MAX] { + for group in [0, 1, u8::MAX] { + for pool in [false, true] { + let bytes = pool_pk.to_bytes(); + let key: &[u8] = if pool { &bytes } else { contract.as_ref() }; + let expected = chia_protocol::compute_plot_id_v2( + strength, + &plot_pk, + pool.then_some(&pool_pk), + (!pool).then_some(&contract), + index, + group, + ); + assert_eq!( + compute_plot_id_v2(strength, &plot_pk, key, index, group), + expected.to_bytes() + ); + } + } + } + } + } + // Same inputs must produce identical plot_id + memo. #[test] fn deterministic_same_seed() { From 5724d6d6fb41a232433201ff1dcda1171a69471c Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 19:40:24 -0500 Subject: [PATCH 03/10] release: package Linux CUDA archives with runtime checks Build explicit Maxwell-through-Blackwell targets with CUDA 12.9.1 on Ubuntu 22.04. Package the executable, source and toolchain metadata, checksums, and dependency licenses through CPack. Test the extracted archive without a development toolkit and keep tagged releases in draft until GPU qualification. --- .github/workflows/release.yml | 69 +++++++++++++++++++++++++++++++++++ CMakeLists.txt | 7 ++++ CONTRIBUTING.md | 32 ++++++++++++++++ INSTALL.md | 21 +++++++++++ README.md | 3 +- ci/release/Containerfile | 18 +++++++++ ci/release/README.txt | 25 +++++++++++++ ci/release/licenses.hbs | 11 ++++++ cmake/Packaging.cmake | 56 ++++++++++++++++++++++++++++ keygen-rs/about.toml | 2 + scripts/build-release.sh | 28 ++++++++++++++ scripts/test/gpu-ci.py | 3 +- scripts/test/release.py | 47 ++++++++++++++++++++++++ 13 files changed, 320 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/release.yml create mode 100644 ci/release/Containerfile create mode 100644 ci/release/README.txt create mode 100644 ci/release/licenses.hbs create mode 100644 cmake/Packaging.cmake create mode 100644 keygen-rs/about.toml create mode 100755 scripts/build-release.sh create mode 100755 scripts/test/release.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b558bb6 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,69 @@ +name: Binary release + +on: + pull_request: + paths: + - .github/workflows/release.yml + - CMakeLists.txt + - cmake/** + - ci/release/** + - keygen-rs/** + - src/** + - tools/** + - contrib/** + - scripts/build-release.sh + - scripts/test/release.py + push: + tags: ['v*-cuda-only'] + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux-cuda: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v7 + - name: Build release image + run: docker build -t xchplot2-release -f ci/release/Containerfile ci/release + - name: Build and package + env: + RELEASE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || '' }} + run: | + docker run --rm -e RELEASE_TAG -v "$PWD:/src" xchplot2-release \ + bash scripts/build-release.sh + - name: Check extracted archive on the oldest supported runtime + run: | + docker run --rm -v "$PWD:/src:ro" -w /src ubuntu:22.04 bash -euc ' + apt-get update + apt-get install -y --no-install-recommends python3 libstdc++6 + python3 scripts/test/release.py build/release/dist/*.tar.gz + ' + - uses: actions/upload-artifact@v7 + with: + name: linux-x86_64-cuda + path: | + build/release/dist/*.tar.gz + build/release/dist/*.sha256 + if-no-files-found: error + + draft: + if: startsWith(github.ref, 'refs/tags/') + needs: linux-cuda + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: linux-x86_64-cuda + path: dist + - name: Attach archives to a draft release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + run: | + gh release create "$RELEASE_TAG" dist/* \ + --verify-tag --draft --title "$RELEASE_TAG" --generate-notes diff --git a/CMakeLists.txt b/CMakeLists.txt index 24e1db3..36dc4a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ if(_xchplot2_nvcc) set(_min_arch 9999) foreach(_a IN LISTS _arches) string(REGEX REPLACE "^(sm_|compute_)" "" _a "${_a}") + string(REGEX REPLACE "-(real|virtual)$" "" _a "${_a}") if(_a MATCHES "^[0-9]+$" AND _a LESS _min_arch) set(_min_arch ${_a}) endif() @@ -633,3 +634,9 @@ foreach(_target IN LISTS _xchplot2_targets) endif() endif() endforeach() + +# Archive packaging is opt-in; ordinary builds need no release tooling. +option(XCHPLOT2_PACKAGE "Configure binary release packaging" OFF) +if(XCHPLOT2_PACKAGE) + include(cmake/Packaging.cmake) +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0c557f1..db521b3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -214,6 +214,38 @@ When moving sections, update incoming links and keep the useful README entry headings. `docs/` is ignored local material; do not publish it as part of a documentation move. +## Binary releases + +The release workflow builds the standalone CMake executable in +`ci/release/Containerfile`: Ubuntu 22.04, CUDA 12.9.1, CMake 3.28.3, and +Rust 1.98.1. CUDA targets are explicit in `scripts/build-release.sh`; keep +the compatibility requirements in `INSTALL.md` and the archive README in +sync when changing them. `cargo-about` collects the Rust dependency licenses +and fails on unresolved licenses. + +Build the same archive locally with Docker or Podman: + +```bash +podman build -t xchplot2-release -f ci/release/Containerfile ci/release +podman run --rm -v "$PWD:/src" xchplot2-release bash scripts/build-release.sh +``` + +The archive and its SHA-256 checksum are written to `build/release/dist/`. +PR and manual runs retain them as workflow artifacts. Pushing a +`vVERSION-cuda-only` tag creates a draft GitHub release; publish it after +qualifying the extracted archive on the supported GPUs. Do not rebuild +between qualification and publication. + +The workflow runs host tests and tests the extracted archive on Ubuntu 22.04 +without a GPU toolkit. Run `scripts/test/release.py ARCHIVE.tar.gz` to repeat +the archive, CPU plotting, and full-proof check. For GPU qualification, use +the packaged executable for k=22 and k=28 CPU byte comparisons, full proofs, +and the tier, spill, and recovery checks described above. Record qualification +in the release notes; keep generated plots and detailed logs out of the tree. +`gpu-ci.py --binary /path/to/extracted/bin/xchplot2` uses the package for +plotting and verification while retaining the build's parity and inventory +tools. Both must come from the same source revision and toolchain. + ## Commit style Short imperative subjects, lowercase scope prefix, no trailing period: diff --git a/INSTALL.md b/INSTALL.md index ce37a38..3901f3e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -9,6 +9,27 @@ git clone --branch cuda-only https://github.com/Jsewill/xchplot2 cd xchplot2 ``` +## Binary archives + +Download the Linux x86-64 CUDA archive and its `.sha256` file from +[GitHub Releases](https://github.com/Jsewill/xchplot2/releases). +Verify it with `sha256sum -c ARCHIVE.tar.gz.sha256`, replacing `ARCHIVE` +with the downloaded filename without `.tar.gz`. Extract it, then run +`./bin/xchplot2 devices` from the extracted directory. Add that `bin` +directory to `PATH` to use `xchplot2` elsewhere. + +The archive requires glibc 2.35+, `libstdc++.so.6` with `GLIBCXX_3.4.30` +(available on updated Ubuntu 22.04), and an x86-64 CPU with AES, SSSE3, +and SSE4.1. It includes native GPU code for +Maxwell through Blackwell and the CUDA runtime; no development toolkit is +needed. Use a compatible NVIDIA driver; 575.57.08+ is recommended for the +pinned CUDA 12.9.1 build. Compiled architecture coverage does not imply that +every card has been tested. Check the release notes for hardware qualification. + +Each archive includes `BUILDINFO.txt` with source and toolchain versions, +and `licenses/` with dependency notices. Releases without an archive require +a source build using the instructions below. + ## Requirements Requires CUDA Toolkit **12.0+** (12.0 is the floor — `cudaGetDeviceProperties_v2`, diff --git a/README.md b/README.md index 7211094..56efd63 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,8 @@ grouping, may require replotting. ## Quick start -Install the [build dependencies](INSTALL.md#requirements) first. +For prebuilt release archives, follow [binary installation](INSTALL.md#binary-archives). +To build from source, install the [build dependencies](INSTALL.md#requirements) first. For containers or Windows, follow [INSTALL.md](INSTALL.md). ```bash diff --git a/ci/release/Containerfile b/ci/release/Containerfile new file mode 100644 index 0000000..1efa001 --- /dev/null +++ b/ci/release/Containerfile @@ -0,0 +1,18 @@ +# CUDA 12.9 retains pre-Turing code generation. Ubuntu 22.04 sets the ABI floor. +FROM docker.io/nvidia/cuda:12.9.1-devel-ubuntu22.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ca-certificates curl git ninja-build python3-pip \ + && rm -rf /var/lib/apt/lists/* +RUN python3 -m pip install --no-cache-dir cmake==3.28.3 +RUN curl --proto '=https' --tlsv1.2 -sSfL \ + --retry 5 --retry-delay 10 --retry-all-errors \ + https://sh.rustup.rs -o /tmp/rustup-init.sh \ + && sh /tmp/rustup-init.sh -y --default-toolchain 1.98.1 --profile minimal \ + && rm /tmp/rustup-init.sh +ENV PATH=/root/.cargo/bin:${PATH} +RUN cargo install --locked --features cli cargo-about --version 0.9.2 +RUN git config --system --add safe.directory /src + +WORKDIR /src diff --git a/ci/release/README.txt b/ci/release/README.txt new file mode 100644 index 0000000..0d22e05 --- /dev/null +++ b/ci/release/README.txt @@ -0,0 +1,25 @@ +xchplot2 — native CUDA binary + +Requirements: + Linux x86_64 with glibc 2.35+ and libstdc++.so.6 with GLIBCXX_3.4.30. + Updated Ubuntu 22.04 or newer provides these runtime libraries. + CPU with AES, SSSE3, and SSE4.1 instructions. + NVIDIA Maxwell or newer GPU with a compatible driver. + NVIDIA driver 575.57.08 or newer is recommended for CUDA 12.9.1. + +Extract the archive, then run: + ./bin/xchplot2 --help + ./bin/xchplot2 devices + +The CUDA runtime is linked into the executable. No CUDA toolkit is needed. +BUILDINFO.txt records the source revisions, compilers, and GPU targets. +GPU targets describe compiled coverage; consult the benchmark report for +hardware measurements and the release notes for qualification results. + +Usage and installation: https://github.com/Jsewill/xchplot2/tree/cuda-only +Command reference: https://github.com/Jsewill/xchplot2/blob/cuda-only/REFERENCE.md +Benchmarks: https://github.com/Jsewill/xchplot2/blob/cuda-only/BENCHMARKS.md +Report issues: https://github.com/Jsewill/xchplot2/issues + +Dependency licenses are in licenses/. The bundled CPU solver includes the +growing-buffer correction in contrib/pos2-solver-candidates.patch. diff --git a/ci/release/licenses.hbs b/ci/release/licenses.hbs new file mode 100644 index 0000000..c0c81f8 --- /dev/null +++ b/ci/release/licenses.hbs @@ -0,0 +1,11 @@ +Rust dependencies included in xchplot2 + +{{#each licenses}} +{{name}} +{{#each used_by}} + {{crate.name}} {{crate.version}} +{{/each}} + +{{{text}}} + +{{/each}} diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake new file mode 100644 index 0000000..f7879b3 --- /dev/null +++ b/cmake/Packaging.cmake @@ -0,0 +1,56 @@ +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux" OR NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + message(FATAL_ERROR "Binary packaging currently supports Linux x86_64") +endif() +if(NOT EXISTS "${XCHPLOT2_LICENSE_DIR}/rust.txt") + message(FATAL_ERROR "Generate the release licenses first: scripts/build-release.sh") +endif() +if(DEFINED ENV{RELEASE_TAG} AND NOT "$ENV{RELEASE_TAG}" STREQUAL "" + AND NOT "$ENV{RELEASE_TAG}" STREQUAL "v${PROJECT_VERSION}-cuda-only") + message(FATAL_ERROR "Release tag must match v${PROJECT_VERSION}-cuda-only") +endif() + +execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + OUTPUT_VARIABLE _release_revision OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY) +execute_process(COMMAND "${GIT_EXECUTABLE}" diff --quiet HEAD -- + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE _release_dirty) +if(NOT _release_dirty EQUAL 0) + string(APPEND _release_revision "-dirty") +endif() +execute_process(COMMAND rustc --version + OUTPUT_VARIABLE _release_rust OUTPUT_STRIP_TRAILING_WHITESPACE + COMMAND_ERROR_IS_FATAL ANY) +file(WRITE "${CMAKE_BINARY_DIR}/BUILDINFO.txt" + "xchplot2 ${PROJECT_VERSION}\n" + "Backend: native CUDA\n" + "Source: ${_release_revision}\n" + "pos2-chip: ${POS2_CHIP_GIT_TAG} (contrib/pos2-solver-candidates.patch applied)\n" + "System: ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}\n" + "C++: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}\n" + "CUDA: ${CMAKE_CUDA_COMPILER_VERSION}\n" + "CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}\n" + "Rust: ${_release_rust}\n") + +install(TARGETS xchplot2 RUNTIME DESTINATION bin) +install(FILES "${CMAKE_BINARY_DIR}/BUILDINFO.txt" ci/release/README.txt + DESTINATION .) +install(FILES LICENSE DESTINATION licenses) +install(DIRECTORY "${XCHPLOT2_LICENSE_DIR}/" DESTINATION licenses) +install(FILES "${POS2_CHIP_DIR}/LICENSE" DESTINATION licenses RENAME pos2-chip.txt) +install(FILES "${POS2_CHIP_DIR}/lib/fse/LICENSE" DESTINATION licenses RENAME fse.txt) +file(READ "${POS2_CHIP_DIR}/src/pos/aes/soft_aes.hpp" _aes_source) +string(REGEX MATCH "^/\\*([^*]|\\*+[^*/])*\\*/" _aes_license "${_aes_source}") +if(NOT _aes_license) + message(FATAL_ERROR "Could not extract the pos2-chip AES license") +endif() +file(WRITE "${CMAKE_BINARY_DIR}/aes-license.txt" "${_aes_license}\n") +install(FILES "${CMAKE_BINARY_DIR}/aes-license.txt" DESTINATION licenses RENAME aes.txt) +set(CPACK_GENERATOR TGZ) +set(CPACK_PACKAGE_NAME xchplot2) +set(CPACK_PACKAGE_VERSION "${PROJECT_VERSION}") +set(CPACK_PACKAGE_FILE_NAME "xchplot2-${PROJECT_VERSION}-linux-x86_64-cuda") +set(CPACK_PACKAGE_CHECKSUM SHA256) +set(CPACK_STRIP_FILES ON) +set(CPACK_SOURCE_GENERATOR "") +include(CPack) diff --git a/keygen-rs/about.toml b/keygen-rs/about.toml new file mode 100644 index 0000000..172f9e0 --- /dev/null +++ b/keygen-rs/about.toml @@ -0,0 +1,2 @@ +accepted = ["Apache-2.0", "MIT", "BSD-2-Clause", "BSD-3-Clause", "ISC", "Unicode-3.0", "Zlib"] +ignore-dev-dependencies = true diff --git a/scripts/build-release.sh b/scripts/build-release.sh new file mode 100755 index 0000000..f019a37 --- /dev/null +++ b/scripts/build-release.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Build the Linux CUDA archive inside ci/release/Containerfile. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +build_dir="${1:-build/release}" +mkdir -p "$build_dir/licenses" +build_dir="$(cd "$build_dir" && pwd)" + +cp /usr/share/doc/cuda-cudart-12-9/copyright "$build_dir/licenses/cuda.txt" +curl --proto '=https' --tlsv1.2 -sSfL --retry 5 \ + https://raw.githubusercontent.com/NVIDIA/cccl/v2.8.2/LICENSE \ + -o "$build_dir/licenses/cuda-cccl.txt" +rust_docs="$(rustc --print sysroot)/share/doc/rust" +mkdir -p "$build_dir/licenses/rust-standard-library" +cp "$rust_docs/COPYRIGHT-library.html" "$build_dir/licenses/rust-standard-library/" +cp -r "$rust_docs/licenses" "$build_dir/licenses/rust-standard-library/" +cargo about generate --locked --fail \ + --manifest-path keygen-rs/Cargo.toml --target x86_64-unknown-linux-gnu \ + --output-file "$build_dir/licenses/rust.txt" ci/release/licenses.hbs +cmake -S . -B "$build_dir" -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CUDA_ARCHITECTURES='50-real;52-real;60-real;61-real;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120' \ + -DCMAKE_CUDA_RUNTIME_LIBRARY=Static \ + -DXCHPLOT2_PACKAGE=ON -DXCHPLOT2_LICENSE_DIR="$build_dir/licenses" +cmake --build "$build_dir" --parallel "${CMAKE_BUILD_PARALLEL_LEVEL:-2}" +ctest --test-dir "$build_dir" --output-on-failure --no-tests=error \ + -R '^(bench_stats_test|numa_topology_test|temp_file_test|spill_engine_test|spill_coverage_test|host_guard_test|host_spill_policy_test|vram_budget_test|cli_host_test|plot_file_parity|solver_filter_parity)$' +cpack --config "$build_dir/CPackConfig.cmake" -B "$build_dir/dist" diff --git a/scripts/test/gpu-ci.py b/scripts/test/gpu-ci.py index 85ecd16..e074c91 100644 --- a/scripts/test/gpu-ci.py +++ b/scripts/test/gpu-ci.py @@ -64,6 +64,7 @@ def tier_caps(info, physical_mib=0): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("build", type=Path) + parser.add_argument("--binary", type=Path, help="Test an extracted release executable") parser.add_argument("--backend", choices=MASKS, required=True) parser.add_argument("--suite", choices=("quick", "vram", "physical"), default="quick") parser.add_argument("--physical-vram-mib", type=int, default=0) @@ -80,7 +81,7 @@ def main(): if not scratch.is_dir(): parser.error("--scratch must be an existing directory on real disk") env["TMPDIR"] = str(scratch) - binary = build / "tools/xchplot2/xchplot2" + binary = args.binary.resolve() if args.binary else build / "tools/xchplot2/xchplot2" def run(label, command, run_env=None, cwd=None): print(f"Running {label}", flush=True) diff --git a/scripts/test/release.py b/scripts/test/release.py new file mode 100755 index 0000000..4215ea0 --- /dev/null +++ b/scripts/test/release.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Exercise an extracted binary archive without a compiler or GPU toolkit.""" +import argparse +import hashlib +import os +import pathlib +import subprocess +import tarfile +import tempfile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("archive", type=pathlib.Path) + args = parser.parse_args() + digest = hashlib.sha256() + with args.archive.open("rb") as archive: + for block in iter(lambda: archive.read(1024 * 1024), b""): + digest.update(block) + checksum = pathlib.Path(str(args.archive) + ".sha256").read_text().split() + assert checksum == [digest.hexdigest(), args.archive.name], "Archive checksum mismatch" + with tempfile.TemporaryDirectory(prefix="xchplot2-release-") as temporary: + work = pathlib.Path(temporary) + with tarfile.open(args.archive) as archive: + archive.extractall(work, filter="data") + packages = list(work.iterdir()) + assert len(packages) == 1 and packages[0].is_dir(), "Expected one package directory" + package = packages[0] + for name in ("BUILDINFO.txt", "README.txt", "licenses/LICENSE", "licenses/rust.txt", + "licenses/pos2-chip.txt", "licenses/fse.txt", "licenses/aes.txt", "licenses/cuda.txt", + "licenses/cuda-cccl.txt", "licenses/rust-standard-library/COPYRIGHT-library.html"): + assert (package / name).stat().st_size > 0, f"Missing or empty {name}" + binary = package / "bin/xchplot2" + subprocess.run([binary, "--help", "--config", os.devnull], check=True, timeout=30) + plot_id, memo = "ab" * 32, "00" * 112 + manifest = work / "cpu.tsv" + manifest.write_text(f"18 2 0 0 0 {plot_id} {memo} . cpu.plot2\n") + subprocess.run([binary, "batch", manifest, "--devices", "cpu", "--config", os.devnull], + cwd=work, check=True, timeout=180) + subprocess.run([binary, "verify", work / "cpu.plot2", "--full", "--trials", "100", + "--config", os.devnull], + check=True, timeout=180) + print("Release archive: extraction, CLI, CPU plotting, and full proofs passed") + + +if __name__ == "__main__": + main() From 23b58eb9ff8d7d40f60dc4270be9ce31c9f5e8a9 Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 21:41:48 -0500 Subject: [PATCH 04/10] windows: port CUDA host I/O and package native releases Preserve private file creation, positional spill I/O, durable publication, and cancellation on native Windows. Build the standalone CLI with MSVC and a matching static Rust runtime, and verify the extracted ZIP with real CPU plotting and recovery checks. --- .github/workflows/release.yml | 48 +++++++++++- CMakeLists.txt | 44 +++++++++-- CONTRIBUTING.md | 23 +++++- INSTALL.md | 105 ++++++++++--------------- REFERENCE.md | 3 +- ci/release/README.windows.txt | 24 ++++++ ci/release/microsoft-runtime.txt | 7 ++ cmake/Packaging.cmake | 29 +++++-- scripts/build-release.ps1 | 40 ++++++++++ scripts/test/release.py | 20 +++-- scripts/test/windows.py | 90 +++++++++++++++++++++ src/host/BatchManifest.cpp | 11 ++- src/host/BatchPlotter.cpp | 22 +++++- src/host/Cancel.cpp | 17 ++++ src/host/PlotFileWriterParallel.cpp | 13 +++- src/host/TempFile.cpp | 117 +++++++++++++++++++++++++++- src/host/TempFile.hpp | 12 +-- src/host/WindowsFile.hpp | 93 ++++++++++++++++++++++ tools/parity/cli_host_test.cpp | 68 ++++++++++++++-- tools/parity/host_guard_test.cpp | 7 ++ tools/parity/spill_engine_test.cpp | 21 +++++ tools/parity/temp_file_test.cpp | 95 +++++++++++++++++----- tools/xchplot2/cli.cpp | 31 +++++++- tools/xchplot2/main.cpp | 7 ++ tools/xchplot2/windows.manifest | 9 +++ 25 files changed, 823 insertions(+), 133 deletions(-) create mode 100644 ci/release/README.windows.txt create mode 100644 ci/release/microsoft-runtime.txt create mode 100644 scripts/build-release.ps1 create mode 100644 scripts/test/windows.py create mode 100644 src/host/WindowsFile.hpp create mode 100644 tools/xchplot2/windows.manifest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b558bb6..7e4e6ab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,9 @@ on: - tools/** - contrib/** - scripts/build-release.sh + - scripts/build-release.ps1 - scripts/test/release.py + - scripts/test/windows.py push: tags: ['v*-cuda-only'] workflow_dispatch: @@ -48,9 +50,50 @@ jobs: build/release/dist/*.sha256 if-no-files-found: error + windows-cuda: + runs-on: windows-2022 + timeout-minutes: 60 + defaults: + run: + shell: pwsh + steps: + - name: Preserve source line endings + run: git config --global core.autocrlf false + - uses: actions/checkout@v7 + - name: Install CUDA 12.9.1 compiler and runtime headers + run: | + $installer = Join-Path $env:RUNNER_TEMP 'cuda.exe' + Invoke-WebRequest 'https://developer.download.nvidia.com/compute/cuda/12.9.1/network_installers/cuda_12.9.1_windows_network.exe' -OutFile $installer + $install = Start-Process $installer -ArgumentList '-s', '-n', 'nvcc_12.9', 'cudart_12.9', 'thrust_12.9' -Wait -PassThru + if ($install.ExitCode -ne 0) { throw "CUDA installer failed: $($install.ExitCode)" } + "CUDA_PATH=$env:ProgramFiles\NVIDIA GPU Computing Toolkit\CUDA\v12.9" >> $env:GITHUB_ENV + - name: Install release Rust tooling + run: | + $PSNativeCommandUseErrorActionPreference = $true + rustup toolchain install 1.98.1 --profile minimal + rustup default 1.98.1 + cargo install --locked --features cli cargo-about --version 0.9.2 + - name: Build and package + env: + RELEASE_TAG: ${{ github.ref_type == 'tag' && github.ref_name || '' }} + run: ./scripts/build-release.ps1 + - name: Check extracted archive without toolkit libraries on PATH + run: | + $python = (Get-Command python.exe).Source + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + $archive = (Get-ChildItem build/release-windows/dist/*.zip).FullName + & $python scripts/test/release.py $archive + - uses: actions/upload-artifact@v7 + with: + name: windows-x86_64-cuda + path: | + build/release-windows/dist/*.zip + build/release-windows/dist/*.sha256 + if-no-files-found: error + draft: if: startsWith(github.ref, 'refs/tags/') - needs: linux-cuda + needs: [linux-cuda, windows-cuda] runs-on: ubuntu-24.04 permissions: contents: write @@ -58,7 +101,8 @@ jobs: - uses: actions/checkout@v7 - uses: actions/download-artifact@v8 with: - name: linux-x86_64-cuda + pattern: '*-x86_64-cuda' + merge-multiple: true path: dist - name: Attach archives to a draft release env: diff --git a/CMakeLists.txt b/CMakeLists.txt index 36dc4a1..7ec4c6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -161,6 +161,14 @@ unset(_xchplot2_nvcc CACHE) project(pos2-gpu VERSION 0.11.0 LANGUAGES C CXX CUDA) +if(MSVC) + # Match the release Rust staticlib's CRT in every CMake configuration. + set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded) + add_compile_definitions(NOMINMAX WIN32_LEAN_AND_MEAN _CRT_SECURE_NO_WARNINGS) + add_compile_options("$<$:/utf-8>" + "$<$:-Xcompiler=/utf-8>") +endif() + # Default to Release: a plain `cmake -B build -S .` otherwise builds # with EMPTY optimization flags — host C++ (FSE compression, writer, # host-side merges) at -O0 silently runs several times slower while @@ -269,6 +277,11 @@ message(STATUS "pos2-chip: ${POS2_CHIP_DIR}") # pos2-chip vendors FSE under lib/fse with its own CMakeLists.txt. # Bring it in so we can link against the same static lib pos2-chip uses. add_subdirectory("${POS2_CHIP_DIR}/lib/fse" "${CMAKE_BINARY_DIR}/fse" EXCLUDE_FROM_ALL) +if(MSVC) + # Upstream's MSVC list has argumentless /wd, /we, and /D options. + # CMake already supplies the optimization and debug flags per configuration. + set_property(TARGET fse PROPERTY COMPILE_OPTIONS /W4) +endif() # Header-only pos2-chip include surface # Keep the upstream checkout intact. Its solver treats a statistical candidate @@ -296,7 +309,7 @@ target_include_directories(pos2_chip_headers INTERFACE ) target_link_libraries(pos2_chip_headers INTERFACE fse) # Match pos2-chip's CPU AES build flags so its AesHash compiles in CPU TUs. -if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") +if(NOT MSVC AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64") target_compile_options(pos2_chip_headers INTERFACE -maes -msse4.1 -mssse3 -msse2) elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") target_compile_options(pos2_chip_headers INTERFACE -march=armv8-a+crypto) @@ -393,7 +406,11 @@ target_link_libraries(pos2_gpu_host PUBLIC pos2_chip_headers pos2_gpu) # ---------------------------------------------------------------------------- set(POS2_KEYGEN_CRATE_DIR "${CMAKE_SOURCE_DIR}/keygen-rs") set(POS2_KEYGEN_TARGET_DIR "${CMAKE_BINARY_DIR}/keygen-rs-target") -set(POS2_KEYGEN_STATIC_LIB "${POS2_KEYGEN_TARGET_DIR}/release/libpos2_keygen.a") +set(POS2_KEYGEN_STATIC_LIB "${POS2_KEYGEN_TARGET_DIR}/release/${CMAKE_STATIC_LIBRARY_PREFIX}pos2_keygen${CMAKE_STATIC_LIBRARY_SUFFIX}") +set(_pos2_keygen_env "CARGO_TARGET_DIR=${POS2_KEYGEN_TARGET_DIR}") +if(MSVC) + list(APPEND _pos2_keygen_env "RUSTFLAGS=$ENV{RUSTFLAGS} -C target-feature=+crt-static") +endif() # Collect the sources so CMake knows when to rebuild. file(GLOB_RECURSE POS2_KEYGEN_SRCS @@ -405,7 +422,7 @@ file(GLOB_RECURSE POS2_KEYGEN_SRCS add_custom_command( OUTPUT "${POS2_KEYGEN_STATIC_LIB}" COMMAND ${CMAKE_COMMAND} -E env - CARGO_TARGET_DIR=${POS2_KEYGEN_TARGET_DIR} + ${_pos2_keygen_env} cargo build --release --locked --manifest-path=${POS2_KEYGEN_CRATE_DIR}/Cargo.toml DEPENDS ${POS2_KEYGEN_SRCS} WORKING_DIRECTORY "${POS2_KEYGEN_CRATE_DIR}" @@ -424,6 +441,8 @@ add_dependencies(pos2_keygen pos2_keygen_build) # Rust staticlibs link in std, which needs these system libs at final link. if(CMAKE_SYSTEM_NAME STREQUAL "Linux") target_link_libraries(pos2_keygen INTERFACE pthread dl m) +elseif(WIN32) + target_link_libraries(pos2_keygen INTERFACE ws2_32 userenv bcrypt ntdll advapi32) endif() # xchplot2_cli: the CLI dispatch logic as a static library, with the @@ -470,8 +489,12 @@ add_executable(xchplot2 tools/xchplot2/main.cpp) # the resolutions before it sees the unresolved references and bails. # Cargo install already passes --start-group via build.rs; the cmake # exe needs the same treatment. -target_link_libraries(xchplot2 PRIVATE - "$") +if(MSVC) + target_link_libraries(xchplot2 PRIVATE xchplot2_cli pos2_gpu_host) +else() + target_link_libraries(xchplot2 PRIVATE + "$") +endif() # pos2-chip's soft_aesenc / soft_aesdec are defined (not just declared) # in headers without `inline`, so any TU that includes the chain — both # PlotFileWriterParallel.cpp and CpuPlotter.cpp do, transitively via @@ -481,7 +504,11 @@ target_link_libraries(xchplot2 PRIVATE # build.rs for an unrelated keygen-rs / libstd duplication; the cmake # exe needs it too once CpuPlotter joined PlotFileWriterParallel as a # pos2-chip-including TU. -target_link_options(xchplot2 PRIVATE LINKER:--allow-multiple-definition) +if(MSVC) + target_link_options(xchplot2 PRIVATE LINKER:/FORCE:MULTIPLE) +else() + target_link_options(xchplot2 PRIVATE LINKER:--allow-multiple-definition) +endif() # Parity tests. Each test gets $ # explicitly: pos2_gpu (INTERFACE) doesn't carry the .o files, so @@ -626,6 +653,11 @@ set_target_properties(gpu_ci_info PROPERTIES include(CTest) get_property(_xchplot2_targets DIRECTORY PROPERTY BUILDSYSTEM_TARGETS) foreach(_target IN LISTS _xchplot2_targets) + get_target_property(_type ${_target} TYPE) + if(WIN32 AND _type STREQUAL "EXECUTABLE") + target_sources(${_target} PRIVATE tools/xchplot2/windows.manifest) + target_link_libraries(${_target} PRIVATE advapi32) + endif() if(_target MATCHES "(_parity|_test)$") set_target_properties(${_target} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tools/parity") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index db521b3..bb5d3d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -230,8 +230,27 @@ podman build -t xchplot2-release -f ci/release/Containerfile ci/release podman run --rm -v "$PWD:/src" xchplot2-release bash scripts/build-release.sh ``` -The archive and its SHA-256 checksum are written to `build/release/dist/`. -PR and manual runs retain them as workflow artifacts. Pushing a +The Linux archive and its SHA-256 checksum are written to `build/release/dist/`. +For native Windows, install the [Windows build tools](INSTALL.md#windows) +and PowerShell 7.3+, then run: + +```powershell +rustup toolchain install 1.98.1 --profile minimal +rustup default 1.98.1 +cargo install --locked --features cli cargo-about --version 0.9.2 +./scripts/build-release.ps1 +python scripts/test/release.py build/release-windows/dist/xchplot2-0.11.0-windows-x86_64-cuda.zip +``` + +The PowerShell script loads the Visual Studio 2022 x64 environment when +needed, builds all targets with CUDA 12.9.1, runs the host CTest subset, and +writes a ZIP and checksum to `build/release-windows/dist/`. The extracted +Windows check also exercises Unicode paths, real key generation, Ctrl-Break, +resume, and publication failure. CI runs it with toolkit libraries removed +from `PATH`. Windows GPU plotting and spill behavior need qualification on +Windows hardware before the archive is advertised for those devices. + +PR and manual runs retain both platforms' archives as workflow artifacts. Pushing a `vVERSION-cuda-only` tag creates a draft GitHub release; publish it after qualifying the extracted archive on the supported GPUs. Do not rebuild between qualification and publication. diff --git a/INSTALL.md b/INSTALL.md index 3901f3e..f314630 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -11,14 +11,14 @@ cd xchplot2 ## Binary archives -Download the Linux x86-64 CUDA archive and its `.sha256` file from +Download the Linux or Windows x86-64 CUDA archive and its `.sha256` file from [GitHub Releases](https://github.com/Jsewill/xchplot2/releases). -Verify it with `sha256sum -c ARCHIVE.tar.gz.sha256`, replacing `ARCHIVE` +On Linux, verify it with `sha256sum -c ARCHIVE.tar.gz.sha256`, replacing `ARCHIVE` with the downloaded filename without `.tar.gz`. Extract it, then run `./bin/xchplot2 devices` from the extracted directory. Add that `bin` directory to `PATH` to use `xchplot2` elsewhere. -The archive requires glibc 2.35+, `libstdc++.so.6` with `GLIBCXX_3.4.30` +The Linux archive requires glibc 2.35+, `libstdc++.so.6` with `GLIBCXX_3.4.30` (available on updated Ubuntu 22.04), and an x86-64 CPU with AES, SSSE3, and SSE4.1. It includes native GPU code for Maxwell through Blackwell and the CUDA runtime; no development toolkit is @@ -26,6 +26,10 @@ needed. Use a compatible NVIDIA driver; 575.57.08+ is recommended for the pinned CUDA 12.9.1 build. Compiled architecture coverage does not imply that every card has been tested. Check the release notes for hardware qualification. +On Windows, compare `(Get-FileHash ARCHIVE.zip -Algorithm SHA256).Hash` +with the first field in `ARCHIVE.zip.sha256`, then extract the ZIP and run +`.\bin\xchplot2.exe devices`. See [Windows](#windows) for its requirements. + Each archive includes `BUILDINFO.txt` with source and toolchain versions, and `licenses/` with dependency notices. Releases without an archive require a source build using the instructions below. @@ -166,68 +170,41 @@ run` step won't see the GPU. ## Windows -Native Windows builds and plotting are experimental and outside the current -hardware test set. WSL2 uses the Linux build instructions. - -Prerequisites: - -- Windows 10 21H2+ or Windows 11, x64 -- [Visual Studio 2022](https://visualstudio.microsoft.com/) Community - with the **"Desktop development with C++"** workload. That workload - bundles MSVC + the Windows SDK; the SDK is non-optional because it - ships `kernel32.lib` / `user32.lib` / etc. that `link.exe` - consumes. If you've trimmed the installer to "C++ build tools" - only, open **Visual Studio Installer → Modify → Individual - components** and tick the latest **Windows 11 SDK** before - retrying. -- [CUDA Toolkit 12.0+](https://developer.nvidia.com/cuda-downloads) — - install **after** Visual Studio so the CUDA installer wires up the - MSBuild integration. 12.8+ required for RTX 50-series (Blackwell, - `sm_120`). -- [Rust](https://www.rust-lang.org/tools/install) using the MSVC - toolchain (`rustup default stable-x86_64-pc-windows-msvc`) -- [CMake 3.26+](https://cmake.org/download/) and [Git for - Windows](https://gitforwindows.org/) - -Launch the **x64 Native Tools Command Prompt for VS 2022** from the -Start menu — there are several similarly-named prompts (x86 / -x86_64 / 2019 / 2022); the one that matters is the x64 for 2022. -That prompt is the one that sets `LIB`, `INCLUDE`, and `PATH` so -`cl.exe`, `link.exe`, `nvcc`, and `cmake` all see each other plus -the Windows SDK. A plain `cmd` / PowerShell / Windows Terminal tab -does **not** do this — running `cargo install` from one of those -produces `LNK1181: cannot open input file 'kernel32.lib'` at the -first link step. - -Quick sanity check in the prompt: - -```cmd -where link.exe -echo %LIB% -``` - -`%LIB%` should include a `...\Windows Kits\10\Lib\...\um\x64` -entry. If it doesn't, you're in the wrong prompt or the Windows SDK -component isn't installed. - -Build: +Native Windows uses the standalone CMake executable. The release ZIP targets +Windows 10 22H2, Windows 11, and Windows Server 2022/2025 on x64. It links +CUDA and the Microsoft C/C++ runtime statically. Runtime use requires a +compatible NVIDIA driver (576.57+ recommended for CUDA 12.9.1) and an +AES/SSSE3/SSE4.1-capable CPU, without installing a development toolkit. + +Use NTFS or ReFS for plots, recovery manifests, and spill files. These files +contain private keys; creation requires filesystem support for access control +lists. Select spill storage with `--temp-dir`. The default configuration is +`%APPDATA%\xchplot2\config.toml`. Ctrl-C or Ctrl-Break drains the current +plots; a second signal aborts. Resume with the automatically saved manifest. +Consult the release notes for tested hardware and drivers. Hosted Windows +checks exercise the CPU path; GPU qualification requires Windows hardware. +WSL2 continues to use the Linux instructions. + +To build from source, install: + +- [Visual Studio 2022](https://visualstudio.microsoft.com/) C++ build tools + with the **Desktop development with C++** workload and Windows SDK. +- [CUDA Toolkit 12.9.1](https://developer.nvidia.com/cuda-12-9-1-download-archive). +- [Rust](https://www.rust-lang.org/tools/install) with the + `stable-x86_64-pc-windows-msvc` toolchain. +- [CMake 3.26+](https://cmake.org/download/), Ninja, and + [Git for Windows](https://gitforwindows.org/). + +In the **x64 Native Tools Command Prompt for VS 2022**, run: ```cmd -set CUDA_ARCHITECTURES=89 -cargo install --git https://github.com/Jsewill/xchplot2 --branch cuda-only --locked -``` - -Or for a local checkout you can iterate on: - -```cmd -git clone -b cuda-only https://github.com/Jsewill/xchplot2 -cd xchplot2 -set CUDA_ARCHITECTURES=89 -cargo install --path . --locked +cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=89 +cmake --build build --parallel 2 +.\build\tools\xchplot2\xchplot2.exe --help +ctest --test-dir build --output-on-failure ``` -Set `CUDA_ARCHITECTURES` to match your card (see the list above). -PowerShell users: use `$env:CUDA_ARCHITECTURES = "89"` instead of -`set`. The CMake path (`cmake -B build -S . && cmake --build build`) -also works inside the same Native Tools prompt if you prefer that over -`cargo install`. +Change `89` to your GPU architecture from the table above. The full CTest +suite needs a GPU. Use the [release recipe](CONTRIBUTING.md#binary-releases) +for the supported archive build and its host-only checks. Windows Cargo +installation is not supported by this release path. diff --git a/REFERENCE.md b/REFERENCE.md index e88ac3e..4adff66 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -125,7 +125,8 @@ Prefer the automatically saved manifest for recovery; see ## Configuration and argument files `--config FILE` loads a configuration file. Without it, xchplot2 looks for -`$HOME/.config/xchplot2/config.toml`. The supported syntax is a small TOML +`$HOME/.config/xchplot2/config.toml` on Linux or +`%APPDATA%\xchplot2\config.toml` on Windows. The supported syntax is a small TOML subset: named sections and scalar `key = value` entries, with double-quoted strings and `#` or `;` comments. Arrays, nested tables, and multiline strings are unsupported. diff --git a/ci/release/README.windows.txt b/ci/release/README.windows.txt new file mode 100644 index 0000000..9ad35e9 --- /dev/null +++ b/ci/release/README.windows.txt @@ -0,0 +1,24 @@ +xchplot2 — native Windows CUDA binary + +Run .\bin\xchplot2.exe --help or .\bin\xchplot2.exe devices. +Add the extracted bin directory to PATH to run the CLI elsewhere. + +Requirements: + Windows 10 22H2, Windows 11, or Windows Server 2022/2025, x86_64. + CPU with AES, SSSE3, and SSE4.1 instructions. + NVIDIA Maxwell or newer GPU; driver 576.57+ recommended for CUDA 12.9.1. + An ACL-capable filesystem (NTFS/ReFS) for plots, manifests, and spill files. + +CUDA and the Microsoft C/C++ runtime are linked statically. No development +toolkit is needed. BUILDINFO.txt records the source and toolchain revisions; +licenses/ contains dependency notices. GPU drivers are not bundled. + +First Ctrl-C or Ctrl-Break finishes the current plot; a second aborts. +Use --resume with the saved manifest to continue an interrupted job. +The default config is %APPDATA%\xchplot2\config.toml. +Use --temp-dir to select a disk with enough free space for spill files. +Consult the release notes for tested hardware and driver versions. + +Usage: https://github.com/Jsewill/xchplot2/tree/cuda-only +Installation: https://github.com/Jsewill/xchplot2/blob/cuda-only/INSTALL.md +Report issues: https://github.com/Jsewill/xchplot2/issues diff --git a/ci/release/microsoft-runtime.txt b/ci/release/microsoft-runtime.txt new file mode 100644 index 0000000..953897c --- /dev/null +++ b/ci/release/microsoft-runtime.txt @@ -0,0 +1,7 @@ +Microsoft Visual C++ runtime +Copyright Microsoft Corporation. All rights reserved. + +The native Windows executable statically links the release C/C++ runtime. +Microsoft's license terms and redistribution documentation are available at: +https://visualstudio.microsoft.com/license-terms/vs2022-cruntime/ +https://learn.microsoft.com/en-us/cpp/windows/redistributing-visual-cpp-files diff --git a/cmake/Packaging.cmake b/cmake/Packaging.cmake index f7879b3..4c3cc77 100644 --- a/cmake/Packaging.cmake +++ b/cmake/Packaging.cmake @@ -1,8 +1,8 @@ -if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux" OR NOT CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - message(FATAL_ERROR "Binary packaging currently supports Linux x86_64") +if(NOT CMAKE_SYSTEM_NAME MATCHES "^(Linux|Windows)$" OR NOT CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$") + message(FATAL_ERROR "Binary packaging supports Linux and Windows x86_64") endif() if(NOT EXISTS "${XCHPLOT2_LICENSE_DIR}/rust.txt") - message(FATAL_ERROR "Generate the release licenses first: scripts/build-release.sh") + message(FATAL_ERROR "Generate the release licenses first with scripts/build-release.sh or scripts/build-release.ps1") endif() if(DEFINED ENV{RELEASE_TAG} AND NOT "$ENV{RELEASE_TAG}" STREQUAL "" AND NOT "$ENV{RELEASE_TAG}" STREQUAL "v${PROJECT_VERSION}-cuda-only") @@ -33,8 +33,13 @@ file(WRITE "${CMAKE_BINARY_DIR}/BUILDINFO.txt" "Rust: ${_release_rust}\n") install(TARGETS xchplot2 RUNTIME DESTINATION bin) -install(FILES "${CMAKE_BINARY_DIR}/BUILDINFO.txt" ci/release/README.txt - DESTINATION .) +install(FILES "${CMAKE_BINARY_DIR}/BUILDINFO.txt" DESTINATION .) +if(WIN32) + install(FILES ci/release/README.windows.txt DESTINATION . RENAME README.txt) + install(FILES ci/release/microsoft-runtime.txt DESTINATION licenses) +else() + install(FILES ci/release/README.txt DESTINATION .) +endif() install(FILES LICENSE DESTINATION licenses) install(DIRECTORY "${XCHPLOT2_LICENSE_DIR}/" DESTINATION licenses) install(FILES "${POS2_CHIP_DIR}/LICENSE" DESTINATION licenses RENAME pos2-chip.txt) @@ -46,11 +51,19 @@ if(NOT _aes_license) endif() file(WRITE "${CMAKE_BINARY_DIR}/aes-license.txt" "${_aes_license}\n") install(FILES "${CMAKE_BINARY_DIR}/aes-license.txt" DESTINATION licenses RENAME aes.txt) -set(CPACK_GENERATOR TGZ) +if(WIN32) + set(CPACK_GENERATOR ZIP) + set(_release_system windows) +else() + set(CPACK_GENERATOR TGZ) + set(_release_system linux) +endif() set(CPACK_PACKAGE_NAME xchplot2) set(CPACK_PACKAGE_VERSION "${PROJECT_VERSION}") -set(CPACK_PACKAGE_FILE_NAME "xchplot2-${PROJECT_VERSION}-linux-x86_64-cuda") +set(CPACK_PACKAGE_FILE_NAME "xchplot2-${PROJECT_VERSION}-${_release_system}-x86_64-cuda") set(CPACK_PACKAGE_CHECKSUM SHA256) -set(CPACK_STRIP_FILES ON) +if(NOT MSVC) + set(CPACK_STRIP_FILES ON) +endif() set(CPACK_SOURCE_GENERATOR "") include(CPack) diff --git a/scripts/build-release.ps1 b/scripts/build-release.ps1 new file mode 100644 index 0000000..d8b3fc6 --- /dev/null +++ b/scripts/build-release.ps1 @@ -0,0 +1,40 @@ +#requires -Version 7.3 +# Build the native Windows CUDA archive with Visual Studio 2022 and CUDA 12.9.1. +param([string]$BuildDir = 'build/release-windows') +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true +Set-Location (Join-Path $PSScriptRoot '..') + +if (-not (Get-Command cl.exe -ErrorAction SilentlyContinue)) { + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio/Installer/vswhere.exe' + $visualStudio = & $vswhere -latest -version '[17.0,18.0)' -products '*' ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if (-not $visualStudio) { throw 'Visual Studio 2022 C++ build tools are required' } + $vcvars = Join-Path $visualStudio 'VC/Auxiliary/Build/vcvars64.bat' + cmd /c "call `"$vcvars`" >nul && set" | ForEach-Object { + if ($_ -match '^([^=]+)=(.*)$') { [Environment]::SetEnvironmentVariable($Matches[1], $Matches[2], 'Process') } + } +} +if (-not $env:CUDA_PATH) { throw 'Set CUDA_PATH to the CUDA 12.9.1 installation' } +$env:PATH = (Join-Path $env:CUDA_PATH 'bin') + ';' + $env:PATH +New-Item -ItemType Directory -Force $BuildDir | Out-Null +$BuildDir = (Resolve-Path $BuildDir).Path +$licenses = Join-Path $BuildDir 'licenses' +New-Item -ItemType Directory -Force $licenses | Out-Null +Copy-Item (Join-Path $env:CUDA_PATH 'EULA.txt') (Join-Path $licenses 'cuda.txt') +Invoke-WebRequest 'https://raw.githubusercontent.com/NVIDIA/cccl/v2.8.2/LICENSE' ` + -OutFile (Join-Path $licenses 'cuda-cccl.txt') +$rustDocs = Join-Path (rustc --print sysroot) 'share/doc/rust' +$rustLicenses = Join-Path $licenses 'rust-standard-library' +New-Item -ItemType Directory -Force $rustLicenses | Out-Null +Copy-Item (Join-Path $rustDocs 'COPYRIGHT-library.html') $rustLicenses +Copy-Item -Recurse -Force (Join-Path $rustDocs 'licenses') $rustLicenses +cargo about generate --locked --fail --manifest-path keygen-rs/Cargo.toml ` + --target x86_64-pc-windows-msvc --output-file (Join-Path $licenses 'rust.txt') ci/release/licenses.hbs +cmake -S . -B $BuildDir -G Ninja -DCMAKE_BUILD_TYPE=Release ` + '-DCMAKE_CUDA_ARCHITECTURES=50-real;52-real;60-real;61-real;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120' ` + -DCMAKE_CUDA_RUNTIME_LIBRARY=Static -DXCHPLOT2_PACKAGE=ON "-DXCHPLOT2_LICENSE_DIR=$licenses" +cmake --build $BuildDir --parallel 2 +ctest --test-dir $BuildDir --output-on-failure --no-tests=error ` + -R '^(bench_stats_test|numa_topology_test|temp_file_test|spill_engine_test|spill_coverage_test|host_guard_test|host_spill_policy_test|vram_budget_test|cli_host_test|plot_file_parity|solver_filter_parity)$' +cpack --config (Join-Path $BuildDir 'CPackConfig.cmake') -B (Join-Path $BuildDir 'dist') diff --git a/scripts/test/release.py b/scripts/test/release.py index 4215ea0..ae27a76 100755 --- a/scripts/test/release.py +++ b/scripts/test/release.py @@ -5,8 +5,10 @@ import os import pathlib import subprocess +import sys import tarfile import tempfile +import zipfile def main(): @@ -19,10 +21,14 @@ def main(): digest.update(block) checksum = pathlib.Path(str(args.archive) + ".sha256").read_text().split() assert checksum == [digest.hexdigest(), args.archive.name], "Archive checksum mismatch" - with tempfile.TemporaryDirectory(prefix="xchplot2-release-") as temporary: + with tempfile.TemporaryDirectory(prefix="xchplot2 release é-") as temporary: work = pathlib.Path(temporary) - with tarfile.open(args.archive) as archive: - archive.extractall(work, filter="data") + if args.archive.suffix == ".zip": + with zipfile.ZipFile(args.archive) as archive: + archive.extractall(work) + else: + with tarfile.open(args.archive) as archive: + archive.extractall(work, filter="data") packages = list(work.iterdir()) assert len(packages) == 1 and packages[0].is_dir(), "Expected one package directory" package = packages[0] @@ -30,16 +36,20 @@ def main(): "licenses/pos2-chip.txt", "licenses/fse.txt", "licenses/aes.txt", "licenses/cuda.txt", "licenses/cuda-cccl.txt", "licenses/rust-standard-library/COPYRIGHT-library.html"): assert (package / name).stat().st_size > 0, f"Missing or empty {name}" - binary = package / "bin/xchplot2" + binary = package / ("bin/xchplot2.exe" if os.name == "nt" else "bin/xchplot2") subprocess.run([binary, "--help", "--config", os.devnull], check=True, timeout=30) plot_id, memo = "ab" * 32, "00" * 112 manifest = work / "cpu.tsv" manifest.write_text(f"18 2 0 0 0 {plot_id} {memo} . cpu.plot2\n") - subprocess.run([binary, "batch", manifest, "--devices", "cpu", "--config", os.devnull], + subprocess.run([binary, "batch", manifest, "--devices", "cpu", "--cpu-workers", "2", "--config", os.devnull], cwd=work, check=True, timeout=180) subprocess.run([binary, "verify", work / "cpu.plot2", "--full", "--trials", "100", "--config", os.devnull], check=True, timeout=180) + if os.name == "nt": + assert (package / "licenses/microsoft-runtime.txt").stat().st_size > 0 + subprocess.run([sys.executable, pathlib.Path(__file__).with_name("windows.py"), binary], + check=True, timeout=360) print("Release archive: extraction, CLI, CPU plotting, and full proofs passed") diff --git a/scripts/test/windows.py b/scripts/test/windows.py new file mode 100644 index 0000000..33547ec --- /dev/null +++ b/scripts/test/windows.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Check native Windows cancellation and durable recovery with real CPU plots.""" +import ctypes +import hashlib +import os +from pathlib import Path +import queue +import shlex +import signal +import subprocess +import sys +import tempfile +import threading + + +def main(): + assert os.name == "nt", "Run this check on native Windows" + binary = str(Path(sys.argv[1]).resolve()) + # Public BLS12-381 generator; all keys and plots in this check are disposable. + farmer = "97f1d3a73197d7942695638c4fa9ac0fc3688c4f9774b905a14e3a3f171bac586c55e83ff97a1aeffb3af00adb22c6bb" + kernel = ctypes.WinDLL("kernel32", use_last_error=True) + allocated_console = kernel.AllocConsole() + if not allocated_console and ctypes.get_last_error() != 5: + raise ctypes.WinError(ctypes.get_last_error()) + try: + with tempfile.TemporaryDirectory(prefix="xchplot2 Windows é-") as directory: + out = Path(directory) + args = [binary, "plot", "--config", os.devnull, "-k", "22", "-n", "12", + "-f", farmer, "--pool-ph", "42" * 32, "-o", str(out), + "--devices", "cpu", "--cpu-workers", "2", "--quiet", "--no-progress"] + with (out / "check.log").open("w", encoding="utf-8") as log: + def run(command, check=True): + return subprocess.run(command, stdout=subprocess.PIPE, stderr=log, text=True, + encoding="utf-8", check=check, timeout=180) + + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=log, text=True, + encoding="utf-8", creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + first_line = queue.Queue() + threading.Thread(target=lambda: first_line.put(process.stdout.readline()), daemon=True).start() + try: + first = first_line.get(timeout=90) + assert first, "No output was published before exit" + process.send_signal(signal.CTRL_BREAK_EVENT) + rest = process.communicate(timeout=180)[0] + finally: + if process.poll() is None: + process.kill() + process.wait() + paths = (first + rest).splitlines() + assert process.returncode == 4 and 0 < len(paths) < 12, (process.returncode, paths) + assert {Path(p) for p in paths} == set(out.glob("*.plot2")) + job, = out.glob("xchplot2-job-*.tsv") + saved = job.read_bytes() + entries = [shlex.split(line) for line in saved.decode("utf-8").splitlines() + if not line.startswith("#")] + expected = {str(Path(e[7]) / e[8]) for e in entries} + hashes = {p: hashlib.sha256(Path(p).read_bytes()).hexdigest() for p in paths} + result = run(args + ["--resume"]) + assert set(result.stdout.splitlines()) == expected + assert job.read_bytes() == saved + assert all(hashlib.sha256(Path(p).read_bytes()).hexdigest() == h for p, h in hashes.items()) + print("Windows Ctrl-Break: drained current plots; resume preserved keys and completed files") + + failed = sorted(expected)[0] + Path(failed).unlink() + Path(failed).mkdir() # force a real publication failure + result = run(args + ["--resume"], check=False) + assert result.returncode == 3 and set(result.stdout.splitlines()) == expected - {failed} + assert not list(out.glob("*.partial.*")) + Path(failed).rmdir() + run(args + ["--resume"]) + assert job.read_bytes() == saved + run([binary, "verify", failed, "--full", "--trials", "100", "--config", os.devnull]) + print("Windows publication failure: correct status, partial cleanup, recovery, and full proofs") + + pool_pk = out / "pool public key" + result = run([binary, "plot", "--config", os.devnull, "-k", "18", "-n", "2", + "-f", farmer, "-p", farmer, "-o", str(pool_pk), "--devices", "cpu", + "--cpu-workers", "2", "--quiet", "--no-progress"]) + assert len(result.stdout.splitlines()) == 2 + for path in result.stdout.splitlines(): + run([binary, "verify", path, "--full", "--trials", "100", "--config", os.devnull]) + print("Windows keygen: pool public key memos and parallel CPU plotting passed") + finally: + if allocated_console: + kernel.FreeConsole() + + +if __name__ == "__main__": + main() diff --git a/src/host/BatchManifest.cpp b/src/host/BatchManifest.cpp index 1c1a309..c3824ac 100644 --- a/src/host/BatchManifest.cpp +++ b/src/host/BatchManifest.cpp @@ -12,6 +12,7 @@ #include #ifdef _WIN32 +#include "host/WindowsFile.hpp" #ifndef NOMINMAX #define NOMINMAX #endif @@ -43,6 +44,11 @@ void validate_batch_entry(BatchEntry const& e) e.out_name.find_first_of("/\\") != std::string::npos || std::filesystem::path(e.out_name).has_root_path()) throw std::invalid_argument("output name must be a filename within the output directory"); +#ifdef _WIN32 + if (e.out_name.find_first_of("<>:\"|?*") != std::string::npos + || e.out_name.back() == '.' || e.out_name.back() == ' ') + throw std::invalid_argument("output name contains characters unsupported by Windows"); +#endif } namespace { @@ -151,10 +157,7 @@ void write_manifest(std::string const& path, std::vector const& entr // Publication must not replace a concurrently saved job. std::string partial = path + ".partial.XXXXXX"; #ifdef _WIN32 - if (_mktemp_s(partial.data(), partial.size() + 1) != 0) - throw std::runtime_error("cannot create temporary manifest: " + path); - int const fd = ::_open(partial.c_str(), _O_CREAT | _O_EXCL | _O_WRONLY | _O_BINARY, - _S_IREAD | _S_IWRITE); + int const fd = create_private_temp(partial); #else int const fd = ::mkstemp(partial.data()); #endif diff --git a/src/host/BatchPlotter.cpp b/src/host/BatchPlotter.cpp index 9a73593..95c447c 100644 --- a/src/host/BatchPlotter.cpp +++ b/src/host/BatchPlotter.cpp @@ -35,7 +35,11 @@ #include #include +#ifdef _WIN32 +#include +#else #include // isatty — in-place progress line only on a TTY +#endif #ifdef __linux__ #include // setpriority / PRIO_PROCESS — see nice_current_thread @@ -44,6 +48,7 @@ #endif #ifdef _WIN32 #include // GlobalMemoryStatusEx — see host_memory_probe +#include #endif namespace pos2gpu { @@ -201,7 +206,11 @@ std::uint64_t host_free_bytes_now() // CpuMemoryGate, which needs the SUM of the two. std::uint64_t self_rss_bytes() { -#if defined(__linux__) +#if defined(_WIN32) + PROCESS_MEMORY_COUNTERS counters{}; + if (!::GetProcessMemoryInfo(::GetCurrentProcess(), &counters, sizeof(counters))) return 0; + return static_cast(counters.WorkingSetSize); +#elif defined(__linux__) std::FILE* fp = std::fopen("/proc/self/statm", "re"); if (!fp) return 0; unsigned long long total_pages = 0; @@ -887,7 +896,11 @@ void emit_progress_line(std::string const& log_prefix, // On a TTY, rewrite one line in place ("\r" + clear-to-EOL); keep // one-line-per-plot when redirected to a file/pipe or when verbose // logging would interleave and garble the in-place line. +#ifdef _WIN32 + static bool const stderr_tty = ::_isatty(::_fileno(stderr)) != 0; +#else static bool const stderr_tty = ::isatty(::fileno(stderr)) != 0; +#endif bool const in_place = stderr_tty && !opts.verbose; // Only surfaces on a resume (--skip-existing). Without it the line counts @@ -2915,7 +2928,12 @@ BatchResult run_batch(std::vector const& entries, // tmpfs guard and die on a raw mkstemp errno minutes into a batch — and the // tmpfs message is exactly what tells a user to reach for this flag. if (!opts.temp_dir.empty()) { - ::setenv("XCHPLOT2_TEMP_DIR", opts.temp_dir.c_str(), /*overwrite=*/1); +#ifdef _WIN32 + int const env_error = ::_putenv_s("XCHPLOT2_TEMP_DIR", opts.temp_dir.c_str()); +#else + int const env_error = ::setenv("XCHPLOT2_TEMP_DIR", opts.temp_dir.c_str(), /*overwrite=*/1); +#endif + if (env_error) throw std::runtime_error("cannot set XCHPLOT2_TEMP_DIR"); std::string const problem = TempFile::dir_problem(opts.temp_dir); if (!problem.empty()) { throw std::runtime_error("--temp-dir " + opts.temp_dir + ": " + problem); diff --git a/src/host/Cancel.cpp b/src/host/Cancel.cpp index 49dc8a5..d6cda22 100644 --- a/src/host/Cancel.cpp +++ b/src/host/Cancel.cpp @@ -7,6 +7,8 @@ #if defined(__unix__) || defined(__APPLE__) # include // write(2) +#elif defined(_WIN32) +# include #endif namespace pos2gpu { @@ -35,6 +37,9 @@ void write_stderr_safe(char const* msg, std::size_t len) noexcept // write(2) is async-signal-safe; std::fprintf is not. ssize_t const rc = ::write(2, msg, len); (void)rc; // nothing useful to do if stderr is gone +#elif defined(_WIN32) + DWORD written = 0; + ::WriteFile(::GetStdHandle(STD_ERROR_HANDLE), msg, static_cast(len), &written, nullptr); #else (void)msg; (void)len; @@ -57,12 +62,24 @@ extern "C" void cancel_handler(int sig) noexcept write_stderr_safe(msg, sizeof(msg) - 1); } +#ifdef _WIN32 +BOOL WINAPI console_cancel_handler(DWORD event) +{ + if (event != CTRL_C_EVENT && event != CTRL_BREAK_EVENT) return FALSE; + cancel_handler(SIGINT); + return TRUE; +} +#endif + } // namespace void install_cancel_signal_handlers() { std::signal(SIGINT, cancel_handler); std::signal(SIGTERM, cancel_handler); +#ifdef _WIN32 + ::SetConsoleCtrlHandler(console_cancel_handler, TRUE); +#endif // SIGHUP — sent when the controlling terminal disappears (SSH // disconnect, terminal closed). Without explicit handling, the // default disposition kills the process immediately, leaving any diff --git a/src/host/PlotFileWriterParallel.cpp b/src/host/PlotFileWriterParallel.cpp index ede7c6a..6a6e4e3 100644 --- a/src/host/PlotFileWriterParallel.cpp +++ b/src/host/PlotFileWriterParallel.cpp @@ -42,6 +42,7 @@ #include #ifdef _WIN32 +#include "host/WindowsFile.hpp" #include #include #include @@ -333,10 +334,7 @@ size_t write_plot_file_parallel( std::vector iobuf(size_t{4} << 20); std::string partial = filename + ".partial.XXXXXX"; #ifdef _WIN32 - if (_mktemp_s(partial.data(), partial.size() + 1) != 0) - throw std::runtime_error("Failed to create temporary name for " + filename); - int const fd = ::_open(partial.c_str(), _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY, - _S_IREAD | _S_IWRITE); + int const fd = create_private_temp(partial); #else int const fd = ::mkstemp(partial.data()); #endif @@ -410,7 +408,14 @@ size_t write_plot_file_parallel( // Preserve the existing replace policy: concurrent successful writers may // replace the destination, but each publishes its own complete file. std::error_code ec; +#ifdef _WIN32 + if (!::MoveFileExW(std::filesystem::path(partial).c_str(), + std::filesystem::path(filename).c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) + ec = std::error_code(static_cast(::GetLastError()), std::system_category()); +#else std::filesystem::rename(partial, filename, ec); +#endif if (ec) throw std::runtime_error("Failed to publish " + filename + ": " + ec.message()); guard.committed = true; fsync_parent_dir_best_effort(filename); diff --git a/src/host/TempFile.cpp b/src/host/TempFile.cpp index a7ae857..9d1167b 100644 --- a/src/host/TempFile.cpp +++ b/src/host/TempFile.cpp @@ -2,32 +2,87 @@ #include "host/TempFile.hpp" +#include #include #include #include #include +#include +#include #include #include #include #include +#ifdef _WIN32 +#include "host/WindowsFile.hpp" +#else #include #include // statvfs — free_space #include // statfs / struct statfs — dir_is_ram_backed #include +#endif namespace pos2gpu { +#ifdef _WIN32 +namespace { + +// Each operation owns an event and offset. Sharing a seek position would race +// when SpillEngine writes disjoint ranges from several worker threads. +DWORD transfer_at(int fd, std::uint64_t offset, void* data, std::size_t bytes, bool write) +{ + OVERLAPPED operation{}; + operation.Offset = static_cast(offset); + operation.OffsetHigh = static_cast(offset >> 32); + operation.hEvent = ::CreateEventW(nullptr, TRUE, FALSE, nullptr); + if (!operation.hEvent) + throw std::system_error(static_cast(::GetLastError()), std::system_category(), "TempFile event"); + struct Cleanup { + HANDLE event; + ~Cleanup() { ::CloseHandle(event); } + } cleanup{operation.hEvent}; + HANDLE const file = reinterpret_cast(::_get_osfhandle(fd)); + DWORD transferred = 0; + DWORD const count = static_cast(std::min(bytes, MAXDWORD)); + BOOL ok = write ? ::WriteFile(file, data, count, &transferred, &operation) + : ::ReadFile(file, data, count, &transferred, &operation); + if (!ok && ::GetLastError() == ERROR_IO_PENDING) + ok = ::GetOverlappedResult(file, &operation, &transferred, TRUE); + if (!ok) { + DWORD const error = ::GetLastError(); + if (!write && error == ERROR_HANDLE_EOF) return 0; + throw std::system_error(static_cast(error), std::system_category(), + write ? "TempFile::pwrite_at" : "TempFile::pread_at"); + } + return transferred; +} + +} // namespace +#endif + std::string TempFile::resolve_dir(std::string_view explicit_dir) { if (!explicit_dir.empty()) return std::string(explicit_dir); if (char const* p = std::getenv("XCHPLOT2_TEMP_DIR"); p && *p) return p; if (char const* p = std::getenv("TMPDIR"); p && *p) return p; +#ifdef _WIN32 + return std::filesystem::temp_directory_path().string(); +#else return "/tmp"; +#endif } bool TempFile::dir_is_ram_backed(std::string const& dir) { +#ifdef _WIN32 + std::error_code error; + auto const path = std::filesystem::absolute(std::filesystem::path(resolve_dir(dir)), error); + if (error) return false; + wchar_t volume[MAX_PATH]{}; + if (!::GetVolumePathNameW(path.c_str(), volume, MAX_PATH)) return false; + return ::GetDriveTypeW(volume) == DRIVE_RAMDISK; +#else std::string const resolved = resolve_dir(dir); struct statfs st {}; if (::statfs(resolved.c_str(), &st) != 0) { @@ -45,6 +100,7 @@ bool TempFile::dir_is_ram_backed(std::string const& dir) return fsmagic == kTmpfsMagic || fsmagic == kRamfsMagic || fsmagic == kHugetlbfsMagic; +#endif } std::string TempFile::dir_problem(std::string const& dir) @@ -84,6 +140,10 @@ TempFile::TempFile(std::string_view dir) if (base.back() == '/') base.pop_back(); std::string templ = base + "/xchplot2-spill-XXXXXX"; std::string buf(templ); +#ifdef _WIN32 + fd_ = create_private_temp(buf, FILE_FLAG_OVERLAPPED | FILE_FLAG_DELETE_ON_CLOSE); + path_ = std::move(buf); +#else fd_ = ::mkstemp(buf.data()); if (fd_ < 0) { int const e = errno; @@ -99,13 +159,18 @@ TempFile::TempFile(std::string_view dir) throw std::runtime_error( "TempFile: unlink(" + path_ + ") failed: " + std::strerror(e)); } +#endif } TempFile::~TempFile() { unmap(); if (fd_ >= 0) { +#ifdef _WIN32 + ::_close(fd_); +#else ::close(fd_); +#endif fd_ = -1; } } @@ -127,7 +192,13 @@ TempFile& TempFile::operator=(TempFile&& other) noexcept { if (this != &other) { unmap(); - if (fd_ >= 0) ::close(fd_); + if (fd_ >= 0) { +#ifdef _WIN32 + ::_close(fd_); +#else + ::close(fd_); +#endif + } fd_ = other.fd_; path_ = std::move(other.path_); high_water_.store(other.high_water_.load(std::memory_order_relaxed), @@ -145,18 +216,36 @@ TempFile& TempFile::operator=(TempFile&& other) noexcept std::uint64_t TempFile::free_space(std::string const& dir) { std::string const resolved = resolve_dir(dir); +#ifdef _WIN32 + ULARGE_INTEGER available{}; + if (!::GetDiskFreeSpaceExW(std::filesystem::path(resolved).c_str(), &available, nullptr, nullptr)) return 0; + return available.QuadPart; +#else struct statvfs st {}; if (::statvfs(resolved.c_str(), &st) != 0) return 0; // unknown // f_bavail, not f_bfree: the latter counts blocks reserved for root, // which this process cannot have. Quoting those would let the check pass // on a filesystem that is already full for everyone but root. return std::uint64_t(st.f_bavail) * std::uint64_t(st.f_frsize); +#endif } void TempFile::preallocate(std::uint64_t bytes) { if (bytes == 0 || fd_ < 0) return; -#if defined(__linux__) +#if defined(_WIN32) + if (bytes > static_cast(std::numeric_limits::max())) + throw std::runtime_error("TempFile::preallocate: size exceeds the file offset range"); + HANDLE const file = reinterpret_cast(::_get_osfhandle(fd_)); + FILE_ALLOCATION_INFO allocation{}; + allocation.AllocationSize.QuadPart = static_cast(bytes); + FILE_END_OF_FILE_INFO end{}; + end.EndOfFile.QuadPart = static_cast(bytes); + if (!::SetFileInformationByHandle(file, FileAllocationInfo, &allocation, sizeof(allocation)) + || !::SetFileInformationByHandle(file, FileEndOfFileInfo, &end, sizeof(end))) + throw std::system_error(static_cast(::GetLastError()), std::system_category(), + "TempFile::preallocate(" + std::to_string(bytes) + "): " + path_); +#elif defined(__linux__) if (::fallocate(fd_, 0, 0, static_cast(bytes)) == 0) return; int const e = errno; // Not every filesystem implements it (network mounts, some FUSE, older @@ -189,6 +278,17 @@ void* TempFile::map(std::size_t bytes) // error here, before the mapping exists. Quietly does nothing where // fallocate is unsupported, which is the old (sparse) behaviour. preallocate(bytes); +#ifdef _WIN32 + HANDLE const file = reinterpret_cast(::_get_osfhandle(fd_)); + HANDLE const mapping = ::CreateFileMappingW(file, nullptr, PAGE_READWRITE, + static_cast(std::uint64_t(bytes) >> 32), static_cast(bytes), nullptr); + if (!mapping) + throw std::system_error(static_cast(::GetLastError()), std::system_category(), "TempFile::map"); + void* p = ::MapViewOfFile(mapping, FILE_MAP_ALL_ACCESS, 0, 0, bytes); + DWORD const error = ::GetLastError(); + ::CloseHandle(mapping); // the view retains the mapping until unmap() + if (!p) throw std::system_error(static_cast(error), std::system_category(), "TempFile::map"); +#else // Size the file so the whole mapping is backed — touching a mapped // page past EOF would raise SIGBUS otherwise. if (::ftruncate(fd_, static_cast(bytes)) != 0) { @@ -205,6 +305,7 @@ void* TempFile::map(std::size_t bytes) "TempFile::mmap(" + std::to_string(bytes) + ") failed: " + std::strerror(e)); } +#endif map_ = p; map_bytes_ = bytes; bump_high_water(bytes); @@ -214,7 +315,11 @@ void* TempFile::map(std::size_t bytes) void TempFile::unmap() noexcept { if (map_) { +#ifdef _WIN32 + ::UnmapViewOfFile(map_); +#else ::munmap(map_, map_bytes_); +#endif map_ = nullptr; map_bytes_ = 0; } @@ -226,6 +331,9 @@ void TempFile::pwrite_at(std::uint64_t offset, void const* data, std::size_t byt std::size_t remaining = bytes; std::uint64_t cur = offset; while (remaining > 0) { +#ifdef _WIN32 + auto const n = transfer_at(fd_, cur, const_cast(p), remaining, true); +#else ssize_t const n = ::pwrite(fd_, p, remaining, static_cast(cur)); if (n < 0) { if (errno == EINTR) continue; @@ -234,6 +342,7 @@ void TempFile::pwrite_at(std::uint64_t offset, void const* data, std::size_t byt "TempFile::pwrite_at(" + std::to_string(offset) + ", " + std::to_string(bytes) + ") failed: " + std::strerror(e)); } +#endif if (n == 0) { throw std::runtime_error( "TempFile::pwrite_at: zero-byte write (disk full?)"); @@ -251,6 +360,9 @@ void TempFile::pread_at(std::uint64_t offset, void* data, std::size_t bytes) std::size_t remaining = bytes; std::uint64_t cur = offset; while (remaining > 0) { +#ifdef _WIN32 + auto const n = transfer_at(fd_, cur, p, remaining, false); +#else ssize_t const n = ::pread(fd_, p, remaining, static_cast(cur)); if (n < 0) { if (errno == EINTR) continue; @@ -259,6 +371,7 @@ void TempFile::pread_at(std::uint64_t offset, void* data, std::size_t bytes) "TempFile::pread_at(" + std::to_string(offset) + ", " + std::to_string(bytes) + ") failed: " + std::strerror(e)); } +#endif if (n == 0) { throw std::runtime_error( "TempFile::pread_at: short read at offset " + diff --git a/src/host/TempFile.hpp b/src/host/TempFile.hpp index 9802fbf..d56863f 100644 --- a/src/host/TempFile.hpp +++ b/src/host/TempFile.hpp @@ -1,14 +1,16 @@ -// TempFile.hpp — POSIX-anonymous temp file with positional read/write. +// TempFile.hpp — automatically removed temp file with positional read/write. // // Task #26 disk-fallback foundation. Self-contained primitive: opens a // unique-named file at construction (mkstemp), unlinks it immediately // so it disappears on process exit even on crash, and supports // thread-safe positional I/O via pread/pwrite. +// Windows uses an exclusive, owner-only file with delete-on-close and +// overlapped I/O offsets; the OS also removes it after process termination. // // Path resolution order (when caller passes empty `dir`): // 1. $XCHPLOT2_TEMP_DIR // 2. $TMPDIR -// 3. /tmp +// 3. /tmp on Linux; the system temporary directory on Windows // // The file is automatically removed when the TempFile destructor runs. // On crash the kernel reclaims the inode at process exit because the @@ -31,7 +33,7 @@ namespace pos2gpu { class TempFile { public: // Open a fresh anonymous temp file. `dir` overrides the env-based - // resolution; pass empty to use $XCHPLOT2_TEMP_DIR / $TMPDIR / /tmp. + // resolution; pass empty to use the environment or platform default above. explicit TempFile(std::string_view dir = ""); ~TempFile(); @@ -104,8 +106,8 @@ class TempFile { return high_water_.load(std::memory_order_relaxed); } - // Underlying file path (unlinked already; useful for diagnostics - // via /proc//fd/ on Linux). + // Underlying file path (already unlinked on Linux; delete-on-close on + // Windows). Useful for diagnostics via /proc//fd/ on Linux. std::string const& path() const noexcept { return path_; } int fd() const noexcept { return fd_; } diff --git a/src/host/WindowsFile.hpp b/src/host/WindowsFile.hpp new file mode 100644 index 0000000..306b5fd --- /dev/null +++ b/src/host/WindowsFile.hpp @@ -0,0 +1,93 @@ +#pragma once + +// Windows counterpart of mkstemp's exclusive creation and owner-only access. +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace pos2gpu { + +inline int create_private_temp(std::string& path, DWORD flags = 0) +{ + // Alternate data streams inherit the existing file's security descriptor. + // They cannot provide the private, newly created file promised here. + if (std::filesystem::path(path).filename().native().find(L':') != std::wstring::npos) + throw std::invalid_argument("alternate data streams are not supported: " + path); + struct Security { + HANDLE token = nullptr; + LPWSTR sid = nullptr; + PSECURITY_DESCRIPTOR descriptor = nullptr; + ~Security() { + if (descriptor) ::LocalFree(descriptor); + if (sid) ::LocalFree(sid); + if (token) ::CloseHandle(token); + } + } security; + auto fail = [&] { + throw std::system_error(static_cast(::GetLastError()), + std::system_category(), "create private file: " + path); + }; + if (!::OpenProcessToken(::GetCurrentProcess(), TOKEN_QUERY, &security.token)) fail(); + DWORD size = 0; + ::GetTokenInformation(security.token, TokenUser, nullptr, 0, &size); + if (!size) fail(); + std::vector token(size); + if (!::GetTokenInformation(security.token, TokenUser, token.data(), size, &size)) fail(); + auto const* user = reinterpret_cast(token.data()); + if (!::ConvertSidToStringSidW(user->User.Sid, &security.sid)) fail(); + // Set the owner explicitly, including for an elevated process, and block + // inherited ACEs. CRT _S_IREAD/_S_IWRITE alone do not restrict other users. + std::wstring const sddl = L"O:" + std::wstring(security.sid) + + L"D:P(A;;FA;;;" + security.sid + L")"; + if (!::ConvertStringSecurityDescriptorToSecurityDescriptorW( + sddl.c_str(), SDDL_REVISION_1, &security.descriptor, nullptr)) fail(); + SECURITY_ATTRIBUTES attributes{sizeof(attributes), security.descriptor, FALSE}; + std::string const prefix = path.substr(0, path.size() - 6); // caller's XXXXXX template + std::random_device random; + for (int attempt = 0; attempt < 32; ++attempt) { + path = prefix + std::to_string((std::uint64_t(random()) << 32) | random()); + auto const native = std::filesystem::path(path); + HANDLE file = ::CreateFileW(native.c_str(), GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_DELETE, &attributes, CREATE_NEW, flags, nullptr); + if (file == INVALID_HANDLE_VALUE) { + if (::GetLastError() == ERROR_FILE_EXISTS) continue; + fail(); + } + struct Cleanup { + HANDLE file; + std::filesystem::path const& path; + ~Cleanup() { + if (file != INVALID_HANDLE_VALUE) { + ::CloseHandle(file); + ::DeleteFileW(path.c_str()); + } + } + } cleanup{file, native}; + DWORD volume_flags = 0; + if (!::GetVolumeInformationByHandleW(file, nullptr, 0, nullptr, nullptr, + &volume_flags, nullptr, 0)) fail(); + if (!(volume_flags & FILE_PERSISTENT_ACLS)) + throw std::runtime_error("private files require an ACL-capable filesystem (NTFS/ReFS): " + path); + int const fd = ::_open_osfhandle(reinterpret_cast(file), _O_RDWR | _O_BINARY); + if (fd < 0) throw std::system_error(errno, std::generic_category(), "open file descriptor: " + path); + cleanup.file = INVALID_HANDLE_VALUE; // descriptor now owns the handle + return fd; + } + throw std::runtime_error("cannot create a unique temporary file: " + path); +} + +} // namespace pos2gpu +#endif diff --git a/tools/parity/cli_host_test.cpp b/tools/parity/cli_host_test.cpp index 1efe8cb..5a7b833 100644 --- a/tools/parity/cli_host_test.cpp +++ b/tools/parity/cli_host_test.cpp @@ -8,11 +8,16 @@ #include #include #include +#include #include #include +#include #include #include -#include +#ifdef _WIN32 +#include +#include +#endif extern "C" int xchplot2_main(int, char**); namespace { @@ -95,13 +100,26 @@ int pos2_keygen_derive_subseed(uint8_t const* seed, uint64_t index, uint8_t* out return POS2_OK; } } -int main() +int main(int argc, char** argv) { - char path[] = "/tmp/xchplot2-cli-test-XXXXXX"; - assert(mkdtemp(path)); - std::filesystem::path const dir(path), config = dir / "config.toml", manifest = dir / "manifest.tsv"; +#ifdef _WIN32 + if (std::string(argv[0]) == "parity-test") { + put(std::getenv("XCHPLOT2_TEST_MARKER"), "executed"); + return 17; + } +#else + (void)argc; (void)argv; +#endif + std::filesystem::path dir; + std::random_device random; + do { dir = std::filesystem::temp_directory_path() / ("xchplot2 cli é-" + std::to_string(random())); } + while (!std::filesystem::create_directory(dir)); + auto const config = dir / "config.toml", manifest = dir / "manifest.tsv"; auto line = [&](std::string const& fields, std::string const& name = "plot.plot2") { - return fields + " " + std::string(64, 'a') + " 00 " + dir.string() + " " + name + "\n"; + std::ostringstream row; + row << fields << ' ' << std::string(64, 'a') << " 00 " + << std::quoted(dir.string()) << ' ' << std::quoted(name) << '\n'; + return row.str(); }; put(manifest, line("18 2 0 0 false")); put(config, ""); @@ -139,10 +157,16 @@ int main() } auto const tests = dir / "space;true # directory"; std::filesystem::create_directory(tests); - auto const failing = tests / "quote'\"_test"; auto const marker = dir / "executed"; +#ifdef _WIN32 + auto const failing = tests / "quote'_test.exe"; + std::filesystem::copy_file(argv[0], failing); + _putenv_s("XCHPLOT2_TEST_MARKER", marker.string().c_str()); +#else + auto const failing = tests / "quote'\"_test"; put(failing, "#!/bin/sh\nprintf executed > '" + marker.string() + "'\nexit 17\n"); std::filesystem::permissions(failing, std::filesystem::perms::owner_all); +#endif put(config, ""); assert(cli({"parity-check", "--config", config.string(), "--dir", tests.string()}) != 0); assert(std::filesystem::exists(marker)); @@ -155,9 +179,26 @@ int main() auto const saved = dir / "saved.tsv"; pos2gpu::write_manifest(saved.string(), {entry}); assert(pos2gpu::parse_manifest(saved.string()) == std::vector{entry}); +#ifdef _WIN32 + PSID owner = nullptr; + PACL acl = nullptr; + PSECURITY_DESCRIPTOR security = nullptr; + assert(::GetNamedSecurityInfoW(saved.c_str(), SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION | DACL_SECURITY_INFORMATION, + &owner, nullptr, &acl, nullptr, &security) == ERROR_SUCCESS); + SECURITY_DESCRIPTOR_CONTROL control{}; + DWORD revision = 0; + assert(::GetSecurityDescriptorControl(security, &control, &revision)); + assert((control & SE_DACL_PROTECTED) && acl && acl->AceCount == 1); + ACCESS_ALLOWED_ACE* ace = nullptr; + assert(::GetAce(acl, 0, reinterpret_cast(&ace))); + assert(ace->Header.AceType == ACCESS_ALLOWED_ACE_TYPE && ::EqualSid(owner, &ace->SidStart)); + assert((ace->Mask & FILE_ALL_ACCESS) == FILE_ALL_ACCESS); + ::LocalFree(security); +#else assert((std::filesystem::status(saved).permissions() & (std::filesystem::perms::group_all | std::filesystem::perms::others_all)) == std::filesystem::perms::none); +#endif std::thread same([&] { pos2gpu::write_manifest(saved.string(), {entry}); }); pos2gpu::write_manifest(saved.string(), {entry}); same.join(); @@ -166,6 +207,19 @@ int main() try { pos2gpu::write_manifest(saved.string(), {different}); } catch (std::exception const&) { refused = true; } assert(refused && pos2gpu::parse_manifest(saved.string()) == std::vector{entry}); +#ifdef _WIN32 + refused = false; + try { pos2gpu::write_manifest(saved.string() + ":keys", {entry}); } + catch (std::invalid_argument const&) { refused = true; } + assert(refused && pos2gpu::parse_manifest(saved.string()) == std::vector{entry}); + for (auto const* name : {"plot:keys", "plot.", "plot ", "plot*"}) { + auto invalid = entry; invalid.out_name = name; + refused = false; + try { pos2gpu::validate_batch_entry(invalid); } + catch (std::invalid_argument const&) { refused = true; } + assert(refused); + } +#endif auto const race = dir / "race.tsv"; auto publish = [&](pos2gpu::BatchEntry const& e) { try { pos2gpu::write_manifest(race.string(), {e}); return true; } diff --git a/tools/parity/host_guard_test.cpp b/tools/parity/host_guard_test.cpp index 5329687..f7ae4d1 100644 --- a/tools/parity/host_guard_test.cpp +++ b/tools/parity/host_guard_test.cpp @@ -20,7 +20,9 @@ #include #include +#ifndef _WIN32 #include +#endif namespace { @@ -59,11 +61,16 @@ int main(int argc, char** argv) { // Re-exec once with the guard enabled (see file header). if (argc < 2 || std::string(argv[1]) != "--armed") { +#ifdef _WIN32 + // No guard instance exists yet, so Windows can enable it in-process. + _putenv_s("XCHPLOT2_HOST_GUARD", "1"); +#else setenv("XCHPLOT2_HOST_GUARD", "1", 1); std::vector av{argv[0], const_cast("--armed"), nullptr}; execv(argv[0], av.data()); std::perror("execv"); return 1; +#endif } auto& g = pos2gpu::HostGuard::instance(); diff --git a/tools/parity/spill_engine_test.cpp b/tools/parity/spill_engine_test.cpp index 2e9c31a..72d33df 100644 --- a/tools/parity/spill_engine_test.cpp +++ b/tools/parity/spill_engine_test.cpp @@ -34,7 +34,11 @@ #include #include +#ifdef _WIN32 +#include +#else #include +#endif namespace { @@ -82,9 +86,17 @@ struct EngineWithThreads explicit EngineWithThreads(int threads) { std::string const n = std::to_string(threads); +#ifdef _WIN32 + ::_putenv_s("XCHPLOT2_SPILL_IO_THREADS", n.c_str()); +#else ::setenv("XCHPLOT2_SPILL_IO_THREADS", n.c_str(), 1); +#endif eng = std::make_unique(ops, /*quiet=*/true); +#ifdef _WIN32 + ::_putenv_s("XCHPLOT2_SPILL_IO_THREADS", ""); +#else ::unsetenv("XCHPLOT2_SPILL_IO_THREADS"); +#endif } pos2gpu::SpillEngine& operator*() { return *eng; } }; @@ -294,13 +306,22 @@ int main() auto const src = ramp(n, 0x66ull); SpillBuffer buf(*e, sizeof(std::uint64_t), n); +#ifdef _WIN32 + int const full = ::_open("NUL", _O_RDONLY | _O_BINARY); +#else int const full = ::open("/dev/full", O_WRONLY); +#endif bool threw = false; if (full < 0) { std::printf("SKIP (no /dev/full) a failing part surfaces as an error\n"); } else { +#ifdef _WIN32 + ::_dup2(full, buf.file.fd()); // writes fail on this read-only handle + ::_close(full); +#else ::dup2(full, buf.file.fd()); // every pwrite now fails ::close(full); +#endif try { buf.write_from_device(src.data(), 0, n); eng.drain(); diff --git a/tools/parity/temp_file_test.cpp b/tools/parity/temp_file_test.cpp index bb8e088..9076d32 100644 --- a/tools/parity/temp_file_test.cpp +++ b/tools/parity/temp_file_test.cpp @@ -10,13 +10,21 @@ #include #include #include +#include +#include #include #include #include #include #include +#ifdef _WIN32 +#include +#include +#include +#else #include +#endif namespace { @@ -31,6 +39,7 @@ bool check(bool cond, char const* what) int main() { bool all_ok = true; + std::string const temp = std::filesystem::temp_directory_path().string(); // Test 1: basic open + write + read round-trip. { @@ -92,11 +101,15 @@ int main() // Test 5: file is unlinked on construction (path no longer in dir). { - pos2gpu::TempFile tf; - std::string const p = tf.path(); - struct stat st{}; - bool const stat_fails = (::stat(p.c_str(), &st) != 0); - all_ok = check(stat_fails, "file unlinked on construction") && all_ok; + std::string path; + { + pos2gpu::TempFile tf; + path = tf.path(); +#ifndef _WIN32 + all_ok = check(!std::filesystem::exists(path), "file unlinked on construction") && all_ok; +#endif + } + all_ok = check(!std::filesystem::exists(path), "file removed on close") && all_ok; } // Test 6: move construction transfers fd; source has -1 fd. @@ -115,13 +128,21 @@ int main() // Test 7: env-based dir resolution. { - ::setenv("XCHPLOT2_TEMP_DIR", "/tmp", 1); +#ifdef _WIN32 + ::_putenv_s("XCHPLOT2_TEMP_DIR", temp.c_str()); +#else + ::setenv("XCHPLOT2_TEMP_DIR", temp.c_str(), 1); +#endif std::string const d = pos2gpu::TempFile::resolve_dir(""); - all_ok = check(d == "/tmp", "resolve_dir uses XCHPLOT2_TEMP_DIR") && all_ok; + all_ok = check(d == temp, "resolve_dir uses XCHPLOT2_TEMP_DIR") && all_ok; std::string const explicit_d = pos2gpu::TempFile::resolve_dir("/var/tmp"); all_ok = check(explicit_d == "/var/tmp", "explicit dir overrides env") && all_ok; +#ifdef _WIN32 + ::_putenv_s("XCHPLOT2_TEMP_DIR", ""); +#else ::unsetenv("XCHPLOT2_TEMP_DIR"); +#endif } // Test 8: dir_problem — the spill guard's usability probe. @@ -132,7 +153,7 @@ int main() // silently restore the original failure — a raw mkstemp errno thrown deep // in the pipeline, minutes into a batch. { - all_ok = check(pos2gpu::TempFile::dir_problem("/tmp").empty(), + all_ok = check(pos2gpu::TempFile::dir_problem(temp).empty(), "dir_problem: usable dir reports no problem") && all_ok; std::string const missing = @@ -142,6 +163,7 @@ int main() // Exists but not writable. Skipped as root, where write permission // is not enforced and the probe would (correctly) succeed. +#ifndef _WIN32 if (::geteuid() != 0) { char tmpl[] = "/tmp/xchplot2-ro-XXXXXX"; if (char const* d = ::mkdtemp(tmpl); d) { @@ -154,18 +176,15 @@ int main() ::rmdir(d); } } +#endif // The probe must not leave its own file behind — it creates one and // relies on TempFile unlinking at construction. { - char tmpl2[] = "/tmp/xchplot2-probe-XXXXXX"; - if (char const* d = ::mkdtemp(tmpl2); d) { - (void) pos2gpu::TempFile::dir_problem(d); - bool const empty_after = (::rmdir(d) == 0); // fails if non-empty - all_ok = check(empty_after, - "dir_problem: leaves nothing behind") && all_ok; - ::rmdir(d); - } + auto const dir = std::filesystem::path(temp) / ("xchplot2-probe-" + std::to_string(std::random_device{}())); + all_ok = check(std::filesystem::create_directory(dir), "create probe directory") && all_ok; + (void) pos2gpu::TempFile::dir_problem(dir.string()); + all_ok = check(std::filesystem::remove(dir), "dir_problem: leaves nothing behind") && all_ok; } } @@ -179,6 +198,13 @@ int main() { pos2gpu::TempFile f; f.preallocate(4u << 20); // 4 MiB +#ifdef _WIN32 + FILE_STANDARD_INFO st{}; + bool const ok = ::GetFileInformationByHandleEx( + reinterpret_cast(::_get_osfhandle(f.fd())), FileStandardInfo, &st, sizeof(st)); + bool const reserved = st.EndOfFile.QuadPart == (4 << 20) && st.AllocationSize.QuadPart >= (4 << 20); + bool const noop = false; +#else struct stat st {}; bool const ok = (::fstat(f.fd(), &st) == 0); // fallocate reserves blocks AND extends i_size (no KEEP_SIZE), so on @@ -190,6 +216,7 @@ int main() bool const reserved = (st.st_size == (4 << 20)) && (std::uint64_t(st.st_blocks) * 512 >= (4u << 20)); bool const noop = (st.st_size == 0) && (st.st_blocks == 0); +#endif all_ok = check(ok && (reserved || noop), "preallocate: reserves blocks, or is a clean no-op") && all_ok; @@ -217,8 +244,8 @@ int main() // free_space answers for a real dir, and returns the documented 0 for // one that cannot be probed. 0 means "unknown" to callers, so a bogus // path must not come back looking like a full disk. - std::uint64_t const here = pos2gpu::TempFile::free_space("/tmp"); - all_ok = check(here > 0, "free_space: reports something for /tmp") + std::uint64_t const here = pos2gpu::TempFile::free_space(temp); + all_ok = check(here > 0, "free_space: reports something for the temp directory") && all_ok; std::uint64_t const nowhere = pos2gpu::TempFile::free_space("/nonexistent-xchplot2-probe"); @@ -226,5 +253,37 @@ int main() && all_ok; } + { + pos2gpu::TempFile file; + auto* mapped = static_cast(file.map(16384)); + mapped[1023] = 0x123456789abcdef0ULL; + bool refused = false; + try { file.map(4096); } catch (std::runtime_error const&) { refused = true; } + all_ok = check(refused, "second live mapping is rejected") && all_ok; + pos2gpu::TempFile moved(std::move(file)); + moved.unmap(); + std::uint64_t value = 0; + moved.pread_at(1023 * sizeof(value), &value, sizeof(value)); + all_ok = check(value == 0x123456789abcdef0ULL, "mapping survives move and unmap") && all_ok; + } + { + pos2gpu::TempFile file; +#ifdef _WIN32 + // Avoid allocating a 4 GiB hole just to exercise OffsetHigh. + DWORD returned = 0; + HANDLE const handle = reinterpret_cast(::_get_osfhandle(file.fd())); + OVERLAPPED operation{}; + BOOL sparse = ::DeviceIoControl(handle, FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, &returned, &operation); + if (!sparse && ::GetLastError() == ERROR_IO_PENDING) + sparse = ::GetOverlappedResult(handle, &operation, &returned, TRUE); + all_ok = check(sparse, "sparse offset test file") && all_ok; +#endif + std::uint64_t const offset = (std::uint64_t{1} << 32) + 8, expected = 12345; + file.pwrite_at(offset, &expected, sizeof(expected)); + std::uint64_t actual = 0; + file.pread_at(offset, &actual, sizeof(actual)); + all_ok = check(actual == expected && file.size() == offset + sizeof(expected), "64-bit positional I/O") && all_ok; + } + return all_ok ? 0 : 1; } diff --git a/tools/xchplot2/cli.cpp b/tools/xchplot2/cli.cpp index 7640bdf..2486d26 100644 --- a/tools/xchplot2/cli.cpp +++ b/tools/xchplot2/cli.cpp @@ -44,12 +44,11 @@ #include #include -#include // isatty — progress defaults to on for interactive runs - #ifdef _WIN32 #include #include #else +#include // isatty — progress defaults to on for interactive runs #include #include extern char** environ; @@ -99,7 +98,11 @@ int run_parity_test(std::string const& path, std::FILE* log) bool resolve_progress(int tri, bool quiet) { if (tri >= 0) return tri != 0; +#ifdef _WIN32 + return !quiet && ::_isatty(::_fileno(stderr)) != 0; +#else return !quiet && ::isatty(::fileno(stderr)) != 0; +#endif } void print_usage(char const* prog) @@ -263,7 +266,11 @@ void print_usage(char const* prog) << "\n" << " Reusable options:\n" << " --config FILE load settings from named command sections. Default:\n" +#ifdef _WIN32 + << " %APPDATA%/xchplot2/config.toml\n" +#else << " $HOME/.config/xchplot2/config.toml\n" +#endif << " @FILE insert whitespace-separated arguments (no shell quoting).\n" << "\n" << " test-mode positional args:\n" @@ -1207,6 +1214,9 @@ std::vector expand_argfiles(int argc, char* argv[]) std::vector out; out.reserve(argc); char const* home = std::getenv("HOME"); +#ifdef _WIN32 + if (!home) home = std::getenv("USERPROFILE"); +#endif auto resolve_path = [&](std::string p) -> std::string { if (home && p.size() >= 2 && p[0] == '~' && p[1] == '/') { return std::string(home) + p.substr(1); @@ -1273,12 +1283,19 @@ extern "C" int xchplot2_main(int argc, char* argv[]) strip_argc = static_cast(argv_stripped.size()); } if (config_path.empty()) { +#ifdef _WIN32 + if (char const* data = std::getenv("APPDATA")) { + auto const default_path = std::filesystem::path(data) / "xchplot2/config.toml"; + if (std::filesystem::exists(default_path)) config_path = default_path.string(); + } +#else if (char const* home = std::getenv("HOME")) { std::string const default_path = std::string(home) + "/.config/xchplot2/config.toml"; std::ifstream probe(default_path); if (probe) config_path = default_path; } +#endif } std::vector config_tokens; if (!config_path.empty()) { @@ -1606,7 +1623,11 @@ extern "C" int xchplot2_main(int argc, char* argv[]) // Fail the bench if a streaming tier outgrows the peak its floor is // derived from — the floors are only honest while that holds. Costs // nothing (no driver calls); setenv does not clobber an explicit 0. +#ifdef _WIN32 + if (!std::getenv("POS2GPU_ASSERT_VRAM")) ::_putenv_s("POS2GPU_ASSERT_VRAM", "1"); +#else setenv("POS2GPU_ASSERT_VRAM", "1", 0); +#endif if (!opts.quiet) { if (worker_count == 1) { @@ -2006,7 +2027,11 @@ extern "C" int xchplot2_main(int argc, char* argv[]) for (auto const& entry : std::filesystem::directory_iterator(dir, ec)) { - auto const name = entry.path().filename().string(); + auto name = entry.path().filename().string(); +#ifdef _WIN32 + if (entry.path().extension() != ".exe") continue; + name = entry.path().stem().string(); +#endif if ((has_suffix(name, "_parity") || has_suffix(name, "_test")) && entry.is_regular_file(ec)) { tests.push_back(entry.path()); diff --git a/tools/xchplot2/main.cpp b/tools/xchplot2/main.cpp index afd20ea..b950343 100644 --- a/tools/xchplot2/main.cpp +++ b/tools/xchplot2/main.cpp @@ -4,7 +4,14 @@ #include "xchplot2_cli.h" +#ifdef _WIN32 +#include +#endif + int main(int argc, char* argv[]) { +#ifdef _WIN32 + ::SetConsoleOutputCP(CP_UTF8); +#endif return xchplot2_main(argc, argv); } diff --git a/tools/xchplot2/windows.manifest b/tools/xchplot2/windows.manifest new file mode 100644 index 0000000..540f6ed --- /dev/null +++ b/tools/xchplot2/windows.manifest @@ -0,0 +1,9 @@ + + + + + UTF-8 + true + + + From 0af08c103daeefe9f2ed15ec6bbb56bd320c8e82 Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 21:52:24 -0500 Subject: [PATCH 05/10] windows: collect the pinned runtime license from its archive The minimal CUDA installer omits EULA.txt. Verify and extract the license from the matching NVIDIA runtime redistribution archive instead. Restore the console code page when the CLI exits. --- scripts/build-release.ps1 | 9 ++++++++- tools/xchplot2/main.cpp | 7 ++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/build-release.ps1 b/scripts/build-release.ps1 index d8b3fc6..6e2239b 100644 --- a/scripts/build-release.ps1 +++ b/scripts/build-release.ps1 @@ -21,7 +21,14 @@ New-Item -ItemType Directory -Force $BuildDir | Out-Null $BuildDir = (Resolve-Path $BuildDir).Path $licenses = Join-Path $BuildDir 'licenses' New-Item -ItemType Directory -Force $licenses | Out-Null -Copy-Item (Join-Path $env:CUDA_PATH 'EULA.txt') (Join-Path $licenses 'cuda.txt') +# The minimal Windows installer omits the license; use the matching runtime archive. +$cudart = Join-Path $BuildDir 'cuda-cudart.zip' +Invoke-WebRequest 'https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/windows-x86_64/cuda_cudart-windows-x86_64-12.9.79-archive.zip' -OutFile $cudart +if ((Get-FileHash $cudart -Algorithm SHA256).Hash -ne '179e9c43b0735ffe67207b3da556eb5a0c50f3047961882b7657d3b822d34ef8') { + throw 'CUDA runtime archive checksum mismatch' +} +Expand-Archive $cudart -DestinationPath (Join-Path $BuildDir 'cudart') -Force +Copy-Item (Join-Path $BuildDir 'cudart/cuda_cudart-windows-x86_64-12.9.79-archive/LICENSE') (Join-Path $licenses 'cuda.txt') Invoke-WebRequest 'https://raw.githubusercontent.com/NVIDIA/cccl/v2.8.2/LICENSE' ` -OutFile (Join-Path $licenses 'cuda-cccl.txt') $rustDocs = Join-Path (rustc --print sysroot) 'share/doc/rust' diff --git a/tools/xchplot2/main.cpp b/tools/xchplot2/main.cpp index b950343..cb82c6e 100644 --- a/tools/xchplot2/main.cpp +++ b/tools/xchplot2/main.cpp @@ -11,7 +11,12 @@ int main(int argc, char* argv[]) { #ifdef _WIN32 + auto const code_page = ::GetConsoleOutputCP(); ::SetConsoleOutputCP(CP_UTF8); #endif - return xchplot2_main(argc, argv); + int const result = xchplot2_main(argc, argv); +#ifdef _WIN32 + if (code_page) ::SetConsoleOutputCP(code_page); +#endif + return result; } From 5b41acfcbb89c962b6250dfa5d767f9a85dcd15f Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 22:10:07 -0500 Subject: [PATCH 06/10] windows: backport the CCCL 64-bit PTX operand fix CUDA 12.9 bundles CCCL headers that use long2 where PTX requires 64-bit operands. Windows long is 32 bits, so Blackwell compilation fails before the plotter can link. Apply the exact upstream fix to a build-local header, following the existing solver patch mechanism, and record it in the package metadata. --- CMakeLists.txt | 27 +++++++++++++++ CONTRIBUTING.md | 4 +++ cmake/Packaging.cmake | 4 +++ contrib/cccl-windows-ptx.patch | 61 ++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) create mode 100644 contrib/cccl-windows-ptx.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ec4c6b..6b700ea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,6 +292,33 @@ foreach(_header Solver.hpp ParallelRadixSort.hpp ProofSolverTimings.hpp) "${_pos2_solver_dir}/solve/${_header}" COPYONLY) endforeach() find_package(Git REQUIRED) +# CCCL 2.x uses long2 for 64-bit PTX operands; Windows long is only 32 bits. +# Backport NVIDIA's fix to a build-local header, as with the solver below. +if(WIN32) + set(_cccl_header_name "cuda/__ptx/instructions/generated/clusterlaunchcontrol.h") + find_file(_cccl_header "${_cccl_header_name}" + PATHS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} NO_DEFAULT_PATH NO_CACHE) + if(_cccl_header) + file(READ "${_cccl_header}" _cccl_text) + if(_cccl_text MATCHES "reinterpret_cast(&__try_cancel_response)).x), +- "l"((*reinterpret_cast(&__try_cancel_response)).y) ++ : "l"((*reinterpret_cast(&__try_cancel_response)).x), ++ "l"((*reinterpret_cast(&__try_cancel_response)).y) + :); + return static_cast(__pred_is_canceled); + # else +@@ -112,8 +112,8 @@ _CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctai + "clusterlaunchcontrol.query_cancel.get_first_ctaid::x.b32.b128 %0, B128_try_cancel_response;\n\t" + "}" + : "=r"(__ret_dim) +- : "l"((*reinterpret_cast(&__try_cancel_response)).x), +- "l"((*reinterpret_cast(&__try_cancel_response)).y) ++ : "l"((*reinterpret_cast(&__try_cancel_response)).x), ++ "l"((*reinterpret_cast(&__try_cancel_response)).y) + :); + return *reinterpret_cast<_B32*>(&__ret_dim); + # else +@@ -150,8 +150,8 @@ _CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctai + "clusterlaunchcontrol.query_cancel.get_first_ctaid::y.b32.b128 %0, B128_try_cancel_response;\n\t" + "}" + : "=r"(__ret_dim) +- : "l"((*reinterpret_cast(&__try_cancel_response)).x), +- "l"((*reinterpret_cast(&__try_cancel_response)).y) ++ : "l"((*reinterpret_cast(&__try_cancel_response)).x), ++ "l"((*reinterpret_cast(&__try_cancel_response)).y) + :); + return *reinterpret_cast<_B32*>(&__ret_dim); + # else +@@ -188,8 +188,8 @@ _CCCL_DEVICE static inline _B32 clusterlaunchcontrol_query_cancel_get_first_ctai + "clusterlaunchcontrol.query_cancel.get_first_ctaid::z.b32.b128 %0, B128_try_cancel_response;\n\t" + "}" + : "=r"(__ret_dim) +- : "l"((*reinterpret_cast(&__try_cancel_response)).x), +- "l"((*reinterpret_cast(&__try_cancel_response)).y) ++ : "l"((*reinterpret_cast(&__try_cancel_response)).x), ++ "l"((*reinterpret_cast(&__try_cancel_response)).y) + :); + return *reinterpret_cast<_B32*>(&__ret_dim); + # else +@@ -227,8 +227,8 @@ clusterlaunchcontrol_query_cancel_get_first_ctaid(_B32 (&__block_dim)[4], _B128 + "clusterlaunchcontrol.query_cancel.get_first_ctaid.v4.b32.b128 {%0, %1, %2, %3}, B128_try_cancel_response;\n\t" + "}" + : "=r"(__block_dim[0]), "=r"(__block_dim[1]), "=r"(__block_dim[2]), "=r"(__block_dim[3]) +- : "l"((*reinterpret_cast(&__try_cancel_response)).x), +- "l"((*reinterpret_cast(&__try_cancel_response)).y) ++ : "l"((*reinterpret_cast(&__try_cancel_response)).x), ++ "l"((*reinterpret_cast(&__try_cancel_response)).y) + :); + # else + // Unsupported architectures will have a linker error with a semi-decent error message From a0b8f016cc3c628a0504f83aa720e6d050caed0f Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 22:30:09 -0500 Subject: [PATCH 07/10] fix(windows): preserve benchmark fallback and path output --- INSTALL.md | 6 +++--- REFERENCE.md | 2 +- SECURITY.md | 6 ++++-- scripts/test/windows.py | 12 ++++++++++++ src/host/TempFile.hpp | 28 ++++++++++++---------------- tools/xchplot2/cli.cpp | 10 ++++++++-- 6 files changed, 40 insertions(+), 24 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index f314630..0c8d7e9 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -39,8 +39,8 @@ a source build using the instructions below. Requires CUDA Toolkit **12.0+** (12.0 is the floor — `cudaGetDeviceProperties_v2`, the v2 ABI we link, and CUDA C++20 dialect all need 12.0; the latest benchmark used 13.3.73), **C++20** host compiler, **CMake ≥ 3.26** (3.26+ knows how to drive nvcc 12.5+; lower works for older nvcc), and a Rust -toolchain new enough to parse `edition2024` (**rustc ≥ 1.85**, i.e. -rustup `stable`; most distro-packaged Rust is too old). +toolchain from rustup `stable`. The release builds use **Rust 1.98.1**; +older distro-packaged Rust may not support the locked dependencies. ### Historical dependency sources @@ -66,7 +66,7 @@ Combinations that **don't** work on a stock install: - **Debian 12 + apt CUDA + apt CMake**: stock CMake 3.25 doesn't know how to drive nvcc 12.5+. Use Kitware's CMake apt repo. - **Ubuntu 22.04/24.04 + apt cargo**: distro-packaged Rust (1.75) can't - parse `edition2024` required by the `chia-client` 0.42 dep tree. + parse `edition2024` used by the locked dependency tree. Install rustup instead. - **WSL**: works the same as native — the only WSL-specific bits are the `libcuda.so` injection at `/usr/lib/wsl/lib` (driver, not diff --git a/REFERENCE.md b/REFERENCE.md index 4adff66..f10dd8b 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -46,7 +46,7 @@ Before plotting starts, `plot` saves the prepared identities and keys in an `xchplot2-job-*.tsv` manifest in the output directory. Use `--manifest FILE` to choose its location. Each new job keeps its own manifest; another job cannot overwrite it. Manifests contain private plot keys and are created -with owner-only permissions on Linux. +with owner-only permissions on Linux and a protected owner-only ACL on Windows. Repeat the same `plot` command with `--resume` (or `--skip-existing`) to recover its saved job, including when no `--seed` was supplied. If several diff --git a/SECURITY.md b/SECURITY.md index c5caf5a..fc02dd4 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,7 +15,7 @@ xchplot2 is a client-side plot builder. It handles: or reused seed lets an attacker who observes plot IDs correlate plots to the same master key. - BLS key parsing via the - [`chia` Rust crate](https://crates.io/crates/chia) through + [`chia-bls` Rust crate](https://crates.io/crates/chia-bls) through `keygen-rs`. - Per-plot private keys, included in plot memos and saved job manifests. - Large file writes into caller-supplied output directories. @@ -29,7 +29,9 @@ while recovery may be needed. Deleting it can prevent recovery of an unseeded job's identities. New manifests and their publication temporaries use owner-only permissions -on Linux and are not allowed to replace another job's manifest. Report +on Linux and a protected owner-only ACL on native Windows. Windows file +creation requires an ACL-capable filesystem such as NTFS or ReFS. +Manifests are not allowed to replace another job's manifest. Report permission, disclosure, or publication failures through the channel above. See [plotting and recovery](REFERENCE.md#plotting-and-recovery) for behavior. diff --git a/scripts/test/windows.py b/scripts/test/windows.py index 33547ec..c80cb0e 100644 --- a/scripts/test/windows.py +++ b/scripts/test/windows.py @@ -81,6 +81,18 @@ def run(command, check=True): for path in result.stdout.splitlines(): run([binary, "verify", path, "--full", "--trials", "100", "--config", os.devnull]) print("Windows keygen: pool public key memos and parallel CPU plotting passed") + + bench = out / "benchmark é" + run([binary, "bench", "--config", os.devnull, "-k", "18", "-n", "1", + "--warmup", "0", "--cpu", "--cpu-workers", "1", "--keep", + "--compute-only", "-o", str(bench)]) + log.flush() + output = (out / "check.log").read_text(encoding="utf-8") + kept = [line.removeprefix("[bench] kept ") for line in output.splitlines() + if line.startswith("[bench] kept ")] + assert len(kept) == 2 and all(Path(p).is_file() for p in kept), kept + assert "no usable tmpfs" in output and "compute+cache" in output + print("Windows benchmark: cache fallback and Unicode kept paths passed") finally: if allocated_console: kernel.FreeConsole() diff --git a/src/host/TempFile.hpp b/src/host/TempFile.hpp index d56863f..2b77556 100644 --- a/src/host/TempFile.hpp +++ b/src/host/TempFile.hpp @@ -1,6 +1,6 @@ // TempFile.hpp — automatically removed temp file with positional read/write. // -// Task #26 disk-fallback foundation. Self-contained primitive: opens a +// Task #26 disk-fallback foundation. On Linux, opens a // unique-named file at construction (mkstemp), unlinks it immediately // so it disappears on process exit even on crash, and supports // thread-safe positional I/O via pread/pwrite. @@ -12,10 +12,6 @@ // 2. $TMPDIR // 3. /tmp on Linux; the system temporary directory on Windows // -// The file is automatically removed when the TempFile destructor runs. -// On crash the kernel reclaims the inode at process exit because the -// directory entry is already unlinked at construction. -// // Backs the host-RAM disk-offload path: SpillEngine/SpillBuffer stream the // cold cap-sized tables (h_t1_meta, h_t3, h_t2_meta, h_t2_xbits) through one // of these, and h_frags takes a map() instead. @@ -42,7 +38,7 @@ class TempFile { TempFile(TempFile&& other) noexcept; TempFile& operator=(TempFile&& other) noexcept; - // Positional write, safe to call CONCURRENTLY on one TempFile: pwrite + // Positional write, safe to call CONCURRENTLY on one TempFile: each operation // carries its own offset, so parallel writers to disjoint ranges need no // lock. SpillEngine relies on exactly that — it splits one 32 MiB chunk // across its worker pool by byte range, so several threads are inside this @@ -57,7 +53,7 @@ class TempFile { // Pageable, file-backed home for a CPU-touched buffer (the host-RAM // disk-offload; see the README's "Host RAM and disk-offload"). - // ftruncate()s the file to `bytes` and MAP_SHARED-maps it, returning a host + // Sizes the file to `bytes` and maps it read/write and shared, returning a host // pointer the CPU (and pageable-host DMA) can use as a drop-in // replacement for a pinned allocation. Unlike pinned pages, these // are reclaimable: under memory pressure the kernel writes dirty @@ -86,15 +82,14 @@ class TempFile { // batch, as "zero-byte write (disk full?)". With it, the failure lands // at table setup with the size that could not be reserved. // - // Does NOT make the file non-sparse: unwritten ranges still read as - // zeros, so SpillCoverage stays load-bearing. Silently does nothing on a - // filesystem without fallocate support (the file simply grows on demand, - // which is the old behaviour); throws only on a real failure such as - // ENOSPC. + // Unwritten ranges still read as zeros, so SpillCoverage stays load-bearing. + // Linux filesystems without fallocate support retain the grow-on-demand + // behavior. Windows reserves allocation space and sets the end of file. + // Real allocation failures throw before a mapping or write begins. void preallocate(std::uint64_t bytes); // Bytes available in `dir` (after resolve_dir) to an unprivileged - // writer, or 0 when statvfs cannot answer. Callers treat 0 as "unknown, + // writer, or 0 when the filesystem query cannot answer. Callers treat 0 as "unknown, // do not block on it" — the same stance dir_is_ram_backed takes, and for // the same reason: an unprobeable filesystem must not veto a spill that // would have worked. @@ -116,13 +111,14 @@ class TempFile { static std::string resolve_dir(std::string_view explicit_dir); // True when the filesystem hosting `dir` (after resolve_dir) keeps file - // contents in RAM — tmpfs, ramfs, or hugetlbfs. Spilling there consumes + // contents in RAM — tmpfs, ramfs, or hugetlbfs on Linux; a RAM-disk volume + // on Windows. Spilling there consumes // the very RAM a --max-host-ram budget is meant to bound, so callers use // this to refuse a RAM-backed spill target before doing any heavy work. - // A zram/zswap SWAP device backing a real disk filesystem is NOT flagged: + // On Linux, a zram/zswap SWAP device backing a real filesystem is NOT flagged: // only the mount's own fs magic is inspected, so files that actually live // on btrfs/ext4 pass even when the system swaps to compressed RAM. - // Returns false if statfs() fails — an unprobeable fs must not block + // Returns false if the filesystem query fails — an unprobeable fs must not block // spilling. That includes a dir that does not exist, so this is NOT a // usability check; pair it with dir_problem(). static bool dir_is_ram_backed(std::string const& dir); diff --git a/tools/xchplot2/cli.cpp b/tools/xchplot2/cli.cpp index 2486d26..9dd0b37 100644 --- a/tools/xchplot2/cli.cpp +++ b/tools/xchplot2/cli.cpp @@ -860,6 +860,7 @@ int batch_exit_code(pos2gpu::BatchResult const& res, std::size_t requested) // 10% of RAM and caps it, while /dev/shm gets 50%, so it reached for the smaller // of the two for a pass that writes the entire plot set at once and deletes // nothing until the end. +#ifndef _WIN32 std::string resolve_tmpfs_dir() { std::string best; @@ -878,6 +879,7 @@ std::string resolve_tmpfs_dir() consider("/dev/shm"); return best; } +#endif // A scratch dir on the roomiest tmpfs that we have actually proven we can write // to, or "" to say there isn't one. @@ -892,6 +894,9 @@ std::string resolve_tmpfs_dir() // write before handing it back. std::string prepare_tmpfs_scratch() { +#ifdef _WIN32 + return {}; // The caller uses its existing compute+cache fallback without tmpfs. +#else std::string const base = resolve_tmpfs_dir(); if (base.empty()) return {}; @@ -913,6 +918,7 @@ std::string prepare_tmpfs_scratch() } std::filesystem::remove(probe, ec); return dir; +#endif } struct BenchMeasurement { @@ -1823,7 +1829,7 @@ extern "C" int xchplot2_main(int argc, char* argv[]) sweep(); if (keep) { for (auto const& p : e2e.paths) { - std::fprintf(stderr, "[bench] kept %s\n", p.c_str()); + std::fprintf(stderr, "[bench] kept %s\n", p.string().c_str()); } // ...but only the ones sweep() actually left behind: the // compute-only set is gone if it lived in the tmpfs scratch. @@ -1831,7 +1837,7 @@ extern "C" int xchplot2_main(int argc, char* argv[]) for (auto const& p : compute.paths) { std::error_code ec; if (std::filesystem::exists(p, ec)) { - std::fprintf(stderr, "[bench] kept %s\n", p.c_str()); + std::fprintf(stderr, "[bench] kept %s\n", p.string().c_str()); } else { ++dropped; } From 7f881a7bb80d6319a118c37f94a640f7911b1f4a Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 22:33:34 -0500 Subject: [PATCH 08/10] fix(windows): keep existing data when reserving spill space --- src/host/TempFile.cpp | 10 ++++++++-- tools/parity/temp_file_test.cpp | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/host/TempFile.cpp b/src/host/TempFile.cpp index 9d1167b..a9aa465 100644 --- a/src/host/TempFile.cpp +++ b/src/host/TempFile.cpp @@ -237,10 +237,16 @@ void TempFile::preallocate(std::uint64_t bytes) if (bytes > static_cast(std::numeric_limits::max())) throw std::runtime_error("TempFile::preallocate: size exceeds the file offset range"); HANDLE const file = reinterpret_cast(::_get_osfhandle(fd_)); + FILE_STANDARD_INFO current{}; + if (!::GetFileInformationByHandleEx(file, FileStandardInfo, ¤t, sizeof(current))) + throw std::system_error(static_cast(::GetLastError()), std::system_category(), + "TempFile::preallocate: " + path_); + // FileAllocationInfo can truncate EOF. Reservation must preserve existing data. FILE_ALLOCATION_INFO allocation{}; - allocation.AllocationSize.QuadPart = static_cast(bytes); + allocation.AllocationSize.QuadPart = std::max({static_cast(bytes), + current.AllocationSize.QuadPart, current.EndOfFile.QuadPart}); FILE_END_OF_FILE_INFO end{}; - end.EndOfFile.QuadPart = static_cast(bytes); + end.EndOfFile.QuadPart = std::max(static_cast(bytes), current.EndOfFile.QuadPart); if (!::SetFileInformationByHandle(file, FileAllocationInfo, &allocation, sizeof(allocation)) || !::SetFileInformationByHandle(file, FileEndOfFileInfo, &end, sizeof(end))) throw std::system_error(static_cast(::GetLastError()), std::system_category(), diff --git a/tools/parity/temp_file_test.cpp b/tools/parity/temp_file_test.cpp index 9076d32..2d90ffd 100644 --- a/tools/parity/temp_file_test.cpp +++ b/tools/parity/temp_file_test.cpp @@ -236,6 +236,11 @@ int main() f.pread_at(1u << 20, &r, sizeof(r)); all_ok = check(r == w, "preallocate: round-trip still works") && all_ok; + f.preallocate(512u << 10); + r = 0; + f.pread_at(1u << 20, &r, sizeof(r)); + all_ok = check(r == w, "preallocate: smaller reservation preserves existing data") && all_ok; + // Zero is a no-op, not an error. f.preallocate(0); all_ok = check(true, "preallocate: zero bytes is a no-op") && all_ok; From 8ccda0863869435d7afff1b01f91271d86c8f4ed Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 22:53:42 -0500 Subject: [PATCH 09/10] fix(windows): use native entropy and extended spill paths --- CMakeLists.txt | 2 +- SECURITY.md | 3 +++ scripts/build-release.ps1 | 10 ++++++++-- src/host/TempFile.cpp | 4 ++++ tools/parity/temp_file_test.cpp | 14 ++++++++++++++ tools/xchplot2/cli.cpp | 19 ++++++++++++++----- 6 files changed, 44 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b700ea..ba4cd6d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -683,7 +683,7 @@ foreach(_target IN LISTS _xchplot2_targets) get_target_property(_type ${_target} TYPE) if(WIN32 AND _type STREQUAL "EXECUTABLE") target_sources(${_target} PRIVATE tools/xchplot2/windows.manifest) - target_link_libraries(${_target} PRIVATE advapi32) + target_link_libraries(${_target} PRIVATE advapi32 bcrypt) endif() if(_target MATCHES "(_parity|_test)$") set_target_properties(${_target} PROPERTIES diff --git a/SECURITY.md b/SECURITY.md index fc02dd4..d358b10 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,6 +20,9 @@ xchplot2 is a client-side plot builder. It handles: - Per-plot private keys, included in plot memos and saved job manifests. - Large file writes into caller-supplied output directories. +Without `--seed`, plot seeds come from the operating system's cryptographic +RNG: `/dev/urandom` on Linux and `BCryptGenRandom` on Windows. + `plot` saves identities before starting, including for unseeded jobs. The `xchplot2-job-*.tsv` files contain the memo and its private plot key material; they are intended persistent recovery data. `--manifest` selects another diff --git a/scripts/build-release.ps1 b/scripts/build-release.ps1 index 6e2239b..4cdf097 100644 --- a/scripts/build-release.ps1 +++ b/scripts/build-release.ps1 @@ -41,7 +41,13 @@ cargo about generate --locked --fail --manifest-path keygen-rs/Cargo.toml ` cmake -S . -B $BuildDir -G Ninja -DCMAKE_BUILD_TYPE=Release ` '-DCMAKE_CUDA_ARCHITECTURES=50-real;52-real;60-real;61-real;70-real;75-real;80-real;86-real;89-real;90-real;100-real;120' ` -DCMAKE_CUDA_RUNTIME_LIBRARY=Static -DXCHPLOT2_PACKAGE=ON "-DXCHPLOT2_LICENSE_DIR=$licenses" -cmake --build $BuildDir --parallel 2 +# Catch host regressions before compiling CUDA for every supported architecture. +$hostTests = 'bench_stats_test', 'numa_topology_test', 'temp_file_test', 'spill_engine_test', ` + 'spill_coverage_test', 'host_guard_test', 'host_spill_policy_test', 'vram_budget_test', ` + 'cli_host_test', 'solver_filter_parity' +cmake --build $BuildDir --parallel 2 --target $hostTests ctest --test-dir $BuildDir --output-on-failure --no-tests=error ` - -R '^(bench_stats_test|numa_topology_test|temp_file_test|spill_engine_test|spill_coverage_test|host_guard_test|host_spill_policy_test|vram_budget_test|cli_host_test|plot_file_parity|solver_filter_parity)$' + -R ('^(' + ($hostTests -join '|') + ')$') +cmake --build $BuildDir --parallel 2 +ctest --test-dir $BuildDir --output-on-failure --no-tests=error -R '^plot_file_parity$' cpack --config (Join-Path $BuildDir 'CPackConfig.cmake') -B (Join-Path $BuildDir 'dist') diff --git a/src/host/TempFile.cpp b/src/host/TempFile.cpp index a9aa465..db05905 100644 --- a/src/host/TempFile.cpp +++ b/src/host/TempFile.cpp @@ -137,8 +137,12 @@ void TempFile::bump_high_water(std::uint64_t end) noexcept TempFile::TempFile(std::string_view dir) { std::string base = resolve_dir(dir); +#ifdef _WIN32 + std::string templ = (std::filesystem::path(base) / "xchplot2-spill-XXXXXX").string(); +#else if (base.back() == '/') base.pop_back(); std::string templ = base + "/xchplot2-spill-XXXXXX"; +#endif std::string buf(templ); #ifdef _WIN32 fd_ = create_private_temp(buf, FILE_FLAG_OVERLAPPED | FILE_FLAG_DELETE_ON_CLOSE); diff --git a/tools/parity/temp_file_test.cpp b/tools/parity/temp_file_test.cpp index 2d90ffd..7393a23 100644 --- a/tools/parity/temp_file_test.cpp +++ b/tools/parity/temp_file_test.cpp @@ -41,6 +41,20 @@ int main() bool all_ok = true; std::string const temp = std::filesystem::temp_directory_path().string(); +#ifdef _WIN32 + { + auto path = std::filesystem::absolute(temp).wstring(); + if (!path.starts_with(L"\\\\?\\")) + path = path.starts_with(L"\\\\") ? L"\\\\?\\UNC\\" + path.substr(2) : L"\\\\?\\" + path; + pos2gpu::TempFile file(std::filesystem::path(path).string()); + std::uint64_t const expected = 12345; + file.pwrite_at(0, &expected, sizeof(expected)); + std::uint64_t actual = 0; + file.pread_at(0, &actual, sizeof(actual)); + all_ok = check(actual == expected, "extended Windows temp directory path") && all_ok; + } +#endif + // Test 1: basic open + write + read round-trip. { pos2gpu::TempFile tf; diff --git a/tools/xchplot2/cli.cpp b/tools/xchplot2/cli.cpp index 9dd0b37..2017b31 100644 --- a/tools/xchplot2/cli.cpp +++ b/tools/xchplot2/cli.cpp @@ -45,6 +45,8 @@ #include #ifdef _WIN32 +#include +#include #include #include #else @@ -151,7 +153,7 @@ void print_usage(char const* prog) << " -S, --seed HEX : optional 64 hex chars of master-SK\n" << " entropy. Per-plot seed = SHA256(seed || i).\n" << " Reproducible across runs. Defaults to\n" - << " fresh /dev/urandom per plot.\n" + << " a fresh random seed per plot.\n" << " -T, --testnet : testnet proof parameters.\n" << " -v, --verbose : per-plot progress on stderr.\n" << " -q, --quiet : suppress info-level stderr output\n" @@ -328,15 +330,22 @@ bool parse_hex(std::string const& s, std::array& out) return true; } -// Read exactly `n` bytes of entropy from /dev/urandom. Throws on failure. -void read_urandom(uint8_t* out, size_t n) +// Read exactly `n` bytes from the OS cryptographic RNG. Throws on failure. +void read_random_bytes(uint8_t* out, size_t n) { +#ifdef _WIN32 + if (n > MAXULONG) throw std::invalid_argument("entropy request exceeds the Windows buffer limit"); + auto const status = ::BCryptGenRandom(nullptr, out, static_cast(n), BCRYPT_USE_SYSTEM_PREFERRED_RNG); + if (status != 0) + throw std::runtime_error("BCryptGenRandom failed: " + std::to_string(status)); +#else std::ifstream f("/dev/urandom", std::ios::binary); if (!f) throw std::runtime_error("cannot open /dev/urandom"); f.read(reinterpret_cast(out), static_cast(n)); if (f.gcount() != static_cast(n)) { throw std::runtime_error("short read from /dev/urandom"); } +#endif } // Parse a --devices value into BatchOptions. @@ -1148,7 +1157,7 @@ std::vector build_bench_entries( e.plot_index = 0; e.meta_group = 0; e.testnet = testnet; - read_urandom(e.plot_id.data(), e.plot_id.size()); + read_random_bytes(e.plot_id.data(), e.plot_id.size()); e.out_dir = out_dir; e.out_name = "bench-" + bytes_to_hex(e.plot_id) + ".plot2"; entries.push_back(std::move(e)); @@ -2361,7 +2370,7 @@ extern "C" int xchplot2_main(int argc, char* argv[]) return 2; } } else { - read_urandom(seed, sizeof(seed)); + read_random_bytes(seed, sizeof(seed)); } uint8_t plot_id[32]; From cfb9d20870b0c5d93f599b03c03799baf35e3a0d Mon Sep 17 00:00:00 2001 From: Abraham Sewill Date: Wed, 9 Sep 2026 23:01:01 -0500 Subject: [PATCH 10/10] fix(windows): use standard limits for entropy buffers --- tools/xchplot2/cli.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/xchplot2/cli.cpp b/tools/xchplot2/cli.cpp index 2017b31..bcceec3 100644 --- a/tools/xchplot2/cli.cpp +++ b/tools/xchplot2/cli.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -334,7 +335,8 @@ bool parse_hex(std::string const& s, std::array& out) void read_random_bytes(uint8_t* out, size_t n) { #ifdef _WIN32 - if (n > MAXULONG) throw std::invalid_argument("entropy request exceeds the Windows buffer limit"); + if (n > std::numeric_limits::max()) + throw std::invalid_argument("entropy request exceeds the Windows buffer limit"); auto const status = ::BCryptGenRandom(nullptr, out, static_cast(n), BCRYPT_USE_SYSTEM_PREFERRED_RNG); if (status != 0) throw std::runtime_error("BCryptGenRandom failed: " + std::to_string(status));