diff --git a/diskann-wide/src/arch/aarch64/double.rs b/diskann-wide/src/arch/aarch64/double.rs index a66e45935..0aeb6ccd3 100644 --- a/diskann-wide/src/arch/aarch64/double.rs +++ b/diskann-wide/src/arch/aarch64/double.rs @@ -243,6 +243,7 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(u8x32, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(u8x32, 0xf24be4e5bc1d95b6, test_neon()); test_utils::ops::test_splitjoin!(u8x32 => u8x16, 0x2e301b7e12090d5c, test_neon()); test_utils::ops::test_zipunzip!(u8x32 => u8x16, 0xa1b2c3d4e5f67890, test_neon()); } @@ -253,6 +254,7 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(u8x64, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(u8x64, 0x7207f32ecac3fbf3, test_neon()); test_utils::ops::test_splitjoin!(u8x64 => u8x32, 0x2e301b7e12090d5c, test_neon()); } @@ -300,6 +302,7 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(i8x32, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(i8x32, 0x45f60286da125e84, test_neon()); test_utils::ops::test_abs!(i8x32, 0xd62d8de09f82ed4e, test_neon()); test_utils::ops::test_splitjoin!(i8x32 => i8x16, 0x2e301b7e12090d5c, test_neon()); test_utils::ops::test_zipunzip!(i8x32 => i8x16, 0xc7e3a92f1d8b5604, test_neon()); @@ -311,6 +314,7 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(i8x64, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(i8x64, 0x73e6c7da94a679ac, test_neon()); test_utils::ops::test_abs!(i8x64, 0xd62d8de09f82ed4e, test_neon()); test_utils::ops::test_splitjoin!(i8x64 => i8x32, 0x2e301b7e12090d5c, test_neon()); } diff --git a/diskann-wide/src/arch/aarch64/i8x16_.rs b/diskann-wide/src/arch/aarch64/i8x16_.rs index 791f99339..e4e8ac774 100644 --- a/diskann-wide/src/arch/aarch64/i8x16_.rs +++ b/diskann-wide/src/arch/aarch64/i8x16_.rs @@ -4,8 +4,8 @@ */ use crate::{ - Emulated, SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDVector, - constant::Const, helpers, + Emulated, SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDPopcount, + SIMDVector, constant::Const, helpers, }; // AArch64 masks @@ -31,6 +31,7 @@ helpers::unsafe_map_binary_op!(i8x16, std::ops::Add, add, vaddq_s8, "neon"); helpers::unsafe_map_binary_op!(i8x16, std::ops::Sub, sub, vsubq_s8, "neon"); helpers::unsafe_map_binary_op!(i8x16, std::ops::Mul, mul, vmulq_s8, "neon"); helpers::unsafe_map_unary_op!(i8x16, SIMDAbs, abs_simd, vabsq_s8, "neon"); +helpers::unsafe_map_unary_op!(i8x16, SIMDPopcount, popcount_simd, vcntq_s8, "neon"); macros::aarch64_define_fma!(i8x16, vmlaq_s8); macros::aarch64_define_cmp!( @@ -102,4 +103,5 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(i8x16, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(i8x16, 0xddff81feaf77563f, test_neon()); } diff --git a/diskann-wide/src/arch/aarch64/i8x8_.rs b/diskann-wide/src/arch/aarch64/i8x8_.rs index adb30dedb..6d325b2fc 100644 --- a/diskann-wide/src/arch/aarch64/i8x8_.rs +++ b/diskann-wide/src/arch/aarch64/i8x8_.rs @@ -4,8 +4,8 @@ */ use crate::{ - Emulated, SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDVector, - constant::Const, helpers, + Emulated, SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDPopcount, + SIMDVector, constant::Const, helpers, }; // AArch64 masks @@ -30,6 +30,7 @@ helpers::unsafe_map_binary_op!(i8x8, std::ops::Add, add, vadd_s8, "neon"); helpers::unsafe_map_binary_op!(i8x8, std::ops::Sub, sub, vsub_s8, "neon"); helpers::unsafe_map_binary_op!(i8x8, std::ops::Mul, mul, vmul_s8, "neon"); helpers::unsafe_map_unary_op!(i8x8, SIMDAbs, abs_simd, vabs_s8, "neon"); +helpers::unsafe_map_unary_op!(i8x8, SIMDPopcount, popcount_simd, vcnt_s8, "neon"); macros::aarch64_define_fma!(i8x8, vmla_s8); macros::aarch64_define_cmp!(i8x8, vceq_s8, (vmvn_u8), vclt_s8, vcle_s8, vcgt_s8, vcge_s8); @@ -92,4 +93,5 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(i8x8, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(i8x8, 0x46c84a6be23bc633, test_neon()); } diff --git a/diskann-wide/src/arch/aarch64/u8x16_.rs b/diskann-wide/src/arch/aarch64/u8x16_.rs index f1109db1f..ea4d65ba1 100644 --- a/diskann-wide/src/arch/aarch64/u8x16_.rs +++ b/diskann-wide/src/arch/aarch64/u8x16_.rs @@ -7,7 +7,7 @@ use crate::{ Emulated, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDPopcount, SIMDVector}, }; // AArch64 masks @@ -33,6 +33,7 @@ macros::aarch64_splitjoin!(u8x16, u8x8, vget_low_u8, vget_high_u8, vcombine_u8); helpers::unsafe_map_binary_op!(u8x16, std::ops::Add, add, vaddq_u8, "neon"); helpers::unsafe_map_binary_op!(u8x16, std::ops::Sub, sub, vsubq_u8, "neon"); helpers::unsafe_map_binary_op!(u8x16, std::ops::Mul, mul, vmulq_u8, "neon"); +helpers::unsafe_map_unary_op!(u8x16, SIMDPopcount, popcount_simd, vcntq_u8, "neon"); macros::aarch64_define_fma!(u8x16, vmlaq_u8); macros::aarch64_define_cmp!( @@ -103,4 +104,5 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(u8x16, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(u8x16, 0xb801a142f098b25d, test_neon()); } diff --git a/diskann-wide/src/arch/aarch64/u8x8_.rs b/diskann-wide/src/arch/aarch64/u8x8_.rs index efdb7787b..9e2baa5df 100644 --- a/diskann-wide/src/arch/aarch64/u8x8_.rs +++ b/diskann-wide/src/arch/aarch64/u8x8_.rs @@ -7,7 +7,7 @@ use crate::{ Emulated, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPartialEq, SIMDPartialOrd, SIMDPopcount, SIMDVector}, }; // AArch64 masks @@ -31,6 +31,7 @@ macros::aarch64_define_loadstore!(u8x8, vld1_u8, internal::load_first::u8x8, vst helpers::unsafe_map_binary_op!(u8x8, std::ops::Add, add, vadd_u8, "neon"); helpers::unsafe_map_binary_op!(u8x8, std::ops::Sub, sub, vsub_u8, "neon"); helpers::unsafe_map_binary_op!(u8x8, std::ops::Mul, mul, vmul_u8, "neon"); +helpers::unsafe_map_unary_op!(u8x8, SIMDPopcount, popcount_simd, vcnt_u8, "neon"); macros::aarch64_define_fma!(u8x8, vmla_u8); macros::aarch64_define_cmp!(u8x8, vceq_u8, (vmvn_u8), vclt_u8, vcle_u8, vcgt_u8, vcge_u8); @@ -92,4 +93,5 @@ mod tests { // Bit ops test_utils::ops::test_bitops!(u8x8, 0xd62d8de09f82ed4e, test_neon()); + test_utils::ops::test_popcount!(u8x8, 0xac222b463efd782c, test_neon()); } diff --git a/diskann-wide/src/arch/x86_64/v4/i16x16_.rs b/diskann-wide/src/arch/x86_64/v4/i16x16_.rs index 28aa1e5d7..75648ed30 100644 --- a/diskann-wide/src/arch/x86_64/v4/i16x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i16x16_.rs @@ -18,7 +18,7 @@ use crate::{ bitmask::BitMask, constant::Const, helpers, - traits::{SIMDAbs, SIMDMulAdd, SIMDVector}, + traits::{SIMDAbs, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; /////////////////// @@ -47,6 +47,13 @@ helpers::unsafe_map_binary_op!(i16x16, std::ops::Add, add, _mm256_add_epi16, "av helpers::unsafe_map_binary_op!(i16x16, std::ops::Sub, sub, _mm256_sub_epi16, "avx2"); helpers::unsafe_map_binary_op!(i16x16, std::ops::Mul, mul, _mm256_mullo_epi16, "avx2"); helpers::unsafe_map_unary_op!(i16x16, SIMDAbs, abs_simd, _mm256_abs_epi16, "avx2"); +helpers::unsafe_map_unary_op!( + i16x16, + SIMDPopcount, + popcount_simd, + _mm256_popcnt_epi16, + "avx512bitalg,avx512vl" +); helpers::unsafe_map_binary_op!(i16x16, std::ops::BitAnd, bitand, _mm256_and_si256, "avx2"); helpers::unsafe_map_binary_op!(i16x16, std::ops::BitOr, bitor, _mm256_or_si256, "avx2"); @@ -142,4 +149,5 @@ mod test_x86_i16 { // Bit ops test_utils::ops::test_bitops!(i16x16, 0xba0be356b04d6427, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i16x16, 0x781e87035cbf3527, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/i16x32_.rs b/diskann-wide/src/arch/x86_64/v4/i16x32_.rs index 19f10f89a..5cdc09b80 100644 --- a/diskann-wide/src/arch/x86_64/v4/i16x32_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i16x32_.rs @@ -17,7 +17,7 @@ use crate::{ bitmask::BitMask, constant::Const, helpers, - traits::{SIMDAbs, SIMDMulAdd, SIMDVector}, + traits::{SIMDAbs, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; /////////////////// @@ -33,6 +33,13 @@ helpers::unsafe_map_binary_op!(i16x32, std::ops::Add, add, _mm512_add_epi16, "av helpers::unsafe_map_binary_op!(i16x32, std::ops::Sub, sub, _mm512_sub_epi16, "avx512bw"); helpers::unsafe_map_binary_op!(i16x32, std::ops::Mul, mul, _mm512_mullo_epi16, "avx512bw"); helpers::unsafe_map_unary_op!(i16x32, SIMDAbs, abs_simd, _mm512_abs_epi16, "avx512bw"); +helpers::unsafe_map_unary_op!( + i16x32, + SIMDPopcount, + popcount_simd, + _mm512_popcnt_epi16, + "avx512bitalg" +); helpers::unsafe_map_binary_op!( i16x32, @@ -126,4 +133,5 @@ mod test_x86_i16 { // Bit ops test_utils::ops::test_bitops!(i16x32, 0xba0be356b04d6427, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i16x32, 0xa787d20c8f94b58b, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/i16x8_.rs b/diskann-wide/src/arch/x86_64/v4/i16x8_.rs index 58683875d..b11db891a 100644 --- a/diskann-wide/src/arch/x86_64/v4/i16x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i16x8_.rs @@ -17,7 +17,7 @@ use crate::{ bitmask::BitMask, constant::Const, helpers, - traits::{SIMDAbs, SIMDMulAdd, SIMDVector}, + traits::{SIMDAbs, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; /////////////////// @@ -33,6 +33,13 @@ helpers::unsafe_map_binary_op!(i16x8, std::ops::Add, add, _mm_add_epi16, "sse2") helpers::unsafe_map_binary_op!(i16x8, std::ops::Sub, sub, _mm_sub_epi16, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::Mul, mul, _mm_mullo_epi16, "sse2"); helpers::unsafe_map_unary_op!(i16x8, SIMDAbs, abs_simd, _mm_abs_epi16, "ssse3"); +helpers::unsafe_map_unary_op!( + i16x8, + SIMDPopcount, + popcount_simd, + _mm_popcnt_epi16, + "avx512bitalg,avx512vl" +); helpers::unsafe_map_binary_op!(i16x8, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); helpers::unsafe_map_binary_op!(i16x8, std::ops::BitOr, bitor, _mm_or_si128, "sse2"); @@ -125,4 +132,5 @@ mod test_x86_i16 { // Bit ops test_utils::ops::test_bitops!(i16x8, 0x2dd8796bda1cb89d, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i16x8, 0x6bc496e226618794, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/i32x16_.rs b/diskann-wide/src/arch/x86_64/v4/i32x16_.rs index 2f4603a5b..f7870f54f 100644 --- a/diskann-wide/src/arch/x86_64/v4/i32x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i32x16_.rs @@ -16,7 +16,10 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDAbs, SIMDDotProduct, SIMDMask, SIMDMulAdd, SIMDSelect, SIMDSumTree, SIMDVector}, + traits::{ + SIMDAbs, SIMDDotProduct, SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDSelect, SIMDSumTree, + SIMDVector, + }, }; ///// @@ -32,6 +35,13 @@ helpers::unsafe_map_binary_op!(i32x16, std::ops::Add, add, _mm512_add_epi32, "av helpers::unsafe_map_binary_op!(i32x16, std::ops::Sub, sub, _mm512_sub_epi32, "avx512f"); helpers::unsafe_map_binary_op!(i32x16, std::ops::Mul, mul, _mm512_mullo_epi32, "avx512f"); helpers::unsafe_map_unary_op!(i32x16, SIMDAbs, abs_simd, _mm512_abs_epi32, "avx512f"); +helpers::unsafe_map_unary_op!( + i32x16, + SIMDPopcount, + popcount_simd, + _mm512_popcnt_epi32, + "avx512vpopcntdq" +); helpers::unsafe_map_binary_op!( i32x16, @@ -170,6 +180,7 @@ mod test_x86_i32 { // Bit ops test_utils::ops::test_bitops!(i32x16, 0xc5f7d8d8df0b7b6c, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i32x16, 0xc8068253462939e5, V4::new_checked_uncached()); // Dot Products test_utils::dot_product::test_dot_product!( diff --git a/diskann-wide/src/arch/x86_64/v4/i32x4_.rs b/diskann-wide/src/arch/x86_64/v4/i32x4_.rs index 66dbec7b5..cb48f4e41 100644 --- a/diskann-wide/src/arch/x86_64/v4/i32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i32x4_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDVector}, + traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; ///// @@ -32,6 +32,13 @@ helpers::unsafe_map_binary_op!(i32x4, std::ops::Add, add, _mm_add_epi32, "sse2") helpers::unsafe_map_binary_op!(i32x4, std::ops::Sub, sub, _mm_sub_epi32, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::Mul, mul, _mm_mullo_epi32, "sse4.1"); helpers::unsafe_map_unary_op!(i32x4, SIMDAbs, abs_simd, _mm_abs_epi32, "ssse3"); +helpers::unsafe_map_unary_op!( + i32x4, + SIMDPopcount, + popcount_simd, + _mm_popcnt_epi32, + "avx512vpopcntdq,avx512vl" +); helpers::unsafe_map_binary_op!(i32x4, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); helpers::unsafe_map_binary_op!(i32x4, std::ops::BitOr, bitor, _mm_or_si128, "sse2"); @@ -109,4 +116,5 @@ mod test_x86_i32 { // Bit ops test_utils::ops::test_bitops!(i32x4, 0x763fc44f8f7cd40c, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i32x4, 0x6b756a0a75b4f2d6, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/i32x8_.rs b/diskann-wide/src/arch/x86_64/v4/i32x8_.rs index 32dcc8d56..7f6a2e0df 100644 --- a/diskann-wide/src/arch/x86_64/v4/i32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i32x8_.rs @@ -17,7 +17,10 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDAbs, SIMDDotProduct, SIMDMask, SIMDMulAdd, SIMDSelect, SIMDSumTree, SIMDVector}, + traits::{ + SIMDAbs, SIMDDotProduct, SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDSelect, SIMDSumTree, + SIMDVector, + }, }; ///// @@ -41,6 +44,13 @@ helpers::unsafe_map_binary_op!(i32x8, std::ops::Add, add, _mm256_add_epi32, "avx helpers::unsafe_map_binary_op!(i32x8, std::ops::Sub, sub, _mm256_sub_epi32, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::Mul, mul, _mm256_mullo_epi32, "avx2"); helpers::unsafe_map_unary_op!(i32x8, SIMDAbs, abs_simd, _mm256_abs_epi32, "avx2"); +helpers::unsafe_map_unary_op!( + i32x8, + SIMDPopcount, + popcount_simd, + _mm256_popcnt_epi32, + "avx512vpopcntdq,avx512vl" +); helpers::unsafe_map_binary_op!(i32x8, std::ops::BitAnd, bitand, _mm256_and_si256, "avx2"); helpers::unsafe_map_binary_op!(i32x8, std::ops::BitOr, bitor, _mm256_or_si256, "avx2"); @@ -167,6 +177,7 @@ mod test_x86_i32 { // Bit ops test_utils::ops::test_bitops!(i32x8, 0xc5f7d8d8df0b7b6c, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i32x8, 0xb24fb02ef70f7345, V4::new_checked_uncached()); // Dot Products test_utils::dot_product::test_dot_product!( diff --git a/diskann-wide/src/arch/x86_64/v4/i8x16_.rs b/diskann-wide/src/arch/x86_64/v4/i8x16_.rs index 55e1491ea..dc3bc58f0 100644 --- a/diskann-wide/src/arch/x86_64/v4/i8x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i8x16_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDVector}, + traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; ////////////////// @@ -31,6 +31,13 @@ macros::x86_retarget!(i8x16 => v3::i8x16); helpers::unsafe_map_binary_op!(i8x16, std::ops::Add, add, _mm_add_epi8, "sse2"); helpers::unsafe_map_binary_op!(i8x16, std::ops::Sub, sub, _mm_sub_epi8, "sse2"); helpers::unsafe_map_unary_op!(i8x16, SIMDAbs, abs_simd, _mm_abs_epi8, "ssse3"); +helpers::unsafe_map_unary_op!( + i8x16, + SIMDPopcount, + popcount_simd, + _mm_popcnt_epi8, + "avx512bitalg,avx512vl" +); impl std::ops::Mul for i8x16 { type Output = Self; @@ -134,4 +141,5 @@ mod test_x86_i8 { // Bit ops test_utils::ops::test_bitops!(i8x16, 0xeb7bb4da5b84ebbe, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i8x16, 0x9d61a6aa25b322cc, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/i8x32_.rs b/diskann-wide/src/arch/x86_64/v4/i8x32_.rs index 8146ed40e..6595a62b8 100644 --- a/diskann-wide/src/arch/x86_64/v4/i8x32_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i8x32_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDVector}, + traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; ////////////////// @@ -59,6 +59,13 @@ impl std::ops::Mul for i8x32 { helpers::unsafe_map_binary_op!(i8x32, std::ops::Add, add, _mm256_add_epi8, "avx2"); helpers::unsafe_map_binary_op!(i8x32, std::ops::Sub, sub, _mm256_sub_epi8, "avx2"); helpers::unsafe_map_unary_op!(i8x32, SIMDAbs, abs_simd, _mm256_abs_epi8, "avx2"); +helpers::unsafe_map_unary_op!( + i8x32, + SIMDPopcount, + popcount_simd, + _mm256_popcnt_epi8, + "avx512bitalg,avx512vl" +); helpers::unsafe_map_binary_op!(i8x32, std::ops::BitAnd, bitand, _mm256_and_si256, "avx2"); helpers::unsafe_map_binary_op!(i8x32, std::ops::BitOr, bitor, _mm256_or_si256, "avx2"); @@ -158,4 +165,5 @@ mod test_x86_i8 { // Bit ops test_utils::ops::test_bitops!(i8x32, 0xd62d8de09f82ed4e, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i8x32, 0xdc54b21bedc25f46, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/i8x64_.rs b/diskann-wide/src/arch/x86_64/v4/i8x64_.rs index b1bf8445f..98d311323 100644 --- a/diskann-wide/src/arch/x86_64/v4/i8x64_.rs +++ b/diskann-wide/src/arch/x86_64/v4/i8x64_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDVector}, + traits::{SIMDAbs, SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; ////////////////// @@ -39,6 +39,13 @@ impl std::ops::Mul for i8x64 { helpers::unsafe_map_binary_op!(i8x64, std::ops::Add, add, _mm512_add_epi8, "avx512bw"); helpers::unsafe_map_binary_op!(i8x64, std::ops::Sub, sub, _mm512_sub_epi8, "avx512bw"); helpers::unsafe_map_unary_op!(i8x64, SIMDAbs, abs_simd, _mm512_abs_epi8, "avx512bw"); +helpers::unsafe_map_unary_op!( + i8x64, + SIMDPopcount, + popcount_simd, + _mm512_popcnt_epi8, + "avx512bitalg" +); helpers::unsafe_map_binary_op!(i8x64, std::ops::BitAnd, bitand, _mm512_and_si512, "avx512f"); helpers::unsafe_map_binary_op!(i8x64, std::ops::BitOr, bitor, _mm512_or_si512, "avx512f"); @@ -131,4 +138,5 @@ mod test_x86_i8 { // Bit ops test_utils::ops::test_bitops!(i8x64, 0xd62d8de09f82ed4e, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(i8x64, 0x50971ea8ed24cedd, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/u32x16_.rs b/diskann-wide/src/arch/x86_64/v4/u32x16_.rs index 113afb863..8a84eeeae 100644 --- a/diskann-wide/src/arch/x86_64/v4/u32x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u32x16_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDSelect, SIMDSumTree, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDSelect, SIMDSumTree, SIMDVector}, }; ///// @@ -30,6 +30,13 @@ macros::x86_splitjoin!(__m512i, u32x16, u32x8); helpers::unsafe_map_binary_op!(u32x16, std::ops::Add, add, _mm512_add_epi32, "avx512f"); helpers::unsafe_map_binary_op!(u32x16, std::ops::Sub, sub, _mm512_sub_epi32, "avx512f"); +helpers::unsafe_map_unary_op!( + u32x16, + SIMDPopcount, + popcount_simd, + _mm512_popcnt_epi32, + "avx512vpopcntdq" +); helpers::unsafe_map_binary_op!(u32x16, std::ops::Mul, mul, _mm512_mullo_epi32, "avx512f"); helpers::unsafe_map_binary_op!( @@ -139,6 +146,7 @@ mod test_x86_u32 { // Bit ops test_utils::ops::test_bitops!(u32x16, 0xc5f7d8d8df0b7b6c, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u32x16, 0x021d781b447eab07, V4::new_checked_uncached()); // Reductions test_utils::ops::test_sumtree!(u32x16, 0xe533708e69ca0117, V4::new_checked_uncached()); diff --git a/diskann-wide/src/arch/x86_64/v4/u32x4_.rs b/diskann-wide/src/arch/x86_64/v4/u32x4_.rs index 938a5ab7e..90120aa6a 100644 --- a/diskann-wide/src/arch/x86_64/v4/u32x4_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u32x4_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDSelect, SIMDSumTree, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDSelect, SIMDSumTree, SIMDVector}, }; ///// @@ -30,6 +30,13 @@ macros::x86_retarget!(u32x4 => v3::u32x4); helpers::unsafe_map_binary_op!(u32x4, std::ops::Add, add, _mm_add_epi32, "sse2"); helpers::unsafe_map_binary_op!(u32x4, std::ops::Sub, sub, _mm_sub_epi32, "sse2"); +helpers::unsafe_map_unary_op!( + u32x4, + SIMDPopcount, + popcount_simd, + _mm_popcnt_epi32, + "avx512vpopcntdq,avx512vl" +); helpers::unsafe_map_binary_op!(u32x4, std::ops::Mul, mul, _mm_mullo_epi32, "sse4.1"); helpers::unsafe_map_binary_op!(u32x4, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); @@ -125,6 +132,7 @@ mod test_x86_u32 { // Bit ops test_utils::ops::test_bitops!(u32x4, 0xbe927713ea310164, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u32x4, 0xd2361d101c9f71a8, V4::new_checked_uncached()); // Reductions test_utils::ops::test_sumtree!(u32x4, 0xb9ac82ab23a855da, V4::new_checked_uncached()); diff --git a/diskann-wide/src/arch/x86_64/v4/u32x8_.rs b/diskann-wide/src/arch/x86_64/v4/u32x8_.rs index 9391383f3..d62441ff3 100644 --- a/diskann-wide/src/arch/x86_64/v4/u32x8_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u32x8_.rs @@ -17,7 +17,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDSelect, SIMDSumTree, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDSelect, SIMDSumTree, SIMDVector}, }; ///// @@ -39,6 +39,13 @@ macros::x86_zipunzip_perm32!(u32x8); helpers::unsafe_map_binary_op!(u32x8, std::ops::Add, add, _mm256_add_epi32, "avx2"); helpers::unsafe_map_binary_op!(u32x8, std::ops::Sub, sub, _mm256_sub_epi32, "avx2"); +helpers::unsafe_map_unary_op!( + u32x8, + SIMDPopcount, + popcount_simd, + _mm256_popcnt_epi32, + "avx512vpopcntdq,avx512vl" +); helpers::unsafe_map_binary_op!(u32x8, std::ops::Mul, mul, _mm256_mullo_epi32, "avx2"); helpers::unsafe_map_binary_op!(u32x8, std::ops::BitAnd, bitand, _mm256_and_si256, "avx2"); @@ -136,6 +143,7 @@ mod test_x86_u32 { // Bit ops test_utils::ops::test_bitops!(u32x8, 0x417f8adb857645a8, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u32x8, 0x393da59c7804c2c1, V4::new_checked_uncached()); // Reductions test_utils::ops::test_sumtree!(u32x8, 0xd6780b08573e203b, V4::new_checked_uncached()); diff --git a/diskann-wide/src/arch/x86_64/v4/u64x2_.rs b/diskann-wide/src/arch/x86_64/v4/u64x2_.rs index e9b799433..1a9738887 100644 --- a/diskann-wide/src/arch/x86_64/v4/u64x2_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u64x2_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDSumTree, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDSumTree, SIMDVector}, }; ///// @@ -30,6 +30,13 @@ macros::x86_retarget!(u64x2 => v3::u64x2); helpers::unsafe_map_binary_op!(u64x2, std::ops::Add, add, _mm_add_epi64, "sse2"); helpers::unsafe_map_binary_op!(u64x2, std::ops::Sub, sub, _mm_sub_epi64, "sse2"); +helpers::unsafe_map_unary_op!( + u64x2, + SIMDPopcount, + popcount_simd, + _mm_popcnt_epi64, + "avx512vpopcntdq,avx512vl" +); helpers::unsafe_map_binary_op!( u64x2, std::ops::Mul, @@ -123,4 +130,5 @@ mod test_x86_u64 { // Bit ops test_utils::ops::test_bitops!(u64x2, 0xf9566b095125ca45, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u64x2, 0xcc60798bf7130d7d, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/u64x4_.rs b/diskann-wide/src/arch/x86_64/v4/u64x4_.rs index f8810f5ee..24fc94788 100644 --- a/diskann-wide/src/arch/x86_64/v4/u64x4_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u64x4_.rs @@ -17,7 +17,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDSumTree, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDSumTree, SIMDVector}, }; ///// @@ -38,6 +38,13 @@ macros::x86_splitjoin!( helpers::unsafe_map_binary_op!(u64x4, std::ops::Add, add, _mm256_add_epi64, "avx2"); helpers::unsafe_map_binary_op!(u64x4, std::ops::Sub, sub, _mm256_sub_epi64, "avx2"); +helpers::unsafe_map_unary_op!( + u64x4, + SIMDPopcount, + popcount_simd, + _mm256_popcnt_epi64, + "avx512vpopcntdq,avx512vl" +); helpers::unsafe_map_binary_op!( u64x4, std::ops::Mul, @@ -132,4 +139,5 @@ mod test_x86_u64 { // Bit ops test_utils::ops::test_bitops!(u64x4, 0xb1ac2e16327a8d5e, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u64x4, 0xf23de3226c0141be, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/u8x16_.rs b/diskann-wide/src/arch/x86_64/v4/u8x16_.rs index 25e72bb41..674584d1f 100644 --- a/diskann-wide/src/arch/x86_64/v4/u8x16_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u8x16_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; //////////////////// @@ -50,6 +50,13 @@ helpers::unsafe_map_binary_op!( _mm_sub_epi8, /* intentional: epu8 does not exist and this does the same thing */ "sse2" ); +helpers::unsafe_map_unary_op!( + u8x16, + SIMDPopcount, + popcount_simd, + _mm_popcnt_epi8, + "avx512bitalg,avx512vl" +); helpers::unsafe_map_binary_op!(u8x16, std::ops::BitAnd, bitand, _mm_and_si128, "sse2"); helpers::unsafe_map_binary_op!(u8x16, std::ops::BitOr, bitor, _mm_or_si128, "sse2"); @@ -141,4 +148,5 @@ mod test_x86_u8 { // Bit ops test_utils::ops::test_bitops!(u8x16, 0xd62d8de09f82ed4e, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u8x16, 0xe67a50869e8b4695, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/u8x32_.rs b/diskann-wide/src/arch/x86_64/v4/u8x32_.rs index 2e0401512..728df0f5b 100644 --- a/diskann-wide/src/arch/x86_64/v4/u8x32_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u8x32_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; //////////////////// @@ -58,6 +58,13 @@ impl std::ops::Mul for u8x32 { helpers::unsafe_map_binary_op!(u8x32, std::ops::Add, add, _mm256_add_epi8, "avx2"); helpers::unsafe_map_binary_op!(u8x32, std::ops::Sub, sub, _mm256_sub_epi8, "avx2"); +helpers::unsafe_map_unary_op!( + u8x32, + SIMDPopcount, + popcount_simd, + _mm256_popcnt_epi8, + "avx512bitalg,avx512vl" +); helpers::unsafe_map_binary_op!(u8x32, std::ops::BitAnd, bitand, _mm256_and_si256, "avx2"); helpers::unsafe_map_binary_op!(u8x32, std::ops::BitOr, bitor, _mm256_or_si256, "avx2"); @@ -156,4 +163,5 @@ mod test_x86_u8 { // Bit ops test_utils::ops::test_bitops!(u8x32, 0xd62d8de09f82ed4e, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u8x32, 0x54a5ef4e4628385f, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/arch/x86_64/v4/u8x64_.rs b/diskann-wide/src/arch/x86_64/v4/u8x64_.rs index e42ac94a6..110a5f5ba 100644 --- a/diskann-wide/src/arch/x86_64/v4/u8x64_.rs +++ b/diskann-wide/src/arch/x86_64/v4/u8x64_.rs @@ -16,7 +16,7 @@ use crate::{ }, constant::Const, helpers, - traits::{SIMDMask, SIMDMulAdd, SIMDVector}, + traits::{SIMDMask, SIMDMulAdd, SIMDPopcount, SIMDVector}, }; //////////////////// @@ -38,6 +38,13 @@ impl std::ops::Mul for u8x64 { helpers::unsafe_map_binary_op!(u8x64, std::ops::Add, add, _mm512_add_epi8, "avx512bw"); helpers::unsafe_map_binary_op!(u8x64, std::ops::Sub, sub, _mm512_sub_epi8, "avx512bw"); +helpers::unsafe_map_unary_op!( + u8x64, + SIMDPopcount, + popcount_simd, + _mm512_popcnt_epi8, + "avx512bitalg" +); helpers::unsafe_map_binary_op!(u8x64, std::ops::BitAnd, bitand, _mm512_and_si512, "avx512f"); helpers::unsafe_map_binary_op!(u8x64, std::ops::BitOr, bitor, _mm512_or_si512, "avx512f"); @@ -129,4 +136,5 @@ mod test_x86_u8 { // Bit ops test_utils::ops::test_bitops!(u8x64, 0xd62d8de09f82ed4e, V4::new_checked_uncached()); + test_utils::ops::test_popcount!(u8x64, 0x6140a4039b289c1b, V4::new_checked_uncached()); } diff --git a/diskann-wide/src/doubled.rs b/diskann-wide/src/doubled.rs index 4f0ecd681..30d08e6cb 100644 --- a/diskann-wide/src/doubled.rs +++ b/diskann-wide/src/doubled.rs @@ -285,6 +285,13 @@ impl SIMDAbs for Doubled { } } +impl crate::SIMDPopcount for Doubled { + #[inline(always)] + fn popcount_simd(self) -> Self { + Self(self.0.popcount_simd(), self.1.popcount_simd()) + } +} + impl SIMDMulAdd for Doubled { #[inline(always)] fn mul_add_simd(self, rhs: Self, accumulator: Self) -> Self { diff --git a/diskann-wide/src/emulated.rs b/diskann-wide/src/emulated.rs index 98a6a24aa..13f998fb5 100644 --- a/diskann-wide/src/emulated.rs +++ b/diskann-wide/src/emulated.rs @@ -10,10 +10,11 @@ use super::{ arch::{self, emulated::Scalar}, bitmask::BitMask, constant::Const, - reference::{ReferenceAbs, ReferenceCast, ReferenceScalarOps, ReferenceShifts, TreeReduce}, + reference::{ReferenceAbs, ReferenceCast, ReferenceIntegerOps, ReferenceScalarOps, TreeReduce}, traits::{ ArrayType, SIMDAbs, SIMDCast, SIMDDotProduct, SIMDMask, SIMDMinMax, SIMDMulAdd, - SIMDPartialEq, SIMDPartialOrd, SIMDReinterpret, SIMDSelect, SIMDSumTree, SIMDVector, + SIMDPartialEq, SIMDPartialOrd, SIMDPopcount, SIMDReinterpret, SIMDSelect, SIMDSumTree, + SIMDVector, }, }; @@ -237,6 +238,16 @@ where } } +impl SIMDPopcount for Emulated +where + T: ReferenceIntegerOps, +{ + #[inline(always)] + fn popcount_simd(self) -> Self { + Self::from_arch_fn(self.1, |i| self.0[i].expected_popcount_()) + } +} + /// SIMDPartialEq impl SIMDPartialEq for Emulated where @@ -328,7 +339,7 @@ where impl std::ops::Shl for Emulated where - T: ReferenceShifts, + T: ReferenceIntegerOps, { type Output = Self; #[inline(always)] @@ -339,7 +350,7 @@ where impl std::ops::Shl for Emulated where - T: ReferenceShifts, + T: ReferenceIntegerOps, { type Output = Self; #[inline(always)] @@ -350,7 +361,7 @@ where impl std::ops::Shr for Emulated where - T: ReferenceShifts, + T: ReferenceIntegerOps, { type Output = Self; #[inline(always)] @@ -361,7 +372,7 @@ where impl std::ops::Shr for Emulated where - T: ReferenceShifts, + T: ReferenceIntegerOps, { type Output = Self; #[inline(always)] @@ -874,11 +885,13 @@ mod test_emulated { test_emulated!($type, $N); test_utils::ops::test_bitops!(Emulated<$type, $N>, 0x14fc7841e66bd162, SC); + test_utils::ops::test_popcount!(Emulated<$type, $N>, 0x78d19fb8aac40131, SC); }; (signed, $type:ty, $N:literal) => { test_emulated!($type, $N); test_utils::ops::test_bitops!(Emulated<$type, $N>, 0x850435f89f86f3b0, SC); + test_utils::ops::test_popcount!(Emulated<$type, $N>, 0x904e10c5fd2d4380, SC); test_utils::ops::test_abs!(Emulated<$type, $N>, 0x1842a2b86dfd9ecb, SC); }; } diff --git a/diskann-wide/src/lib.rs b/diskann-wide/src/lib.rs index 202772774..4f6ca5c92 100644 --- a/diskann-wide/src/lib.rs +++ b/diskann-wide/src/lib.rs @@ -144,8 +144,8 @@ pub use reference::{cast_f16_to_f32, cast_f32_to_f16}; mod traits; pub use traits::{ AsSIMD, SIMDAbs, SIMDCast, SIMDDotProduct, SIMDFloat, SIMDMask, SIMDMinMax, SIMDMulAdd, - SIMDPartialEq, SIMDPartialOrd, SIMDReinterpret, SIMDSelect, SIMDSigned, SIMDSumTree, - SIMDUnsigned, SIMDVector, ZipUnzip, + SIMDPartialEq, SIMDPartialOrd, SIMDPopcount, SIMDReinterpret, SIMDSelect, SIMDSigned, + SIMDSumTree, SIMDUnsigned, SIMDVector, ZipUnzip, }; mod splitjoin; diff --git a/diskann-wide/src/reference.rs b/diskann-wide/src/reference.rs index 801eeddb1..8f1fe4a98 100644 --- a/diskann-wide/src/reference.rs +++ b/diskann-wide/src/reference.rs @@ -3,14 +3,15 @@ * Licensed under the MIT license. */ -pub(crate) trait ReferenceShifts: Copy { +pub(crate) trait ReferenceIntegerOps: Copy { fn expected_shr_(self, rhs: Self) -> Self; fn expected_shl_(self, rhs: Self) -> Self; + fn expected_popcount_(self) -> Self; } -macro_rules! impl_shifts_unsigned { +macro_rules! impl_integer_ops_unsigned { ($type:ty) => { - impl ReferenceShifts for $type { + impl ReferenceIntegerOps for $type { #[inline(always)] fn expected_shr_(self, rhs: Self) -> Self { if (rhs as usize) >= 8 * std::mem::size_of::() { @@ -28,13 +29,18 @@ macro_rules! impl_shifts_unsigned { self << rhs } } + + #[inline(always)] + fn expected_popcount_(self) -> Self { + self.count_ones() as Self + } } }; } -macro_rules! impl_shifts_signed { +macro_rules! impl_integer_ops_signed { ($type:ty) => { - impl ReferenceShifts for $type { + impl ReferenceIntegerOps for $type { #[inline(always)] fn expected_shr_(self, rhs: Self) -> Self { if rhs < 0 || rhs >= ((8 * std::mem::size_of::()) as $type) { @@ -52,19 +58,24 @@ macro_rules! impl_shifts_signed { self << rhs } } + + #[inline(always)] + fn expected_popcount_(self) -> Self { + self.count_ones() as Self + } } }; } -impl_shifts_unsigned!(u8); -impl_shifts_unsigned!(u16); -impl_shifts_unsigned!(u32); -impl_shifts_unsigned!(u64); +impl_integer_ops_unsigned!(u8); +impl_integer_ops_unsigned!(u16); +impl_integer_ops_unsigned!(u32); +impl_integer_ops_unsigned!(u64); -impl_shifts_signed!(i8); -impl_shifts_signed!(i16); -impl_shifts_signed!(i32); -impl_shifts_signed!(i64); +impl_integer_ops_signed!(i8); +impl_integer_ops_signed!(i16); +impl_integer_ops_signed!(i32); +impl_integer_ops_signed!(i64); /// This is the ground truth for how operations behave. /// diff --git a/diskann-wide/src/test_utils/ops.rs b/diskann-wide/src/test_utils/ops.rs index 24da31892..fc15661f1 100644 --- a/diskann-wide/src/test_utils/ops.rs +++ b/diskann-wide/src/test_utils/ops.rs @@ -10,7 +10,7 @@ use super::common::{self, ScalarTraits}; use crate::{ BitMask, Const, SIMDMask, SIMDMinMax, SIMDPartialEq, SIMDPartialOrd, SIMDSumTree, SIMDVector, SplitJoin, SupportedLaneCount, ZipUnzip, arch, - reference::{ReferenceScalarOps, ReferenceShifts, TreeReduce}, + reference::{ReferenceIntegerOps, ReferenceScalarOps, TreeReduce}, }; fn identity(x: T) -> T { @@ -563,6 +563,37 @@ macro_rules! test_abs { }; } +macro_rules! test_popcount { + ($wide:ident $(< $($ps:tt),+ >)?, $seed:literal, $arch:expr) => { + paste::paste! { + #[test] + fn []() { + use crate::{SIMDPopcount, SIMDVector, reference::ReferenceIntegerOps}; + + type T = $wide $(< $($ps),+>)?; + + if let Some(arch) = $arch { + let f = move |input: &[::Scalar]| { + let got = ::from_array( + arch, + input.try_into().unwrap() + ).popcount_simd().to_array(); + + $crate::test_utils::test_unary_op( + input, + &got, + &|x| x.expected_popcount_(), + "population count", + ) + }; + let n: usize = T::LANES; + $crate::test_utils::driver::drive_unary(&f, n, $seed); + } + } + } + }; +} + macro_rules! test_select { ($wide:ident $(< $($ps:tt),+ >)?, $seed:literal, $arch:expr) => { paste::paste! { @@ -776,7 +807,7 @@ impl BitOps for T where pub(crate) fn test_bitops_impl(arch: A, a: &[T], b: &[T]) where A: arch::Sealed, - T: BitOps + Debug + Copy + Eq + ReferenceShifts + ScalarTraits, + T: BitOps + Debug + Copy + Eq + ReferenceIntegerOps + ScalarTraits, Const: SupportedLaneCount, BitMask: SIMDMask, V: SIMDVector> + BitOps, @@ -816,7 +847,7 @@ where pub(crate) fn test_scalar_shift_impl(arch: A, a: &[T], b: &[T]) where A: arch::Sealed, - T: BitOps + Debug + Copy + Eq + ReferenceShifts + ScalarTraits, + T: BitOps + Debug + Copy + Eq + ReferenceIntegerOps + ScalarTraits, Const: SupportedLaneCount, BitMask: SIMDMask, V: SIMDVector> @@ -1114,6 +1145,7 @@ pub(crate) use test_fma; pub(crate) use test_lossless_convert; pub(crate) use test_minmax; pub(crate) use test_mul; +pub(crate) use test_popcount; pub(crate) use test_select; pub(crate) use test_splitjoin; pub(crate) use test_sub; diff --git a/diskann-wide/src/traits.rs b/diskann-wide/src/traits.rs index d581406c9..8d163efc6 100644 --- a/diskann-wide/src/traits.rs +++ b/diskann-wide/src/traits.rs @@ -524,6 +524,14 @@ pub trait SIMDAbs { fn abs_simd(self) -> Self; } +/// Count the set bits in each lane. +/// +/// This trait is only implemented for vector/architecture pairs that have native support +/// for popcounts. [`crate::Emulated`] is the exception and computes popcounts in scalar code. +pub trait SIMDPopcount { + fn popcount_simd(self) -> Self; +} + /// A SIMD equivalent of `std::cmp::PartialEq`. /// /// Instead of a boolean, return `Self::Mask` containing the result of the element-wise