diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index d9dd051..e570eb6 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -3,6 +3,7 @@ use napi::bindgen_prelude::Buffer; use napi::{Env, Error, Status, Unknown}; use napi_derive::napi; +use std::collections::BTreeMap; #[napi(object)] pub struct TextRange { @@ -224,19 +225,26 @@ fn js_range(range: datafog_core::TextRange) -> napi::Result { }) } -fn js_utf16_range(text: &str, range: datafog_core::TextRange) -> napi::Result { - datafog_core::utf16_range(text, range) +fn js_utf16_range( + index: &mut datafog_core::TextIndex<'_>, + range: datafog_core::TextRange, +) -> napi::Result { + index + .utf16_range(range) .map_err(|error| Error::new(Status::GenericFailure, error.to_string())) .and_then(js_range) } -fn js_finding(text: &str, finding: datafog_core::Finding) -> napi::Result { +fn js_finding( + index: &mut datafog_core::TextIndex<'_>, + finding: datafog_core::Finding, +) -> napi::Result { Ok(Finding { entity_type: finding.entity_type, matched_text: finding.matched_text, byte_range: js_range(finding.byte_range)?, codepoint_range: js_range(finding.codepoint_range)?, - utf16_range: js_utf16_range(text, finding.byte_range)?, + utf16_range: js_utf16_range(index, finding.byte_range)?, confidence: finding.confidence.map(f64::from), detector_name: finding.detector_name, detector_version: finding.detector_version, @@ -282,50 +290,51 @@ fn js_transform_result( source_text: &str, result: datafog_core::TransformResult, ) -> napi::Result { - let output_text = &result.text; + let mut source_index = datafog_core::TextIndex::new(source_text); + let mut output_index = datafog_core::TextIndex::new(&result.text); Ok(TransformResult { transformations: result .transformations .into_iter() .map(|transformation| { - Ok(Transformation { - entity_type: transformation.entity_type, - source_byte_range: js_range(transformation.source_byte_range)?, - source_codepoint_range: js_range(transformation.source_codepoint_range)?, - source_utf16_range: js_utf16_range( - source_text, - transformation.source_byte_range, - )?, - confidence: transformation.confidence.map(f64::from), - detector_name: transformation.detector_name, - detector_version: transformation.detector_version, - strategy: match transformation.strategy { - datafog_core::TransformationStrategy::Redact => "redact".to_owned(), - datafog_core::TransformationStrategy::Remove => "remove".to_owned(), - datafog_core::TransformationStrategy::Mask(_) => "mask".to_owned(), - datafog_core::TransformationStrategy::Pseudonymize(_) => { - "pseudonymize".to_owned() - } - datafog_core::TransformationStrategy::Tokenize(_) => "tokenize".to_owned(), - }, - replacement: transformation.replacement, - output_byte_range: js_range(transformation.output_byte_range)?, - output_codepoint_range: js_range(transformation.output_codepoint_range)?, - output_utf16_range: js_utf16_range( - output_text, - transformation.output_byte_range, - )?, - key_ref: transformation.key_ref, - resolved_key_version: transformation.resolved_key_version, - token_ref: transformation.token_ref, - resolved_token_version: transformation.resolved_token_version, - }) + js_transformation(&mut source_index, &mut output_index, transformation) }) .collect::>>()?, text: result.text, }) } +fn js_transformation( + source_index: &mut datafog_core::TextIndex<'_>, + output_index: &mut datafog_core::TextIndex<'_>, + transformation: datafog_core::Transformation, +) -> napi::Result { + Ok(Transformation { + entity_type: transformation.entity_type, + source_byte_range: js_range(transformation.source_byte_range)?, + source_codepoint_range: js_range(transformation.source_codepoint_range)?, + source_utf16_range: js_utf16_range(source_index, transformation.source_byte_range)?, + confidence: transformation.confidence.map(f64::from), + detector_name: transformation.detector_name, + detector_version: transformation.detector_version, + strategy: match transformation.strategy { + datafog_core::TransformationStrategy::Redact => "redact".to_owned(), + datafog_core::TransformationStrategy::Remove => "remove".to_owned(), + datafog_core::TransformationStrategy::Mask(_) => "mask".to_owned(), + datafog_core::TransformationStrategy::Pseudonymize(_) => "pseudonymize".to_owned(), + datafog_core::TransformationStrategy::Tokenize(_) => "tokenize".to_owned(), + }, + replacement: transformation.replacement, + output_byte_range: js_range(transformation.output_byte_range)?, + output_codepoint_range: js_range(transformation.output_codepoint_range)?, + output_utf16_range: js_utf16_range(output_index, transformation.output_byte_range)?, + key_ref: transformation.key_ref, + resolved_key_version: transformation.resolved_key_version, + token_ref: transformation.token_ref, + resolved_token_version: transformation.resolved_token_version, + }) +} + fn core_token_results(results: Vec) -> Vec { results .into_iter() @@ -343,28 +352,35 @@ fn js_restore_result( source_text: &str, result: datafog_core::RestoreResult, ) -> napi::Result { - let output_text = &result.text; + let mut source_index = datafog_core::TextIndex::new(source_text); + let mut output_index = datafog_core::TextIndex::new(&result.text); Ok(RestoreResult { restorations: result .restorations .into_iter() - .map(|record| { - Ok(Restoration { - source_byte_range: js_range(record.source_byte_range)?, - source_codepoint_range: js_range(record.source_codepoint_range)?, - source_utf16_range: js_utf16_range(source_text, record.source_byte_range)?, - output_byte_range: js_range(record.output_byte_range)?, - output_codepoint_range: js_range(record.output_codepoint_range)?, - output_utf16_range: js_utf16_range(output_text, record.output_byte_range)?, - token_ref: record.token_ref, - resolved_token_version: record.resolved_token_version, - }) - }) + .map(|record| js_restoration(&mut source_index, &mut output_index, record)) .collect::>>()?, text: result.text, }) } +fn js_restoration( + source_index: &mut datafog_core::TextIndex<'_>, + output_index: &mut datafog_core::TextIndex<'_>, + record: datafog_core::Restoration, +) -> napi::Result { + Ok(Restoration { + source_byte_range: js_range(record.source_byte_range)?, + source_codepoint_range: js_range(record.source_codepoint_range)?, + source_utf16_range: js_utf16_range(source_index, record.source_byte_range)?, + output_byte_range: js_range(record.output_byte_range)?, + output_codepoint_range: js_range(record.output_codepoint_range)?, + output_utf16_range: js_utf16_range(output_index, record.output_byte_range)?, + token_ref: record.token_ref, + resolved_token_version: record.resolved_token_version, + }) +} + fn js_key_selectors(selectors: &[datafog_core::KeySelector]) -> napi::Result> { selectors .iter() @@ -414,9 +430,10 @@ pub fn scan( } else { datafog_core::ScanConfig::default() }; + let mut index = datafog_core::TextIndex::new(&text); datafog_core::scan_with_config(&text, &config) .into_iter() - .map(|finding| js_finding(&text, finding)) + .map(|finding| js_finding(&mut index, finding)) .collect() } @@ -498,10 +515,11 @@ pub fn prepare_scan_and_transform( let selectors = datafog_core::required_key_selectors(&text, &findings, config.transformation_config()) .map_err(js_privacy_error)?; + let mut index = datafog_core::TextIndex::new(&text); Ok(PreparedScanAndTransform { findings: findings .into_iter() - .map(|finding| js_finding(&text, finding)) + .map(|finding| js_finding(&mut index, finding)) .collect::>>()?, selectors: js_key_selectors(&selectors)?, }) @@ -683,20 +701,7 @@ pub fn scan_structured( let result = datafog_core::structured::scan(&data, &config).map_err(js_privacy_error)?; Ok(StructuredScanResult { mappings: result.mappings.into_iter().map(js_field_mapping).collect(), - findings: result - .findings - .into_iter() - .map(|located| { - let text = data - .pointer(&located.path) - .and_then(serde_json::Value::as_str) - .ok_or_else(|| js_privacy_error(datafog_core::structured::invalid_data()))?; - Ok(StructuredFinding { - path: located.path, - finding: js_finding(text, located.finding)?, - }) - }) - .collect::>()?, + findings: js_structured_findings(&data, result.findings)?, }) } @@ -745,25 +750,46 @@ fn structured_text<'a>(data: &'a serde_json::Value, path: &str) -> napi::Result< .ok_or_else(|| js_privacy_error(datafog_core::structured::invalid_data())) } +fn js_structured_findings( + data: &serde_json::Value, + findings: Vec, +) -> napi::Result> { + let mut indices = BTreeMap::new(); + findings + .into_iter() + .map(|located| { + let text = structured_text(data, &located.path)?; + let index = indices + .entry(located.path.clone()) + .or_insert_with(|| datafog_core::TextIndex::new(text)); + Ok(StructuredFinding { + finding: js_finding(index, located.finding)?, + path: located.path, + }) + }) + .collect() +} + fn js_structured_transform_result( data: &serde_json::Value, result: datafog_core::structured::StructuredTransformResult, ) -> napi::Result { let mut transformations = Vec::new(); + let mut indices = BTreeMap::new(); for record in result.transformations { - let converted = js_transform_result( - structured_text(data, &record.path)?, - datafog_core::TransformResult { - text: structured_text(&result.data, &record.path)?.into(), - transformations: vec![record.transformation], - }, - )?; - for transformation in converted.transformations { - transformations.push(StructuredTransformation { - path: record.path.clone(), - transformation, + let source = structured_text(data, &record.path)?; + let output = structured_text(&result.data, &record.path)?; + let (source_index, output_index) = + indices.entry(record.path.clone()).or_insert_with(|| { + ( + datafog_core::TextIndex::new(source), + datafog_core::TextIndex::new(output), + ) }); - } + transformations.push(StructuredTransformation { + path: record.path, + transformation: js_transformation(source_index, output_index, record.transformation)?, + }); } Ok(NativeStructuredTransformResult { data_json: result.data.to_string(), @@ -776,20 +802,21 @@ fn js_structured_restore_result( result: datafog_core::structured::StructuredRestoreResult, ) -> napi::Result { let mut restorations = Vec::new(); + let mut indices = BTreeMap::new(); for record in result.restorations { - let converted = js_restore_result( - structured_text(data, &record.path)?, - datafog_core::RestoreResult { - text: structured_text(&result.data, &record.path)?.into(), - restorations: vec![record.restoration], - }, - )?; - for restoration in converted.restorations { - restorations.push(StructuredRestoration { - path: record.path.clone(), - restoration, + let source = structured_text(data, &record.path)?; + let output = structured_text(&result.data, &record.path)?; + let (source_index, output_index) = + indices.entry(record.path.clone()).or_insert_with(|| { + ( + datafog_core::TextIndex::new(source), + datafog_core::TextIndex::new(output), + ) }); - } + restorations.push(StructuredRestoration { + path: record.path, + restoration: js_restoration(source_index, output_index, record.restoration)?, + }); } Ok(NativeStructuredRestoreResult { data_json: result.data.to_string(), @@ -999,14 +1026,6 @@ pub fn prepare_structured_scan_and_transform( .map_err(js_privacy_error)?; Ok(PreparedStructuredScan { selectors: js_key_selectors(&selectors)?, - findings: findings - .into_iter() - .map(|located| { - Ok(StructuredFinding { - finding: js_finding(structured_text(&data, &located.path)?, located.finding)?, - path: located.path, - }) - }) - .collect::>()?, + findings: js_structured_findings(&data, findings)?, }) } diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index 16c6ba7..843f484 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -77,19 +77,26 @@ struct TransformResult { transformations: Vec, } -fn utf16_range(text: &str, range: datafog_core::TextRange) -> Result { - datafog_core::utf16_range(text, range) +fn utf16_range( + index: &mut datafog_core::TextIndex<'_>, + range: datafog_core::TextRange, +) -> Result { + index + .utf16_range(range) .map(TextRange::from) .map_err(|error| JsValue::from_str(&error.to_string())) } -fn finding_from_core(text: &str, finding: datafog_core::Finding) -> Result { +fn finding_from_core( + index: &mut datafog_core::TextIndex<'_>, + finding: datafog_core::Finding, +) -> Result { Ok(Finding { entity_type: finding.entity_type, matched_text: finding.matched_text, byte_range: finding.byte_range.into(), codepoint_range: finding.codepoint_range.into(), - utf16_range: utf16_range(text, finding.byte_range)?, + utf16_range: utf16_range(index, finding.byte_range)?, confidence: finding.confidence, detector_name: finding.detector_name, detector_version: finding.detector_version, @@ -117,13 +124,14 @@ fn result_to_js( source_text: &str, result: datafog_core::TransformResult, ) -> Result { - let output_text = &result.text; + let mut source_index = datafog_core::TextIndex::new(source_text); + let mut output_index = datafog_core::TextIndex::new(&result.text); let result = TransformResult { transformations: result .transformations .into_iter() .map(|transformation| { - transformation_from_core(source_text, output_text, transformation) + transformation_from_core(&mut source_index, &mut output_index, transformation) }) .collect::, JsValue>>()?, text: result.text, @@ -140,9 +148,10 @@ pub fn scan(text: &str, config: Option) -> Result { } else { datafog_core::ScanConfig::default() }; + let mut index = datafog_core::TextIndex::new(text); let findings: Vec = datafog_core::scan_with_config(text, &config) .into_iter() - .map(|finding| finding_from_core(text, finding)) + .map(|finding| finding_from_core(&mut index, finding)) .collect::>()?; serde_wasm_bindgen::to_value(&findings).map_err(|error| JsValue::from_str(&error.to_string())) @@ -288,6 +297,7 @@ pub fn scan_structured(data_json: &str, config: Option) -> Result) -> Result, JsValue>>()?; @@ -310,15 +323,15 @@ pub fn scan_structured(data_json: &str, config: Option) -> Result, + output_index: &mut datafog_core::TextIndex<'_>, transformation: datafog_core::Transformation, ) -> Result { Ok(Transformation { entity_type: transformation.entity_type, source_byte_range: transformation.source_byte_range.into(), source_codepoint_range: transformation.source_codepoint_range.into(), - source_utf16_range: utf16_range(source_text, transformation.source_byte_range)?, + source_utf16_range: utf16_range(source_index, transformation.source_byte_range)?, confidence: transformation.confidence, detector_name: transformation.detector_name, detector_version: transformation.detector_version, @@ -332,7 +345,7 @@ fn transformation_from_core( replacement: transformation.replacement, output_byte_range: transformation.output_byte_range.into(), output_codepoint_range: transformation.output_codepoint_range.into(), - output_utf16_range: utf16_range(output_text, transformation.output_byte_range)?, + output_utf16_range: utf16_range(output_index, transformation.output_byte_range)?, key_ref: transformation.key_ref, resolved_key_version: transformation.resolved_key_version, token_ref: transformation.token_ref, @@ -388,6 +401,7 @@ fn structured_result_to_js( data_json: String, transformations: Vec, } + let mut indices = std::collections::BTreeMap::new(); let transformations = result .transformations .into_iter() @@ -401,9 +415,20 @@ fn structured_result_to_js( .pointer(&record.path) .and_then(serde_json::Value::as_str) .ok_or_else(|| privacy_error(datafog_core::structured::invalid_data()))?; + let (source_index, output_index) = + indices.entry(record.path.clone()).or_insert_with(|| { + ( + datafog_core::TextIndex::new(source), + datafog_core::TextIndex::new(output), + ) + }); Ok(Record { path: record.path, - transformation: transformation_from_core(source, output, record.transformation)?, + transformation: transformation_from_core( + source_index, + output_index, + record.transformation, + )?, }) }) .collect::, JsValue>>()?; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 1ab2349..b8c4634 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,7 +1,9 @@ //! Core PII scanning API for DataFog. +mod offsets; pub mod structured; use base64::Engine; use hmac::{Hmac, Mac}; +pub use offsets::TextIndex; use regex::{Regex, RegexSet, RegexSetBuilder}; use sha2::Sha256; use std::collections::{BTreeMap, BTreeSet}; @@ -36,6 +38,14 @@ impl std::error::Error for Utf16RangeError {} /// Convert a UTF-8 byte range into zero-based, end-exclusive UTF-16 code-unit /// offsets for JavaScript consumers. pub fn utf16_range(text: &str, byte_range: TextRange) -> Result { + validate_utf8_range(text, byte_range)?; + Ok(TextRange { + start: text[..byte_range.start].encode_utf16().count(), + end: text[..byte_range.end].encode_utf16().count(), + }) +} + +fn validate_utf8_range(text: &str, byte_range: TextRange) -> Result<(), Utf16RangeError> { if byte_range.start > byte_range.end { return Err(Utf16RangeError); } @@ -46,10 +56,7 @@ pub fn utf16_range(text: &str, byte_range: TextRange) -> Result Result, PrivacyError> { + let mut offsets = TextIndex::new(text); for (finding_index, finding) in findings.iter().enumerate() { - if let Err(kind) = validate_finding(text, finding) { + if let Err(kind) = validate_finding_with_index(text, finding, &mut offsets) { return Err(PrivacyError::invalid_finding(finding_index, kind)); } } @@ -2457,11 +2465,14 @@ fn apply_transformations( let mut output = String::with_capacity(text.len()); let mut transformations = Vec::with_capacity(selected_findings.len()); let mut source_byte_cursor = 0; + let mut output_codepoints = 0; for (finding_index, finding) in selected_findings.iter().enumerate() { - output.push_str(&text[source_byte_cursor..finding.byte_range.start]); + let unchanged = &text[source_byte_cursor..finding.byte_range.start]; + output.push_str(unchanged); + output_codepoints += unchanged.chars().count(); let output_byte_start = output.len(); - let output_codepoint_start = output.chars().count(); + let output_codepoint_start = output_codepoints; let strategy = config.strategy_for(finding); let mut key_ref = None; let mut resolved_key_version = None; @@ -2512,6 +2523,7 @@ fn apply_transformations( }; output.push_str(&replacement); + output_codepoints += replacement.chars().count(); transformations.push(Transformation { entity_type: finding.entity_type.clone(), source_byte_range: finding.byte_range, @@ -2527,7 +2539,7 @@ fn apply_transformations( }, output_codepoint_range: TextRange { start: output_codepoint_start, - end: output.chars().count(), + end: output_codepoints, }, key_ref, resolved_key_version, @@ -2819,7 +2831,16 @@ fn overlap_preference(left: &Finding, right: &Finding) -> std::cmp::Ordering { .then_with(|| left.detector_version.cmp(&right.detector_version)) } +#[cfg(test)] fn validate_finding(text: &str, finding: &Finding) -> Result<(), FindingValidationError> { + validate_finding_with_index(text, finding, &mut TextIndex::new(text)) +} + +fn validate_finding_with_index( + text: &str, + finding: &Finding, + offsets: &mut TextIndex<'_>, +) -> Result<(), FindingValidationError> { if finding.byte_range.start >= finding.byte_range.end { return Err(FindingValidationError::EmptyOrReversedByteRange); } @@ -2835,11 +2856,12 @@ fn validate_finding(text: &str, finding: &Finding) -> Result<(), FindingValidati return Err(FindingValidationError::EmptyOrReversedCodepointRange); } - let Some(codepoint_start_byte) = byte_offset_at_codepoint(text, finding.codepoint_range.start) + let Some(codepoint_start_byte) = + offsets.byte_offset_at_codepoint(finding.codepoint_range.start) else { return Err(FindingValidationError::CodepointRangeOutOfBounds); }; - let Some(codepoint_end_byte) = byte_offset_at_codepoint(text, finding.codepoint_range.end) + let Some(codepoint_end_byte) = offsets.byte_offset_at_codepoint(finding.codepoint_range.end) else { return Err(FindingValidationError::CodepointRangeOutOfBounds); }; @@ -2860,6 +2882,7 @@ fn validate_finding(text: &str, finding: &Finding) -> Result<(), FindingValidati Ok(()) } +#[cfg(test)] fn byte_offset_at_codepoint(text: &str, codepoint_offset: usize) -> Option { text.char_indices() .map(|(byte_offset, _)| byte_offset) @@ -2882,6 +2905,7 @@ fn finalize(text: &str, mut candidates: Vec) -> Vec { }); let is_ascii = text.is_ascii(); + let mut offsets = TextIndex::new(text); candidates .into_iter() @@ -2892,13 +2916,13 @@ fn finalize(text: &str, mut candidates: Vec) -> Vec { let start = if is_ascii { candidate.start_byte } else { - code_point_offset(text, candidate.start_byte) + offsets.codepoint_offset(candidate.start_byte) }; let end = if is_ascii { candidate.end_byte } else { - code_point_offset(text, candidate.end_byte) + offsets.codepoint_offset(candidate.end_byte) }; Finding { @@ -2917,10 +2941,6 @@ fn finalize(text: &str, mut candidates: Vec) -> Vec { .collect() } -fn code_point_offset(text: &str, byte_offset: usize) -> usize { - text[..byte_offset].chars().count() -} - static PHONE_RE: LazyLock = LazyLock::new(|| { Regex::new( r"(?x) diff --git a/crates/core/src/offsets.rs b/crates/core/src/offsets.rs new file mode 100644 index 0000000..7d3a841 --- /dev/null +++ b/crates/core/src/offsets.rs @@ -0,0 +1,203 @@ +use crate::{TextRange, Utf16RangeError, validate_utf8_range}; + +const CHECKPOINT_INTERVAL: usize = 256; + +#[derive(Clone, Copy, Default)] +struct Position { + byte: usize, + codepoint: usize, + utf16: usize, +} + +#[derive(Clone, Copy)] +enum Coordinate { + Byte, + Codepoint, +} + +impl Coordinate { + fn offset(self, position: Position) -> usize { + match self { + Self::Byte => position.byte, + Self::Codepoint => position.codepoint, + } + } +} + +/// Reusable offset conversion for one immutable string. +/// +/// The index walks text lazily and keeps one checkpoint per 256 code points. +/// Reuse it when converting many ranges, including overlapping or unordered +/// ranges. Short strings require no checkpoint allocation. No text is copied. +pub struct TextIndex<'a> { + text: &'a str, + frontier: Position, + checkpoints: Vec, +} + +impl<'a> TextIndex<'a> { + /// Create an empty index borrowing the exact source string. + pub fn new(text: &'a str) -> Self { + Self { + text, + frontier: Position::default(), + checkpoints: Vec::new(), + } + } + + /// Validate UTF-8 byte boundaries and convert to UTF-16 code-unit offsets. + /// Empty ranges are accepted; malformed ranges return `Utf16RangeError`. + pub fn utf16_range(&mut self, range: TextRange) -> Result { + validate_utf8_range(self.text, range)?; + Ok(TextRange { + start: self.locate(range.start, Coordinate::Byte).utf16, + end: self.locate(range.end, Coordinate::Byte).utf16, + }) + } + + pub(crate) fn byte_offset_at_codepoint(&mut self, offset: usize) -> Option { + let position = self.locate(offset, Coordinate::Codepoint); + (position.codepoint == offset).then_some(position.byte) + } + + // Core detector/token spans already identify valid UTF-8 boundaries. + pub(crate) fn codepoint_offset(&mut self, byte: usize) -> usize { + debug_assert!(self.text.is_char_boundary(byte)); + self.locate(byte, Coordinate::Byte).codepoint + } + + fn locate(&mut self, offset: usize, coordinate: Coordinate) -> Position { + let frontier_offset = coordinate.offset(self.frontier); + if frontier_offset == offset { + return self.frontier; + } + let mut position = if offset > frontier_offset { + self.frontier + } else { + let end = self + .checkpoints + .partition_point(|&position| coordinate.offset(position) <= offset); + end.checked_sub(1) + .map(|index| self.checkpoints[index]) + .unwrap_or_default() + }; + for character in self.text[position.byte..].chars() { + if coordinate.offset(position) >= offset { + break; + } + position.byte += character.len_utf8(); + position.codepoint += 1; + position.utf16 += character.len_utf16(); + if position.byte > self.frontier.byte { + self.frontier = position; + if position.codepoint % CHECKPOINT_INTERVAL == 0 { + self.checkpoints.push(position); + } + } + } + position + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unordered_offsets_match_direct_unicode_counts_across_checkpoints() { + for text in ["", "ascii", "aπŸ‘‹e\u{301}δΈ­"].map(|part| part.repeat(300)) { + let boundaries: Vec<_> = text + .char_indices() + .map(|(byte, _)| byte) + .chain(std::iter::once(text.len())) + .collect(); + let mut index = TextIndex::new(&text); + assert!(index.checkpoints.is_empty()); + for codepoint in (0..boundaries.len()).rev().chain(0..boundaries.len()) { + let byte = boundaries[codepoint]; + assert_eq!(index.byte_offset_at_codepoint(codepoint), Some(byte)); + assert_eq!(index.codepoint_offset(byte), codepoint); + let range = TextRange { + start: byte, + end: text.len(), + }; + assert_eq!(index.utf16_range(range), crate::utf16_range(&text, range)); + } + assert_eq!(index.byte_offset_at_codepoint(usize::MAX), None); + assert_eq!(index.byte_offset_at_codepoint(boundaries.len()), None); + assert_eq!(index.byte_offset_at_codepoint(0), Some(0)); + assert_eq!( + index.checkpoints.len(), + (boundaries.len() - 1) / CHECKPOINT_INTERVAL + ); + } + } + + #[test] + fn invalid_and_empty_ranges_match_the_scalar_converter() { + let text = "aπŸ‘‹δΈ­"; + let mut index = TextIndex::new(text); + for start in (0..=text.len() + 1).chain(std::iter::once(usize::MAX)) { + for end in (0..=text.len() + 1).chain(std::iter::once(usize::MAX)) { + let range = TextRange { start, end }; + assert_eq!(index.utf16_range(range), crate::utf16_range(text, range)); + } + } + assert!(index.checkpoints.is_empty()); + } + + #[test] + fn cached_validation_preserves_first_error_and_original_finding_index() { + use crate::{ + Finding, FindingValidationError, PrivacyError, TransformationConfig, + TransformationStrategy, + }; + let text = format!("{}may@example.test", "πŸ‘‹e\u{301} ".repeat(300)); + let late = crate::scan(&text).pop().unwrap(); + let early = Finding { + entity_type: "CUSTOM".to_owned(), + matched_text: "πŸ‘‹".to_owned(), + byte_range: TextRange { start: 0, end: 4 }, + codepoint_range: TextRange { start: 0, end: 1 }, + confidence: None, + detector_name: "test".to_owned(), + detector_version: None, + }; + let config = TransformationConfig::new(TransformationStrategy::Redact) + .with_entities(vec!["UNSELECTED".to_owned()]) + .unwrap(); + for (byte_range, codepoint_range, expected) in [ + ( + TextRange { start: 0, end: 4 }, + TextRange { start: 1, end: 2 }, + FindingValidationError::InconsistentRanges, + ), + ( + TextRange { start: 0, end: 4 }, + TextRange { + start: 0, + end: usize::MAX, + }, + FindingValidationError::CodepointRangeOutOfBounds, + ), + ( + TextRange { start: 1, end: 4 }, + TextRange { + start: 0, + end: usize::MAX, + }, + FindingValidationError::InvalidUtf8Boundary, + ), + ] { + let invalid = Finding { + byte_range, + codepoint_range, + ..early.clone() + }; + assert_eq!( + crate::select_findings(&text, &[late.clone(), early.clone(), invalid], &config), + Err(PrivacyError::invalid_finding(2, expected)), + ); + } + } +} diff --git a/crates/core/src/structured.rs b/crates/core/src/structured.rs index cb6b442..2be4a7e 100644 --- a/crates/core/src/structured.rs +++ b/crates/core/src/structured.rs @@ -321,6 +321,7 @@ fn selected_leaves<'a>( ) -> Result>, PrivacyError> { let all = leaves(data)?; let mut grouped: BTreeMap<&str, Vec> = BTreeMap::new(); + let mut offsets = BTreeMap::new(); for (index, located) in findings.iter().enumerate() { if validate_pointer(&located.path, "").is_err() || data @@ -340,7 +341,10 @@ fn selected_leaves<'a>( .pointer(&located.path) .and_then(Value::as_str) .ok_or_else(invalid_data)?; - validate_finding(text, &located.finding).map_err(|kind| { + let field_offsets = offsets + .entry(located.path.as_str()) + .or_insert_with(|| TextIndex::new(text)); + validate_finding_with_index(text, &located.finding, field_offsets).map_err(|kind| { let mut error = PrivacyError::invalid_finding(index, kind); error.path = error.path.map(|path| { path.replacen( diff --git a/docs/.mintignore b/docs/.mintignore index 61159d1..ec9182a 100644 --- a/docs/.mintignore +++ b/docs/.mintignore @@ -4,3 +4,4 @@ privacy-operations-roadmap.md person-detection-plan.md structured-performance.md finding-selection-performance.md +bookkeeping-performance.md diff --git a/docs/bookkeeping-performance.md b/docs/bookkeeping-performance.md new file mode 100644 index 0000000..ff65adc --- /dev/null +++ b/docs/bookkeeping-performance.md @@ -0,0 +1,125 @@ +# Slice Eleven: offset calculation and record conversion + +This is a focused follow-up to [Slice Ten](finding-selection-performance.md). +The baseline is the merge of PR #13, commit +`083eaaab6bcfcfba9a31bc8ce1cfc99f1c2a3cae`. The change is a candidate for the +planned 0.3.0 release; package versions stay at 0.2.0 until release preparation. + +## Scope and behavior + +Dense inputs still performed repeated prefix walks after overlap selection was +optimized. Core converted each finding's code-point positions independently, +recounted non-ASCII prefixes when producing scan findings, and recounted the +growing output when producing transformation records. JavaScript bindings also +recounted prefixes for UTF-16 ranges. Node structured result conversion copied +the entire output field for each transformation or restoration record. + +This change: + +- Adds Core's reusable `TextIndex`, which borrows one immutable string and + calculates byte, code-point, and UTF-16 positions using lazy checkpoints. +- Reuses indexes during Core finding validation and scan finalization, and + during Node/WASM conversion of findings and transformation records. Node + restoration records use the same conversion helper. Structured indexes are + scoped to individual fields, including when supplied findings are unordered. +- Maintains a running output code-point count while applying transformations. +- Converts individual Node structured records directly against borrowed field + text, eliminating the per-record copy of that text. + +Existing operation signatures, configurations, output shapes, detector coverage, +selection policies, error precedence, finding indices, and provider sequencing +are unchanged. `TextIndex::new` and `TextIndex::utf16_range` are additive Rust +helpers shared by the JavaScript bindings; the existing scalar `utf16_range` +function remains available. No dependencies, model files, or dictionaries are +added. Indexes are local to an operation and are not cached across requests. + +## Index cost + +An index advances through newly requested text once, storing one checkpoint per +256 code points. A lookup behind the furthest visited position searches the +checkpoints and walks at most 256 code points. For `C` visited code points and +`m` lookups, this costs `O(C + m(log(C/256 + 1) + 256))`, with `O(C/256)` stored +checkpoints. Ordered forward lookups take `O(C + m)` work. This replaces repeated +prefix walks that could cost `O(Cm)`. + +Each checkpoint stores three `usize` offsets (24 bytes on a 64-bit target), plus +the vector's spare capacity. Strings shorter than 256 code points need no +checkpoint allocation, and text beyond the furthest requested position is not +indexed. Running output counts visit newly appended text instead of revisiting +the entire output. This is an algorithmic accounting, not a process-level memory +benchmark or a bound on every part of a complete request. + +## Reproduce the comparison + +Use Node 24 and the same Rust toolchain for both builds. From the candidate +repository, create a separate baseline checkout and build both native packages: + +```sh +git worktree add --detach ../datafog-core-bookkeeping-baseline 083eaaab6bcfcfba9a31bc8ce1cfc99f1c2a3cae +npm ci --prefix ../datafog-core-bookkeeping-baseline/bindings/node +npm run build --prefix ../datafog-core-bookkeeping-baseline/bindings/node +npm ci --prefix bindings/node +npm run build --prefix bindings/node +node scripts/benchmark-bookkeeping.mjs ../datafog-core-bookkeeping-baseline/bindings/node/index.js bindings/node/index.js target/bookkeeping-results.json +``` + +The [benchmark](../scripts/benchmark-bookkeeping.mjs) loads both native packages +in one process and compares complete results before timing. It measures scan, +transformation of supplied findings, and combined scan-and-transform separately. +Both implementations are warmed and measured in alternating order over seven +rounds; the JSON report contains every sample and its median. Timings include +JavaScript input checks, JSON transport, Core work, and result conversion. They +exclude package import and provider I/O. Baseline findings are supplied to both +standalone transformations. The policy redacts by default and masks emails. + +## Local results + +Recorded 2026-09-04 on macOS ARM64, Rust 1.88.0, Node 24.19.0. The baseline native +package was copied from the unchanged baseline build before editing Core. The +table shows median **combined scan-and-transform** time, in milliseconds: + +| Workload | Findings | Baseline | Candidate | Speedup | +| --- | ---: | ---: | ---: | ---: | +| One customer record | 3 | 0.0179 | 0.0180 | 0.99Γ— | +| 100 customer records | 300 | 1.430 | 1.454 | 0.98Γ— | +| Long Unicode field, one email | 1 | 2.555 | 1.345 | 1.90Γ— | +| Dense Unicode field | 128 | 4.109 | 0.393 | 10.45Γ— | +| Dense Unicode field | 512 | 61.189 | 1.564 | 39.12Γ— | +| Dense Unicode field | 1,024 | 242.311 | 3.204 | 75.63Γ— | +| Dense ASCII field | 1,024 | 137.840 | 3.265 | 42.22Γ— | + +Doubling dense Unicode findings from 512 to 1,024 took 3.96Γ— as long on the +baseline and 2.05Γ— on the candidate. Standalone transformation of the 1,024 +Unicode findings improved from 243.682 ms to 5.453 ms (44.69Γ—); scanning improved +from 8.987 ms to 1.291 ms (6.96Γ—). + +There are tradeoffs. Across the small/customer workloads, the candidate was +approximately 0–4% slower depending on the operation. Scanning the long sparse +Unicode field was 0.837 ms versus 0.762 ms (about 10% slower), although its +combined operation improved. These shared-machine measurements are not latency +guarantees and do not establish that every workload improves. The measured +benefit is strongest when a field contains many findings. + +## Verification and remaining work + +- Rust formatting, Clippy with warnings denied, and all workspace tests pass: + 84 tests; the Slice Ten manual selection benchmark remains ignored by default. +- Index tests compare ascending and descending lookups with direct Unicode + counts across checkpoint boundaries, including emoji, combining characters, + ASCII, empty strings, malformed byte boundaries, and oversized offsets. +- Cached validation retains the first error and original finding index even + after looking up later source positions and when entity filtering excludes + the findings. +- Installed Python, Node, and browser WASM conformance tests pass. Additional + Node/WASM tests verify dense multi-field byte/code-point/UTF-16 output ranges, + reversed findings, redaction, removal, and emoji masking. Node also verifies + a dense structured tokenization/restoration round trip. +- Every benchmark workload returns identical complete findings and transformed + results on the baseline and candidate. + +Validation and selection can still repeat across preparation/completion stages. +JSON transport, structured token routing, Core restoration prefix counts, and +the mixed-confidence overlap fallback are not redesigned here. The next pass +should profile representative remaining costs before introducing reusable +prepared-request state. This change does not claim an end-to-end complexity +bound for arbitrary requests or a complete removal of internal bookkeeping. diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 872914b..233222f 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -244,7 +244,7 @@ completed privacy-operation slices or promise general prose name recognition. ## Slice 10: Finding-selection performance -**Status: implemented; review and combined release pending.** +**Status: merged; combined release pending.** Replace linear duplicate searches with indexed groups and repeated overlap scans with ordered interval selection. Preserve validation, filtering, @@ -256,6 +256,17 @@ behavior comparisons, reproducible scaling benchmarks, and remaining performance limits. Prepare one 0.3.0 release containing structured PERSON and this change after review; individual feature PRs do not bump or publish package versions. +## Slice 11: Offset calculation and record conversion + +**Status: implemented; review pending for inclusion in 0.3.0.** + +Reuse lazy text indexes for finding validation and Unicode range conversion, +maintain running output positions, and remove per-record field copies from Node +structured result conversion. Preserve existing validation and selection rules. +The [bookkeeping measurements](bookkeeping-performance.md) record complete +request comparisons, correctness checks, and remaining costs. Reusing validated +request state across operation stages remains separate work. + ## Acceptance bar Every completed slice must have: diff --git a/docs/reference/rust.mdx b/docs/reference/rust.mdx index f026f7a..ad7c7d5 100644 --- a/docs/reference/rust.mdx +++ b/docs/reference/rust.mdx @@ -72,6 +72,12 @@ optional confidence, and detector provenance. `TransformResult` contains transformed `text` and ordered `transformations`. `RestoreResult` contains restored `text` and ordered `restorations`. +For repeated UTF-16 conversions on the same string, reuse +`TextIndex::new(text)` and call its `utf16_range(byte_range)` method. It accepts +overlapping or unordered ranges, validates UTF-8 boundaries, and returns the +same ranges and errors as the standalone `utf16_range` helper. The index borrows +the string and builds sparse checkpoints as needed. + ## Provider-backed manager `PrivacyManager` composes key and token provider capabilities. diff --git a/scripts/benchmark-bookkeeping.mjs b/scripts/benchmark-bookkeeping.mjs new file mode 100644 index 0000000..13fb075 --- /dev/null +++ b/scripts/benchmark-bookkeeping.mjs @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import { pathToFileURL } from "node:url"; + +// Pass separately built baseline/candidate Node package entry points. Both +// native modules remain loaded so measurements can alternate in one process. +const [baselinePath, candidatePath, outputPath] = process.argv.slice(2); +if (!baselinePath || !candidatePath) { + throw new Error("Usage: node scripts/benchmark-bookkeeping.mjs BASELINE/index.js CANDIDATE/index.js [report.json]"); +} +const baseline = await import(pathToFileURL(path.resolve(baselinePath)).href); +const candidate = await import(pathToFileURL(path.resolve(candidatePath)).href); +const policy = { default: { strategy: "redact" }, overrides: { EMAIL: { strategy: "mask" } } }; +const customer = () => ({ first_name: "May", contact: "may@example.test", phone: "(212) 555-0100", note: "Order is ready." }); +const workloads = [ + { name: "one_customer", data: customer(), iterations: 1000 }, + { name: "100_customers", data: Array.from({ length: 100 }, customer), iterations: 30 }, + { name: "sparse_unicode", data: { note: "πŸ‘‹ plain text. ".repeat(5000) + "may@example.test" }, iterations: 30 }, + ...[128, 512, 1024].map(count => ({ name: `dense_unicode_${count}`, data: { note: "πŸ‘‹ may@example.test ".repeat(count) }, iterations: count === 128 ? 10 : 2 })), + { name: "dense_ascii_1024", data: { note: "may@example.test ".repeat(1024) }, iterations: 2 }, +]; +const median = values => [...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]; +const results = []; +let sink = 0; +for (const { name, data, iterations } of workloads) { + const findings = baseline.scanStructured(data).findings; + const operations = { + scan: api => api.scanStructured(data), + transform: api => api.transformStructured(data, findings, policy), + combined: api => api.scanAndTransformStructured(data, { transform: policy }), + }; + for (const [operation, run] of Object.entries(operations)) { + assert.deepEqual(run(candidate), run(baseline), `${name}/${operation}: full result parity`); + const timings = { baseline: [], candidate: [] }; + const implementations = { baseline, candidate }; + for (const api of Object.values(implementations)) { + for (let iteration = 0; iteration < Math.min(iterations, 10); iteration++) { + const result = run(api); + sink += result.findings?.length ?? result.transformations.length; + } + } + for (let round = 0; round < 7; round++) { + for (const label of round % 2 ? ["candidate", "baseline"] : ["baseline", "candidate"]) { + const started = performance.now(); + for (let iteration = 0; iteration < iterations; iteration++) { + const result = run(implementations[label]); + sink += result.findings?.length ?? result.transformations.length; + } + timings[label].push((performance.now() - started) * 1000 / iterations); + } + } + const before = median(timings.baseline); + const after = median(timings.candidate); + const row = { workload: name, operation, bytes: Buffer.byteLength(JSON.stringify(data)), findings: findings.length, iterations, baseline_us: before, candidate_us: after, speedup: before / after, samples_us: timings }; + results.push(row); + console.log(`${name}/${operation}: ${before.toFixed(3)} -> ${after.toFixed(3)} Β΅s (${row.speedup.toFixed(2)}x)`); + } +} +if (outputPath) { + writeFileSync(outputPath, JSON.stringify({ runtime: process.version, platform: process.platform, arch: process.arch, rounds: 7, baselinePath, candidatePath, sink, results }, null, 2) + "\n"); +} diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index e5d192b..a37b226 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -126,6 +126,26 @@ for (const record of structuredTransformRecords) { } } +const denseData = {a:"πŸ‘‹ may@example.test ".repeat(80), b:"δΈ­ é other@example.test ".repeat(80)}; +const denseFindings = scanStructured(denseData).findings; +assert.equal(denseFindings.length, 160); +for (const {path, finding} of denseFindings) verifyContract(pointerValue(denseData, path), finding); +for (const strategy of [{strategy:"redact"}, {strategy:"remove"}, {strategy:"mask",character:"πŸ”’"}]) { + const config = {default:strategy}; + const result = transformStructured(denseData, [...denseFindings].reverse(), config); + assert.deepEqual(result, scanAndTransformStructured(denseData, {transform:config})); + assert.equal(result.transformations.length, 160); + for (const {path, transformation:t} of result.transformations) { + const source = pointerValue(denseData, path); + const output = pointerValue(result.data, path); + const expectedSource = path === "/a" ? "may@example.test" : "other@example.test"; + assert.equal(source.slice(t.sourceUtf16Range.start,t.sourceUtf16Range.end), expectedSource); + assert.equal(Array.from(output).slice(t.outputCodepointRange.start,t.outputCodepointRange.end).join(""), t.replacement); + assert.equal(output.slice(t.outputUtf16Range.start,t.outputUtf16Range.end), t.replacement); + assert.equal(Buffer.from(output).subarray(t.outputByteRange.start,t.outputByteRange.end).toString(), t.replacement); + } +} + const emojiFinding = scan("πŸ‘‹ jane@example.com")[0]; assert.deepEqual(emojiFinding.byteRange, { start: 5, end: 21 }); assert.deepEqual(emojiFinding.codepointRange, { start: 2, end: 18 }); @@ -419,6 +439,17 @@ assert.throws( error.path === "/overides", ); +const denseTokens = await tokenManager.scanAndTransformStructured(denseData, {transform:{default:{strategy:"tokenize",token_ref:"dense"}}}, tokenContext); +const denseRestored = await tokenManager.restoreStructured(denseTokens.data, tokenContext); +assert.deepEqual(denseRestored.data, denseData); +assert.equal(denseRestored.restorations.length, 160); +for (const {path, restoration:r} of denseRestored.restorations) { + const source = pointerValue(denseTokens.data, path); + const output = pointerValue(denseRestored.data, path); + assert.ok(source.slice(r.sourceUtf16Range.start,r.sourceUtf16Range.end).startsWith("DFTOKENv1(")); + assert.equal(output.slice(r.outputUtf16Range.start,r.outputUtf16Range.end), path === "/a" ? "may@example.test" : "other@example.test"); +} + console.log("Installed @datafog/node package matches fixtures and transform contracts."); `.trimStart(), ); diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index 735a578..ed407ce 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -334,6 +334,26 @@ for (const record of structuredTransformRecords) { } } +const denseData = {a:"πŸ‘‹ may@example.test ".repeat(80), b:"δΈ­ é other@example.test ".repeat(80)}; +const denseFindings = scanStructured(denseData).findings; +if (denseFindings.length !== 160) throw new Error("dense finding count"); +for (const {path, finding} of denseFindings) verifyContract(pointerValue(denseData, path), finding); +for (const strategy of [{strategy:"redact"}, {strategy:"remove"}, {strategy:"mask",character:"πŸ”’"}]) { + const config = {default:strategy}; + const result = transformStructured(denseData, [...denseFindings].reverse(), config); + if (JSON.stringify(result) !== JSON.stringify(scanAndTransformStructured(denseData, {transform:config}))) throw new Error("dense explicit mismatch"); + if (result.transformations.length !== 160) throw new Error("dense transformation count"); + for (const {path, transformation:t} of result.transformations) { + const source = pointerValue(denseData, path); + const output = pointerValue(result.data, path); + const expectedSource = path === "/a" ? "may@example.test" : "other@example.test"; + if (source.slice(t.sourceUtf16Range.start,t.sourceUtf16Range.end) !== expectedSource) throw new Error("dense source range"); + if (Array.from(output).slice(t.outputCodepointRange.start,t.outputCodepointRange.end).join("") !== t.replacement) throw new Error("dense codepoint range"); + if (output.slice(t.outputUtf16Range.start,t.outputUtf16Range.end) !== t.replacement) throw new Error("dense UTF-16 range"); + if (new TextDecoder().decode(new TextEncoder().encode(output).subarray(t.outputByteRange.start,t.outputByteRange.end)) !== t.replacement) throw new Error("dense byte range"); + } +} + for (const strategy of [{strategy:"pseudonymize",key_ref:"names"},{strategy:"tokenize",token_ref:"names"}]) { let rejected=false; try { scanAndTransformStructured({first_name:"May"},{transform:{default:strategy}}); } catch(e) { rejected=e.code === "unsupported_strategy"; }