diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index c73bd5d..1ab2349 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -2350,30 +2350,50 @@ fn select_findings( } } + Ok(select_validated_findings(findings, config)) +} + +fn select_validated_findings(findings: &[Finding], config: &TransformationConfig) -> Vec { let mut selected_findings: Vec = Vec::with_capacity(findings.len()); + let mut duplicate_indices = BTreeMap::new(); for finding in findings .iter() .filter(|finding| config.includes(finding) && !config.allows(finding)) { - if let Some(existing) = selected_findings - .iter_mut() - .find(|existing| findings_are_duplicates(existing, finding)) - { - if duplicate_preference(finding, existing).is_lt() { - *existing = finding.clone(); + // Validation against the same text makes the byte range determine both + // the matched text and the code-point range. Preserve encounter order + // within each group: mixed confidence is not a sortable preference. + let identity = ( + finding.entity_type.as_str(), + finding.byte_range.start, + finding.byte_range.end, + ); + match duplicate_indices.entry(identity) { + std::collections::btree_map::Entry::Occupied(entry) => { + let existing = &mut selected_findings[*entry.get()]; + if duplicate_preference(finding, existing).is_lt() { + *existing = finding.clone(); + } + } + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(selected_findings.len()); + selected_findings.push(finding.clone()); } - } else { - selected_findings.push(finding.clone()); } } - selected_findings.sort_by_key(|finding| { + selected_findings.sort_by(|left, right| { ( - finding.codepoint_range.start, - finding.codepoint_range.end, - finding.entity_type.clone(), + left.codepoint_range.start, + left.codepoint_range.end, + &left.entity_type, ) + .cmp(&( + right.codepoint_range.start, + right.codepoint_range.end, + &right.entity_type, + )) }); - Ok(resolve_overlaps(selected_findings)) + resolve_overlaps(selected_findings) } fn key_selectors(config: &TransformationConfig, selected_findings: &[Finding]) -> Vec { @@ -2678,13 +2698,6 @@ pub fn scan_and_transform( .map_err(|error| error.prefixed("/transform")) } -fn findings_are_duplicates(left: &Finding, right: &Finding) -> bool { - left.entity_type == right.entity_type - && left.matched_text == right.matched_text - && left.byte_range == right.byte_range - && left.codepoint_range == right.codepoint_range -} - fn duplicate_preference(candidate: &Finding, existing: &Finding) -> std::cmp::Ordering { if let (Some(candidate_confidence), Some(existing_confidence)) = (candidate.confidence, existing.confidence) @@ -2699,7 +2712,51 @@ fn duplicate_preference(candidate: &Finding, existing: &Finding) -> std::cmp::Or .cmp(&(&existing.detector_name, &existing.detector_version)) } -fn resolve_overlaps(mut remaining: Vec) -> Vec { +// Input is validated, deduplicated, and sorted in source order. +fn resolve_overlaps(mut findings: Vec) -> Vec { + if findings + .windows(2) + .all(|pair| pair[0].byte_range.end <= pair[1].byte_range.start) + { + return findings; + } + + // Containment agrees with descending code-point length on validated ranges. + // Within one length, confidence is comparable only if all values are present + // or all are absent. Mixing them can make the preference cyclic, so retain + // the original pairwise selection in that case. + let mut confidence_by_length = BTreeMap::new(); + for finding in &findings { + let length = finding.codepoint_range.end - finding.codepoint_range.start; + let has_confidence = finding.confidence.is_some(); + let previous = confidence_by_length.entry(length).or_insert(has_confidence); + if *previous != has_confidence { + return resolve_overlaps_pairwise(findings); + } + } + + findings.sort_by(overlap_preference); + let mut selected: BTreeMap = BTreeMap::new(); + for finding in findings { + let start = finding.byte_range.start; + // Accepted intervals never overlap. The closest predecessor and + // successor suffice, and tree insertion avoids shifting a sorted Vec. + let overlaps_previous = selected + .range(..=start) + .next_back() + .is_some_and(|(_, previous)| previous.byte_range.end > start); + let overlaps_next = selected + .range(start..) + .next() + .is_some_and(|(&next_start, _)| next_start < finding.byte_range.end); + if !overlaps_previous && !overlaps_next { + selected.insert(start, finding); + } + } + selected.into_values().collect() +} + +fn resolve_overlaps_pairwise(mut remaining: Vec) -> Vec { let mut selected = Vec::with_capacity(remaining.len()); while !remaining.is_empty() { let mut preferred_index = 0; @@ -3453,6 +3510,9 @@ fn detect_ip_address(text: &str, candidates: &mut Vec) { } } +#[cfg(test)] +mod selection_tests; + #[cfg(test)] mod tests { use super::{ diff --git a/crates/core/src/selection_tests.rs b/crates/core/src/selection_tests.rs new file mode 100644 index 0000000..5e638c5 --- /dev/null +++ b/crates/core/src/selection_tests.rs @@ -0,0 +1,320 @@ +use super::*; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +// Frozen selection algorithm from 4275002, before Slice Ten. Keep the linear +// duplicate search and repeated overlap scans as an independent reference. +// The preference comparators themselves are unchanged by this optimization. +fn reference_selection(findings: &[Finding], config: &TransformationConfig) -> Vec { + let mut remaining: Vec = Vec::with_capacity(findings.len()); + for finding in findings + .iter() + .filter(|finding| config.includes(finding) && !config.allows(finding)) + { + if let Some(existing) = remaining.iter_mut().find(|existing| { + existing.entity_type == finding.entity_type + && existing.matched_text == finding.matched_text + && existing.byte_range == finding.byte_range + && existing.codepoint_range == finding.codepoint_range + }) { + if duplicate_preference(finding, existing).is_lt() { + *existing = finding.clone(); + } + } else { + remaining.push(finding.clone()); + } + } + remaining.sort_by_key(|finding| { + ( + finding.codepoint_range.start, + finding.codepoint_range.end, + finding.entity_type.clone(), + ) + }); + let mut selected = Vec::with_capacity(remaining.len()); + while !remaining.is_empty() { + let mut preferred_index = 0; + for candidate_index in 1..remaining.len() { + if overlap_preference(&remaining[candidate_index], &remaining[preferred_index]).is_lt() + { + preferred_index = candidate_index; + } + } + let preferred = remaining.remove(preferred_index); + remaining.retain(|finding| !findings_overlap(&preferred, finding)); + selected.push(preferred); + } + selected.sort_by(|left, right| { + left.byte_range + .start + .cmp(&right.byte_range.start) + .then_with(|| left.byte_range.end.cmp(&right.byte_range.end)) + .then_with(|| left.entity_type.cmp(&right.entity_type)) + }); + selected +} + +fn supplied_finding(text: &str, start: usize, end: usize, confidence: Option) -> Finding { + let byte_start = byte_offset_at_codepoint(text, start).unwrap(); + let byte_end = byte_offset_at_codepoint(text, end).unwrap(); + Finding { + entity_type: "PERSON".to_owned(), + matched_text: text[byte_start..byte_end].to_owned(), + byte_range: TextRange { + start: byte_start, + end: byte_end, + }, + codepoint_range: TextRange { start, end }, + confidence, + detector_name: "test".to_owned(), + detector_version: None, + } +} + +#[test] +fn cyclic_overlap_preferences_preserve_pairwise_winner() { + let text = "abcdef"; + let findings = vec![ + supplied_finding(text, 0, 4, Some(0.2)), + supplied_finding(text, 1, 5, None), + supplied_finding(text, 2, 6, Some(0.8)), + ]; + assert!(overlap_preference(&findings[0], &findings[1]).is_lt()); + assert!(overlap_preference(&findings[1], &findings[2]).is_lt()); + assert!(overlap_preference(&findings[2], &findings[0]).is_lt()); + let config = TransformationConfig::new(TransformationStrategy::Redact); + assert_eq!( + select_findings(text, &findings, &config).unwrap(), + vec![findings[2].clone()] + ); + assert_eq!( + transform(text, &findings, &config).unwrap().text, + "ab[PERSON]" + ); +} + +#[test] +fn duplicate_confidence_and_provenance_preserve_encounter_order() { + let text = "José"; + let mut findings = vec![ + supplied_finding(text, 0, 4, Some(0.2)), + supplied_finding(text, 0, 4, None), + supplied_finding(text, 0, 4, Some(0.8)), + ]; + for (finding, detector) in findings.iter_mut().zip(["a", "b", "c"]) { + finding.detector_name = detector.to_owned(); + } + let config = TransformationConfig::new(TransformationStrategy::Redact); + assert_eq!( + select_findings(text, &findings, &config).unwrap(), + vec![findings[2].clone()] + ); + findings.rotate_left(1); + assert_eq!( + select_findings(text, &findings, &config).unwrap(), + vec![findings[2].clone()] + ); +} + +#[test] +fn validation_still_precedes_filtering_and_duplicate_collapse() { + let text = "José"; + let valid = supplied_finding(text, 0, 4, None); + let mut invalid = valid.clone(); + invalid.matched_text = "wrong".to_owned(); + let mut later_invalid = valid.clone(); + later_invalid.byte_range.end = text.len() + 1; + for config in [ + TransformationConfig::new(TransformationStrategy::Redact), + TransformationConfig::new(TransformationStrategy::Redact) + .with_entities(vec!["EMAIL".to_owned()]) + .unwrap(), + TransformationConfig::new(TransformationStrategy::Redact) + .with_exact_allowlist("PERSON", vec![text.to_owned(), "wrong".to_owned()]) + .unwrap(), + ] { + assert_eq!( + select_findings( + text, + &[valid.clone(), invalid.clone(), later_invalid.clone()], + &config + ), + Err(PrivacyError::invalid_finding( + 1, + FindingValidationError::MatchedTextMismatch + )), + ); + } +} + +struct Random(u64); + +impl Random { + fn below(&mut self, limit: usize) -> usize { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1); + ((self.0 >> 32) as usize) % limit + } + + fn shuffle(&mut self, findings: &mut [Finding]) { + for index in (1..findings.len()).rev() { + findings.swap(index, self.below(index + 1)); + } + } +} + +#[test] +fn selection_matches_reference_across_unicode_ranges_policies_and_input_orders() { + let text = "a👋é中e\u{301} z".repeat(16); + let codepoints = text.chars().count(); + let mut random = Random(0x5eed); + let configs = [ + TransformationConfig::new(TransformationStrategy::Redact), + TransformationConfig::new(TransformationStrategy::Remove) + .with_entities(vec!["PERSON".to_owned(), "EMAIL".to_owned()]) + .unwrap() + .with_exact_allowlist("PERSON", vec!["a👋".to_owned()]) + .unwrap(), + TransformationConfig::new(TransformationStrategy::Redact) + .with_regex_allowlist("PERSON", vec![RegexAllowRule::new("a.*", true)]) + .unwrap(), + ]; + for trial in 0..3000 { + let mut findings: Vec = Vec::new(); + for _ in 0..random.below(96) { + let start = random.below(codepoints - 1); + let length = 1 + random.below((codepoints - start).min(12)); + let score = [0.0, -0.0, 0.2, 0.8, 1.0][random.below(5)]; + let confidence = match trial % 4 { + 0 => None, + 1 => Some(score), + 2 => (length % 2 == 0).then_some(score), + _ => (random.below(2) == 0).then_some(score), + }; + let mut finding = supplied_finding(&text, start, start + length, confidence); + if !findings.is_empty() && random.below(4) == 0 { + finding = findings[random.below(findings.len())].clone(); + if trial % 4 == 3 { + finding.confidence = confidence; + } + } else { + finding.entity_type = ["PERSON", "EMAIL", "CUSTOM"][random.below(3)].to_owned(); + } + finding.detector_name = ["a", "b", "c"][random.below(3)].to_owned(); + finding.detector_version = + [None, Some("1"), Some("2")][random.below(3)].map(str::to_owned); + findings.push(finding); + } + for permutation in 0..3 { + random.shuffle(&mut findings); + let config = &configs[trial % configs.len()]; + assert_eq!( + select_findings(&text, &findings, config).unwrap(), + reference_selection(&findings, config), + "trial {trial}, permutation {permutation}", + ); + } + } +} + +#[test] +fn ordered_interval_selection_handles_touching_nested_and_partial_spans() { + let text = "a👋é中e\u{301} z".repeat(16); + let config = TransformationConfig::new(TransformationStrategy::Redact); + let mut random = Random(42); + for confidence in [None, Some(0.5)] { + for ranges in [ + vec![(0, 4), (4, 8), (8, 12), (20, 22)], + vec![(0, 12), (1, 11), (2, 10), (3, 9)], + vec![(0, 5), (4, 9), (8, 13), (12, 17)], + vec![(0, 2), (10, 20), (19, 29), (5, 15), (30, 32)], + ] { + let mut findings: Vec<_> = ranges + .into_iter() + .map(|(start, end)| supplied_finding(&text, start, end, confidence)) + .collect(); + for _ in 0..20 { + random.shuffle(&mut findings); + assert_eq!( + select_findings(&text, &findings, &config).unwrap(), + reference_selection(&findings, &config) + ); + } + } + } +} + +fn benchmark_findings(count: usize, workload: &str) -> Vec { + (0..count) + .map(|index| { + let (start, length) = match workload { + "disjoint" => (index * 8, 4), + "overlap_clusters" => ((index / 4) * 16 + (index % 4) * 2, 4), + "duplicates" => ((index / 4) * 8, 4), + _ => unreachable!("unknown test workload"), + }; + Finding { + entity_type: "PERSON".to_owned(), + matched_text: "x".repeat(length), + byte_range: TextRange { + start, + end: start + length, + }, + codepoint_range: TextRange { + start, + end: start + length, + }, + confidence: None, + detector_name: format!("detector-{}", index % 4), + detector_version: None, + } + }) + .collect() +} + +fn measure(selection: impl FnOnce() -> Vec) -> Duration { + let started = Instant::now(); + black_box(selection()); + started.elapsed() +} + +#[test] +#[ignore = "manual release-mode selection benchmark; no wall-clock assertions"] +fn finding_selection_benchmark() { + let config = TransformationConfig::new(TransformationStrategy::Redact); + println!("workload,findings,selected,reference_us,optimized_us,speedup"); + for workload in ["disjoint", "overlap_clusters", "duplicates"] { + for count in [256, 512, 1024, 2048, 4096] { + let findings = benchmark_findings(count, workload); + let text = "x".repeat(count * 8 + 16); + for finding in &findings { + validate_finding(&text, finding).unwrap(); + } + let expected = reference_selection(&findings, &config); + assert_eq!(select_validated_findings(&findings, &config), expected); + let reference = || reference_selection(black_box(&findings), black_box(&config)); + let optimized = || select_validated_findings(black_box(&findings), black_box(&config)); + black_box(reference()); + black_box(optimized()); + let mut before = Vec::new(); + let mut after = Vec::new(); + for round in 0..7 { + if round % 2 == 0 { + before.push(measure(reference)); + after.push(measure(optimized)); + } else { + after.push(measure(optimized)); + before.push(measure(reference)); + } + } + before.sort(); + after.sort(); + let before = before[3].as_secs_f64() * 1_000_000.0; + let after = after[3].as_secs_f64() * 1_000_000.0; + println!( + "{workload},{count},{},{before:.3},{after:.3},{:.2}", + expected.len(), + before / after + ); + } + } +} diff --git a/docs/.mintignore b/docs/.mintignore index 34a1cad..61159d1 100644 --- a/docs/.mintignore +++ b/docs/.mintignore @@ -3,3 +3,4 @@ privacy-capability-matrix.md privacy-operations-roadmap.md person-detection-plan.md structured-performance.md +finding-selection-performance.md diff --git a/docs/concepts/findings-and-ranges.mdx b/docs/concepts/findings-and-ranges.mdx index a85ef23..dda2a0f 100644 --- a/docs/concepts/findings-and-ranges.mdx +++ b/docs/concepts/findings-and-ranges.mdx @@ -74,6 +74,20 @@ length, confidence when both values are present, source position, entity type, and detector provenance. Selected transformations are returned in source document order. +### Missing confidence + +Omitted confidence means unknown; Core does not assign it a score of zero. +Comparing confidence only when both values are present can produce conflicting +preferences across three or more overlapping findings. Core preserves its +existing pairwise selection behavior in these cases rather than changing which +text gets protected through sorting. + +When overlaps exist and an equal-length group mixes scored and unscored +findings, overlap selection uses a compatibility fallback that can take +quadratic time. Length here means Unicode code points. Disjoint findings and +findings produced exclusively by the built-in detectors, including structured +PERSON, use the faster selection path. + ## Structured findings `scan_structured` / `scanStructured` returns located findings with `path` and diff --git a/docs/finding-selection-performance.md b/docs/finding-selection-performance.md new file mode 100644 index 0000000..5582c9f --- /dev/null +++ b/docs/finding-selection-performance.md @@ -0,0 +1,156 @@ +# Slice Ten: finding-selection performance + +This change addresses [issue #10](https://github.com/DataFog/datafog-core/issues/10) +in the shared Rust transformation path, following the structured PERSON work. +Public APIs, policies, detection coverage, offsets, and package versions are +unchanged. PERSON and this optimization are intended for a combined 0.3.0 +release after review; this change does not publish packages. + +## Algorithm and compatibility + +Every supplied finding is still validated before entity filtering or allowlists, +including findings that will ultimately be discarded. The first invalid finding +retains its original index and error. Selection then: + +1. Collapses duplicates through a `BTreeMap` keyed by entity type and byte range. + Validation against the same input ensures that matched text and code-point + range are also identical. Each duplicate group keeps the original encounter + order and confidence/provenance preference. +2. Sorts by source position. If all intervals are disjoint or merely touching, + returns them directly. +3. For sortable preferences, sorts once by the existing overlap priority and + accepts each candidate only if it misses the nearest accepted interval on + either side. A `BTreeMap` provides logarithmic lookup and insertion; results + are returned in source order. + +For `m` supplied findings, duplicate indexing takes `O(m log m)` comparisons. +Overlap selection takes `O(u log u)` comparisons for `u` unique candidates in +the sortable case, with an `O(u)` disjoint check after source sorting. Additional +index storage is `O(m)`. These bounds exclude validation, filtering, text cloning, +and variable-length string comparison costs. + +The existing preference is **not always a total order**: confidence is compared +only when both findings provide it. For equal-length spans on `abcdef`, let +`A = [0,4), confidence 0.2`, `B = [1,5), no confidence`, and +`C = [2,6), confidence 0.8`. A beats B by position, B beats C by position, and C +beats A by confidence. Sorting that cycle could change the protected output. + +When any overlaps exist and an equal-code-point-length group mixes present and +absent confidence, selection conservatively keeps the original pairwise overlap +algorithm. This fallback remains `O(u²)` in the worst case. Different lengths +can safely use different confidence-presence modes because length takes priority. +All current built-in detectors, including structured PERSON, omit confidence +and therefore use the faster algorithm. Disjoint inputs take the fast path +regardless of confidence. Duplicate preferences can also be cyclic, which is why +indexing preserves the original per-group fold rather than sorting duplicates. + +## Policy rationale and comparable tools + +The fallback preserves the confidence rule in +[ADR 001](adr/001-privacy-core-contract.md#duplicates-and-overlaps). It is a +compatibility choice for this performance change, not a requirement for every +future overlap policy. The public +[finding guide](concepts/findings-and-ranges.mdx#missing-confidence) explains +what callers supplying optional confidence should expect. + +The following comparison was checked on 2026-09-04. These are policy examples; +they do not establish equivalent outputs or performance across the tools. + +| Tool | Confidence and overlap policy | Relevance to Core | +| --- | --- | --- | +| Presidio | Recognizer results expect a numeric score, and regex patterns require an assigned score. Its anonymizer prefers higher scores for identical spans and the larger span for containment. It also merges overlapping findings of the same type and handles partial intersections differently from Core. | A score can be assigned by a rule author without running a model. The supported contract avoids our distinction between scored and unscored findings, but adopting the full policy would change Core behavior. | +| spaCy EntityRuler | Among overlapping rule matches, prefers the span with more tokens, then the earlier position. Confidence is not used for this selection. | Structural priority avoids the missing-confidence cycle. Core measures length in Unicode code points, so its ranges and length metric would remain different. | +| Google Sensitive Data Protection | Uses defined likelihood levels. A custom detector defaults to `VERY_LIKELY` when its likelihood is omitted. Explicit exclusion rules can suppress domain matches that overlap email matches. | Defaults give omitted configuration a defined meaning, and explicit rules handle particular overlap relationships. This does not establish Google's complete internal winner-selection algorithm. | + +Sources: + +- Presidio: [result score contract](https://github.com/data-privacy-stack/presidio/blob/e9b5795ff9302fc8a306eee9a73b87ab00426166/presidio-analyzer/presidio_analyzer/recognizer_result.py), + [regex pattern scores](https://presidio.dataprivacystack.org/tutorial/02_regex/), + [documented overlap behavior](https://presidio.dataprivacystack.org/anonymizer/#handling-overlaps-between-entities), + and [same-type merging implementation](https://github.com/data-privacy-stack/presidio/blob/e9b5795ff9302fc8a306eee9a73b87ab00426166/presidio-anonymizer/presidio_anonymizer/anonymizer_engine.py). +- spaCy: [EntityRuler overlap rules](https://spacy.io/api/entityruler#call). +- Google: [custom-detector likelihood default](https://docs.cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/InspectConfig#CustomInfoType) + and [overlap-exclusion example](https://docs.cloud.google.com/sensitive-data-protection/docs/samples/dlp-inspect-string-without-overlap). + +### Proposed future direction — not implemented + +For a future simplification, prefer a consistent structural ordering: longer +Unicode code-point span, earlier source position, then stable entity/provenance +tie-breakers. Retain confidence as metadata without using it to rank overlaps. +This would keep confidence optional while allowing all overlap candidates to be +sorted consistently, including mixed-confidence inputs. + +This proposal would change some winners for caller-supplied scored findings. +It needs a separate behavior decision and focused PR updating ADR 001, public +documentation, and regression fixtures. That work must also explicitly decide +whether confidence still chooses provenance within duplicate groups; changing +overlap ordering alone does not remove their encounter-order dependence. +No confidence policy option, inferred score, or change to protection behavior +is introduced in Slice Ten. The fallback stays in place for this PR. + +## Reproducible measurement + +Run from the repository root: + +```sh +cargo test -p datafog-core --release finding_selection_benchmark -- --ignored --nocapture +``` + +The benchmark and differential tests live in +[selection_tests.rs](../crates/core/src/selection_tests.rs). The reference copies +the selection algorithm from commit `4275002833b45d846fcc75f3a8cd083310f89970`; +both versions use the unchanged preference comparators and run in the same +release build. Inputs are prevalidated. Timing includes policy filtering, +duplicate collapse, overlap selection, result allocation, and destruction. It +excludes scanning, range validation, replacement generation, bindings, and +provider I/O. No public benchmarking API or dependency was added. + +Recorded on macOS ARM64 with Rust 1.88.0, 2026-09-04. Each implementation is +warmed, then measured seven times with alternating execution order; values below +are medians. Exact selected findings are compared before timing. Workloads use +unscored four-character spans: disjoint intervals; clusters of four partially +overlapping intervals (half survive); and groups of four duplicates (one quarter +survive). These are synthetic local measurements, not a product latency SLA. + +| Workload | Input findings | Previous selection (µs) | New selection (µs) | Speedup | +| --- | ---: | ---: | ---: | ---: | +| Disjoint | 256 | 461.875 | 52.042 | 8.88× | +| Disjoint | 512 | 1,527.041 | 97.541 | 15.66× | +| Disjoint | 1,024 | 3,557.000 | 133.750 | 26.59× | +| Disjoint | 2,048 | 15,150.042 | 261.750 | 57.88× | +| Disjoint | 4,096 | 60,289.583 | 561.000 | 107.47× | +| Overlap clusters | 256 | 190.834 | 36.625 | 5.21× | +| Overlap clusters | 512 | 734.667 | 88.542 | 8.30× | +| Overlap clusters | 1,024 | 2,904.125 | 180.917 | 16.05× | +| Overlap clusters | 2,048 | 12,108.334 | 419.417 | 28.87× | +| Overlap clusters | 4,096 | 51,017.417 | 877.375 | 58.15× | +| Duplicates | 256 | 41.583 | 13.250 | 3.14× | +| Duplicates | 512 | 139.083 | 27.750 | 5.01× | +| Duplicates | 1,024 | 562.916 | 60.583 | 9.29× | +| Duplicates | 2,048 | 2,206.708 | 142.792 | 15.45× | +| Duplicates | 4,096 | 8,114.958 | 313.625 | 25.87× | + +Doubling disjoint findings from 2,048 to 4,096 took 3.98× as long previously +and 2.14× with this change. The algorithm establishes the complexity bound; +these measurements demonstrate its effect on the tested inputs. + +## Correctness evidence and limits + +Five additional regression tests cover the confidence cycle, order-sensitive +duplicates, validation before filtering/deduplication, interval boundaries, and +3,000 seeded randomized Unicode cases with three shuffled input orders each. +All 9,000 randomized comparisons preserve exact findings and provenance across +absent, present, mixed, and length-dependent confidence; entity filters; and +exact/regex allowlists. Existing transformation and provider tests cover output +ranges, policy application, and validation before provider calls. + +Required Rust formatting, Clippy, and workspace tests pass: 81 tests, plus the +separately run manual benchmark. Installed Python, Node, and browser WASM +packages also pass the existing text, structured, and transformation fixtures. + +The mixed-confidence fallback is intentionally still quadratic. Repeated Unicode +prefix walks during validation and output-range construction, repeated validation +across operation layers, and binding conversions are separate costs. This PR +does not establish an end-to-end `O(m log m)` bound or promise the selection-only +speedups for complete requests. Optimizing those costs requires separate evidence +and a separate focused change. diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 0c506ba..872914b 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -242,6 +242,20 @@ and its bindings ship together before downstream MCP adoption. This workstream adds schema-guided PERSON coverage. It does not change the completed privacy-operation slices or promise general prose name recognition. +## Slice 10: Finding-selection performance + +**Status: implemented; review and combined release pending.** + +Replace linear duplicate searches with indexed groups and repeated overlap +scans with ordered interval selection. Preserve validation, filtering, +confidence/provenance preferences, and source order for all labels. Retain a +compatibility fallback where mixed confidence prevents safe sorting. + +The [selection measurements](finding-selection-performance.md) document exact +behavior comparisons, reproducible scaling benchmarks, and remaining performance +limits. Prepare one 0.3.0 release containing structured PERSON and this change +after review; individual feature PRs do not bump or publish package versions. + ## Acceptance bar Every completed slice must have: