From 8b2a8cadf2fafefeacf9cc49110409cf70c36d66 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:52:01 -0700 Subject: [PATCH] feat: add structured PERSON discovery and protection --- README.md | 6 + bindings/node/dts-header.d.ts | 24 + bindings/node/index.d.ts | 92 ++ bindings/node/index.js | 122 ++- bindings/node/src/lib.rs | 396 +++++++ bindings/python/src/lib.rs | 419 +++++++- bindings/python/tests/test_installed.py | 72 ++ bindings/wasm/index.d.ts | 30 + bindings/wasm/index.js | 74 ++ bindings/wasm/src/lib.rs | 244 ++++- crates/core/examples/scan_benchmark.rs | 88 ++ crates/core/src/lib.rs | 1 + crates/core/src/structured.rs | 1035 +++++++++++++++++++ docs/.mintignore | 2 + docs/adr/002-structured-person-discovery.md | 145 +++ docs/concepts/findings-and-ranges.mdx | 8 + docs/docs.json | 3 +- docs/guides/person-discovery.mdx | 153 +++ docs/person-detection-plan.md | 239 +++++ docs/privacy-capability-matrix.md | 2 + docs/privacy-operations-roadmap.md | 13 + docs/reference/browser-wasm.mdx | 14 + docs/reference/node.mdx | 14 + docs/reference/python.mdx | 13 + docs/reference/rust.mdx | 12 + docs/structured-performance.md | 129 +++ fixtures/structured-transform.jsonl | 7 + fixtures/structured.jsonl | 13 + scripts/test-node-package.mjs | 96 +- scripts/test-wasm-package.mjs | 66 +- 30 files changed, 3484 insertions(+), 48 deletions(-) create mode 100644 crates/core/examples/scan_benchmark.rs create mode 100644 crates/core/src/structured.rs create mode 100644 docs/adr/002-structured-person-discovery.md create mode 100644 docs/guides/person-discovery.mdx create mode 100644 docs/person-detection-plan.md create mode 100644 docs/structured-performance.md create mode 100644 fixtures/structured-transform.jsonl create mode 100644 fixtures/structured.jsonl diff --git a/README.md b/README.md index fc15769..75dc9c1 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ entity type, matched text, byte range, code-point range, optional confidence, detector name, optional detector version ``` +Structured JSON scanning additionally discovers `PERSON` from documented name-field +aliases or explicit JSON Pointer mappings. It scans every string value with the +existing detectors and returns field paths plus string-local findings. No model +or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx) +for `scan_structured` / `scanStructured` and structured transformation APIs. + Both ranges use zero-based, end-exclusive offsets. The byte range addresses the UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based detectors currently report no confidence score. Node.js and browser WASM also diff --git a/bindings/node/dts-header.d.ts b/bindings/node/dts-header.d.ts index d6402d4..2e9210f 100644 --- a/bindings/node/dts-header.d.ts +++ b/bindings/node/dts-header.d.ts @@ -138,6 +138,10 @@ export interface PrivacyManagerProviders { } export declare class PrivacyManager { + transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context?: PrivacyContext): Promise; + scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig, context?: PrivacyContext): Promise; + restoreStructured(data: JsonDocument, context: PrivacyContext): Promise; + constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider); transform( text: string, @@ -152,3 +156,23 @@ export declare class PrivacyManager { ): Promise; restore(text: string, context: PrivacyContext): Promise; } + +/** JSON input uses finite numbers; integer values must be JavaScript-safe. */ +export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; +export type JsonDocument = JsonValue[] | { [key: string]: JsonValue }; +export interface StructuredScanConfig { + readonly locale?: string; + readonly discover_person?: boolean; + readonly mappings?: Readonly>; + readonly exclude?: readonly string[]; +} + +export interface StructuredScanAndTransformConfig { + readonly scan?: StructuredScanConfig; + readonly transform: TransformationConfig; +} +export declare function transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): StructuredTransformResult; +export declare function scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig): StructuredTransformResult; + +export interface StructuredTransformResult { readonly data: JsonDocument; readonly transformations: StructuredTransformation[]; } +export interface StructuredRestoreResult { readonly data: JsonDocument; readonly restorations: StructuredRestoration[]; } diff --git a/bindings/node/index.d.ts b/bindings/node/index.d.ts index b39558e..58e6474 100644 --- a/bindings/node/index.d.ts +++ b/bindings/node/index.d.ts @@ -138,6 +138,10 @@ export interface PrivacyManagerProviders { } export declare class PrivacyManager { + transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context?: PrivacyContext): Promise; + scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig, context?: PrivacyContext): Promise; + restoreStructured(data: JsonDocument, context: PrivacyContext): Promise; + constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider); transform( text: string, @@ -152,6 +156,35 @@ export declare class PrivacyManager { ): Promise; restore(text: string, context: PrivacyContext): Promise; } + +/** JSON input uses finite numbers; integer values must be JavaScript-safe. */ +export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; +export type JsonDocument = JsonValue[] | { [key: string]: JsonValue }; +export interface StructuredScanConfig { + readonly locale?: string; + readonly discover_person?: boolean; + readonly mappings?: Readonly>; + readonly exclude?: readonly string[]; +} + +export interface StructuredScanAndTransformConfig { + readonly scan?: StructuredScanConfig; + readonly transform: TransformationConfig; +} +export declare function transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): StructuredTransformResult; +export declare function scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig): StructuredTransformResult; + +export interface StructuredTransformResult { readonly data: JsonDocument; readonly transformations: StructuredTransformation[]; } +export interface StructuredRestoreResult { readonly data: JsonDocument; readonly restorations: StructuredRestoration[]; } +export declare function discoverFields(dataJson: JsonDocument, config?: StructuredScanConfig | undefined): Array + +export interface FieldMapping { + path: string + entityType: string + source: string + rule: string +} + export interface Finding { readonly entityType: EntityType readonly matchedText: string @@ -180,13 +213,30 @@ export interface KeySelector { readonly path: string } +export interface NativeStructuredRestoreResult { + dataJson: string + restorations: Array +} + +export interface NativeStructuredTransformResult { + dataJson: string + transformations: Array +} + export interface PreparedScanAndTransform { readonly findings: Array readonly selectors: Array } +export interface PreparedStructuredScan { + findings: Array + selectors: Array +} + export declare function prepareScanAndTransform(text: string, config: ScanAndTransformConfig): PreparedScanAndTransform +export declare function prepareStructuredScanAndTransform(text: string, config: unknown): PreparedStructuredScan + export declare function requiredKeySelectors(text: string, findings: FindingInput[], config: TransformationConfig): Array export declare function requiredRestoreItems(text: string, context: PrivacyContext): Array @@ -235,6 +285,48 @@ export declare function scan(text: string, config?: ScanConfig | undefined): Arr /** Scan text and transform the detected findings. */ export declare function scanAndTransform(text: string, config: ScanAndTransformConfig): TransformResult +export declare function scanStructured(dataJson: JsonDocument, config?: StructuredScanConfig | undefined): StructuredScanResult + +export interface StructuredFinding { + path: string + finding: Finding +} + +export interface StructuredFindingInput { + path: string + finding: FindingInput +} + +/** Scan text and transform the detected findings. */ +export declare function structuredRequiredKeySelectors(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): Array + +export declare function structuredRequiredRestoreItems(text: JsonDocument, context: PrivacyContext): Array + +export declare function structuredRequiredTokenizationItems(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context?: PrivacyContext | undefined): Array + +export interface StructuredRestoration { + path: string + restoration: Restoration +} + +export declare function structuredRestoreWithResults(text: JsonDocument, context: PrivacyContext, results: Array): NativeStructuredRestoreResult + +export declare function structuredScanAndTransform(text: JsonDocument, config: StructuredScanAndTransformConfig): NativeStructuredTransformResult + +export interface StructuredScanResult { + mappings: Array + findings: Array +} + +export declare function structuredTransform(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): NativeStructuredTransformResult + +export interface StructuredTransformation { + path: string + transformation: Transformation +} + +export declare function structuredTransformWithProviderResults(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array, tokenResults: Array): NativeStructuredTransformResult + export interface TextRange { readonly start: number readonly end: number diff --git a/bindings/node/index.js b/bindings/node/index.js index 1279f21..ed1a12b 100644 --- a/bindings/node/index.js +++ b/bindings/node/index.js @@ -1,5 +1,15 @@ import { Buffer } from "node:buffer"; import { + structuredTransform as nativeStructuredTransform, + structuredScanAndTransform as nativeStructuredScanAndTransform, + structuredRequiredKeySelectors as nativeStructuredRequiredKeySelectors, + structuredRequiredTokenizationItems as nativeStructuredRequiredTokenizationItems, + structuredTransformWithProviderResults as nativeStructuredTransformWithProviderResults, + structuredRequiredRestoreItems as nativeStructuredRequiredRestoreItems, + structuredRestoreWithResults as nativeStructuredRestoreWithResults, + prepareStructuredScanAndTransform as nativePrepareStructuredScanAndTransform, + discoverFields as nativeDiscoverFields, + scanStructured as nativeScanStructured, prepareScanAndTransform as nativePrepareScanAndTransform, requiredKeySelectors as nativeRequiredKeySelectors, requiredRestoreItems as nativeRequiredRestoreItems, @@ -167,7 +177,14 @@ export class PrivacyManager { })); } - async transform(text, findings, config, context) { + async transform(text, findings, config, context) { return this.#transform(text, findings, config, context); } + async scanAndTransform(text, config, context) { return this.#scanAndTransform(text, config, context); } + async restore(text, context) { return this.#restore(text, context); } + async transformStructured(data, findings, config, context) { return this.#transform(structuredJson(data), structuredOptions(findings), structuredOptions(config), structuredOptions(context), true); } + async scanAndTransformStructured(data, config, context) { return this.#scanAndTransform(structuredJson(data), structuredOptions(config), structuredOptions(context), true); } + async restoreStructured(data, context) { return this.#restore(structuredJson(data), structuredOptions(context), true); } + + async #transform(text, findings, config, context, structured = false) { if (typeof text !== "string") { throw new TypeError("transform text must be a string"); } @@ -176,11 +193,15 @@ export class PrivacyManager { } let resolved = []; try { - const selectors = nativeRequiredKeySelectors(text, findings, config); + const selectors = (structured ? nativeStructuredRequiredKeySelectors : nativeRequiredKeySelectors)(text, findings, config); + const preparedItems = structured + ? nativeStructuredRequiredTokenizationItems(text, findings, config, context) + : undefined; resolved = await this.#resolve(selectors); - const items = nativeRequiredTokenizationItems(text, findings, config, context); + const items = preparedItems ?? nativeRequiredTokenizationItems(text, findings, config, context); const tokens = await this.#tokenize(items, context); - return nativeTransformWithProviderResults(text, findings, config, context, resolved, tokens); + const result = (structured ? nativeStructuredTransformWithProviderResults : nativeTransformWithProviderResults)(text, findings, config, context, resolved, tokens); + return structured ? structuredResult(result) : result; } catch (error) { if (error instanceof DataFogError) throw error; throw normalizeError(error, "invalid_configuration"); @@ -189,21 +210,29 @@ export class PrivacyManager { } } - async scanAndTransform(text, config, context) { + async #scanAndTransform(text, config, context, structured = false) { if (typeof text !== "string") { throw new TypeError("scanAndTransform text must be a string"); } let prepared; try { - prepared = nativePrepareScanAndTransform(text, config); + prepared = (structured ? nativePrepareStructuredScanAndTransform : nativePrepareScanAndTransform)(text, config); } catch (error) { throw normalizeError(error, "invalid_configuration"); } + let preparedItems; + if (structured) { + try { + preparedItems = nativeStructuredRequiredTokenizationItems(text, prepared.findings, config.transform, context); + } catch (error) { + throw withPathPrefix(error, "/transform"); + } + } const resolved = await this.#resolve(prepared.selectors, "/transform"); try { - const items = nativeRequiredTokenizationItems(text, prepared.findings, config.transform, context); + const items = preparedItems ?? nativeRequiredTokenizationItems(text, prepared.findings, config.transform, context); const tokens = await this.#tokenize(items, context); - return nativeTransformWithProviderResults( + const result = (structured ? nativeStructuredTransformWithProviderResults : nativeTransformWithProviderResults)( text, prepared.findings, config.transform, @@ -211,6 +240,7 @@ export class PrivacyManager { resolved, tokens, ); + return structured ? structuredResult(result) : result; } catch (error) { throw withPathPrefix(error, "/transform"); } finally { @@ -219,17 +249,20 @@ export class PrivacyManager { } - async restore(text, context) { + async #restore(text, context, structured = false) { if (typeof text !== "string") { throw new TypeError("restore text must be a string"); } let items; try { - items = nativeRequiredRestoreItems(text, context); + items = (structured ? nativeStructuredRequiredRestoreItems : nativeRequiredRestoreItems)(text, context); } catch (error) { throw normalizeError(error, "invalid_configuration"); } - if (items.length === 0) return nativeRestoreWithResults(text, context, []); + if (items.length === 0) { + const result = (structured ? nativeStructuredRestoreWithResults : nativeRestoreWithResults)(text, context, []); + return structured ? structuredResult(result) : result; + } if (!this.#tokenProvider) { throw new DataFogError({ code: "token_provider_required", @@ -242,7 +275,8 @@ export class PrivacyManager { } catch (error) { throw normalizeProviderError(error, undefined, "token_provider_error"); } - return nativeRestoreWithResults(text, context, Array.isArray(results) ? results : []); + const result = (structured ? nativeStructuredRestoreWithResults : nativeRestoreWithResults)(text, context, Array.isArray(results) ? results : []); + return structured ? structuredResult(result) : result; } } @@ -287,3 +321,67 @@ export function scanAndTransform(text, config) { throw normalizeError(error, "invalid_configuration"); } } + +function structuredJson(data, omitUndefinedOptions = false) { + try { + if (data === null || typeof data !== "object") throw new TypeError(); + const pending = [data]; + const seen = new Set(); + while (pending.length) { + const value = pending.pop(); + if (value === null || typeof value === "string" || typeof value === "boolean") continue; + if (typeof value === "number") { + if (!Number.isFinite(value) || (Number.isInteger(value) && !Number.isSafeInteger(value))) throw new TypeError(); + continue; + } + if (typeof value !== "object") throw new TypeError(); + if (seen.has(value)) continue; + seen.add(value); + const array = Array.isArray(value); + if (!array && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new TypeError(); + if (array && Object.keys(value).length !== value.length) throw new TypeError(); + for (const key of Reflect.ownKeys(value)) { + if (array && key === "length") continue; + if (typeof key !== "string") throw new TypeError(); + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor.enumerable || !("value" in descriptor)) throw new TypeError(); + if (array && (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= value.length)) throw new TypeError(); + if (omitUndefinedOptions && !array && descriptor.value === undefined) continue; + pending.push(descriptor.value); + } + } + return JSON.stringify(data); + } catch { + throw new DataFogError({code:"invalid_configuration", reason:"invalid_type", path:"/data", message:"data must be a JSON object or array with finite numbers and safe integers"}); + } +} + +export function discoverFields(data, config) { + const json = structuredJson(data); + try { return nativeDiscoverFields(json, structuredOptions(config)); } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} + +export function scanStructured(data, config) { + const json = structuredJson(data); + try { return nativeScanStructured(json, structuredOptions(config)); } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} + +export function transformStructured(data, findings, config) { + const json = structuredJson(data); + try { return structuredResult(nativeStructuredTransform(json, structuredOptions(findings), structuredOptions(config))); } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} +export function scanAndTransformStructured(data, config) { + const json = structuredJson(data); + try { return structuredResult(nativeStructuredScanAndTransform(json, structuredOptions(config))); } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} + +function structuredResult({dataJson, ...records}) { + return {data:JSON.parse(dataJson), ...records}; +} + +function structuredOptions(value) { + if (value === undefined) return undefined; + try { return JSON.parse(structuredJson(value, true)); } catch { + throw new DataFogError({code:"invalid_configuration", reason:"invalid_type", path:"", message:"structured request options must be JSON-compatible"}); + } +} diff --git a/bindings/node/src/lib.rs b/bindings/node/src/lib.rs index e511233..d9dd051 100644 --- a/bindings/node/src/lib.rs +++ b/bindings/node/src/lib.rs @@ -614,3 +614,399 @@ pub fn restore_with_results( .map_err(js_privacy_error) .and_then(|result| js_restore_result(&text, result)) } + +#[napi(object)] +pub struct FieldMapping { + pub path: String, + pub entity_type: String, + pub source: String, + pub rule: String, +} + +#[napi(object, object_from_js = false)] +pub struct StructuredFinding { + pub path: String, + pub finding: Finding, +} + +#[napi(object, object_from_js = false)] +pub struct StructuredScanResult { + pub mappings: Vec, + pub findings: Vec, +} + +fn js_field_mapping(mapping: datafog_core::structured::FieldMapping) -> FieldMapping { + FieldMapping { + path: mapping.path, + entity_type: mapping.entity_type, + source: mapping.source, + rule: mapping.rule, + } +} + +fn structured_config( + env: Env, + config: Option>, +) -> napi::Result { + match config { + Some(config) => datafog_core::structured::parse_scan_config(&env.from_js_value(config)?) + .map_err(js_privacy_error), + None => Ok(datafog_core::structured::StructuredScanConfig::default()), + } +} + +#[napi(strict, catch_unwind)] +pub fn discover_fields( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] data_json: String, + #[napi(ts_arg_type = "StructuredScanConfig | undefined")] config: Option>, +) -> napi::Result> { + let data = + datafog_core::structured::parse_document_json(&data_json).map_err(js_privacy_error)?; + let config = structured_config(env, config)?; + Ok(datafog_core::structured::discover_fields(&data, &config) + .map_err(js_privacy_error)? + .into_iter() + .map(js_field_mapping) + .collect()) +} + +#[napi(strict, catch_unwind)] +pub fn scan_structured( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] data_json: String, + #[napi(ts_arg_type = "StructuredScanConfig | undefined")] config: Option>, +) -> napi::Result { + let data = + datafog_core::structured::parse_document_json(&data_json).map_err(js_privacy_error)?; + let config = structured_config(env, config)?; + let result = datafog_core::structured::scan(&data, &config).map_err(js_privacy_error)?; + Ok(StructuredScanResult { + mappings: result.mappings.into_iter().map(js_field_mapping).collect(), + findings: result + .findings + .into_iter() + .map(|located| { + let text = data + .pointer(&located.path) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| js_privacy_error(datafog_core::structured::invalid_data()))?; + Ok(StructuredFinding { + path: located.path, + finding: js_finding(text, located.finding)?, + }) + }) + .collect::>()?, + }) +} + +#[napi(object, object_to_js = false)] +pub struct StructuredFindingInput { + pub path: String, + pub finding: FindingInput, +} + +#[napi(object, object_from_js = false)] +pub struct StructuredTransformation { + pub path: String, + pub transformation: Transformation, +} + +#[napi(object, object_from_js = false)] +pub struct NativeStructuredTransformResult { + pub data_json: String, + pub transformations: Vec, +} + +#[napi(object, object_from_js = false)] +pub struct StructuredRestoration { + pub path: String, + pub restoration: Restoration, +} + +#[napi(object, object_from_js = false)] +pub struct NativeStructuredRestoreResult { + pub data_json: String, + pub restorations: Vec, +} + +fn core_structured_finding( + located: StructuredFindingInput, +) -> datafog_core::structured::StructuredFinding { + datafog_core::structured::StructuredFinding { + path: located.path, + finding: core_finding(located.finding), + } +} + +fn structured_text<'a>(data: &'a serde_json::Value, path: &str) -> napi::Result<&'a str> { + data.pointer(path) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| js_privacy_error(datafog_core::structured::invalid_data())) +} + +fn js_structured_transform_result( + data: &serde_json::Value, + result: datafog_core::structured::StructuredTransformResult, +) -> napi::Result { + let mut transformations = Vec::new(); + for record in result.transformations { + let converted = js_transform_result( + structured_text(data, &record.path)?, + datafog_core::TransformResult { + text: structured_text(&result.data, &record.path)?.into(), + transformations: vec![record.transformation], + }, + )?; + for transformation in converted.transformations { + transformations.push(StructuredTransformation { + path: record.path.clone(), + transformation, + }); + } + } + Ok(NativeStructuredTransformResult { + data_json: result.data.to_string(), + transformations, + }) +} + +fn js_structured_restore_result( + data: &serde_json::Value, + result: datafog_core::structured::StructuredRestoreResult, +) -> napi::Result { + let mut restorations = Vec::new(); + for record in result.restorations { + let converted = js_restore_result( + structured_text(data, &record.path)?, + datafog_core::RestoreResult { + text: structured_text(&result.data, &record.path)?.into(), + restorations: vec![record.restoration], + }, + )?; + for restoration in converted.restorations { + restorations.push(StructuredRestoration { + path: record.path.clone(), + restoration, + }); + } + } + Ok(NativeStructuredRestoreResult { + data_json: result.data.to_string(), + restorations, + }) +} + +#[napi(strict, catch_unwind)] +pub fn structured_transform( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] text: String, + #[napi(ts_arg_type = "StructuredFindingInput[]")] findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, +) -> napi::Result { + let text = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let config: serde_json::Value = env.from_js_value(config)?; + let config = datafog_core::parse_transformation_config(&config).map_err(js_privacy_error)?; + let findings = findings + .into_iter() + .map(core_structured_finding) + .collect::>(); + datafog_core::structured::transform(&text, &findings, &config) + .map_err(js_privacy_error) + .and_then(|result| js_structured_transform_result(&text, result)) +} + +/// Scan text and transform the detected findings. + +#[napi(strict, catch_unwind)] +pub fn structured_required_key_selectors( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] text: String, + #[napi(ts_arg_type = "StructuredFindingInput[]")] findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, +) -> napi::Result> { + let text = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let config: serde_json::Value = env.from_js_value(config)?; + let config = datafog_core::parse_transformation_config(&config).map_err(js_privacy_error)?; + let findings = findings + .into_iter() + .map(core_structured_finding) + .collect::>(); + let selectors = datafog_core::structured::required_key_selectors(&text, &findings, &config) + .map_err(js_privacy_error)?; + js_key_selectors(&selectors) +} + +#[napi(strict, catch_unwind)] +pub fn structured_required_tokenization_items( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] text: String, + #[napi(ts_arg_type = "StructuredFindingInput[]")] findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, + #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option>, +) -> napi::Result> { + let text = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let config_value: serde_json::Value = env.from_js_value(config)?; + let config = + datafog_core::parse_transformation_config(&config_value).map_err(js_privacy_error)?; + let context = context + .map(|value| -> napi::Result { env.from_js_value(value) }) + .transpose()? + .map(|value| datafog_core::parse_privacy_context(&value)) + .transpose() + .map_err(js_privacy_error)?; + let findings = findings + .into_iter() + .map(core_structured_finding) + .collect::>(); + datafog_core::structured::required_tokenization_items( + &text, + &findings, + &config, + context.as_ref(), + ) + .map_err(js_privacy_error) + .map(|items| { + items + .into_iter() + .map(|item| TokenizeItem { + id: item.id().to_owned(), + exact_value: item.exact_value().to_owned(), + token_ref: item.token_ref().to_owned(), + }) + .collect() + }) +} + +#[napi(strict, catch_unwind)] +pub fn structured_transform_with_provider_results( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] text: String, + #[napi(ts_arg_type = "StructuredFindingInput[]")] findings: Vec, + #[napi(ts_arg_type = "TransformationConfig")] config: Unknown<'_>, + #[napi(ts_arg_type = "PrivacyContext | undefined")] context: Option>, + resolved_keys: Vec, + token_results: Vec, +) -> napi::Result { + let text = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let config_value: serde_json::Value = env.from_js_value(config)?; + let config = + datafog_core::parse_transformation_config(&config_value).map_err(js_privacy_error)?; + let context = context + .map(|value| -> napi::Result { env.from_js_value(value) }) + .transpose()? + .map(|value| datafog_core::parse_privacy_context(&value)) + .transpose() + .map_err(js_privacy_error)?; + let findings = findings + .into_iter() + .map(core_structured_finding) + .collect::>(); + let selectors = datafog_core::structured::required_key_selectors(&text, &findings, &config) + .map_err(js_privacy_error)?; + let keys = core_key_bindings(selectors, resolved_keys)?; + datafog_core::structured::transform_with_provider_results( + &text, + &findings, + &config, + context.as_ref(), + keys, + core_token_results(token_results), + ) + .map_err(js_privacy_error) + .and_then(|result| js_structured_transform_result(&text, result)) +} + +#[napi(strict, catch_unwind)] +pub fn structured_required_restore_items( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] text: String, + #[napi(ts_arg_type = "PrivacyContext")] context: Unknown<'_>, +) -> napi::Result> { + let text = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let value: serde_json::Value = env.from_js_value(context)?; + let context = datafog_core::parse_privacy_context(&value).map_err(js_privacy_error)?; + datafog_core::structured::required_restore_items(&text, &context) + .map_err(js_privacy_error) + .map(|items| { + items + .into_iter() + .map(|item| RestoreItem { + id: item.id().to_owned(), + token_ref: item.token_ref().to_owned(), + resolved_version: item.resolved_version().to_owned(), + payload: Buffer::from(item.payload()), + }) + .collect() + }) +} + +#[napi(strict, catch_unwind)] +pub fn structured_restore_with_results( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] text: String, + #[napi(ts_arg_type = "PrivacyContext")] context: Unknown<'_>, + results: Vec, +) -> napi::Result { + let text = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let value: serde_json::Value = env.from_js_value(context)?; + let context = datafog_core::parse_privacy_context(&value).map_err(js_privacy_error)?; + let results = results + .into_iter() + .map(|result| datafog_core::RestoredValue::new(result.id, result.value)) + .collect(); + datafog_core::structured::restore_with_results(&text, &context, results) + .map_err(js_privacy_error) + .and_then(|result| js_structured_restore_result(&text, result)) +} + +#[napi(strict, catch_unwind)] +pub fn structured_scan_and_transform( + env: Env, + #[napi(ts_arg_type = "JsonDocument")] text: String, + #[napi(ts_arg_type = "StructuredScanAndTransformConfig")] config: Unknown<'_>, +) -> napi::Result { + let data = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let config = + datafog_core::structured::parse_scan_and_transform_config(&env.from_js_value(config)?) + .map_err(js_privacy_error)?; + let result = + datafog_core::structured::scan_and_transform(&data, &config).map_err(js_privacy_error)?; + js_structured_transform_result(&data, result) +} + +#[napi(object, object_from_js = false)] +pub struct PreparedStructuredScan { + pub findings: Vec, + pub selectors: Vec, +} + +#[napi(strict, catch_unwind)] +pub fn prepare_structured_scan_and_transform( + env: Env, + text: String, + config: Unknown<'_>, +) -> napi::Result { + let data = datafog_core::structured::parse_document_json(&text).map_err(js_privacy_error)?; + let config = + datafog_core::structured::parse_scan_and_transform_config(&env.from_js_value(config)?) + .map_err(js_privacy_error)?; + let findings = datafog_core::structured::scan(&data, &config.scan) + .map_err(js_privacy_error)? + .findings; + let selectors = + datafog_core::structured::required_key_selectors(&data, &findings, &config.transform) + .map_err(js_privacy_error)?; + Ok(PreparedStructuredScan { + selectors: js_key_selectors(&selectors)?, + findings: findings + .into_iter() + .map(|located| { + Ok(StructuredFinding { + finding: js_finding(structured_text(&data, &located.path)?, located.finding)?, + path: located.path, + }) + }) + .collect::>()?, + }) +} diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs index bb3774e..cfe0a7e 100644 --- a/bindings/python/src/lib.rs +++ b/bindings/python/src/lib.rs @@ -320,14 +320,7 @@ impl From for RestoreResult { restorations: result .restorations .into_iter() - .map(|record| Restoration { - source_byte_range: record.source_byte_range.into(), - source_codepoint_range: record.source_codepoint_range.into(), - output_byte_range: record.output_byte_range.into(), - output_codepoint_range: record.output_codepoint_range.into(), - token_ref: record.token_ref, - resolved_token_version: record.resolved_token_version, - }) + .map(Restoration::from) .collect(), } } @@ -571,6 +564,124 @@ struct PrivacyManager { #[pymethods] impl PrivacyManager { + #[pyo3(signature = (data, findings, config, context=None))] + fn transform_structured<'py>( + &self, + py: Python<'py>, + data: Py, + findings: Vec>, + config: Py, + context: Option>, + ) -> PyResult> { + let data = structured_data(py, data.bind(py))?; + let config_value = structured_options(py, config.bind(py), "")?; + let config = core::parse_transformation_config(&config_value) + .map_err(|error| privacy_error(py, error))?; + let findings = findings + .iter() + .map(|finding| finding.bind(py).borrow().to_core()) + .collect::>(); + let context = context + .map(|context| structured_options(py, context.bind(py), "")) + .transpose()? + .map(|value| core::parse_privacy_context(&value)) + .transpose() + .map_err(|error| privacy_error(py, error))?; + let key_provider = self + .key_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + let token_provider = self + .token_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let manager = core::PrivacyManager::new(PythonKeyProvider { + provider: key_provider, + }) + .with_token_provider(PythonTokenProvider { + provider: token_provider, + }); + let result = manager + .transform_structured(&data, &findings, &config, context.as_ref()) + .await + .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; + Python::attach(|py| Py::new(py, structured_transform_result(py, result)?)) + }) + } + #[pyo3(signature = (data, config, context=None))] + fn scan_and_transform_structured<'py>( + &self, + py: Python<'py>, + data: Py, + config: Py, + context: Option>, + ) -> PyResult> { + let data = structured_data(py, data.bind(py))?; + let config_value = structured_options(py, config.bind(py), "")?; + let config = core::structured::parse_scan_and_transform_config(&config_value) + .map_err(|error| privacy_error(py, error))?; + let context = context + .map(|context| structured_options(py, context.bind(py), "")) + .transpose()? + .map(|value| core::parse_privacy_context(&value)) + .transpose() + .map_err(|error| privacy_error(py, error))?; + let key_provider = self + .key_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + let token_provider = self + .token_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let manager = core::PrivacyManager::new(PythonKeyProvider { + provider: key_provider, + }) + .with_token_provider(PythonTokenProvider { + provider: token_provider, + }); + let result = manager + .scan_and_transform_structured(&data, &config, context.as_ref()) + .await + .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; + Python::attach(|py| Py::new(py, structured_transform_result(py, result)?)) + }) + } + fn restore_structured<'py>( + &self, + py: Python<'py>, + data: Py, + context: Py, + ) -> PyResult> { + let data = structured_data(py, data.bind(py))?; + let context = structured_options(py, context.bind(py), "")?; + let context = + core::parse_privacy_context(&context).map_err(|error| privacy_error(py, error))?; + let key_provider = self + .key_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + let token_provider = self + .token_provider + .as_ref() + .map(|provider| provider.clone_ref(py)); + pyo3_async_runtimes::tokio::future_into_py(py, async move { + let manager = core::PrivacyManager::new(PythonKeyProvider { + provider: key_provider, + }) + .with_token_provider(PythonTokenProvider { + provider: token_provider, + }); + let result = manager + .restore_structured(&data, &context) + .await + .map_err(|error| Python::attach(|py| privacy_error(py, error)))?; + Python::attach(|py| Py::new(py, structured_restore_result(py, result)?)) + }) + } + #[new] #[pyo3(signature = (provider=None, token_provider=None))] fn new( @@ -899,6 +1010,260 @@ fn scan_and_transform( .map_err(|error| privacy_error(py, error)) } +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +struct FieldMapping { + #[pyo3(get)] + path: String, + #[pyo3(get)] + entity_type: String, + #[pyo3(get)] + source: String, + #[pyo3(get)] + rule: String, +} +impl From for FieldMapping { + fn from(mapping: core::structured::FieldMapping) -> Self { + Self { + path: mapping.path, + entity_type: mapping.entity_type, + source: mapping.source, + rule: mapping.rule, + } + } +} + +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +struct StructuredFinding { + #[pyo3(get)] + path: String, + #[pyo3(get)] + finding: Finding, +} + +#[pyclass(frozen, skip_from_py_object)] +struct StructuredScanResult { + #[pyo3(get)] + mappings: Vec, + #[pyo3(get)] + findings: Vec, +} + +fn structured_data(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { + // The stdlib encoder rejects cycles/deep recursion safely. Check dictionaries + // and sequences first so it cannot silently coerce keys or tuple values. + let mut pending = vec![value.clone()]; + let mut seen = std::collections::BTreeSet::new(); + while let Some(value) = pending.pop() { + if value.is_none() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + || value.is_instance_of::() + { + continue; + } + if !seen.insert(value.as_ptr() as usize) { + // Repeated containers are valid JSON trees after serialization; + // the encoder below distinguishes shared references from cycles. + continue; + } + if let Ok(object) = value.cast::() { + for (key, child) in object.iter() { + if !key.is_instance_of::() { + return Err(privacy_error(py, core::structured::invalid_data())); + } + pending.push(child); + } + } else if let Ok(array) = value.cast::() { + pending.extend(array.iter()); + } else { + return Err(privacy_error(py, core::structured::invalid_data())); + } + } + let kwargs = PyDict::new(py); + kwargs.set_item("allow_nan", false)?; + let json: String = py + .import("json")? + .call_method("dumps", (value,), Some(&kwargs)) + .and_then(|value| value.extract()) + .map_err(|_| privacy_error(py, core::structured::invalid_data()))?; + core::structured::parse_document_json(&json).map_err(|error| privacy_error(py, error)) +} + +fn structured_config( + py: Python<'_>, + config: Option<&Bound<'_, PyAny>>, +) -> PyResult { + match config { + Some(config) => core::structured::parse_scan_config(&structured_options(py, config, "")?) + .map_err(|error| privacy_error(py, error)), + None => Ok(core::structured::StructuredScanConfig::default()), + } +} + +#[pyfunction] +#[pyo3(signature = (data, config=None))] +fn discover_fields( + py: Python<'_>, + data: &Bound<'_, PyAny>, + config: Option<&Bound<'_, PyAny>>, +) -> PyResult> { + core::structured::discover_fields(&structured_data(py, data)?, &structured_config(py, config)?) + .map(|mappings| mappings.into_iter().map(FieldMapping::from).collect()) + .map_err(|error| privacy_error(py, error)) +} + +#[pyfunction] +#[pyo3(signature = (data, config=None))] +fn scan_structured( + py: Python<'_>, + data: &Bound<'_, PyAny>, + config: Option<&Bound<'_, PyAny>>, +) -> PyResult { + let result = + core::structured::scan(&structured_data(py, data)?, &structured_config(py, config)?) + .map_err(|error| privacy_error(py, error))?; + Ok(StructuredScanResult { + mappings: result + .mappings + .into_iter() + .map(FieldMapping::from) + .collect(), + findings: result + .findings + .into_iter() + .map(|located| StructuredFinding { + path: located.path, + finding: Finding::from(located.finding), + }) + .collect(), + }) +} + +#[pymethods] +impl StructuredFinding { + #[new] + fn new(path: String, finding: PyRef<'_, Finding>) -> Self { + Self { + path, + finding: finding.clone(), + } + } +} +impl StructuredFinding { + fn to_core(&self) -> core::structured::StructuredFinding { + core::structured::StructuredFinding { + path: self.path.clone(), + finding: self.finding.to_core(), + } + } +} + +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +struct StructuredTransformation { + #[pyo3(get)] + path: String, + #[pyo3(get)] + transformation: Transformation, +} +#[pyclass(frozen, skip_from_py_object)] +struct StructuredTransformResult { + #[pyo3(get)] + data: Py, + #[pyo3(get)] + transformations: Vec, +} +#[pyclass(frozen, skip_from_py_object)] +#[derive(Clone)] +struct StructuredRestoration { + #[pyo3(get)] + path: String, + #[pyo3(get)] + restoration: Restoration, +} +#[pyclass(frozen, skip_from_py_object)] +struct StructuredRestoreResult { + #[pyo3(get)] + data: Py, + #[pyo3(get)] + restorations: Vec, +} +fn structured_transform_result( + py: Python<'_>, + result: core::structured::StructuredTransformResult, +) -> PyResult { + Ok(StructuredTransformResult { + data: py + .import("json")? + .call_method1("loads", (result.data.to_string(),))? + .unbind(), + transformations: result + .transformations + .into_iter() + .map(|record| StructuredTransformation { + path: record.path, + transformation: Transformation::from(record.transformation), + }) + .collect(), + }) +} +fn structured_restore_result( + py: Python<'_>, + result: core::structured::StructuredRestoreResult, +) -> PyResult { + Ok(StructuredRestoreResult { + data: py + .import("json")? + .call_method1("loads", (result.data.to_string(),))? + .unbind(), + restorations: result + .restorations + .into_iter() + .map(|record| StructuredRestoration { + path: record.path, + restoration: Restoration::from(record.restoration), + }) + .collect(), + }) +} + +#[pyfunction] +fn transform_structured( + py: Python<'_>, + data: &Bound<'_, PyAny>, + findings: Vec>, + config: &Bound<'_, PyAny>, +) -> PyResult { + let data = structured_data(py, data)?; + let config = core::parse_transformation_config(&structured_options(py, config, "")?) + .map_err(|error| privacy_error(py, error))?; + let findings: Vec<_> = findings + .iter() + .map(|finding| finding.bind(py).borrow().to_core()) + .collect(); + let result = core::structured::transform(&data, &findings, &config) + .map_err(|error| privacy_error(py, error))?; + structured_transform_result(py, result) +} + +#[pyfunction] +fn scan_and_transform_structured( + py: Python<'_>, + data: &Bound<'_, PyAny>, + config: &Bound<'_, PyAny>, +) -> PyResult { + let data = structured_data(py, data)?; + let config = + core::structured::parse_scan_and_transform_config(&structured_options(py, config, "")?) + .map_err(|error| privacy_error(py, error))?; + let result = core::structured::scan_and_transform(&data, &config) + .map_err(|error| privacy_error(py, error))?; + structured_transform_result(py, result) +} + #[pymodule] fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add( @@ -917,6 +1282,17 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { "DataFogKeyProviderError", module.py().get_type::(), )?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_class::()?; + module.add_function(wrap_pyfunction!(transform_structured, module)?)?; + module.add_function(wrap_pyfunction!(scan_and_transform_structured, module)?)?; + module.add_function(wrap_pyfunction!(discover_fields, module)?)?; + module.add_function(wrap_pyfunction!(scan_structured, module)?)?; module.add_class::()?; module.add_class::()?; module.add_class::()?; @@ -929,3 +1305,30 @@ fn datafog_core(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(scan_and_transform, module)?)?; Ok(()) } + +impl From for Restoration { + fn from(record: core::Restoration) -> Self { + Restoration { + source_byte_range: record.source_byte_range.into(), + source_codepoint_range: record.source_codepoint_range.into(), + output_byte_range: record.output_byte_range.into(), + output_codepoint_range: record.output_codepoint_range.into(), + token_ref: record.token_ref, + resolved_token_version: record.resolved_token_version, + } + } +} + +fn structured_options( + py: Python<'_>, + value: &Bound<'_, PyAny>, + path: &str, +) -> PyResult { + structured_data(py, value).map_err(|_| { + configuration_conversion_error( + py, + path, + "structured request options must be JSON-compatible", + ) + }) +} diff --git a/bindings/python/tests/test_installed.py b/bindings/python/tests/test_installed.py index 379be76..2b54ee0 100644 --- a/bindings/python/tests/test_installed.py +++ b/bindings/python/tests/test_installed.py @@ -14,6 +14,10 @@ PrivacyManager, TextRange, scan, + scan_structured, + transform_structured, + scan_and_transform_structured, + discover_fields, scan_and_transform, transform, ) @@ -64,7 +68,54 @@ def verify_fixture(name: str) -> None: verify_contract(record["text"]) +def verify_structured() -> None: + for line in (ROOT / "fixtures" / "structured.jsonl").read_text().splitlines(): + record = json.loads(line) + result = scan_structured(record["data"], record.get("config")) + mappings = [dict(path=m.path, entity_type=m.entity_type, source=m.source, rule=m.rule) for m in result.mappings] + assert mappings == record["mappings"], record["id"] + discovered = discover_fields(record["data"], record.get("config")) + assert [m.path for m in discovered] == [m.path for m in result.mappings] + actual = [] + for located in result.findings: + text = record["data"] + for part in located.path[1:].split("/"): + key = part.replace("~1", "/").replace("~0", "~") + text = text[int(key)] if isinstance(text, list) else text[key] + f = located.finding + assert text.encode()[f.byte_range.start:f.byte_range.end].decode() == f.matched_text + assert text[f.codepoint_range.start:f.codepoint_range.end] == f.matched_text + assert f.confidence is None + actual.append(dict(path=located.path, label=f.entity_type, text=f.matched_text, start=f.codepoint_range.start, end=f.codepoint_range.end)) + assert actual == record["findings"], record["id"] + for line in (ROOT / "fixtures" / "structured-transform.jsonl").read_text().splitlines(): + record = json.loads(line) + result = scan_and_transform_structured(record["data"],record["config"]) + explicit = transform_structured(record["data"],scan_structured(record["data"]).findings,record["config"]["transform"]) + assert result.data == record["expected_data"], record["id"] + assert explicit.data == result.data + assert all(not hasattr(r.transformation,"matched_text") for r in result.transformations) + cycle = {} + cycle["cycle"] = cycle + try: + scan_structured({}, cycle) + except DataFogConfigurationError: + pass + else: + raise AssertionError("cyclic options accepted") + for data in [None, "secret-value", {1: "name"}, {"n": 2**100}, {"n": float("nan")}, {"n": float("inf")}, {"tuple": (1, 2)}, cycle]: + try: + scan_structured(data) + except DataFogConfigurationError as error: + assert error.code == "invalid_configuration" + assert error.path == "/data" + assert "secret-value" not in str(error) + else: + raise AssertionError("invalid structured input accepted") + + def main() -> None: + verify_structured() verify_fixture("development.jsonl") verify_fixture("final.jsonl") emoji_finding = scan("👋 jane@example.com")[0] @@ -307,6 +358,27 @@ async def restore_batch(self, scope: str, items: list[dict[str, object]]): results.append({"id": item["id"], "value": record[3]}) return results + async def structured_round_trip(): + original = {"users":[{"first_name":"👋 José"},{"full_name":"May"}],"count":2} + provider = Provider() + pseudonyms = await PrivacyManager(provider).scan_and_transform_structured(original,{"transform":pseudonym_config}) + assert len(provider.calls) == 1 + assert pseudonyms.data != original + manager = PrivacyManager(None,TokenProvider()) + context = {"scope":"tenant/α"} + config = {"transform":{"default":{"strategy":"tokenize","token_ref":"names"}}} + tokens = await manager.scan_and_transform_structured(original,config,context) + restored = await manager.restore_structured(tokens.data,context) + assert restored.data == original + assert len(restored.restorations) == 2 + try: + await manager.restore_structured(tokens.data,{"scope":"wrong"}) + except DataFogKeyProviderError as error: + assert error.code == "token_access_denied" + else: + raise AssertionError("wrong scope was accepted") + asyncio.run(structured_round_trip()) + async def token_round_trip(): manager = PrivacyManager(None, TokenProvider()) context = {"scope": "tenant/α"} diff --git a/bindings/wasm/index.d.ts b/bindings/wasm/index.d.ts index 5833f86..6ace05d 100644 --- a/bindings/wasm/index.d.ts +++ b/bindings/wasm/index.d.ts @@ -138,3 +138,33 @@ export interface Restoration { } export interface RestoreResult { readonly text: string; readonly restorations: Restoration[]; } export function restore(text: string, context: PrivacyContext): RestoreResult; + +/** JSON input uses finite numbers; integer values must be JavaScript-safe. */ +export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue }; +export type JsonDocument = JsonValue[] | { [key: string]: JsonValue }; +export interface StructuredScanConfig { + readonly locale?: string; + readonly discover_person?: boolean; + readonly mappings?: Readonly>; + readonly exclude?: readonly string[]; +} + +export interface FieldMapping { + readonly path: string; + readonly entityType: "PERSON"; + readonly source: "field_alias" | "explicit_mapping"; + readonly rule: string; +} +export interface StructuredFinding { readonly path: string; readonly finding: Finding; } +export interface StructuredScanResult { readonly mappings: FieldMapping[]; readonly findings: StructuredFinding[]; } +export function discoverFields(data: JsonDocument, config?: StructuredScanConfig): FieldMapping[]; +export function scanStructured(data: JsonDocument, config?: StructuredScanConfig): StructuredScanResult; + +export interface StructuredScanAndTransformConfig { readonly scan?: StructuredScanConfig; readonly transform: TransformationConfig; } +export interface StructuredFindingInput { readonly path: string; readonly finding: FindingInput; } +export interface StructuredTransformation { readonly path: string; readonly transformation: Transformation; } +export interface StructuredTransformResult { readonly data: JsonDocument; readonly transformations: StructuredTransformation[]; } +export function transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): StructuredTransformResult; +export function scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig): StructuredTransformResult; +/** Always rejects with unsupported_strategy after input validation. */ +export function restoreStructured(data: JsonDocument, context: PrivacyContext): never; diff --git a/bindings/wasm/index.js b/bindings/wasm/index.js index 6b53ea1..ae915db 100644 --- a/bindings/wasm/index.js +++ b/bindings/wasm/index.js @@ -1,4 +1,9 @@ import initWasm, { + transform_structured as nativeTransformStructured, + scan_and_transform_structured as nativeScanAndTransformStructured, + restore_structured as nativeRestoreStructured, + discover_fields as nativeDiscoverFields, + scan_structured as nativeScanStructured, scan as scanWasm, scan_and_transform as scanAndTransformWasm, restore as restoreWasm, @@ -123,3 +128,72 @@ export function restore(text, context) { throw normalizeError(error, "invalid_configuration"); } } + +function structuredJson(data, omitUndefinedOptions = false) { + try { + if (data === null || typeof data !== "object") throw new TypeError(); + const pending = [data]; + const seen = new Set(); + while (pending.length) { + const value = pending.pop(); + if (value === null || typeof value === "string" || typeof value === "boolean") continue; + if (typeof value === "number") { + if (!Number.isFinite(value) || (Number.isInteger(value) && !Number.isSafeInteger(value))) throw new TypeError(); + continue; + } + if (typeof value !== "object") throw new TypeError(); + if (seen.has(value)) continue; + seen.add(value); + const array = Array.isArray(value); + if (!array && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new TypeError(); + if (array && Object.keys(value).length !== value.length) throw new TypeError(); + for (const key of Reflect.ownKeys(value)) { + if (array && key === "length") continue; + if (typeof key !== "string") throw new TypeError(); + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor.enumerable || !("value" in descriptor)) throw new TypeError(); + if (array && (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= value.length)) throw new TypeError(); + if (omitUndefinedOptions && !array && descriptor.value === undefined) continue; + pending.push(descriptor.value); + } + } + return JSON.stringify(data); + } catch { + throw new DataFogError({code:"invalid_configuration", reason:"invalid_type", path:"/data", message:"data must be a JSON object or array with finite numbers and safe integers"}); + } +} + +export function discoverFields(data, config) { + assertInitialized("discoverFields"); + const json = structuredJson(data); + try { return nativeDiscoverFields(json, structuredOptions(config)); } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} + +export function scanStructured(data, config) { + assertInitialized("scanStructured"); + const json = structuredJson(data); + try { return nativeScanStructured(json, structuredOptions(config)); } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} + +export function transformStructured(data, findings, config) { + assertInitialized("transformStructured"); + const json = structuredJson(data); + try { const {dataJson, transformations} = nativeTransformStructured(json, structuredOptions(findings), structuredOptions(config)); return {data:JSON.parse(dataJson), transformations}; } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} +export function scanAndTransformStructured(data, config) { + assertInitialized("scanAndTransformStructured"); + const json = structuredJson(data); + try { const {dataJson, transformations} = nativeScanAndTransformStructured(json, structuredOptions(config)); return {data:JSON.parse(dataJson), transformations}; } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} +export function restoreStructured(data, context) { + assertInitialized("restoreStructured"); + const json = structuredJson(data); + try { return nativeRestoreStructured(json, structuredOptions(context)); } catch (error) { throw normalizeError(error, "invalid_configuration"); } +} + +function structuredOptions(value) { + if (value === undefined) return undefined; + try { return JSON.parse(structuredJson(value, true)); } catch { + throw new DataFogError({code:"invalid_configuration", reason:"invalid_type", path:"", message:"structured request options must be JSON-compatible"}); + } +} diff --git a/bindings/wasm/src/lib.rs b/bindings/wasm/src/lib.rs index 880e26b..16c6ba7 100644 --- a/bindings/wasm/src/lib.rs +++ b/bindings/wasm/src/lib.rs @@ -123,30 +123,7 @@ fn result_to_js( .transformations .into_iter() .map(|transformation| { - Ok(Transformation { - entity_type: transformation.entity_type, - source_byte_range: transformation.source_byte_range.into(), - source_codepoint_range: transformation.source_codepoint_range.into(), - source_utf16_range: utf16_range(source_text, transformation.source_byte_range)?, - confidence: transformation.confidence, - detector_name: transformation.detector_name, - detector_version: transformation.detector_version, - strategy: match transformation.strategy { - datafog_core::TransformationStrategy::Redact => "redact", - datafog_core::TransformationStrategy::Remove => "remove", - datafog_core::TransformationStrategy::Mask(_) => "mask", - datafog_core::TransformationStrategy::Pseudonymize(_) => "pseudonymize", - datafog_core::TransformationStrategy::Tokenize(_) => "tokenize", - }, - replacement: transformation.replacement, - output_byte_range: transformation.output_byte_range.into(), - output_codepoint_range: transformation.output_codepoint_range.into(), - output_utf16_range: utf16_range(output_text, transformation.output_byte_range)?, - key_ref: transformation.key_ref, - resolved_key_version: transformation.resolved_key_version, - token_ref: transformation.token_ref, - resolved_token_version: transformation.resolved_token_version, - }) + transformation_from_core(source_text, output_text, transformation) }) .collect::, JsValue>>()?, text: result.text, @@ -253,3 +230,222 @@ pub fn restore(text: &str, context: JsValue) -> Result { datafog_core::PrivacyError::unsupported_strategy("/restore"), )) } + +#[derive(Serialize)] +struct StructuredFinding { + path: String, + finding: Finding, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct FieldMapping { + path: String, + entity_type: String, + source: String, + rule: String, +} + +fn field_mapping(mapping: datafog_core::structured::FieldMapping) -> FieldMapping { + FieldMapping { + path: mapping.path, + entity_type: mapping.entity_type, + source: mapping.source, + rule: mapping.rule, + } +} + +fn structured_config( + config: Option, +) -> Result { + match config { + Some(config) => datafog_core::structured::parse_scan_config(&config_value(config)?) + .map_err(privacy_error), + None => Ok(datafog_core::structured::StructuredScanConfig::default()), + } +} + +#[wasm_bindgen] +pub fn discover_fields(data_json: &str, config: Option) -> Result { + let data = datafog_core::structured::parse_document_json(data_json).map_err(privacy_error)?; + let mappings: Vec<_> = + datafog_core::structured::discover_fields(&data, &structured_config(config)?) + .map_err(privacy_error)? + .into_iter() + .map(field_mapping) + .collect(); + serde_wasm_bindgen::to_value(&mappings) + .map_err(|_| privacy_error(datafog_core::structured::invalid_data())) +} + +#[wasm_bindgen] +pub fn scan_structured(data_json: &str, config: Option) -> Result { + #[derive(Serialize)] + struct ResultValue { + mappings: Vec, + findings: Vec, + } + let data = datafog_core::structured::parse_document_json(data_json).map_err(privacy_error)?; + let result = datafog_core::structured::scan(&data, &structured_config(config)?) + .map_err(privacy_error)?; + let findings = result + .findings + .into_iter() + .map(|located| { + let text = data + .pointer(&located.path) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| privacy_error(datafog_core::structured::invalid_data()))?; + Ok(StructuredFinding { + path: located.path, + finding: finding_from_core(text, located.finding)?, + }) + }) + .collect::, JsValue>>()?; + serde_wasm_bindgen::to_value(&ResultValue { + mappings: result.mappings.into_iter().map(field_mapping).collect(), + findings, + }) + .map_err(|_| privacy_error(datafog_core::structured::invalid_data())) +} + +fn transformation_from_core( + source_text: &str, + output_text: &str, + transformation: datafog_core::Transformation, +) -> Result { + Ok(Transformation { + entity_type: transformation.entity_type, + source_byte_range: transformation.source_byte_range.into(), + source_codepoint_range: transformation.source_codepoint_range.into(), + source_utf16_range: utf16_range(source_text, transformation.source_byte_range)?, + confidence: transformation.confidence, + detector_name: transformation.detector_name, + detector_version: transformation.detector_version, + strategy: match transformation.strategy { + datafog_core::TransformationStrategy::Redact => "redact", + datafog_core::TransformationStrategy::Remove => "remove", + datafog_core::TransformationStrategy::Mask(_) => "mask", + datafog_core::TransformationStrategy::Pseudonymize(_) => "pseudonymize", + datafog_core::TransformationStrategy::Tokenize(_) => "tokenize", + }, + replacement: transformation.replacement, + output_byte_range: transformation.output_byte_range.into(), + output_codepoint_range: transformation.output_codepoint_range.into(), + output_utf16_range: utf16_range(output_text, transformation.output_byte_range)?, + key_ref: transformation.key_ref, + resolved_key_version: transformation.resolved_key_version, + token_ref: transformation.token_ref, + resolved_token_version: transformation.resolved_token_version, + }) +} + +#[derive(Deserialize)] +struct StructuredFindingInput { + path: String, + finding: Finding, +} + +fn structured_findings( + value: JsValue, +) -> Result, JsValue> { + let findings: Vec = serde_wasm_bindgen::from_value(value) + .map_err(|_| privacy_error(datafog_core::structured::invalid_data()))?; + Ok(findings + .into_iter() + .map(|located| datafog_core::structured::StructuredFinding { + path: located.path, + finding: located.finding.into(), + }) + .collect()) +} + +fn structured_operation_error(error: datafog_core::PrivacyError) -> JsValue { + if matches!( + error.code(), + datafog_core::PrivacyErrorCode::KeyProviderRequired + | datafog_core::PrivacyErrorCode::TokenProviderRequired + ) { + return privacy_error(datafog_core::PrivacyError::unsupported_strategy( + error.path().unwrap_or("/default"), + )); + } + privacy_error(error) +} + +fn structured_result_to_js( + data: &serde_json::Value, + result: datafog_core::structured::StructuredTransformResult, +) -> Result { + #[derive(Serialize)] + struct Record { + path: String, + transformation: Transformation, + } + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct ResultValue { + data_json: String, + transformations: Vec, + } + let transformations = result + .transformations + .into_iter() + .map(|record| { + let source = data + .pointer(&record.path) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| privacy_error(datafog_core::structured::invalid_data()))?; + let output = result + .data + .pointer(&record.path) + .and_then(serde_json::Value::as_str) + .ok_or_else(|| privacy_error(datafog_core::structured::invalid_data()))?; + Ok(Record { + path: record.path, + transformation: transformation_from_core(source, output, record.transformation)?, + }) + }) + .collect::, JsValue>>()?; + serde_wasm_bindgen::to_value(&ResultValue { + data_json: result.data.to_string(), + transformations, + }) + .map_err(|_| privacy_error(datafog_core::structured::invalid_data())) +} + +#[wasm_bindgen] +pub fn transform_structured( + data_json: &str, + findings: JsValue, + config: JsValue, +) -> Result { + let data = datafog_core::structured::parse_document_json(data_json).map_err(privacy_error)?; + let findings = structured_findings(findings)?; + let config = + datafog_core::parse_transformation_config(&config_value(config)?).map_err(privacy_error)?; + let result = datafog_core::structured::transform(&data, &findings, &config) + .map_err(structured_operation_error)?; + structured_result_to_js(&data, result) +} + +#[wasm_bindgen] +pub fn scan_and_transform_structured(data_json: &str, config: JsValue) -> Result { + let data = datafog_core::structured::parse_document_json(data_json).map_err(privacy_error)?; + let config = datafog_core::structured::parse_scan_and_transform_config(&config_value(config)?) + .map_err(privacy_error)?; + let result = datafog_core::structured::scan_and_transform(&data, &config) + .map_err(structured_operation_error)?; + structured_result_to_js(&data, result) +} + +#[wasm_bindgen] +pub fn restore_structured(data_json: &str, context: JsValue) -> Result { + let data = datafog_core::structured::parse_document_json(data_json).map_err(privacy_error)?; + let context = + datafog_core::parse_privacy_context(&config_value(context)?).map_err(privacy_error)?; + datafog_core::structured::required_restore_items(&data, &context).map_err(privacy_error)?; + Err(privacy_error( + datafog_core::PrivacyError::unsupported_strategy("/restore"), + )) +} diff --git a/crates/core/examples/scan_benchmark.rs b/crates/core/examples/scan_benchmark.rs new file mode 100644 index 0000000..21fe8b1 --- /dev/null +++ b/crates/core/examples/scan_benchmark.rs @@ -0,0 +1,88 @@ +//! Reproducible text-scanner baseline. Iterations are a measurement parameter, +//! not a performance acceptance threshold. +use std::hint::black_box; +use std::time::Instant; + +fn main() -> Result<(), Box> { + let iterations: usize = std::env::args() + .nth(1) + .ok_or("provide iterations")? + .parse()?; + if iterations == 0 { + return Err("iterations must be positive".into()); + } + if std::env::args().nth(2).as_deref() == Some("structured") { + let corpus: Vec = include_str!("../../../fixtures/structured.jsonl") + .lines() + .map(serde_json::from_str) + .collect::>()?; + let config = datafog_core::structured::StructuredScanConfig::default(); + let transform = datafog_core::structured::parse_scan_and_transform_config( + &serde_json::json!({"transform":{"default":{"strategy":"redact"}}}), + )?; + for operation in ["discover", "scan", "protect"] { + let start = Instant::now(); + let mut records = 0; + for _ in 0..iterations { + for row in &corpus { + let data = black_box(&row["data"]); + records += match operation { + "discover" => { + black_box(datafog_core::structured::discover_fields(data, &config)?) + .len() + } + "scan" => black_box(datafog_core::structured::scan(data, &config)?) + .findings + .len(), + _ => black_box(datafog_core::structured::scan_and_transform( + data, &transform, + )?) + .transformations + .len(), + }; + } + } + let seconds = start.elapsed().as_secs_f64(); + println!( + "operation={} documents={} iterations={} records={} elapsed_s={:.6} us_per_document={:.3}", + operation, + corpus.len(), + iterations, + records, + seconds, + seconds * 1e6 / (corpus.len() * iterations) as f64 + ); + } + return Ok(()); + } + let corpus: Vec = include_str!("../../../fixtures/development.jsonl") + .lines() + .map(serde_json::from_str) + .collect::>()?; + let texts: Vec<&str> = corpus + .iter() + .map(|row| row["text"].as_str().ok_or("missing text")) + .collect::>()?; + let cold = Instant::now(); + black_box(datafog_core::scan("Email jane@example.test")); + println!("cold_scan_us={:.3}", cold.elapsed().as_secs_f64() * 1e6); + let start = Instant::now(); + let mut findings = 0; + for _ in 0..iterations { + for text in &texts { + findings += black_box(datafog_core::scan(black_box(text))).len(); + } + } + let seconds = start.elapsed().as_secs_f64(); + let bytes: usize = texts.iter().map(|text| text.len()).sum(); + println!( + "documents={} iterations={} bytes_per_iteration={} findings={} elapsed_s={:.6} mib_s={:.3}", + texts.len(), + iterations, + bytes, + findings, + seconds, + bytes as f64 * iterations as f64 / seconds / 1048576.0 + ); + Ok(()) +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index bef29ad..c73bd5d 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,4 +1,5 @@ //! Core PII scanning API for DataFog. +pub mod structured; use base64::Engine; use hmac::{Hmac, Mac}; use regex::{Regex, RegexSet, RegexSetBuilder}; diff --git a/crates/core/src/structured.rs b/crates/core/src/structured.rs new file mode 100644 index 0000000..cb6b442 --- /dev/null +++ b/crates/core/src/structured.rs @@ -0,0 +1,1035 @@ +//! Schema-guided scanning of JSON string values. Finding offsets are local to +//! the decoded string at the reported RFC 6901 JSON Pointer. +use super::*; +use serde_json::Value; + +/// Reusable structured scan policy. Explicit PERSON mappings still apply when +/// automatic discovery is disabled. Exclusions affect PERSON discovery only. +#[derive(Debug, Clone, Default)] +pub struct StructuredScanConfig { + scan: ScanConfig, + disable_person_discovery: bool, + mappings: BTreeSet, + exclude: BTreeSet, +} + +/// Evidence identifying a name field; contains no field value. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct FieldMapping { + pub path: String, + pub entity_type: String, + /// `field_alias` or `explicit_mapping`. + pub source: String, + /// Canonical alias, or `explicit_mapping` for a caller-declared field. + pub rule: String, +} + +/// A text finding located inside a JSON string value. +#[derive(Debug, Clone, PartialEq)] +pub struct StructuredFinding { + pub path: String, + pub finding: Finding, +} + +/// Discovered name fields and all text findings, in deterministic field order. +#[derive(Debug, Clone, PartialEq)] +pub struct StructuredScanResult { + pub mappings: Vec, + pub findings: Vec, +} + +/// A structured argument could not be represented as supported JSON data. +/// Bindings use this constructor to avoid leaking values in conversion errors. +pub fn invalid_data() -> PrivacyError { + PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidType, + "/data", + "data must be a JSON object or array with finite numbers and safe integers", + ) +} + +/// Decode JSON using serde_json's default nesting limit. Conversion failures +/// deliberately omit parser excerpts, which may contain sensitive values. +pub fn parse_document_json(text: &str) -> Result { + let data: Value = serde_json::from_str(text).map_err(|_| invalid_data())?; + // Reject unsupported numeric representations before any findings are exposed. + leaves(&data)?; + Ok(data) +} + +fn validate_pointer(pointer: &str, path: &str) -> Result<(), PrivacyError> { + let mut bytes = pointer.bytes(); + let valid_prefix = bytes.next() == Some(b'/'); + let mut valid_escapes = true; + while let Some(byte) = bytes.next() { + if byte == b'~' && !matches!(bytes.next(), Some(b'0' | b'1')) { + valid_escapes = false; + break; + } + } + if !valid_prefix || !valid_escapes { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + path, + "field path must be a non-root RFC 6901 JSON Pointer", + )); + } + Ok(()) +} + +/// Parse `{ locale?, discover_person?, mappings?: {pointer: "PERSON"}, exclude?: [pointer] }`. +pub fn parse_scan_config(value: &Value) -> Result { + let object = require_object(value, "", "structured scan configuration must be an object")?; + reject_unknown_fields( + object, + &["locale", "discover_person", "mappings", "exclude"], + "", + )?; + let mut config = StructuredScanConfig::default(); + if let Some(locale) = object.get("locale") { + config.scan = config.scan.with_locale(require_string( + locale, + "/locale", + "locale must be a string", + )?)?; + } + if let Some(enabled) = object.get("discover_person") { + config.disable_person_discovery = !enabled.as_bool().ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidType, + "/discover_person", + "discover_person must be a boolean", + ) + })?; + } + if let Some(mappings) = object.get("mappings") { + for (pointer, entity) in + require_object(mappings, "/mappings", "mappings must be an object")? + { + let path = format!("/mappings/{}", json_pointer_segment(pointer)); + validate_pointer(pointer, &path)?; + if require_string(entity, &path, "mapping entity must be a string")? != "PERSON" { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + path, + "structured mappings currently support PERSON only", + )); + } + config.mappings.insert(pointer.clone()); + } + } + if let Some(exclude) = object.get("exclude") { + let values = exclude.as_array().ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidType, + "/exclude", + "exclude must be an array", + ) + })?; + for (index, pointer) in values.iter().enumerate() { + let path = format!("/exclude/{index}"); + let pointer = require_string(pointer, &path, "excluded path must be a string")?; + validate_pointer(&pointer, &path)?; + if config.mappings.contains(&pointer) { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::InvalidValue, + path, + "a field cannot be both explicitly mapped and excluded", + )); + } + if !config.exclude.insert(pointer) { + return Err(PrivacyError::invalid_configuration( + PrivacyErrorReason::DuplicateValue, + path, + "excluded field paths must be unique", + )); + } + } + } + Ok(config) +} + +fn person_alias(key: &str) -> Option<&'static str> { + // Explicit spelling variants avoid substring/fuzzy matches on arbitrary keys. + const ALIASES: [(&str, &str, &str); 6] = [ + ("first_name", "firstName", "FirstName"), + ("given_name", "givenName", "GivenName"), + ("last_name", "lastName", "LastName"), + ("family_name", "familyName", "FamilyName"), + ("full_name", "fullName", "FullName"), + ("surname", "surname", "Surname"), + ]; + ALIASES.iter().find_map(|(snake, camel, pascal)| { + (key.eq_ignore_ascii_case(snake) || key == *camel || key == *pascal).then_some(*snake) + }) +} + +struct Leaf<'a> { + path: String, + key: Option<&'a str>, + text: &'a str, +} + +fn leaves(data: &Value) -> Result>, PrivacyError> { + if !data.is_object() && !data.is_array() { + return Err(invalid_data()); + } + let mut pending = vec![(String::new(), None, data, 1usize)]; + let mut leaves = Vec::new(); + while let Some((path, key, value, depth)) = pending.pop() { + // Match serde_json's default parser: at most 127 nested containers. + if depth >= 128 && (value.is_object() || value.is_array()) { + return Err(invalid_data()); + } + match value { + Value::Object(object) => { + let mut entries: Vec<_> = object.iter().collect(); + entries.sort_unstable_by(|(left, _), (right, _)| left.cmp(right)); + for (key, value) in entries.into_iter().rev() { + pending.push(( + format!("{path}/{}", json_pointer_segment(key)), + Some(key.as_str()), + value, + depth + 1, + )); + } + } + Value::Array(array) => { + for (index, value) in array.iter().enumerate().rev() { + pending.push((format!("{path}/{index}"), None, value, depth + 1)); + } + } + Value::String(text) => leaves.push(Leaf { path, key, text }), + Value::Number(number) => { + // JavaScript Number.MAX_SAFE_INTEGER is the portable integer + // boundary shared by the native and browser bindings. + let number = number.as_f64().ok_or_else(invalid_data)?; + if !number.is_finite() + || (number.fract() == 0.0 && number.abs() > 9_007_199_254_740_991.0) + { + return Err(invalid_data()); + } + } + Value::Null | Value::Bool(_) => {} + } + } + Ok(leaves) +} + +fn field_mapping(leaf: &Leaf<'_>, config: &StructuredScanConfig) -> Option { + let explicit = config.mappings.contains(&leaf.path); + let (source, rule) = if explicit { + ("explicit_mapping", "explicit_mapping") + } else { + if config.disable_person_discovery || config.exclude.contains(&leaf.path) { + return None; + } + ("field_alias", person_alias(leaf.key?)?) + }; + Some(FieldMapping { + path: leaf.path.clone(), + entity_type: "PERSON".into(), + source: source.into(), + rule: rule.into(), + }) +} + +/// Discover name fields without scanning their contents for other entities. +/// Empty string fields can have a mapping while producing no PERSON finding. +pub fn discover_fields( + data: &Value, + config: &StructuredScanConfig, +) -> Result, PrivacyError> { + Ok(leaves(data)? + .iter() + .filter_map(|leaf| field_mapping(leaf, config)) + .collect()) +} + +/// Scan every JSON string leaf and use name-field evidence to emit PERSON. +pub fn scan( + data: &Value, + config: &StructuredScanConfig, +) -> Result { + let mut result = StructuredScanResult { + mappings: Vec::new(), + findings: Vec::new(), + }; + for leaf in leaves(data)? { + let mut findings = scan_with_config(leaf.text, &config.scan); + if let Some(mapping) = field_mapping(&leaf, config) { + if !leaf.text.trim().is_empty() { + findings.push(Finding { + entity_type: "PERSON".into(), + matched_text: leaf.text.into(), + byte_range: TextRange { + start: 0, + end: leaf.text.len(), + }, + codepoint_range: TextRange { + start: 0, + end: leaf.text.chars().count(), + }, + confidence: None, + detector_name: format!("datafog-core/person/{}", mapping.source), + detector_version: Some(env!("CARGO_PKG_VERSION").into()), + }); + } + result.mappings.push(mapping); + } + findings.sort_by(|left, right| { + left.byte_range + .start + .cmp(&right.byte_range.start) + .then_with(|| right.byte_range.end.cmp(&left.byte_range.end)) + .then_with(|| left.entity_type.cmp(&right.entity_type)) + }); + result + .findings + .extend(findings.into_iter().map(|finding| StructuredFinding { + path: leaf.path.clone(), + finding, + })); + } + Ok(result) +} + +/// A transformation inside a JSON string value. +#[derive(Debug, Clone, PartialEq)] +pub struct StructuredTransformation { + pub path: String, + pub transformation: Transformation, +} + +/// A transformed JSON document and field-relative replacement records. +#[derive(Debug, Clone, PartialEq)] +pub struct StructuredTransformResult { + pub data: Value, + pub transformations: Vec, +} + +struct SelectedLeaf<'a> { + path: String, + text: &'a str, + findings: Vec, +} + +fn selected_leaves<'a>( + data: &'a Value, + findings: &[StructuredFinding], + config: &TransformationConfig, +) -> Result>, PrivacyError> { + let all = leaves(data)?; + let mut grouped: BTreeMap<&str, Vec> = BTreeMap::new(); + for (index, located) in findings.iter().enumerate() { + if validate_pointer(&located.path, "").is_err() + || data + .pointer(&located.path) + .and_then(Value::as_str) + .is_none() + { + return Err(PrivacyError { + code: PrivacyErrorCode::InvalidFinding, + reason: Some(PrivacyErrorReason::InvalidValue), + path: Some(format!("/findings/{index}/path")), + finding_index: Some(index), + message: "finding path must select an existing string value".into(), + }); + } + let text = data + .pointer(&located.path) + .and_then(Value::as_str) + .ok_or_else(invalid_data)?; + validate_finding(text, &located.finding).map_err(|kind| { + let mut error = PrivacyError::invalid_finding(index, kind); + error.path = error.path.map(|path| { + path.replacen( + &format!("/findings/{index}/"), + &format!("/findings/{index}/finding/"), + 1, + ) + }); + error + })?; + grouped + .entry(&located.path) + .or_default() + .push(located.finding.clone()); + } + all.into_iter() + .filter_map(|leaf| { + grouped.remove(leaf.path.as_str()).map(|findings| { + select_findings(leaf.text, &findings, config).map(|findings| SelectedLeaf { + path: leaf.path, + text: leaf.text, + findings, + }) + }) + }) + .collect() +} + +fn leaf_selectors(leaves: &[SelectedLeaf<'_>], config: &TransformationConfig) -> Vec { + let all: Vec<_> = leaves + .iter() + .flat_map(|leaf| leaf.findings.iter().cloned()) + .collect(); + key_selectors(config, &all) +} + +fn leaf_token_items( + leaves: &[SelectedLeaf<'_>], + config: &TransformationConfig, + context: Option<&PrivacyContext>, +) -> Result, PrivacyError> { + let mut items = Vec::new(); + for (index, leaf) in leaves.iter().enumerate() { + for mut item in + super::required_tokenization_items(leaf.text, &leaf.findings, config, context)? + { + item.id = format!("{index}:{}", item.id); + items.push(item); + } + } + Ok(items) +} + +/// Resolve each distinct key once across the whole document. +pub fn required_key_selectors( + data: &Value, + findings: &[StructuredFinding], + config: &TransformationConfig, +) -> Result, PrivacyError> { + Ok(leaf_selectors( + &selected_leaves(data, findings, config)?, + config, + )) +} + +/// Collect one request-wide tokenization batch after validating all findings. +pub fn required_tokenization_items( + data: &Value, + findings: &[StructuredFinding], + config: &TransformationConfig, + context: Option<&PrivacyContext>, +) -> Result, PrivacyError> { + leaf_token_items(&selected_leaves(data, findings, config)?, config, context) +} + +/// Transform all selected fields using complete provider results. Input data is +/// never mutated; failure returns no partial structured result. +pub fn transform_with_provider_results( + data: &Value, + findings: &[StructuredFinding], + config: &TransformationConfig, + context: Option<&PrivacyContext>, + keys: Vec, + token_results: Vec, +) -> Result { + let leaves = selected_leaves(data, findings, config)?; + let keys = validate_resolved_keys(leaf_selectors(&leaves, config), keys)?; + let tokens = + validate_tokenize_results(&leaf_token_items(&leaves, config, context)?, token_results)?; + let mut output = data.clone(); + let mut transformations = Vec::new(); + for (index, leaf) in leaves.iter().enumerate() { + let prefix = format!("{index}:"); + let local_tokens = tokens + .iter() + .filter_map(|(id, token)| { + id.strip_prefix(&prefix) + .map(|id| (id.to_owned(), token.clone())) + }) + .collect(); + let result = + apply_transformations(leaf.text, &leaf.findings, config, &keys, &local_tokens)?; + *output + .pointer_mut(&leaf.path) + .ok_or_else(|| PrivacyError::internal("structured field disappeared"))? = + Value::String(result.text); + transformations.extend(result.transformations.into_iter().map(|transformation| { + StructuredTransformation { + path: leaf.path.clone(), + transformation, + } + })); + } + Ok(StructuredTransformResult { + data: output, + transformations, + }) +} + +/// Transform explicit structured findings without invoking detection. +pub fn transform( + data: &Value, + findings: &[StructuredFinding], + config: &TransformationConfig, +) -> Result { + if let Some(selector) = required_key_selectors(data, findings, config)? + .into_iter() + .next() + { + return Err(PrivacyError::provider_required(selector.path)); + } + transform_with_provider_results(data, findings, config, None, Vec::new(), Vec::new()) +} + +/// Reusable detection and transformation configuration for structured data. +#[derive(Debug, Clone)] +pub struct StructuredScanAndTransformConfig { + pub scan: StructuredScanConfig, + pub transform: TransformationConfig, +} + +pub fn parse_scan_and_transform_config( + value: &Value, +) -> Result { + let object = require_object( + value, + "", + "scan-and-transform configuration must be an object", + )?; + reject_unknown_fields(object, &["scan", "transform"], "")?; + let scan = object + .get("scan") + .map(parse_scan_config) + .transpose() + .map_err(|error| error.prefixed("/scan"))? + .unwrap_or_default(); + let transform = object.get("transform").ok_or_else(|| { + PrivacyError::invalid_configuration( + PrivacyErrorReason::MissingField, + "/transform", + "transformation configuration is required", + ) + })?; + let transform = + parse_transformation_config(transform).map_err(|error| error.prefixed("/transform"))?; + Ok(StructuredScanAndTransformConfig { scan, transform }) +} + +pub fn scan_and_transform( + data: &Value, + config: &StructuredScanAndTransformConfig, +) -> Result { + let findings = scan(data, &config.scan)?.findings; + transform(data, &findings, &config.transform).map_err(|error| error.prefixed("/transform")) +} + +impl PrivacyManager { + /// Validate the whole document, resolve keys once, and issue one token batch. + pub async fn transform_structured( + &self, + data: &Value, + findings: &[StructuredFinding], + config: &TransformationConfig, + context: Option<&PrivacyContext>, + ) -> Result { + let leaves = selected_leaves(data, findings, config)?; + let items = leaf_token_items(&leaves, config, context)?; + let mut keys = Vec::new(); + for selector in leaf_selectors(&leaves, config) { + if !self.provider.is_configured() { + return Err(PrivacyError::provider_required(selector.path)); + } + let key = self + .provider + .resolve_key(selector.clone()) + .await + .map_err(|error| PrivacyError::from_provider_error(selector.path.clone(), error))?; + validate_resolved_key(&selector, &key)?; + keys.push(ResolvedKeyBinding::new(selector, key)); + } + let tokens = if items.is_empty() { + Vec::new() + } else { + if !self.token_provider.is_configured() { + return Err(PrivacyError::token_provider_required("/context/scope")); + } + let context = + context.ok_or_else(|| PrivacyError::token_provider_required("/context/scope"))?; + self.token_provider + .tokenize_batch(context.scope(), items) + .await + .map_err(PrivacyError::from_token_provider_error)? + }; + transform_with_provider_results(data, findings, config, context, keys, tokens) + } + + pub async fn scan_and_transform_structured( + &self, + data: &Value, + config: &StructuredScanAndTransformConfig, + context: Option<&PrivacyContext>, + ) -> Result { + let findings = scan(data, &config.scan)?.findings; + self.transform_structured(data, &findings, &config.transform, context) + .await + .map_err(|error| error.prefixed("/transform")) + } +} + +/// A restoration record local to one JSON string value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StructuredRestoration { + pub path: String, + pub restoration: Restoration, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StructuredRestoreResult { + pub data: Value, + pub restorations: Vec, +} + +fn restore_inventory(data: &Value) -> Result, PrivacyError> { + let mut inventory = BTreeMap::new(); + for leaf in leaves(data)? { + for token in parse_tokens(leaf.text)? { + inventory.entry(token.envelope).or_insert(RestoreItem { + id: String::new(), + token_ref: token.token_ref, + resolved_version: token.resolved_version, + payload: token.payload, + }); + } + } + for (index, item) in inventory.values_mut().enumerate() { + item.id = index.to_string(); + } + Ok(inventory) +} + +pub fn required_restore_items( + data: &Value, + _context: &PrivacyContext, +) -> Result, PrivacyError> { + Ok(restore_inventory(data)?.into_values().collect()) +} + +pub fn restore_with_results( + data: &Value, + context: &PrivacyContext, + results: Vec, +) -> Result { + let inventory = restore_inventory(data)?; + let expected: BTreeSet<_> = inventory.values().map(|item| item.id.as_str()).collect(); + let mut values = BTreeMap::new(); + for result in results { + if !expected.contains(result.id.as_str()) + || values.insert(result.id, result.value).is_some() + { + return Err(PrivacyError::invalid_token_material()); + } + } + if values.len() != expected.len() { + return Err(PrivacyError::invalid_token_material()); + } + let mut output = data.clone(); + let mut restorations = Vec::new(); + for leaf in leaves(data)? { + let mut local = BTreeMap::new(); + let mut local_results = Vec::new(); + for token in parse_tokens(leaf.text)? { + if local.contains_key(&token.envelope) { + continue; + } + let id = local.len().to_string(); + let item = inventory + .get(&token.envelope) + .ok_or_else(PrivacyError::invalid_token_material)?; + let value = values + .get(&item.id) + .ok_or_else(PrivacyError::invalid_token_material)?; + local.insert(token.envelope, id.clone()); + local_results.push(RestoredValue::new(id, value.clone())); + } + let result = super::restore_with_results(leaf.text, context, local_results)?; + *output + .pointer_mut(&leaf.path) + .ok_or_else(|| PrivacyError::internal("structured field disappeared"))? = + Value::String(result.text); + restorations.extend(result.restorations.into_iter().map(|restoration| { + StructuredRestoration { + path: leaf.path.clone(), + restoration, + } + })); + } + Ok(StructuredRestoreResult { + data: output, + restorations, + }) +} + +impl PrivacyManager { + pub async fn restore_structured( + &self, + data: &Value, + context: &PrivacyContext, + ) -> Result { + let items = required_restore_items(data, context)?; + let results = if items.is_empty() { + Vec::new() + } else { + if !self.token_provider.is_configured() { + return Err(PrivacyError::token_provider_required("/restore")); + } + self.token_provider + .restore_batch(context.scope(), items) + .await + .map_err(PrivacyError::from_token_provider_error)? + }; + restore_with_results(data, context, results) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn shared_structured_fixtures() { + for line in include_str!("../../../fixtures/structured.jsonl").lines() { + let case: Value = serde_json::from_str(line).unwrap(); + let config = parse_scan_config(case.get("config").unwrap_or(&json!({}))).unwrap(); + let result = scan(&case["data"], &config).unwrap(); + assert_eq!( + serde_json::to_value(&result.mappings).unwrap(), + case["mappings"], + "{}", + case["id"] + ); + assert_eq!( + discover_fields(&case["data"], &config).unwrap(), + result.mappings + ); + let projected: Vec<_> = result.findings.iter().map(|located| { + let finding = &located.finding; + let text = case["data"].pointer(&located.path).unwrap().as_str().unwrap(); + assert_eq!(&text[finding.byte_range.start..finding.byte_range.end], finding.matched_text); + assert_eq!(text.chars().skip(finding.codepoint_range.start).take(finding.codepoint_range.end - finding.codepoint_range.start).collect::(), finding.matched_text); + assert_eq!(finding.confidence, None); + json!({"path": located.path, "label": finding.entity_type, "text": finding.matched_text, "start": finding.codepoint_range.start, "end": finding.codepoint_range.end}) + }).collect(); + assert_eq!(json!(projected), case["findings"], "{}", case["id"]); + } + } + + #[test] + fn aliases_are_exact_and_do_not_classify_prose() { + for alias in [ + "first_name", + "firstName", + "FirstName", + "FIRST_NAME", + "given_name", + "givenName", + "GivenName", + "last_name", + "lastName", + "LastName", + "family_name", + "familyName", + "FamilyName", + "full_name", + "fullName", + "FullName", + "surname", + "Surname", + "SURNAME", + ] { + assert_eq!( + scan(&json!({alias: "May"}), &StructuredScanConfig::default()) + .unwrap() + .findings + .len(), + 1, + "{alias}" + ); + } + for alias in [ + "firstname", + "name", + "first-name", + "name_of_customer", + "first_name_backup", + "customer.firstName", + ] { + assert!( + scan(&json!({alias: "May"}), &StructuredScanConfig::default()) + .unwrap() + .findings + .is_empty(), + "{alias}" + ); + } + assert!(crate::scan(r#"{"first_name":"May"}"#).is_empty()); + } + + #[test] + fn configuration_errors_are_strict_and_value_free() { + for (value, path) in [ + (json!({"discover_person":null}), "/discover_person"), + (json!({"unknown":true}), "/unknown"), + (json!({"mappings":{"name":"PERSON"}}), "/mappings/name"), + ( + json!({"mappings":{"/name~2":"PERSON"}}), + "/mappings/~1name~02", + ), + (json!({"mappings":{"/name":"EMAIL"}}), "/mappings/~1name"), + ( + json!({"mappings":{"/name":"PERSON"},"exclude":["/name"]}), + "/exclude/0", + ), + (json!({"exclude":["/name","/name"]}), "/exclude/1"), + (json!({"locale":null}), "/locale"), + ] { + let error = parse_scan_config(&value).unwrap_err(); + assert_eq!(error.code(), PrivacyErrorCode::InvalidConfiguration); + assert_eq!(error.path(), Some(path)); + } + for value in [ + json!(null), + json!("secret-value"), + json!(4), + json!({"n":9007199254740992u64}), + ] { + let error = scan(&value, &StructuredScanConfig::default()).unwrap_err(); + assert_eq!(error.path(), Some("/data")); + assert!(!error.to_string().contains("secret-value")); + } + } + #[test] + fn shared_structured_transform_fixtures() { + for line in include_str!("../../../fixtures/structured-transform.jsonl").lines() { + let case: Value = serde_json::from_str(line).unwrap(); + let config = parse_scan_and_transform_config(&case["config"]).unwrap(); + let result = scan_and_transform(&case["data"], &config).unwrap(); + assert_eq!(result.data, case["expected_data"], "{}", case["id"]); + let findings = scan(&case["data"], &config.scan).unwrap().findings; + assert_eq!( + transform(&case["data"], &findings, &config.transform).unwrap(), + result + ); + } + } + + #[test] + fn structured_transformations_preserve_structure_and_local_ranges() { + let data = json!({"a/b": {"full_name": "👋 José", "note": "mail a@example.test"}, "count": 3, "name": "Acme", "empty": null}); + let findings = scan(&data, &StructuredScanConfig::default()) + .unwrap() + .findings; + let config = + parse_transformation_config(&json!({"default":{"strategy":"redact"}})).unwrap(); + let result = transform(&data, &findings, &config).unwrap(); + assert_eq!( + result.data, + json!({"a/b": {"full_name":"[PERSON]", "note":"mail [EMAIL]"}, "count":3, "name":"Acme", "empty":null}) + ); + for record in &result.transformations { + let source = data.pointer(&record.path).unwrap().as_str().unwrap(); + let output = result.data.pointer(&record.path).unwrap().as_str().unwrap(); + let record = &record.transformation; + assert!( + !source[record.source_byte_range.start..record.source_byte_range.end].is_empty() + ); + assert_eq!( + &output[record.output_byte_range.start..record.output_byte_range.end], + record.replacement + ); + } + let mask = parse_transformation_config( + &json!({"default":{"strategy":"mask"}, "entities":["PERSON"]}), + ) + .unwrap(); + assert_eq!( + transform(&data, &findings, &mask).unwrap().data["a/b"]["full_name"], + "******" + ); + let remove = parse_transformation_config( + &json!({"default":{"strategy":"remove"}, "entities":["PERSON"]}), + ) + .unwrap(); + assert_eq!( + transform(&data, &findings, &remove).unwrap().data["a/b"]["full_name"], + "" + ); + let allow = parse_transformation_config(&json!({"default":{"strategy":"redact"}, "allow":{"exact":{"PERSON":["👋 José"]}, "regex":{"EMAIL":[{"pattern":".+@example\\.test"}]}}})).unwrap(); + assert_eq!(transform(&data, &findings, &allow).unwrap().data, data); + } + + #[test] + fn malformed_later_finding_fails_before_provider_work() { + use std::sync::atomic::{AtomicUsize, Ordering}; + struct Keys(AtomicUsize); + impl KeyProvider for Keys { + fn resolve_key(&self, _selector: KeySelector) -> KeyProviderFuture<'_> { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(ResolvedKey::new(vec![7; 32], "v1")) }) + } + } + let data = json!({"first_name":"May", "last_name":"Chen"}); + let mut findings = scan(&data, &StructuredScanConfig::default()) + .unwrap() + .findings; + findings[1].finding.byte_range.end = 999; + let config = parse_transformation_config( + &json!({"default":{"strategy":"pseudonymize","key_ref":"names"}}), + ) + .unwrap(); + let manager = PrivacyManager::new(Keys(AtomicUsize::new(0))); + let error = futures::executor::block_on( + manager.transform_structured(&data, &findings, &config, None), + ) + .unwrap_err(); + assert_eq!(error.finding_index(), Some(1)); + assert_eq!(error.path(), Some("/findings/1/finding/byte_range")); + assert_eq!(manager.provider().0.load(Ordering::SeqCst), 0); + findings[1].path = "/absent".into(); + assert_eq!( + transform(&data, &findings, &config).unwrap_err().path(), + Some("/findings/1/path") + ); + } + + #[test] + fn structured_keys_are_deduplicated_across_fields() { + use std::sync::atomic::{AtomicUsize, Ordering}; + struct Keys(AtomicUsize); + impl KeyProvider for Keys { + fn resolve_key(&self, _: KeySelector) -> KeyProviderFuture<'_> { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async { Ok(ResolvedKey::new(vec![3; 32], "v1")) }) + } + } + let data = json!({"first_name":"May", "last_name":"May"}); + let config = parse_scan_and_transform_config( + &json!({"transform":{"default":{"strategy":"pseudonymize","key_ref":"names"}}}), + ) + .unwrap(); + let manager = PrivacyManager::new(Keys(AtomicUsize::new(0))); + let result = futures::executor::block_on( + manager.scan_and_transform_structured(&data, &config, None), + ) + .unwrap(); + assert_eq!(result.data["first_name"], result.data["last_name"]); + assert_ne!(result.data["first_name"], "May"); + assert_eq!(manager.provider().0.load(Ordering::SeqCst), 1); + } + + #[test] + fn token_batches_are_document_wide_and_restore_is_deduplicated() { + use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }; + #[derive(Default)] + struct Vault { + stored: Mutex, String>>, + calls: AtomicUsize, + restore_items: AtomicUsize, + } + impl TokenProvider for Vault { + fn tokenize_batch( + &self, + scope: &str, + items: Vec, + ) -> TokenizeProviderFuture<'_> { + let scope = scope.to_owned(); + Box::pin(async move { + assert_eq!(scope, "test-scope"); + self.calls.fetch_add(1, Ordering::SeqCst); + assert_eq!(items.len(), 2); + let mut stored = self.stored.lock().unwrap(); + Ok(items + .into_iter() + .enumerate() + .map(|(index, item)| { + let payload = index.to_string().into_bytes(); + stored.insert(payload.clone(), item.exact_value.clone()); + TokenizeResult::new(item.id, payload, "v1") + }) + .collect()) + }) + } + fn restore_batch( + &self, + scope: &str, + items: Vec, + ) -> RestoreProviderFuture<'_> { + let scope = scope.to_owned(); + Box::pin(async move { + assert_eq!(scope, "test-scope"); + self.restore_items.store(items.len(), Ordering::SeqCst); + let stored = self.stored.lock().unwrap(); + Ok(items + .into_iter() + .map(|item| RestoredValue::new(item.id, stored[&item.payload].clone())) + .collect()) + }) + } + } + let data = json!({"first_name":"May", "last_name":"May"}); + let context = PrivacyContext::new("test-scope").unwrap(); + let manager = PrivacyManager::token_provider_only(Vault::default()); + let config = parse_scan_and_transform_config( + &json!({"transform":{"default":{"strategy":"tokenize","token_ref":"names"}}}), + ) + .unwrap(); + let result = futures::executor::block_on(manager.scan_and_transform_structured( + &data, + &config, + Some(&context), + )) + .unwrap(); + assert_eq!(manager.token_provider().calls.load(Ordering::SeqCst), 1); + assert_ne!(result.data["first_name"], result.data["last_name"]); + let mut repeated = result.data.clone(); + repeated["copy"] = result.data["first_name"].clone(); + let restored = + futures::executor::block_on(manager.restore_structured(&repeated, &context)).unwrap(); + assert_eq!( + restored.data, + json!({"first_name":"May", "last_name":"May", "copy":"May"}) + ); + assert_eq!( + manager + .token_provider() + .restore_items + .load(Ordering::SeqCst), + 2 + ); + assert_eq!(restored.restorations.len(), 3); + assert!(restore_with_results(&repeated, &context, vec![]).is_err()); + assert_eq!( + scan_and_transform(&data, &config).unwrap_err().code(), + PrivacyErrorCode::TokenProviderRequired + ); + } + + #[test] + fn structured_limits_follow_json_transport_and_invalid_values_are_atomic() { + let mut data = json!({"first_name":"May"}); + for _ in 0..127 { + data = json!([data]); + } + assert!(scan(&data, &StructuredScanConfig::default()).is_err()); + assert!(parse_document_json(r#"{"first_name":"May","n":1e999}"#).is_err()); + let data = json!({"first_name":"a@example.test"}); + let findings = scan(&data, &StructuredScanConfig::default()) + .unwrap() + .findings; + let config = parse_transformation_config( + &json!({"default":{"strategy":"redact"},"entities":["PERSON"]}), + ) + .unwrap(); + assert_eq!( + transform(&data, &findings, &config).unwrap().data["first_name"], + "[PERSON]" + ); + } +} diff --git a/docs/.mintignore b/docs/.mintignore index ecc8053..34a1cad 100644 --- a/docs/.mintignore +++ b/docs/.mintignore @@ -1,3 +1,5 @@ adr/ privacy-capability-matrix.md privacy-operations-roadmap.md +person-detection-plan.md +structured-performance.md diff --git a/docs/adr/002-structured-person-discovery.md b/docs/adr/002-structured-person-discovery.md new file mode 100644 index 0000000..b27a1ad --- /dev/null +++ b/docs/adr/002-structured-person-discovery.md @@ -0,0 +1,145 @@ +# ADR 002: Structured PERSON discovery and protection + +- **Status:** Implemented; pending release +- **Date:** 2026-09-04 +- **Extends:** [ADR 001](001-privacy-core-contract.md) + +## Decision + +Add structured JSON operations in `datafog_core::structured`. Keep the existing +text operations unchanged. All detection, traversal, mapping rules, selection, +and transformation semantics live in Rust. Bindings convert JSON inputs and +finding/record representations and coordinate application providers. + +The input is a parsed JSON object or array. Scan string values; preserve numeric, +boolean, and null values. Do not scan object keys or coerce non-string values. +Binding transport uses JSON serialization with strict conversion checks: reject +cycles, non-JSON objects, non-string object keys, non-finite numbers, and integers +outside JavaScript's safe range, ±(2^53−1). This is a shared portability limit, +not a detector threshold. Python accepts dict/list containers; tuples are rejected. +JavaScript accepts plain objects and dense arrays, without accessors, symbol +keys, undefined values, or custom serialization methods. Repeated references +are serialized as independent JSON subtrees; cycles are rejected. +JavaScript options omit object properties whose value is `undefined`, including +optional finding metadata; document data still rejects undefined values. + +Use serde_json's default nesting contract: fewer than 128 nested containers. +Apply the same limit to parsed Rust values before cloning/transformation. +Conversion failures produce value-free `invalid_configuration` errors. Input +serialization preserves JSON values, not original formatting or source bytes. + +## Discovery and configuration + +The structured scan configuration is: + +```json +{ + "locale": "en-US", + "discover_person": true, + "mappings": {"/customer/name": "PERSON"}, + "exclude": ["/example/first_name"] +} +``` + +All members are optional. Automatic discovery defaults to enabled. `locale` +retains the existing scanner behavior; it does not enable other field-label +languages. Mappings currently support PERSON only. Explicit mappings remain +active when `discover_person` is false. Exclusions suppress automatic PERSON +classification only; other detectors continue to run. An explicit mapping and +exclusion on the same path are an error. Repeated exclusions are errors. Empty +maps and arrays are accepted; unknown options and explicit null options are not. + +Paths are non-root RFC 6901 JSON Pointers. There is no wildcard, suffix, dotted +path, or cross-record matching. Missing paths and non-string targets have no +mapping or finding. Map a string array element by its concrete index. Do not +inherit an alias from a container into its descendant values. + +Built-in canonical aliases are `first_name`, `given_name`, `last_name`, +`family_name`, `full_name`, and `surname`. Snake-case aliases accept ASCII case +variants. The two-word aliases also accept their exact camelCase and PascalCase +spellings. `Surname` is covered by the case-insensitive single-word alias. +There is no substring, separator-removal, fuzzy, or dictionary matching. +`name`, `customer.name`, `package.name`, and ordinary prose do not infer PERSON. + +A mapping reports `path`, `entity_type: PERSON`, `source`, and `rule`. Source is +`field_alias` or `explicit_mapping`; rule is the canonical alias or +`explicit_mapping`. It contains no field value, although application field names +can themselves be sensitive. Empty string values can have mappings but do not +have PERSON findings. Null and non-string values have neither. + +For a string containing any non-whitespace character, a mapped PERSON finding +selects the entire original string, including surrounding whitespace. It does +not infer which substrings are given/family names. Detector names are +`datafog-core/person/field_alias` and `datafog-core/person/explicit_mapping`. +Confidence remains absent, and detector version is the Core package version. + +## Operations and results + +Rust exposes `discover_fields`, `scan`, `transform`, `scan_and_transform`, and +provider coordination functions in the `structured` module. Python exposes +`discover_fields`, `scan_structured`, `transform_structured`, and +`scan_and_transform_structured`. JavaScript uses `discoverFields`, +`scanStructured`, `transformStructured`, and `scanAndTransformStructured`. + +Scan returns `{ mappings, findings }`. Each finding is `{ path, finding }`, +where the nested finding uses the existing text finding contract. Discovery is +also callable alone; ordinary structured scanning includes discovery automatically. + +Every string leaf still runs the seven existing text detectors. Findings from +PERSON and existing detectors can overlap. Existing transformation selection +and overlap rules apply unchanged; selecting only PERSON can choose it over an +otherwise overlapping EMAIL finding. Callers must not interpret an unresolved +field as safe. + +Object keys are traversed in lexicographic Rust string order and arrays in +numeric index order, depth first. Findings within a leaf sort by start byte, +then decreasing end byte, then entity type. Mapping order follows leaf order. +No global schema cache or propagation between heterogeneous records is added. + +Byte/code-point ranges address the exact decoded string at the finding's path. +JavaScript also exposes UTF-16 ranges. No range addresses serialized JSON bytes. +The transformation result is `{ data, transformations }`; each record is +`{ path, transformation }` with the existing source/output range and provenance +fields. Unrelated values and container structure are preserved. Results have +no plaintext mapping or original matched text in transformation records. + +Explicit transformation validates all supplied paths/findings before selection. +Invalid findings carry their original list index and a path such as +`/findings/1/path` or `/findings/1/finding/byte_range`. No partial structured +result accompanies an error and input data is not mutated. + +Scan-and-transform uses `{ scan?: StructuredScanConfig, transform: +TransformationConfig }`. Scan and transformation settings stay separate. Its +transformation-stage errors receive the same `/transform` prefix as text calls. + +## Providers + +Rust, Python, and Node managers expose structured transform, scan-and-transform, +and restore methods. Validate the complete request before provider work. Resolve +each distinct key once, then issue one tokenization batch for the document. +Repeated values remain separate tokenization items. Provider correlation IDs +are opaque and must not be interpreted as field paths or indices. + +Restoration validates all token envelopes before calling the provider, +deduplicates identical envelopes across fields, and issues one restoration +batch. Every token occurrence is restored or no structured result is returned. +Provider scope checks, errors, and exact-value semantics remain those of ADR 001. +Result atomicity does not imply rollback of external provider side effects. +Node snapshots structured inputs and options before asynchronous provider calls. + +Browser WASM supports stateless structured operations. Selected provider-backed +strategies and structured restoration produce `unsupported_strategy`; input and +configuration validation still applies. Core makes no network calls and adds +no model, dictionary, or runtime dependency for discovery. + +## Proof and limits + +Shared structured detection and transformation fixtures run through Rust and +installed bindings. Additional tests cover strict conversion, Unicode ranges, +invalid later findings before provider work, key deduplication, document-wide +token batches, scoped restoration, and preservation of special JSON keys. + +See [the implementation plan](../person-detection-plan.md) and +[performance measurements](../structured-performance.md). Coverage is +schema-guided PERSON protection. Arbitrary prose recognition, learned schemas, +additional formats, and cross-session known-name matching are separate work. diff --git a/docs/concepts/findings-and-ranges.mdx b/docs/concepts/findings-and-ranges.mdx index 18754fb..a85ef23 100644 --- a/docs/concepts/findings-and-ranges.mdx +++ b/docs/concepts/findings-and-ranges.mdx @@ -73,3 +73,11 @@ Overlapping findings are resolved deterministically using structural span, length, confidence when both values are present, source position, entity type, and detector provenance. Selected transformations are returned in source document order. + +## Structured findings + +`scan_structured` / `scanStructured` returns located findings with `path` and +`finding` fields. The path is an RFC 6901 JSON Pointer. Every range in the nested +finding addresses the decoded string value at that path, not serialized JSON. +Structured transformation/restoration records follow the same field-local rule +for source and output strings. See [person-field discovery](/guides/person-discovery). diff --git a/docs/docs.json b/docs/docs.json index 00770ee..185417a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -31,7 +31,8 @@ "guides/migrating-from-datafog-python", "guides/configuration", "guides/pseudonymization", - "guides/tokenization-and-restoration" + "guides/tokenization-and-restoration", + "guides/person-discovery" ] }, { diff --git a/docs/guides/person-discovery.mdx b/docs/guides/person-discovery.mdx new file mode 100644 index 0000000..a10bff1 --- /dev/null +++ b/docs/guides/person-discovery.mdx @@ -0,0 +1,153 @@ +--- +title: "Discover and protect person fields" +description: "Detect names from JSON field context without downloading a model." +icon: "user-shield" +--- + + + Structured PERSON support is implemented and tested locally. Its package + release is pending. + + +Structured scanning discovers person-name fields automatically and runs the +existing text detectors on every string value. It uses field context, so names +do not need to appear in a dictionary. This capability recognizes supported +schemas; it does not recognize arbitrary person names in prose. + +## Scan and protect a record + +```js +import { discoverFields, scanStructured, scanAndTransformStructured } from "@datafog/node"; + +const data = { + customer: { firstName: "May", last_name: "Nguyễn", email: "may@example.test" }, + package: { name: "Rose" }, +}; + +const analysis = scanStructured(data); +// analysis.mappings identifies /customer/firstName and /customer/last_name. +// analysis.findings also includes the email in /customer/email. + +const result = scanAndTransformStructured(data, { + transform: { default: { strategy: "redact" } }, +}); +// result.data.customer is: +// { firstName: "[PERSON]", last_name: "[PERSON]", email: "[EMAIL]" } +// result.data.package.name remains "Rose". +``` + +Use `discoverFields(data)` when you need mapping evidence alone. No separate +mapping-discovery call or approval step is required before `scanStructured`. +The browser package exposes the same stateless methods after `await init()`. + +```python +from datafog_core import scan_structured, scan_and_transform_structured + +record = {"first_name": "May", "last_name": "Nguyễn"} +analysis = scan_structured(record) +result = scan_and_transform_structured( + record, {"transform": {"default": {"strategy": "redact"}}} +) +assert result.data == {"first_name": "[PERSON]", "last_name": "[PERSON]"} +``` + +Rust uses the `datafog_core::structured` module and `serde_json::Value`: + +```rust +use datafog_core::structured; +use serde_json::json; + +fn main() -> Result<(), datafog_core::PrivacyError> { + let data = json!({"first_name": "May"}); + let analysis = structured::scan(&data, &Default::default())?; + let config = structured::parse_scan_and_transform_config(&json!({ + "transform": {"default": {"strategy": "redact"}} + }))?; + let result = structured::scan_and_transform(&data, &config)?; + assert_eq!(result.data["first_name"], "[PERSON]"); + Ok(()) +} +``` + +## Supported field names + +Canonical aliases are `first_name`, `given_name`, `last_name`, `family_name`, +`full_name`, and `surname`. Snake-case aliases accept ASCII case variants such +as `FIRST_NAME`. Two-word aliases also accept exact camelCase and PascalCase +spellings, such as `firstName` and `FirstName`. + +Generic `name` fields remain unresolved, including `customer.name`: a customer +could be a company. There is no fuzzy matching; `firstname`, `first-name`, and +`first_name_backup` are not aliases. Field-label coverage is this explicit list; +name values can contain any valid Unicode. Locale does not translate field names. + +## Explicit mappings and exclusions + +```js +const scan = { + mappings: { "/customer/name": "PERSON" }, + exclude: ["/example/first_name"], +}; +const analysis = scanStructured(data, scan); +``` + +Paths are concrete JSON Pointers. `/users/0/name` addresses an array element; +`/a~1b/~0name` addresses keys `a/b` and `~name`. Dots are literal key characters. +Wildcards and inherited container mappings are not supported. Missing paths and +non-string targets have no effect. Explicit mappings currently support PERSON. + +Set `discover_person: false` to disable automatic aliases while retaining +explicit mappings and the existing detectors. Exclusions suppress only automatic +PERSON discovery. They do not exempt an EMAIL or other finding in the same field. +Use transformation allowlists for value exemptions. Mapping and excluding the +same path is an error, as are duplicate exclusions and unknown options. + +Mappings report a path, entity type, source (`field_alias` or `explicit_mapping`), +and rule. They contain no field values. Empty strings can have a mapping without +a PERSON finding. Whitespace-only strings have no PERSON finding; otherwise the +entire original string is selected, including surrounding whitespace. + +## Findings and transformations + +A structured finding contains `path` and `finding`. Its ranges apply to the +**decoded string at that path**, using the existing byte, code-point, and +JavaScript UTF-16 coordinate systems. They are not serialized-document offsets. +See [Findings and ranges](/concepts/findings-and-ranges). + +`transformStructured(data, analysis.findings, policy)` transforms explicit +findings. `scanAndTransformStructured(data, { scan, transform: policy })` +performs both operations. Results contain `data` and records shaped as +`{ path, transformation }`, with source and output ranges local to that field. +Input data is not mutated. An invalid finding fails the complete request. + +Selection, allowlists, masking, overrides, and overlap rules are shared with +text transformations. PERSON can overlap another detector's finding. Use +`entities: ["PERSON"]` when you want only name-field protection. + +Rust, Python, and Node `PrivacyManager` instances also support structured +pseudonymization, tokenization, and restoration. JavaScript methods are +`transformStructured`, `scanAndTransformStructured`, and `restoreStructured`; +Python and Rust use snake_case. Supply providers and request scope as described +in [Tokenization and restoration](/guides/tokenization-and-restoration). +Keys resolve once per selector and tokenization uses one document-wide batch. +Restoration deduplicates identical tokens across fields. Browser WASM rejects +provider-backed operations with `unsupported_strategy`. + +## Input boundaries + +Supply a JSON object or array. String values are scanned; keys, numbers, +booleans, and null values are not scanned. Null/empty names produce no PERSON +finding. Unresolved fields still receive the existing text detectors, but a +clean result does not establish that they contain no names. + +Inputs require finite numbers and integers in JavaScript's safe range, +±(2^53−1), consistently across bindings. Nesting follows serde_json's default +limit of fewer than 128 containers. Cycles and unsupported runtime values are +rejected instead of silently coerced. Python accepts dict/list containers with +string keys; JavaScript accepts plain objects and dense arrays without accessors, +symbol keys, or undefined values. Serialized formatting and object key order +are not preserved. Special keys such as `__proto__` remain ordinary data. + +The existing `scan(text)` API is unchanged. Passing raw JSON text to it does +not enable field discovery. CSV, SQL schemas, logs, and prose name recognition +are outside this structured API's coverage. diff --git a/docs/person-detection-plan.md b/docs/person-detection-plan.md new file mode 100644 index 0000000..6683cc7 --- /dev/null +++ b/docs/person-detection-plan.md @@ -0,0 +1,239 @@ +# PERSON detection with automatic field discovery + +**Status: Core and binding implementation complete; release and downstream adoption pending.** + +The implemented contract is [ADR 002](adr/002-structured-person-discovery.md). +Verification and measurement results are recorded in [structured performance](structured-performance.md). + +## Outcome and release proof + +A caller submits a parsed JSON object or array. Core discovers documented +person-name fields, returns path-qualified `PERSON` findings, and supports +protecting the selected values through its existing transformation semantics. +Explicit mappings and exclusions handle application-specific schemas. No model, +dictionary download, or network access is required for detection. + +The first release is proven by positive and negative shared fixtures across +Rust, Python, Node, and browser WASM; unchanged text-scanner conformance; +structured transformation and error tests; and a measured performance and size +comparison with the baseline. Published coverage must describe field discovery, +not arbitrary recognition of names in prose. + +Core and its bindings ship together. The current MCP server and other consumers +adopt the verified package afterward. The historical Claude plugin is not the +architecture or API reference for this work. + +## Recommended first-release behavior + +| Input context | PERSON behavior | +| --- | --- | +| Documented unambiguous aliases such as `first_name`, `firstName`, `given_name`, `last_name`, `family_name`, and `full_name` | Infer a mapping and select the non-empty string value. | +| Caller-supplied field mapping | Apply the declared classification, with explicit provenance. | +| Caller exclusion from automatic PERSON discovery | Do not infer PERSON for that field; continue existing detectors. | +| `name` or `customer.name` without an explicit mapping | Leave unresolved; customers may be organizations. | +| `package.name`, `file.name`, and name-like words inside ordinary text | Do not infer PERSON. | +| Unknown fields | Run the existing text detectors on string values; lack of a PERSON mapping does not imply safety. | +| Null, empty, or whitespace-only name values | Produce no PERSON span. | +| Objects, arrays, numbers, and booleans | Traverse containers; preserve non-string leaves without coercing them into names. | + +Resolve aliases through a documented finite list and explicit naming-convention +normalization. Do not use substring matches, fuzzy spelling, or broad synonyms. +Values are selected because of their schema context, not dictionary membership; +uncommon names and non-Latin names therefore remain eligible. Do not infer a +given/family-name split from the contents of a full-name field. For any value +containing a non-whitespace character, select the entire original string, +including surrounding whitespace; whitespace-only values have no finding. + +An inferred mapping records an unambiguous field path and the rule that produced +it. Keep mapping evidence separate from `Finding.confidence`; deterministic +rules continue to omit a numeric confidence. Mapping summaries need not contain +the field's value. Treat paths as potentially sensitive metadata when downstream +applications decide what to log. + +Automatic PERSON discovery is enabled in the new structured flow. Existing +`scan(text)`, `transform(text, ...)`, and text scan-and-transform calls retain +their current contracts. A raw JSON string passed to `scan(text)` does not gain +implicit JSON parsing. + +## Core boundary and coordinate systems + +Put traversal, alias rules, mapping precedence, and PERSON classification in +`crates/core`. Use the existing `serde_json` dependency where appropriate. +Bindings translate supported JSON values and results; they do not duplicate +classification rules. Start with parsed JSON data. CSV, SQL schema discovery, +source-code parsing, and original serialized-JSON byte preservation are outside +this release. + +A structured finding consists of a field path plus the existing finding for +that string leaf. Its byte and code-point ranges address the exact decoded +string passed to the detector; JavaScript bindings additionally expose UTF-16 +ranges. These are not offsets into the serialized JSON document. Source/output +ranges in structured transformation records use the corresponding leaf strings. + +Use concrete RFC 6901 JSON Pointers to address object keys and array indices. +Escape keys containing `/` or `~`; dots in keys are literal. Do not introduce +implicit wildcard or suffix path matching. Discover mappings for each input; +do not add a global cache or automatically generalize one record's mapping to +other records. Explicit missing paths have no effect on that document, while +malformed paths and conflicting declarations are configuration errors. Define +mapping-versus-exclusion conflicts as errors rather than silently choosing one. + +Keep discovery inspectable, and compose discovery, scanning, and transformation +in a convenient structured workflow. Public names and signatures are specified +in ADR 002. Preserve explicit-findings transformation and the +separate scan-then-transform convenience pattern from ADR 001. + +Structured transformation must validate the entire request before returning +changes, preserve unrelated values and container structure, and return no +partial result on failure. Preserve existing entity selection, allowlists, +overlap resolution, overrides, and provider restrictions. Document and test +overlapping PERSON and existing detector findings rather than changing priority +rules incidentally. + +## Implementation sequence + +### 1. Contract, fixtures, and baseline + +**Status: implemented.** + +- Write a proposed ADR extending [ADR 001](adr/001-privacy-core-contract.md) + with the structured operation signatures, JSON input representation, result + shapes, mapping provenance, error paths, and deterministic traversal order. +- Finalize the alias list and normalization examples. Document PERSON-only + automatic-discovery exclusions separately from transformation allowlists. +- Specify how invalid non-JSON inputs, numeric representation, malformed + mappings, missing paths, and non-string mapped values behave consistently + across bindings. Avoid silent value loss during binding conversion. +- Add expected structured fixture cases without changing the old text fixtures. +- Capture release-build baselines for existing scanning, binding startup, and + package size, plus representative structured payloads for the new workflow. + +**Exit evidence:** precise examples and expected outcomes cover the contract; +baseline commands and environment are reproducible. Proposed APIs are clearly +distinguished from shipped functionality. + +### 2. Automatic discovery and scanning vertical slice + +**Status: implemented and verified through installed bindings.** + +- Implement JSON traversal, alias discovery, explicit mappings/exclusions, and + PERSON findings in Core as distinct logical components where needed. +- Scan every string leaf with the existing detectors, including unresolved + fields. Keep non-string values intact and document their coverage boundary. +- Expose the same capability through Python, Node, and WASM in this slice. +- Run the shared structured fixtures through Rust and installed bindings; + retain the existing text fixture suite as regression evidence. +- Add discovery/scanning API documentation with examples and limitations. + +**Exit evidence:** common name fields work without caller mappings; ambiguous +and unrelated fields do not become PERSON; explicit mappings and exclusions +behave identically across runtimes; every returned path and range selects the +reported value. + +### 3. Structured protection and Core release + +**Status: implementation and local verification complete. Publishing remains pending.** + +- Compose the findings with existing transformation semantics and reconstruct + the structured result in Core. Support explicit-findings transformation and + the convenience workflow without duplicating detector logic. +- Prove stateless transformations across all bindings. Integrate provider-backed + transformations/restoration through the existing manager boundary in Rust, + Python, and Node; browser WASM retains its unsupported-strategy behavior. +- Define provider collection, batching, and validation for a whole structured + request before implementation. Core result atomicity does not promise rollback + of external provider side effects. +- Complete the public guides, binding references, release notes, and benchmarks. +- Publish compatible Core and binding packages after the required checks pass. + +**Exit evidence:** the complete discover/scan/protect workflow passes across +supported runtimes, provider behavior preserves the existing contract, and +measured overhead and known limitations are reported with the release. + +### 4. Downstream adoption + +**Status: pending verified release and identification of the current MCP consumer.** + +- Update the current MCP consumer to the verified Core package. It supplies + structured payloads, application mappings, and transformation policy. +- Keep allow/warn/block decisions and tool interception downstream. Do not copy + name rules or structured transformation semantics into that integration. +- Test the complete customer-record flow from tool payload to model-visible + output, including unrelated fields, error responses, and integration-specific + handling of scanner failure. +- Exercise any text-only tool output path explicitly: the new structured + capability does not claim name recognition in arbitrary logs or prose. + +Downstream development may use the settled API and a prerelease; its public +release depends on a verified Core/binding release. No change to the historical +Claude plugin is implied by this plan. + +## Required test matrix + +| Area | Proof | +| --- | --- | +| Discovery positives | Every documented alias and naming convention; nested records; arrays; uncommon and non-Latin names. | +| Discovery negatives | Generic `name`; organization/customer ambiguity; package and file names; key substrings; ordinary prose; aliases appearing only in values. | +| Mapping policy | Explicit mapping, exclusion, provenance, conflicting declarations, malformed pointers, absent paths, and non-string targets. | +| Structure | Empty objects/arrays, nulls, mixed arrays, heterogeneous records, nested containers, and keys containing dots, slashes, or tildes. No mapping propagation between unrelated records. | +| Ranges | Apostrophes, hyphens, combining marks, emoji, whitespace, and escaped input examples. Byte/code-point/UTF-16 slices select the exact decoded value. | +| Transformations | Redact, mask, remove, selection, overrides, exact/regex allowlists, overlapping findings, unrelated-value preservation, output ranges, and atomic errors. | +| Providers | Pseudonymization consistency, scoped tokenization/restoration, repeated values across fields, whole-request error behavior, and WASM restrictions. | +| Regression | Existing seven-detector fixtures, text APIs, strict configuration validation, and installed-package behavior remain valid. | + +Use shared synthetic fixtures for deterministic conformance. Maintain separate +evaluation examples covering realistic schemas to report false mappings and +missed mappings; do not claim general prose-name recall from field fixtures. +Test fallible paths without silently skipping failed fields or panicking on +user-controlled input. Reuse existing test runners instead of creating a +parallel binding test framework. + +Before merging Rust changes, run the repository-required commands: + +```sh +cargo fmt --all --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test --workspace --all-features +``` + +Run installed Python, Node, and browser/WASM suites using +[Development](development.mdx), plus the documentation validation commands +there when the public documentation changes. + +## Performance and documentation + +Measure cold-start latency, repeated-scan throughput, memory/allocation behavior, +binding conversion and per-field costs, and native/WASM package size. Include +ordinary code/log strings, Unicode names, nested customer records, and repeated +records with and without discoverable fields. Verify offline runtime operation +without downloaded assets. Do not add a dependency or a mapping cache without +evidence that the current implementation needs it. + +No numerical performance budget is established yet. Use the baseline and actual +product latency requirements to establish any acceptance budget before using it +as a release gate; do not invent a percentage or latency promise. + +Ship a person-discovery guide, configuration/mapping reference, structured range +examples, all binding API references, README coverage updates, and release notes. +Document built-in aliases, override/exclusion rules, unresolved fields, language +coverage of field labels, non-string limitations, and the difference between +schema-guided protection and names recognized in prose. Update the capability +matrix and roadmap status only as functionality lands. + +## Work outside this release + +Known-name matching across a session, model-based or dictionary-based prose +recognition, fuzzy schema inference, additional document formats, and credential +detectors are independent follow-on work. None is required to establish this +release's automatic field-discovery contract. + +## Draft release note + +Add model-free PERSON discovery and protection for parsed JSON records across +Rust, Python, Node, and browser WASM. Common given/family/full-name field labels +work automatically; explicit JSON Pointer mappings and exclusions handle custom +schemas. Findings and transformation records include field paths and local +ranges. Rust, Python, and Node managers support structured provider operations. +Existing text APIs retain their coverage and behavior. See the person-discovery +guide for supported aliases, input limits, and the distinction from prose-name +recognition. Package versions and publication are pending. diff --git a/docs/privacy-capability-matrix.md b/docs/privacy-capability-matrix.md index e24e21f..950df02 100644 --- a/docs/privacy-capability-matrix.md +++ b/docs/privacy-capability-matrix.md @@ -7,6 +7,8 @@ for a PII detection and transformation engine. | Capability | Treatment | Core direction | | --- | --- | --- | | PII scanning | Preserve | Return validated findings with explicit byte and code-point ranges, optional confidence, and detector provenance; JavaScript bindings also expose explicit UTF-16 ranges. | +| Structured PERSON discovery | New; pending release | Discover exact name-field aliases, apply explicit JSON Pointer mappings/exclusions, scan string leaves, and return path-qualified findings; preserve text APIs. | +| Structured transformations | New; pending release | Reuse transformation and provider semantics across selected string leaves with document-wide validation and batching. | | Typed redaction | Preserve | Replace findings with typed, document-local placeholders after deterministic overlap resolution. | | Character masking | Preserve | Mask Unicode code points with a validated character and explicit leading- or trailing-reveal semantics. | | Entity-type selection | Preserve | Select canonical entity types through transformation configuration. | diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index d47ffd6..0c506ba 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -229,6 +229,19 @@ and browser WASM continues to reject key- and token-provider work explicitly. Pseudonymization and reversible token storage are not promised in browser WASM without a separately accepted host-managed key-custody boundary. +## Next workstream: PERSON detection with automatic field discovery + +**Status: Core and bindings implemented; release and downstream adoption pending.** + +The [PERSON detection plan](person-detection-plan.md) extends Core with +conservative automatic name-field discovery for structured JSON data, explicit +mapping overrides, and path-qualified findings. It includes shared tests, +transformation integration, performance measurement, and documentation. Core +and its bindings ship together before downstream MCP adoption. + +This workstream adds schema-guided PERSON coverage. It does not change the +completed privacy-operation slices or promise general prose name recognition. + ## Acceptance bar Every completed slice must have: diff --git a/docs/reference/browser-wasm.mdx b/docs/reference/browser-wasm.mdx index ac8163a..ad88306 100644 --- a/docs/reference/browser-wasm.mdx +++ b/docs/reference/browser-wasm.mdx @@ -53,3 +53,17 @@ credentials and key custody require a separately designed host integration. The JavaScript wrapper throws `DataFogError` with the same stable fields as the Node.js binding. See [Errors](/reference/errors). + +## Structured JSON and PERSON fields + +Use `discoverFields(data, config?)`, `scanStructured(data, config?)`, +`transformStructured(data, findings, config)`, and +`scanAndTransformStructured(data, config)`. Scan returns `{ mappings, findings }`; +transformation returns `{ data, transformations }`. Located findings contain +`path` and `finding`; located records contain `path` and `transformation`. +Call `await init()` first. Selected provider-backed strategies and +`restoreStructured` reject with `unsupported_strategy`. + +Paths are JSON Pointers. All ranges, including JavaScript UTF-16 ranges, +address the decoded string at that path. See [person-field discovery](/guides/person-discovery) +for aliases, policy configuration, input limits, and examples. diff --git a/docs/reference/node.mdx b/docs/reference/node.mdx index a258343..795820b 100644 --- a/docs/reference/node.mdx +++ b/docs/reference/node.mdx @@ -73,3 +73,17 @@ The package build is configured for: JavaScript operations throw `DataFogError` with stable `code`, optional `reason`, optional RFC 6901 `path`, and optional `findingIndex` properties. + +## Structured JSON and PERSON fields + +Use `discoverFields(data, config?)`, `scanStructured(data, config?)`, +`transformStructured(data, findings, config)`, and +`scanAndTransformStructured(data, config)`. Scan returns `{ mappings, findings }`; +transformation returns `{ data, transformations }`. Located findings contain +`path` and `finding`; located records contain `path` and `transformation`. +`PrivacyManager` adds asynchronous structured transform, scan-and-transform, +and restore methods with the same provider and scope contracts as text calls. + +Paths are JSON Pointers. All ranges, including JavaScript UTF-16 ranges, +address the decoded string at that path. See [person-field discovery](/guides/person-discovery) +for aliases, policy configuration, input limits, and examples. diff --git a/docs/reference/python.mdx b/docs/reference/python.mdx index 99a19fb..7a59dfb 100644 --- a/docs/reference/python.mdx +++ b/docs/reference/python.mdx @@ -90,3 +90,16 @@ async def restore_batch(scope: str, items: list[dict]) -> list[dict]: Exceptions expose stable `code`, optional `reason`, optional `path`, and optional `finding_index` attributes. See [Errors](/reference/errors). + +## Structured JSON and PERSON fields + +Use `discover_fields(data, config=None)`, `scan_structured(data, config=None)`, +`transform_structured(data, findings, config)`, and +`scan_and_transform_structured(data, config)`. Results expose `.mappings` and +`.findings`, or `.data` and `.transformations`. `PrivacyManager` also exposes +`transform_structured`, `scan_and_transform_structured`, and `restore_structured` +for provider-backed work. + +Paths are JSON Pointers. All ranges, including JavaScript UTF-16 ranges, +address the decoded string at that path. See [person-field discovery](/guides/person-discovery) +for aliases, policy configuration, input limits, and examples. diff --git a/docs/reference/rust.mdx b/docs/reference/rust.mdx index ced88e0..f026f7a 100644 --- a/docs/reference/rust.mdx +++ b/docs/reference/rust.mdx @@ -93,3 +93,15 @@ DataFog Core ships no cloud-, vault-, or database-specific provider. For exact public definitions, see the [crate source](https://github.com/DataFog/datafog-core/blob/main/crates/core/src/lib.rs). + +## Structured JSON and PERSON fields + +The `datafog_core::structured` module exposes `discover_fields`, `scan`, +`transform`, and `scan_and_transform`. Use `StructuredScanConfig::default()` +for automatic discovery or `structured::parse_scan_config` for explicit mappings. +`PrivacyManager` adds `transform_structured`, `scan_and_transform_structured`, +and `restore_structured` with the existing provider and scope contracts. + +Paths are JSON Pointers. All ranges, including JavaScript UTF-16 ranges, +address the decoded string at that path. See [person-field discovery](/guides/person-discovery) +for aliases, policy configuration, input limits, and examples. diff --git a/docs/structured-performance.md b/docs/structured-performance.md new file mode 100644 index 0000000..c669c0b --- /dev/null +++ b/docs/structured-performance.md @@ -0,0 +1,129 @@ +# Structured PERSON implementation: verification and measurements + +Recorded 2026-09-04 for the local implementation of +[ADR 002](adr/002-structured-person-discovery.md). Packages are not published. +The baseline is commit `9db22844f6828a4f592c3aed82c75475710e867b`. + +## Verification + +- `cargo fmt --all --check` passed. +- `cargo clippy --workspace --all-targets --all-features -- -D warnings` passed. +- `cargo test --workspace --all-features` passed: 76 tests. +- Installed Python wheel, packed Node package, and packed browser WASM package + passed their existing text fixtures and the shared structured fixtures. +- Node and WASM consumer TypeScript checks cover the new public APIs; the WASM + runtime suite runs in Chromium. +- Mintlify build validation and public-document link checks passed; relative + links in the engineering plan, ADR, and measurement notes also resolve. +- Positive and negative aliases, explicit mappings, exclusions, JSON Pointers, + Unicode ranges, invalid inputs, stateless protection, and special JSON keys + are covered. Provider tests cover request validation before provider work, + key deduplication, token batches, restoration, and scope rejection. Node also + verifies that asynchronous operations use a snapshot of caller inputs. + +There are no new third-party dependencies, downloaded dictionaries, or model +assets. Detection uses Core's finite field-alias rules and existing detectors. +The seven text-detector fixture expectations are unchanged. + +## Environment and method + +macOS on Apple Silicon; Rust/Cargo 1.88.0; Python 3.14.6; Node 24.19.0 for package +tests and structured timings. Raw native-import comparisons below use the same +Node 25.8.2 executable for both artifacts. All native artifacts are release +builds for macOS ARM64. These measurements do not cover other deployment targets. + +The benchmark is [scan_benchmark.rs](../crates/core/examples/scan_benchmark.rs). +Run from the repository root: + +```sh +cargo build -p datafog-core --release --example scan_benchmark +/usr/bin/time -l target/release/examples/scan_benchmark 10000 +/usr/bin/time -l target/release/examples/scan_benchmark 10000 structured +``` + +The baseline used the same text-only benchmark before the structured branch +was added. To repeat against the baseline commit, copy the example to a separate +baseline checkout and omit its `if ... == Some("structured")` block; that block +references the new APIs. Keep the text corpus and measurement loop identical. + +Text timings use 100 development fixtures, 6,606 UTF-8 bytes per iteration, +10,000 iterations, and 1,000,000 findings. Structured timings use the 13 input +documents from `fixtures/structured.jsonl`, including nested data, arrays, +Unicode names, empty values, and unrelated fields. Timing uses default discovery +and redaction for every document, independently of fixture-specific policies. +Core timings start with parsed values and exclude JSON parsing and bindings. + +## Existing text scanner + +| Measurement | Baseline | Candidate | +| --- | ---: | ---: | +| Repeated-scan elapsed time | 0.765740 s | 0.786930 s | +| Throughput | 82.273 MiB/s | 80.058 MiB/s | +| First scan in process | 857.125 µs | 2,161.458 µs | +| Maximum resident set size | 4,341,760 bytes | 4,554,752 bytes | +| Peak memory footprint | 3,064,144 bytes | 3,096,912 bytes | + +These are single runs on a shared machine. The first-scan difference is not an +isolated startup-regression estimate; repeated throughput also has no measured +confidence interval. The evidence establishes behavior parity and records an +initial cost comparison. It does not establish a numerical performance SLA. + +## Structured operations + +| Mean time per small document | Rust Core | Python binding | Node binding | +| --- | ---: | ---: | ---: | +| Discover mappings | 0.545 µs | 3.008 µs | 2.454 µs | +| Discover and scan | 0.953 µs | 3.428 µs | 4.618 µs | +| Discover, scan, and redact | 4.353 µs | 10.634 µs | 11.534 µs | + +Core used 10,000 iterations over the 13 documents. It returned 140,000 mapping +records, 180,000 findings, and 160,000 replacements respectively. The structured +benchmark process reached 4,653,056 bytes maximum RSS. No allocation-count +profiling was performed. + +Bindings used 1,000 iterations over the same parsed documents, calling +`discover_fields`/`discoverFields`, `scan_structured`/`scanStructured`, and +`scan_and_transform_structured`/`scanAndTransformStructured` with default +redaction. Timings include runtime validation, JSON transport, Core work, and +result conversion. They exclude package import, file loading, and provider I/O. +These are end-to-end binding costs, not an isolated measurement of serialization. + +## Native import and artifact size + +| Artifact | Baseline bytes | Candidate bytes | Change | +| --- | ---: | ---: | ---: | +| Python extension | 3,578,112 | 4,167,472 | +589,360 (+16.5%) | +| Node native module | 2,960,192 | 3,222,832 | +262,640 (+8.9%) | +| Browser WASM | — | 1,943,549 | Comparable clean baseline not captured | + +Sizes are uncompressed native payloads, not wheel/tarball transfer sizes or +installed footprints. Python and Node include the whole new structured API, +transformation records, and provider coordination, as well as discovery. + +Five fresh subprocesses loaded each baseline/candidate native extension from +separate directories. Python measured `import datafog_core` with +`time.perf_counter()`; Node measured `require("./datafog.node")` with +`performance.now()`. The median of all five samples was: + +| Native import | Baseline | Candidate | +| --- | ---: | ---: | +| Python | 0.656 ms | 0.726 ms | +| Node | 1.695 ms | 1.884 ms | + +The filesystem and OS loader caches were not cleared. First loads of copied +candidate binaries took 161.326 ms (Python) and 123.274 ms (Node), versus 8.615 ms +and 9.084 ms for previously loaded baseline copies. Earlier initial baseline +loads took 231.823 ms and 218.047 ms. This variation makes these measurements +unsuitable for a cold-start guarantee. Native-import timings omit the JavaScript +wrapper and Python runtime startup. + +## Open claims and release work + +No product latency or package-size budget has been established. Large payloads, +production-schema precision/recall, browser startup, other architectures, and +provider latency remain unmeasured. Synthetic field fixtures cannot establish +general person-name recognition in prose. + +Release/version selection, publication, and migration of the current MCP +consumer remain pending. The current MCP repository must be identified before +its customer-record flow can be tested against these APIs. diff --git a/fixtures/structured-transform.jsonl b/fixtures/structured-transform.jsonl new file mode 100644 index 0000000..6e86756 --- /dev/null +++ b/fixtures/structured-transform.jsonl @@ -0,0 +1,7 @@ +{"id": "redact", "data": {"firstName": "May", "last_name": "Nguyễn", "name": "Acme", "n": 3, "null": null}, "config": {"transform": {"default": {"strategy": "redact"}}}, "expected_data": {"firstName": "[PERSON]", "last_name": "[PERSON]", "name": "Acme", "n": 3, "null": null}} +{"id": "mask", "data": [{"full_name": "👋 José", "email": "a@example.test"}], "config": {"transform": {"default": {"strategy": "mask"}, "entities": ["PERSON"]}}, "expected_data": [{"full_name": "******", "email": "a@example.test"}]} +{"id": "remove", "data": {"surname": "李", "note": "Hello"}, "config": {"transform": {"default": {"strategy": "remove"}}}, "expected_data": {"surname": "", "note": "Hello"}} +{"id": "allow", "data": {"first_name": "May", "last_name": "Chen", "email": "a@example.test"}, "config": {"transform": {"default": {"strategy": "redact"}, "allow": {"exact": {"PERSON": ["May"]}, "regex": {"EMAIL": [{"pattern": ".+@example\\.test"}]}}}}, "expected_data": {"first_name": "May", "last_name": "[PERSON]", "email": "a@example.test"}} +{"id": "override", "data": {"first_name": "May", "email": "a@example.test"}, "config": {"transform": {"default": {"strategy": "redact"}, "overrides": {"PERSON": {"strategy": "mask", "reveal": {"direction": "last", "count": 1}}}}}, "expected_data": {"first_name": "**y", "email": "[EMAIL]"}} +{"id": "overlap", "data": {"first_name": "a@example.test"}, "config": {"transform": {"default": {"strategy": "redact"}, "entities": ["PERSON"]}}, "expected_data": {"first_name": "[PERSON]"}} +{"id": "special-keys", "data": {"__proto__": {"first_name": "May"}, "a/b": {"fullName": "José"}, "~x.y": false}, "config": {"transform": {"default": {"strategy": "redact"}}}, "expected_data": {"__proto__": {"first_name": "[PERSON]"}, "a/b": {"fullName": "[PERSON]"}, "~x.y": false}} diff --git a/fixtures/structured.jsonl b/fixtures/structured.jsonl new file mode 100644 index 0000000..81205d6 --- /dev/null +++ b/fixtures/structured.jsonl @@ -0,0 +1,13 @@ +{"id": "names-and-email", "data": {"customer": {"firstName": "May", "last_name": "Nguyễn", "email": "may@example.test"}, "package": {"name": "Rose"}}, "mappings": [{"path": "/customer/firstName", "entity_type": "PERSON", "source": "field_alias", "rule": "first_name"}, {"path": "/customer/last_name", "entity_type": "PERSON", "source": "field_alias", "rule": "last_name"}], "findings": [{"path": "/customer/email", "label": "EMAIL", "text": "may@example.test", "start": 0, "end": 16}, {"path": "/customer/firstName", "label": "PERSON", "text": "May", "start": 0, "end": 3}, {"path": "/customer/last_name", "label": "PERSON", "text": "Nguyễn", "start": 0, "end": 6}]} +{"id": "ambiguous-negative", "data": {"name": "Jane Chen", "customer": {"name": "Acme"}, "file": {"name": "Mark"}, "first_names": ["May"], "first_name_backup": "Rose", "text": "first_name: May; Mark fixed this."}, "mappings": [], "findings": []} +{"id": "nested-array", "data": {"users": [{"given_name": "李", "familyName": "王"}, {"name": "May"}, {"full_name": " José O’Neill "}, null, false, 42]}, "mappings": [{"path": "/users/0/familyName", "entity_type": "PERSON", "source": "field_alias", "rule": "family_name"}, {"path": "/users/0/given_name", "entity_type": "PERSON", "source": "field_alias", "rule": "given_name"}, {"path": "/users/2/full_name", "entity_type": "PERSON", "source": "field_alias", "rule": "full_name"}], "findings": [{"path": "/users/0/familyName", "label": "PERSON", "text": "王", "start": 0, "end": 1}, {"path": "/users/0/given_name", "label": "PERSON", "text": "李", "start": 0, "end": 1}, {"path": "/users/2/full_name", "label": "PERSON", "text": " José O’Neill ", "start": 0, "end": 16}]} +{"id": "explicit-escaped", "data": {"a/b": {"~key.name": "👋 Zoë"}, "name": "Jane"}, "config": {"discover_person": false, "mappings": {"/a~1b/~0key.name": "PERSON", "/missing": "PERSON"}}, "mappings": [{"path": "/a~1b/~0key.name", "entity_type": "PERSON", "source": "explicit_mapping", "rule": "explicit_mapping"}], "findings": [{"path": "/a~1b/~0key.name", "label": "PERSON", "text": "👋 Zoë", "start": 0, "end": 6}]} +{"id": "exclusion-keeps-email", "data": {"first_name": "may@example.test", "last_name": "Chen", "name": "May"}, "config": {"exclude": ["/first_name"], "mappings": {"/name": "PERSON"}}, "mappings": [{"path": "/last_name", "entity_type": "PERSON", "source": "field_alias", "rule": "last_name"}, {"path": "/name", "entity_type": "PERSON", "source": "explicit_mapping", "rule": "explicit_mapping"}], "findings": [{"path": "/first_name", "label": "EMAIL", "text": "may@example.test", "start": 0, "end": 16}, {"path": "/last_name", "label": "PERSON", "text": "Chen", "start": 0, "end": 4}, {"path": "/name", "label": "PERSON", "text": "May", "start": 0, "end": 3}]} +{"id": "empty-and-non-string", "data": {"first_name": "", "given_name": " \n", "last_name": null, "full_name": false, "family_name": 7, "surname": {"name": "Jane"}}, "mappings": [{"path": "/first_name", "entity_type": "PERSON", "source": "field_alias", "rule": "first_name"}, {"path": "/given_name", "entity_type": "PERSON", "source": "field_alias", "rule": "given_name"}], "findings": []} +{"id": "disabled", "data": {"first_name": "May", "email": "a@example.test"}, "config": {"discover_person": false}, "mappings": [], "findings": [{"path": "/email", "label": "EMAIL", "text": "a@example.test", "start": 0, "end": 14}]} +{"id": "root-array", "data": [{"FIRST_NAME": "May"}, "a@example.test", {"Surname": "Rose"}, {"first_name": ["Jane"]}], "mappings": [{"path": "/0/FIRST_NAME", "entity_type": "PERSON", "source": "field_alias", "rule": "first_name"}, {"path": "/2/Surname", "entity_type": "PERSON", "source": "field_alias", "rule": "surname"}], "findings": [{"path": "/0/FIRST_NAME", "label": "PERSON", "text": "May", "start": 0, "end": 3}, {"path": "/1", "label": "EMAIL", "text": "a@example.test", "start": 0, "end": 14}, {"path": "/2/Surname", "label": "PERSON", "text": "Rose", "start": 0, "end": 4}]} +{"id": "explicit-wins-evidence", "data": {"first_name": "May"}, "config": {"mappings": {"/first_name": "PERSON"}}, "mappings": [{"path": "/first_name", "entity_type": "PERSON", "source": "explicit_mapping", "rule": "explicit_mapping"}], "findings": [{"path": "/first_name", "label": "PERSON", "text": "May", "start": 0, "end": 3}]} +{"id": "overlapping-types", "data": {"first_name": "a@example.test"}, "mappings": [{"path": "/first_name", "entity_type": "PERSON", "source": "field_alias", "rule": "first_name"}], "findings": [{"path": "/first_name", "label": "EMAIL", "text": "a@example.test", "start": 0, "end": 14}, {"path": "/first_name", "label": "PERSON", "text": "a@example.test", "start": 0, "end": 14}]} +{"id": "embedded-existing", "data": {"note": "👋 email a@example.test"}, "mappings": [], "findings": [{"path": "/note", "label": "EMAIL", "text": "a@example.test", "start": 8, "end": 22}]} +{"id": "empty-object", "data": {}, "mappings": [], "findings": []} +{"id": "empty-array", "data": [], "mappings": [], "findings": []} diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index b883431..e5d192b 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -30,7 +30,7 @@ function writeConsumerTest() { import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import path from "node:path"; -import { DataFogError, PrivacyManager, scan, scanAndTransform, transform } from "@datafog/node"; +import { DataFogError, PrivacyManager, scan, scanAndTransform, transform, scanStructured, discoverFields, transformStructured, scanAndTransformStructured } from "@datafog/node"; const fixturesDirectory = process.argv[2]; @@ -83,6 +83,49 @@ for (const name of ["development.jsonl", "final.jsonl"]) { } } +const structuredRecords = readFileSync(path.join(fixturesDirectory, "structured.jsonl"), "utf8").trim().split("\\n").map(JSON.parse); + +function pointerValue(data, pointer) { + return pointer.slice(1).split("/").reduce((value, key) => value[key.replaceAll("~1", "/").replaceAll("~0", "~")], data); +} +for (const record of structuredRecords) { + const result = scanStructured(record.data, record.config); + const mappings = result.mappings.map(m => ({path:m.path, entity_type:m.entityType, source:m.source, rule:m.rule})); + if (JSON.stringify(mappings) !== JSON.stringify(record.mappings)) throw new Error("structured mappings: " + record.id); + if (JSON.stringify(discoverFields(record.data, record.config)) !== JSON.stringify(result.mappings)) throw new Error("discovery mismatch"); + const actual = result.findings.map(({path, finding}) => { + verifyContract(pointerValue(record.data, path), finding); + return {path, ...legacyProjection(finding)}; + }); + if (JSON.stringify(actual) !== JSON.stringify(record.findings)) throw new Error("structured findings: " + record.id); +} +const cycle = {}; cycle.self = cycle; +for (const input of [null, "secret-value", {n:NaN}, {n:Infinity}, {n:2 ** 53}, {n:1n}, {n:undefined}, {n:new Date()}, {n:new Map()}, {n:[,]}, cycle]) { + let failed = false; + try { scanStructured(input); } catch (error) { + failed = error instanceof DataFogError && error.code === "invalid_configuration" && error.path === "/data" && !error.message.includes("secret-value"); + } + if (!failed) throw new Error("invalid structured input accepted"); +} + +const structuredTransformRecords = readFileSync(path.join(fixturesDirectory, "structured-transform.jsonl"), "utf8").trim().split("\\n").map(JSON.parse); + +for (const record of structuredTransformRecords) { + const result = scanAndTransformStructured(record.data, record.config); + const explicit = transformStructured(record.data, scanStructured(record.data, record.config.scan).findings, record.config.transform); + // Compare recursively without depending on object insertion order. + const ordered = value => Array.isArray(value) ? value.map(ordered) : value && typeof value === "object" ? Object.fromEntries(Object.keys(value).sort().map(k => [k,ordered(value[k])])) : value; + if (JSON.stringify(ordered(result.data)) !== JSON.stringify(ordered(record.expected_data))) throw new Error("structured transform: " + record.id); + if (JSON.stringify(result) !== JSON.stringify(explicit)) throw new Error("structured explicit mismatch"); + for (const {path, transformation:t} of result.transformations) { + const source = pointerValue(record.data, path); + const output = pointerValue(result.data, path); + if (output.slice(t.outputUtf16Range.start,t.outputUtf16Range.end) !== t.replacement) throw new Error("structured output range"); + if (!source.slice(t.sourceUtf16Range.start,t.sourceUtf16Range.end)) throw new Error("structured source range"); + if ("matchedText" in t) throw new Error("structured record echoes plaintext"); + } +} + const emojiFinding = scan("👋 jane@example.com")[0]; assert.deepEqual(emojiFinding.byteRange, { start: 5, end: 21 }); assert.deepEqual(emojiFinding.codepointRange, { start: 2, end: 18 }); @@ -267,6 +310,32 @@ await assert.rejects( error.path === "/transform/default/key_ref", ); + +const structuredPseudonyms = await manager.scanAndTransformStructured({first_name:"May",last_name:"May"}, {transform:pseudonymConfig}); +assert.equal(structuredPseudonyms.data.first_name, structuredPseudonyms.data.last_name); +assert.notEqual(structuredPseudonyms.data.first_name,"May"); +assert.equal(providerCalls.length,2); +const invalidContextData = {first_name:"May"}; +await assert.rejects( + manager.transformStructured(invalidContextData, scanStructured(invalidContextData).findings, pseudonymConfig, {scope:""}), + error => error instanceof DataFogError && error.code === "invalid_configuration", +); +await assert.rejects( + manager.scanAndTransformStructured(invalidContextData, {transform:pseudonymConfig}, {scope:""}), + error => error instanceof DataFogError && error.code === "invalid_configuration", +); +assert.equal(providerCalls.length,2, "invalid structured context must fail before resolving keys"); +const mutableData = {first_name:"May"}; +const mutableConfig = {transform:{default:{strategy:"pseudonymize",key_ref:"names"}}}; +const snapshotManager = new PrivacyManager({async resolveKey() { + mutableData.first_name="changed"; + mutableConfig.transform.default.strategy="remove"; + return {key: new Uint8Array(32),resolvedVersion:"v1"}; +}}); +const snapshotResult = await snapshotManager.scanAndTransformStructured(mutableData,mutableConfig); +assert.notEqual(snapshotResult.data.first_name,""); +assert.notEqual(snapshotResult.data.first_name,"changed"); +assert.equal(snapshotResult.transformations[0].transformation.strategy,"pseudonymize"); const tokenRecords = new Map(); let tokenCounter = 0; const tokenProvider = { @@ -304,6 +373,18 @@ const tokenized = await tokenManager.scanAndTransform( assert.notEqual(tokenized.transformations[0].replacement, tokenized.transformations[1].replacement); assert.equal(tokenized.transformations[0].tokenRef, "customers/default"); assert.equal(tokenized.transformations[0].resolvedTokenVersion, "active-1"); + +const structuredOriginal = {users:[{first_name:"👋 José"},{full_name:"May"}], count:2}; +const structuredTokens = await tokenManager.scanAndTransformStructured(structuredOriginal, {transform:{default:{strategy:"tokenize",token_ref:"names"}}}, tokenContext); +const structuredRestored = await tokenManager.restoreStructured(structuredTokens.data, tokenContext); +assert.deepEqual(structuredRestored.data, structuredOriginal); +assert.equal(structuredRestored.restorations.length,2); +await assert.rejects(tokenManager.restoreStructured(structuredTokens.data,{scope:"wrong"}), e => e.code === "token_access_denied"); +const invalidStructured = scanStructured(structuredOriginal).findings; +invalidStructured[1].finding.byteRange.end = 999; +const callsBefore = tokenCounter; +await assert.rejects(tokenManager.transformStructured(structuredOriginal,invalidStructured,{default:{strategy:"tokenize",token_ref:"names"}},tokenContext), e => e.code === "invalid_finding" && e.findingIndex === 1); +assert.equal(tokenCounter,callsBefore); const restored = await tokenManager.restore(tokenized.text, tokenContext); assert.equal(restored.text, "👋 jane@example.com jane@example.com"); assert.equal(restored.restorations.length, 2); @@ -404,6 +485,19 @@ void explicit; void convenience; void masked; void pseudonymized; + +import { discoverFields, scanStructured, transformStructured, scanAndTransformStructured, type StructuredScanResult, type StructuredTransformResult } from "@datafog/node"; +const document = { first_name: "May", count: 1 }; +const discovered = discoverFields(document, { mappings: { "/first_name": "PERSON" } }); +const structured: StructuredScanResult = scanStructured(document); +const protectedDocument: StructuredTransformResult = transformStructured(document, structured.findings, maskConfig); +const protectedTogether: StructuredTransformResult = scanAndTransformStructured(document, { transform: maskConfig }); +void discovered; +void protectedDocument; +void protectedTogether; +const structuredManagerResult: Promise = new PrivacyManager(provider).transformStructured(document, structured.findings, maskConfig); +void structuredManagerResult; + `.trimStart(), ); diff --git a/scripts/test-wasm-package.mjs b/scripts/test-wasm-package.mjs index c6fbba1..735a578 100644 --- a/scripts/test-wasm-package.mjs +++ b/scripts/test-wasm-package.mjs @@ -131,6 +131,17 @@ void transformed; void scannedAndTransformed; void masked; void unchanged; + +import { discoverFields, scanStructured, transformStructured, scanAndTransformStructured, type StructuredScanResult, type StructuredTransformResult } from "@datafog/wasm"; +const document = { first_name: "May", count: 1 }; +const discovered = discoverFields(document, { mappings: { "/first_name": "PERSON" } }); +const structured: StructuredScanResult = scanStructured(document); +const protectedDocument: StructuredTransformResult = transformStructured(document, structured.findings, maskConfig); +const protectedTogether: StructuredTransformResult = scanAndTransformStructured(document, { transform: maskConfig }); +void discovered; +void protectedDocument; +void protectedTogether; + `.trimStart(), ); @@ -190,7 +201,7 @@ try { temporaryDirectory, ); - for (const fixture of ["development.jsonl", "final.jsonl"]) { + for (const fixture of ["development.jsonl", "final.jsonl", "structured.jsonl", "structured-transform.jsonl"]) { writeFileSync( path.join(temporaryDirectory, fixture), readFileSync(path.join(fixturesDirectory, fixture)), @@ -204,7 +215,7 @@ try { await page.goto(serverInfo.url); await page.evaluate(async () => { - const { DataFogError, init, restore, scan, scanAndTransform, transform } = await import( + const { DataFogError, init, restore, scan, scanAndTransform, transform, scanStructured, discoverFields, transformStructured, scanAndTransformStructured, restoreStructured } = await import( "/node_modules/@datafog/wasm/index.js" ); @@ -280,6 +291,57 @@ try { } } +const structuredRecords = (await fetch("/structured.jsonl").then(r => r.text())).trim().split("\n").map(JSON.parse); + +function pointerValue(data, pointer) { + return pointer.slice(1).split("/").reduce((value, key) => value[key.replaceAll("~1", "/").replaceAll("~0", "~")], data); +} +for (const record of structuredRecords) { + const result = scanStructured(record.data, record.config); + const mappings = result.mappings.map(m => ({path:m.path, entity_type:m.entityType, source:m.source, rule:m.rule})); + if (JSON.stringify(mappings) !== JSON.stringify(record.mappings)) throw new Error("structured mappings: " + record.id); + if (JSON.stringify(discoverFields(record.data, record.config)) !== JSON.stringify(result.mappings)) throw new Error("discovery mismatch"); + const actual = result.findings.map(({path, finding}) => { + verifyContract(pointerValue(record.data, path), finding); + return {path, ...legacyProjection(finding)}; + }); + if (JSON.stringify(actual) !== JSON.stringify(record.findings)) throw new Error("structured findings: " + record.id); +} +const cycle = {}; cycle.self = cycle; +for (const input of [null, "secret-value", {n:NaN}, {n:Infinity}, {n:2 ** 53}, {n:1n}, {n:undefined}, {n:new Date()}, {n:new Map()}, {n:[,]}, cycle]) { + let failed = false; + try { scanStructured(input); } catch (error) { + failed = error instanceof DataFogError && error.code === "invalid_configuration" && error.path === "/data" && !error.message.includes("secret-value"); + } + if (!failed) throw new Error("invalid structured input accepted"); +} + +const structuredTransformRecords = (await fetch("/structured-transform.jsonl").then(r => r.text())).trim().split("\n").map(JSON.parse); + +for (const record of structuredTransformRecords) { + const result = scanAndTransformStructured(record.data, record.config); + const explicit = transformStructured(record.data, scanStructured(record.data, record.config.scan).findings, record.config.transform); + // Compare recursively without depending on object insertion order. + const ordered = value => Array.isArray(value) ? value.map(ordered) : value && typeof value === "object" ? Object.fromEntries(Object.keys(value).sort().map(k => [k,ordered(value[k])])) : value; + if (JSON.stringify(ordered(result.data)) !== JSON.stringify(ordered(record.expected_data))) throw new Error("structured transform: " + record.id); + if (JSON.stringify(result) !== JSON.stringify(explicit)) throw new Error("structured explicit mismatch"); + for (const {path, transformation:t} of result.transformations) { + const source = pointerValue(record.data, path); + const output = pointerValue(result.data, path); + if (output.slice(t.outputUtf16Range.start,t.outputUtf16Range.end) !== t.replacement) throw new Error("structured output range"); + if (!source.slice(t.sourceUtf16Range.start,t.sourceUtf16Range.end)) throw new Error("structured source range"); + if ("matchedText" in t) throw new Error("structured record echoes plaintext"); + } +} + +for (const strategy of [{strategy:"pseudonymize",key_ref:"names"},{strategy:"tokenize",token_ref:"names"}]) { + let rejected=false; + try { scanAndTransformStructured({first_name:"May"},{transform:{default:strategy}}); } catch(e) { rejected=e.code === "unsupported_strategy"; } + if (!rejected) throw new Error("structured provider operation accepted in WASM"); +} +let restoreRejected = false; +try { restoreStructured({}, {scope:"test"}); } catch(e) { restoreRejected=e.code === "unsupported_strategy"; } +if (!restoreRejected) throw new Error("structured restore accepted in WASM"); const emojiFinding = scan("👋 jane@example.com")[0]; if ( JSON.stringify(emojiFinding.byteRange) !== JSON.stringify({ start: 5, end: 21 }) ||