Skip to content

Implement Slice Nine structured PERSON discovery and protection #11

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sidmohan0 wants to merge 1 commit into from
Closed

Implement Slice Nine structured PERSON discovery and protection #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ entity type, matched text, byte range, code-point range,
optional confidence, detector name, optional detector version
```

Structured JSON scanning additionally discovers `PERSON` from documented name-field
aliases or explicit JSON Pointer mappings. It scans every string value with the
existing detectors and returns field paths plus string-local findings. No model
or dictionary download is needed. See [person-field discovery](docs/guides/person-discovery.mdx)
for `scan_structured` / `scanStructured` and structured transformation APIs.

Both ranges use zero-based, end-exclusive offsets. The byte range addresses the
UTF-8 input; the code-point range addresses Unicode scalar values. Rule-based
detectors currently report no confidence score. Node.js and browser WASM also
Expand Down
24 changes: 24 additions & 0 deletions bindings/node/dts-header.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,10 @@ export interface PrivacyManagerProviders {
}

export declare class PrivacyManager {
transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context?: PrivacyContext): Promise<StructuredTransformResult>;
scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig, context?: PrivacyContext): Promise<StructuredTransformResult>;
restoreStructured(data: JsonDocument, context: PrivacyContext): Promise<StructuredRestoreResult>;

constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider);
transform(
text: string,
Expand All @@ -152,3 +156,23 @@ export declare class PrivacyManager {
): Promise<TransformResult>;
restore(text: string, context: PrivacyContext): Promise<RestoreResult>;
}

/** JSON input uses finite numbers; integer values must be JavaScript-safe. */
export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue };
export type JsonDocument = JsonValue[] | { [key: string]: JsonValue };
export interface StructuredScanConfig {
readonly locale?: string;
readonly discover_person?: boolean;
readonly mappings?: Readonly<Record<string, "PERSON">>;
readonly exclude?: readonly string[];
}

export interface StructuredScanAndTransformConfig {
readonly scan?: StructuredScanConfig;
readonly transform: TransformationConfig;
}
export declare function transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): StructuredTransformResult;
export declare function scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig): StructuredTransformResult;

export interface StructuredTransformResult { readonly data: JsonDocument; readonly transformations: StructuredTransformation[]; }
export interface StructuredRestoreResult { readonly data: JsonDocument; readonly restorations: StructuredRestoration[]; }
92 changes: 92 additions & 0 deletions bindings/node/index.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,10 @@ export interface PrivacyManagerProviders {
}

export declare class PrivacyManager {
transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context?: PrivacyContext): Promise<StructuredTransformResult>;
scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig, context?: PrivacyContext): Promise<StructuredTransformResult>;
restoreStructured(data: JsonDocument, context: PrivacyContext): Promise<StructuredRestoreResult>;

constructor(provider: KeyProvider | PrivacyManagerProviders, tokenProvider?: TokenProvider);
transform(
text: string,
Expand All @@ -152,6 +156,35 @@ export declare class PrivacyManager {
): Promise<TransformResult>;
restore(text: string, context: PrivacyContext): Promise<RestoreResult>;
}

/** JSON input uses finite numbers; integer values must be JavaScript-safe. */
export type JsonValue = null | boolean | number | string | JsonValue[] | { [key: string]: JsonValue };
export type JsonDocument = JsonValue[] | { [key: string]: JsonValue };
export interface StructuredScanConfig {
readonly locale?: string;
readonly discover_person?: boolean;
readonly mappings?: Readonly<Record<string, "PERSON">>;
readonly exclude?: readonly string[];
}

export interface StructuredScanAndTransformConfig {
readonly scan?: StructuredScanConfig;
readonly transform: TransformationConfig;
}
export declare function transformStructured(data: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): StructuredTransformResult;
export declare function scanAndTransformStructured(data: JsonDocument, config: StructuredScanAndTransformConfig): StructuredTransformResult;

export interface StructuredTransformResult { readonly data: JsonDocument; readonly transformations: StructuredTransformation[]; }
export interface StructuredRestoreResult { readonly data: JsonDocument; readonly restorations: StructuredRestoration[]; }
export declare function discoverFields(dataJson: JsonDocument, config?: StructuredScanConfig | undefined): Array<FieldMapping>

export interface FieldMapping {
path: string
entityType: string
source: string
rule: string
}

export interface Finding {
readonly entityType: EntityType
readonly matchedText: string
Expand Down Expand Up @@ -180,13 +213,30 @@ export interface KeySelector {
readonly path: string
}

export interface NativeStructuredRestoreResult {
dataJson: string
restorations: Array<StructuredRestoration>
}

export interface NativeStructuredTransformResult {
dataJson: string
transformations: Array<StructuredTransformation>
}

export interface PreparedScanAndTransform {
readonly findings: Array<Finding>
readonly selectors: Array<KeySelector>
}

export interface PreparedStructuredScan {
findings: Array<StructuredFinding>
selectors: Array<KeySelector>
}

export declare function prepareScanAndTransform(text: string, config: ScanAndTransformConfig): PreparedScanAndTransform

export declare function prepareStructuredScanAndTransform(text: string, config: unknown): PreparedStructuredScan

export declare function requiredKeySelectors(text: string, findings: FindingInput[], config: TransformationConfig): Array<KeySelector>

export declare function requiredRestoreItems(text: string, context: PrivacyContext): Array<RestoreItem>
Expand Down Expand Up @@ -235,6 +285,48 @@ export declare function scan(text: string, config?: ScanConfig | undefined): Arr
/** Scan text and transform the detected findings. */
export declare function scanAndTransform(text: string, config: ScanAndTransformConfig): TransformResult

export declare function scanStructured(dataJson: JsonDocument, config?: StructuredScanConfig | undefined): StructuredScanResult

export interface StructuredFinding {
path: string
finding: Finding
}

export interface StructuredFindingInput {
path: string
finding: FindingInput
}

/** Scan text and transform the detected findings. */
export declare function structuredRequiredKeySelectors(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): Array<KeySelector>

export declare function structuredRequiredRestoreItems(text: JsonDocument, context: PrivacyContext): Array<RestoreItem>

export declare function structuredRequiredTokenizationItems(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context?: PrivacyContext | undefined): Array<TokenizeItem>

export interface StructuredRestoration {
path: string
restoration: Restoration
}

export declare function structuredRestoreWithResults(text: JsonDocument, context: PrivacyContext, results: Array<RestoredValueInput>): NativeStructuredRestoreResult

export declare function structuredScanAndTransform(text: JsonDocument, config: StructuredScanAndTransformConfig): NativeStructuredTransformResult

export interface StructuredScanResult {
mappings: Array<FieldMapping>
findings: Array<StructuredFinding>
}

export declare function structuredTransform(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig): NativeStructuredTransformResult

export interface StructuredTransformation {
path: string
transformation: Transformation
}

export declare function structuredTransformWithProviderResults(text: JsonDocument, findings: StructuredFindingInput[], config: TransformationConfig, context: PrivacyContext | undefined, resolvedKeys: Array<ResolvedKeyInput>, tokenResults: Array<TokenizeResultInput>): NativeStructuredTransformResult

export interface TextRange {
readonly start: number
readonly end: number
Expand Down
122 changes: 110 additions & 12 deletions bindings/node/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
import { Buffer } from "node:buffer";
import {
structuredTransform as nativeStructuredTransform,
structuredScanAndTransform as nativeStructuredScanAndTransform,
structuredRequiredKeySelectors as nativeStructuredRequiredKeySelectors,
structuredRequiredTokenizationItems as nativeStructuredRequiredTokenizationItems,
structuredTransformWithProviderResults as nativeStructuredTransformWithProviderResults,
structuredRequiredRestoreItems as nativeStructuredRequiredRestoreItems,
structuredRestoreWithResults as nativeStructuredRestoreWithResults,
prepareStructuredScanAndTransform as nativePrepareStructuredScanAndTransform,
discoverFields as nativeDiscoverFields,
scanStructured as nativeScanStructured,
prepareScanAndTransform as nativePrepareScanAndTransform,
requiredKeySelectors as nativeRequiredKeySelectors,
requiredRestoreItems as nativeRequiredRestoreItems,
Expand Down Expand Up @@ -167,7 +177,14 @@ export class PrivacyManager {
}));
}

async transform(text, findings, config, context) {
async transform(text, findings, config, context) { return this.#transform(text, findings, config, context); }
async scanAndTransform(text, config, context) { return this.#scanAndTransform(text, config, context); }
async restore(text, context) { return this.#restore(text, context); }
async transformStructured(data, findings, config, context) { return this.#transform(structuredJson(data), structuredOptions(findings), structuredOptions(config), structuredOptions(context), true); }
async scanAndTransformStructured(data, config, context) { return this.#scanAndTransform(structuredJson(data), structuredOptions(config), structuredOptions(context), true); }
async restoreStructured(data, context) { return this.#restore(structuredJson(data), structuredOptions(context), true); }

async #transform(text, findings, config, context, structured = false) {
if (typeof text !== "string") {
throw new TypeError("transform text must be a string");
}
Expand All @@ -176,11 +193,15 @@ export class PrivacyManager {
}
let resolved = [];
try {
const selectors = nativeRequiredKeySelectors(text, findings, config);
const selectors = (structured ? nativeStructuredRequiredKeySelectors : nativeRequiredKeySelectors)(text, findings, config);
const preparedItems = structured
? nativeStructuredRequiredTokenizationItems(text, findings, config, context)
: undefined;
resolved = await this.#resolve(selectors);
const items = nativeRequiredTokenizationItems(text, findings, config, context);
const items = preparedItems ?? nativeRequiredTokenizationItems(text, findings, config, context);
const tokens = await this.#tokenize(items, context);
return nativeTransformWithProviderResults(text, findings, config, context, resolved, tokens);
const result = (structured ? nativeStructuredTransformWithProviderResults : nativeTransformWithProviderResults)(text, findings, config, context, resolved, tokens);
return structured ? structuredResult(result) : result;
} catch (error) {
if (error instanceof DataFogError) throw error;
throw normalizeError(error, "invalid_configuration");
Expand All @@ -189,28 +210,37 @@ export class PrivacyManager {
}
}

async scanAndTransform(text, config, context) {
async #scanAndTransform(text, config, context, structured = false) {
if (typeof text !== "string") {
throw new TypeError("scanAndTransform text must be a string");
}
let prepared;
try {
prepared = nativePrepareScanAndTransform(text, config);
prepared = (structured ? nativePrepareStructuredScanAndTransform : nativePrepareScanAndTransform)(text, config);
} catch (error) {
throw normalizeError(error, "invalid_configuration");
}
let preparedItems;
if (structured) {
try {
preparedItems = nativeStructuredRequiredTokenizationItems(text, prepared.findings, config.transform, context);
} catch (error) {
throw withPathPrefix(error, "/transform");
}
}
const resolved = await this.#resolve(prepared.selectors, "/transform");
try {
const items = nativeRequiredTokenizationItems(text, prepared.findings, config.transform, context);
const items = preparedItems ?? nativeRequiredTokenizationItems(text, prepared.findings, config.transform, context);
const tokens = await this.#tokenize(items, context);
return nativeTransformWithProviderResults(
const result = (structured ? nativeStructuredTransformWithProviderResults : nativeTransformWithProviderResults)(
text,
prepared.findings,
config.transform,
context,
resolved,
tokens,
);
return structured ? structuredResult(result) : result;
} catch (error) {
throw withPathPrefix(error, "/transform");
} finally {
Expand All @@ -219,17 +249,20 @@ export class PrivacyManager {
}


async restore(text, context) {
async #restore(text, context, structured = false) {
if (typeof text !== "string") {
throw new TypeError("restore text must be a string");
}
let items;
try {
items = nativeRequiredRestoreItems(text, context);
items = (structured ? nativeStructuredRequiredRestoreItems : nativeRequiredRestoreItems)(text, context);
} catch (error) {
throw normalizeError(error, "invalid_configuration");
}
if (items.length === 0) return nativeRestoreWithResults(text, context, []);
if (items.length === 0) {
const result = (structured ? nativeStructuredRestoreWithResults : nativeRestoreWithResults)(text, context, []);
return structured ? structuredResult(result) : result;
}
if (!this.#tokenProvider) {
throw new DataFogError({
code: "token_provider_required",
Expand All @@ -242,7 +275,8 @@ export class PrivacyManager {
} catch (error) {
throw normalizeProviderError(error, undefined, "token_provider_error");
}
return nativeRestoreWithResults(text, context, Array.isArray(results) ? results : []);
const result = (structured ? nativeStructuredRestoreWithResults : nativeRestoreWithResults)(text, context, Array.isArray(results) ? results : []);
return structured ? structuredResult(result) : result;
}
}

Expand Down Expand Up @@ -287,3 +321,67 @@ export function scanAndTransform(text, config) {
throw normalizeError(error, "invalid_configuration");
}
}

function structuredJson(data, omitUndefinedOptions = false) {
try {
if (data === null || typeof data !== "object") throw new TypeError();
const pending = [data];
const seen = new Set();
while (pending.length) {
const value = pending.pop();
if (value === null || typeof value === "string" || typeof value === "boolean") continue;
if (typeof value === "number") {
if (!Number.isFinite(value) || (Number.isInteger(value) && !Number.isSafeInteger(value))) throw new TypeError();
continue;
}
if (typeof value !== "object") throw new TypeError();
if (seen.has(value)) continue;
seen.add(value);
const array = Array.isArray(value);
if (!array && Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) throw new TypeError();
if (array && Object.keys(value).length !== value.length) throw new TypeError();
for (const key of Reflect.ownKeys(value)) {
if (array && key === "length") continue;
if (typeof key !== "string") throw new TypeError();
const descriptor = Object.getOwnPropertyDescriptor(value, key);
if (!descriptor.enumerable || !("value" in descriptor)) throw new TypeError();
if (array && (!/^(0|[1-9][0-9]*)$/.test(key) || Number(key) >= value.length)) throw new TypeError();
if (omitUndefinedOptions && !array && descriptor.value === undefined) continue;
pending.push(descriptor.value);
}
}
return JSON.stringify(data);
} catch {
throw new DataFogError({code:"invalid_configuration", reason:"invalid_type", path:"/data", message:"data must be a JSON object or array with finite numbers and safe integers"});
}
}

export function discoverFields(data, config) {
const json = structuredJson(data);
try { return nativeDiscoverFields(json, structuredOptions(config)); } catch (error) { throw normalizeError(error, "invalid_configuration"); }
}

export function scanStructured(data, config) {
const json = structuredJson(data);
try { return nativeScanStructured(json, structuredOptions(config)); } catch (error) { throw normalizeError(error, "invalid_configuration"); }
}

export function transformStructured(data, findings, config) {
const json = structuredJson(data);
try { return structuredResult(nativeStructuredTransform(json, structuredOptions(findings), structuredOptions(config))); } catch (error) { throw normalizeError(error, "invalid_configuration"); }
}
export function scanAndTransformStructured(data, config) {
const json = structuredJson(data);
try { return structuredResult(nativeStructuredScanAndTransform(json, structuredOptions(config))); } catch (error) { throw normalizeError(error, "invalid_configuration"); }
}

function structuredResult({dataJson, ...records}) {
return {data:JSON.parse(dataJson), ...records};
}

function structuredOptions(value) {
if (value === undefined) return undefined;
try { return JSON.parse(structuredJson(value, true)); } catch {
throw new DataFogError({code:"invalid_configuration", reason:"invalid_type", path:"", message:"structured request options must be JSON-compatible"});
}
}
Loading
Loading