From 940a11391530a4a1a7c8c312e83633e71341aea1 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:58:42 -0700 Subject: [PATCH] Prepare 0.3.0 and enable trusted npm publishing from GitHub --- .github/workflows/publish-node.yml | 37 +++++++++-- .github/workflows/publish-python.yml | 2 + .github/workflows/publish-rust.yml | 1 + .github/workflows/publish-wasm.yml | 20 ++++-- Cargo.lock | 8 +-- README.md | 2 +- bindings/node/Cargo.toml | 2 +- bindings/node/package-lock.json | 4 +- bindings/node/package.json | 2 +- bindings/python/Cargo.toml | 2 +- bindings/python/pyproject.toml | 2 +- bindings/wasm/Cargo.toml | 2 +- bindings/wasm/package-lock.json | 4 +- bindings/wasm/package.json | 2 +- crates/core/Cargo.toml | 2 +- docs/bookkeeping-performance.md | 5 +- docs/development.mdx | 49 +++++++++++++++ docs/docs.json | 4 ++ docs/finding-selection-performance.md | 4 +- docs/get-started/installation.mdx | 3 + docs/guides/migrating-from-datafog-python.mdx | 18 +++--- docs/guides/person-discovery.mdx | 3 +- docs/person-detection-plan.md | 2 +- docs/privacy-operations-roadmap.md | 10 +-- docs/reference/browser-wasm.mdx | 5 -- docs/reference/node.mdx | 5 -- docs/releases/0-3-0.mdx | 62 +++++++++++++++++++ scripts/check-release.py | 38 ++++++++++++ scripts/test-node-package.mjs | 18 +++++- 29 files changed, 261 insertions(+), 57 deletions(-) create mode 100644 docs/releases/0-3-0.mdx create mode 100644 scripts/check-release.py diff --git a/.github/workflows/publish-node.yml b/.github/workflows/publish-node.yml index f6d12f7..acf2f05 100644 --- a/.github/workflows/publish-node.yml +++ b/.github/workflows/publish-node.yml @@ -5,6 +5,11 @@ on: tags: - "node-v*" workflow_dispatch: + inputs: + dry_run: + description: Build and verify packages without publishing + type: boolean + default: true permissions: contents: read @@ -34,6 +39,7 @@ jobs: build_args: "" steps: - uses: actions/checkout@v4 + - run: python3 scripts/check-release.py node - uses: dtolnay/rust-toolchain@stable with: toolchain: 1.88.0 @@ -50,6 +56,14 @@ jobs: name: bindings-${{ matrix.target }} path: bindings/node/datafog.*.node if-no-files-found: error + - uses: actions/upload-artifact@v4 + if: matrix.target == 'x86_64-unknown-linux-gnu' + with: + name: node-wrapper + path: | + bindings/node/native.js + bindings/node/index.d.ts + if-no-files-found: error publish: name: Publish to npm @@ -58,25 +72,40 @@ jobs: environment: npm permissions: contents: read + id-token: write steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 24 registry-url: https://registry.npmjs.org - cache: npm - cache-dependency-path: bindings/node/package-lock.json + - run: npm install --global npm@11.17.0 - run: npm ci --prefix bindings/node - uses: actions/download-artifact@v4 with: + pattern: bindings-* path: bindings/node/artifacts + - uses: actions/download-artifact@v4 + with: + name: node-wrapper + path: bindings/node - name: Assemble platform packages working-directory: bindings/node run: | npx napi create-npm-dirs npx napi artifacts --output-dir artifacts + npx napi prepublish -t npm --no-gh-release --skip-optional-publish + - name: Verify assembled package and native dependency + working-directory: bindings/node + run: npm run test:package -- --prebuilt --platform-package npm/linux-x64-gnu + - name: Check all package contents without publishing + working-directory: bindings/node + run: | + for package in npm/*; do + (cd "$package" && npm publish --dry-run --ignore-scripts) + done + npm publish --dry-run --ignore-scripts - name: Publish + if: github.event_name == 'push' || !inputs.dry_run working-directory: bindings/node run: npm publish - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/publish-python.yml b/.github/workflows/publish-python.yml index e4f0168..03836f1 100644 --- a/.github/workflows/publish-python.yml +++ b/.github/workflows/publish-python.yml @@ -31,6 +31,7 @@ jobs: target: x86_64-pc-windows-msvc steps: - uses: actions/checkout@v4 + - run: python3 scripts/check-release.py python - uses: PyO3/maturin-action@v1 with: command: build @@ -47,6 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - run: python3 scripts/check-release.py python - uses: PyO3/maturin-action@v1 with: command: sdist diff --git a/.github/workflows/publish-rust.yml b/.github/workflows/publish-rust.yml index 991689d..1d31817 100644 --- a/.github/workflows/publish-rust.yml +++ b/.github/workflows/publish-rust.yml @@ -16,6 +16,7 @@ jobs: environment: crates-io steps: - uses: actions/checkout@v4 + - run: python3 scripts/check-release.py rust - uses: dtolnay/rust-toolchain@stable - run: cargo publish --package datafog-core --locked env: diff --git a/.github/workflows/publish-wasm.yml b/.github/workflows/publish-wasm.yml index 136622a..741c5a2 100644 --- a/.github/workflows/publish-wasm.yml +++ b/.github/workflows/publish-wasm.yml @@ -5,6 +5,11 @@ on: tags: - "wasm-v*" workflow_dispatch: + inputs: + dry_run: + description: Build and verify the package without publishing + type: boolean + default: true permissions: contents: read @@ -14,8 +19,12 @@ jobs: name: Publish @datafog/wasm to npm runs-on: ubuntu-latest environment: npm + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@v4 + - run: python3 scripts/check-release.py wasm - uses: dtolnay/rust-toolchain@stable with: toolchain: 1.88.0 @@ -24,12 +33,13 @@ jobs: with: node-version: 24 registry-url: https://registry.npmjs.org - cache: npm - cache-dependency-path: bindings/wasm/package-lock.json + - run: npm install --global npm@11.17.0 - run: cargo install wasm-bindgen-cli --version 0.2.127 --locked - run: npm ci --prefix bindings/wasm - - run: npm run build --prefix bindings/wasm + - run: npx --prefix bindings/wasm playwright install --with-deps chromium + - run: npm run test:package --prefix bindings/wasm + - run: npm publish --dry-run --ignore-scripts + working-directory: bindings/wasm - run: npm publish + if: github.event_name == 'push' || !inputs.dry_run working-directory: bindings/wasm - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 5d45705..d5fa492 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" [[package]] name = "datafog-core" -version = "0.2.0" +version = "0.3.0" dependencies = [ "base64", "futures", @@ -94,7 +94,7 @@ dependencies = [ [[package]] name = "datafog-core-python" -version = "0.2.0" +version = "0.3.0" dependencies = [ "datafog-core", "pyo3", @@ -104,7 +104,7 @@ dependencies = [ [[package]] name = "datafog-node" -version = "0.2.0" +version = "0.3.0" dependencies = [ "datafog-core", "napi", @@ -115,7 +115,7 @@ dependencies = [ [[package]] name = "datafog-wasm" -version = "0.2.0" +version = "0.3.0" dependencies = [ "datafog-core", "serde", diff --git a/README.md b/README.md index 75dc9c1..9c4c95e 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ detection settings remain separate from transformation policy. DataFog Core is a separate distribution and canonical API, not a drop-in replacement for the established `datafog` Python package. -| DataFog Python 4.8.x | DataFog Core 0.2.x | +| DataFog Python 4.8.x | DataFog Core 0.3.x | | --- | --- | | `pip install datafog` | `pip install datafog-core` | | `from datafog.engine import ...` | `from datafog_core import ...` | diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index 2b53c06..19970ce 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-node" -version = "0.2.0" +version = "0.3.0" edition = "2024" rust-version = "1.88" description = "Node.js bindings for datafog-core" diff --git a/bindings/node/package-lock.json b/bindings/node/package-lock.json index eb2430c..2f0cc09 100644 --- a/bindings/node/package-lock.json +++ b/bindings/node/package-lock.json @@ -1,12 +1,12 @@ { "name": "@datafog/node", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@datafog/node", - "version": "0.2.0", + "version": "0.3.0", "license": "MIT", "devDependencies": { "@napi-rs/cli": "3.8.6", diff --git a/bindings/node/package.json b/bindings/node/package.json index 3ab1dd1..75ca81b 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@datafog/node", - "version": "0.2.0", + "version": "0.3.0", "description": "Fast structured PII detection for Node.js, powered by Rust", "license": "MIT", "repository": { diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 0286332..b8e43bb 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-core-python" -version = "0.2.0" +version = "0.3.0" edition = "2024" rust-version = "1.88" description = "Python bindings for datafog-core" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 1b1bd74..7cdcdda 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "datafog-core" -version = "0.2.0" +version = "0.3.0" description = "Fast structured PII detection implemented in Rust" readme = "../../README.md" license = "MIT" diff --git a/bindings/wasm/Cargo.toml b/bindings/wasm/Cargo.toml index b0e3abf..359a132 100644 --- a/bindings/wasm/Cargo.toml +++ b/bindings/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-wasm" -version = "0.2.0" +version = "0.3.0" edition = "2024" rust-version = "1.88" description = "WebAssembly bindings for datafog-core" diff --git a/bindings/wasm/package-lock.json b/bindings/wasm/package-lock.json index a404182..5d9d753 100644 --- a/bindings/wasm/package-lock.json +++ b/bindings/wasm/package-lock.json @@ -1,12 +1,12 @@ { "name": "@datafog/wasm", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@datafog/wasm", - "version": "0.2.0", + "version": "0.3.0", "license": "MIT", "devDependencies": { "playwright": "1.62.1", diff --git a/bindings/wasm/package.json b/bindings/wasm/package.json index 5611653..18b45a5 100644 --- a/bindings/wasm/package.json +++ b/bindings/wasm/package.json @@ -1,6 +1,6 @@ { "name": "@datafog/wasm", - "version": "0.2.0", + "version": "0.3.0", "description": "Fast structured PII detection for browsers, powered by Rust and WebAssembly", "license": "MIT", "repository": { diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 0a5fc5b..c42a3d2 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-core" -version = "0.2.0" +version = "0.3.0" edition = "2024" rust-version = "1.88" description = "Fast structured PII detection library" diff --git a/docs/bookkeeping-performance.md b/docs/bookkeeping-performance.md index ff65adc..cdae997 100644 --- a/docs/bookkeeping-performance.md +++ b/docs/bookkeeping-performance.md @@ -2,8 +2,9 @@ This is a focused follow-up to [Slice Ten](finding-selection-performance.md). The baseline is the merge of PR #13, commit -`083eaaab6bcfcfba9a31bc8ce1cfc99f1c2a3cae`. The change is a candidate for the -planned 0.3.0 release; package versions stay at 0.2.0 until release preparation. +`083eaaab6bcfcfba9a31bc8ce1cfc99f1c2a3cae`. The change is included in the +0.3.0 release. The measurements below compare the two implementations before +the release version bump. ## Scope and behavior diff --git a/docs/development.mdx b/docs/development.mdx index 92b8a34..ef6517f 100644 --- a/docs/development.mdx +++ b/docs/development.mdx @@ -70,3 +70,52 @@ bindings/wasm/ Browser WebAssembly binding fixtures/ Shared conformance fixtures docs/ Architecture records and Mintlify documentation ``` + +## Publish a release from GitHub Actions + +Keep the four Cargo package versions, Python project version, both npm +package versions, and their lockfiles aligned. The publishing workflows run +`python3 scripts/check-release.py ` and reject inconsistent versions +or a release tag that does not match the package version. + +Run the quality gates and installed-package tests above on the release commit. +Merge the release preparation PR, then create these tags on that same commit +for version 0.3.0: + +| Tag | Workflow | Destination | +| --- | --- | --- | +| `rust-v0.3.0` | `publish-rust.yml` | crates.io | +| `python-v0.3.0` | `publish-python.yml` | PyPI | +| `node-v0.3.0` | `publish-node.yml` | npm: Node and five native platform packages | +| `wasm-v0.3.0` | `publish-wasm.yml` | npm: browser WASM | + +Each tag push starts its publishing workflow. Check all four runs and install +the published versions before announcing the combined release. Registry +publication is not atomic across packages; inspect any partial result before +retrying. Never move a tag that has already published a package. + +The npm workflows also accept a manual `dry_run` input, which defaults to +`true`. A dry run builds and verifies package contents without publishing. +For an actual manual publish, select the intended release tag and set +`dry_run` to `false`. + +### npm trusted publisher setup + +Configure [npm trusted publishing](https://docs.npmjs.com/trusted-publishers/) +once in each package's npm settings. Use GitHub organization **DataFog**, +repository **datafog-core**, environment **npm**, and allow direct +`npm publish`: + +| Packages | Workflow filename | +| --- | --- | +| `@datafog/node`, `@datafog/node-darwin-arm64`, `@datafog/node-darwin-x64`, `@datafog/node-linux-x64-gnu`, `@datafog/node-linux-arm64-gnu`, `@datafog/node-win32-x64-msvc` | `publish-node.yml` | +| `@datafog/wasm` | `publish-wasm.yml` | + +The publish jobs request GitHub's OIDC identity with `id-token: write` and use +npm 11.17.0 on Node 24. They do not use `NPM_TOKEN` or a local machine's npm +session. npm generates provenance for these public packages automatically. + +The Node workflow builds all five native binaries, transfers the generated +JavaScript loader and types to the publish job, and runs the installed-package +suite against the assembled package and its Linux native dependency before +publishing. The WASM workflow runs the browser package suite before publishing. diff --git a/docs/docs.json b/docs/docs.json index 185417a..cd3a263 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -45,6 +45,10 @@ "reference/errors" ] }, + { + "group": "Releases", + "pages": ["releases/0-3-0"] + }, { "group": "Contributing", "pages": ["development"] diff --git a/docs/finding-selection-performance.md b/docs/finding-selection-performance.md index 5582c9f..91f2897 100644 --- a/docs/finding-selection-performance.md +++ b/docs/finding-selection-performance.md @@ -3,8 +3,8 @@ This change addresses [issue #10](https://github.com/DataFog/datafog-core/issues/10) in the shared Rust transformation path, following the structured PERSON work. Public APIs, policies, detection coverage, offsets, and package versions are -unchanged. PERSON and this optimization are intended for a combined 0.3.0 -release after review; this change does not publish packages. +unchanged. PERSON and this optimization are included in the combined 0.3.0 +release; the original feature PR did not publish packages. ## Algorithm and compatibility diff --git a/docs/get-started/installation.mdx b/docs/get-started/installation.mdx index 443b97b..8d92834 100644 --- a/docs/get-started/installation.mdx +++ b/docs/get-started/installation.mdx @@ -6,6 +6,9 @@ icon: "download" ## Package availability +The structured PERSON APIs require version **0.3.0 or newer** in every +binding. See the [0.3.0 release notes](/releases/0-3-0). + | Runtime | Distribution | Import | Availability | | --- | --- | --- | --- | | Rust | `datafog-core` | `datafog_core` | Published on crates.io | diff --git a/docs/guides/migrating-from-datafog-python.mdx b/docs/guides/migrating-from-datafog-python.mdx index 8e740e4..3706193 100644 --- a/docs/guides/migrating-from-datafog-python.mdx +++ b/docs/guides/migrating-from-datafog-python.mdx @@ -1,6 +1,6 @@ --- title: "Migrate from DataFog Python" -description: "Move from datafog 4.8.x to the canonical DataFog Core 0.2.x Python API." +description: "Move from datafog 4.8.x to the canonical DataFog Core 0.3.x Python API." icon: "arrow-right-arrow-left" --- @@ -15,7 +15,7 @@ DataFog Core makes detection results, text ranges, transformation policy, and provider-backed operations consistent across Rust, Python, Node.js, and browser/WASM. That consistency requires a few deliberate API changes in Python. -DataFog Core 0.2 does not replace the legacy package's optional spaCy, GLiNER, +DataFog Core 0.3 does not replace the legacy package's optional spaCy, GLiNER, OCR, distributed-processing, CLI, or application-guardrail features. Keep the established package for those workloads while adopting Core where its smaller, cross-runtime contract fits. @@ -28,7 +28,7 @@ cross-runtime contract fits. python -m pip install datafog ``` -```bash DataFog Core 0.2.x +```bash DataFog Core 0.3.x python -m pip install datafog-core ``` @@ -40,7 +40,7 @@ python -m pip install datafog-core from datafog.engine import Entity, redact, scan, scan_and_redact ``` -```python DataFog Core 0.2.x +```python DataFog Core 0.3.x from datafog_core import Finding, scan, scan_and_transform, transform ``` @@ -70,7 +70,7 @@ for entity in result.entities: print(entity.type, entity.text, entity.start, entity.end) ``` -```python DataFog Core 0.2.x +```python DataFog Core 0.3.x from datafog_core import scan findings = scan("Email jane@example.com") @@ -90,7 +90,7 @@ Use `codepoint_range` when slicing a Python string. Use `byte_range` when addressing the UTF-8 encoded input. Both ranges are zero-based and end-exclusive. -| DataFog Python 4.8.x | DataFog Core 0.2.x | +| DataFog Python 4.8.x | DataFog Core 0.3.x | | --- | --- | | `ScanResult.entities` | The return value from `scan` | | `Entity.type` | `Finding.entity_type` | @@ -124,7 +124,7 @@ result = scan_and_redact( print(result.redacted_text) ``` -```python DataFog Core 0.2.x +```python DataFog Core 0.3.x from datafog_core import scan_and_transform result = scan_and_transform( @@ -142,7 +142,7 @@ print(result.text) -| DataFog Python 4.8.x | DataFog Core 0.2.x | +| DataFog Python 4.8.x | DataFog Core 0.3.x | | --- | --- | | `scan_and_redact(...)` | `scan_and_transform(text, {"scan": ..., "transform": ...})` | | `redact(text, entities, ...)` | `transform(text, findings, config)` | @@ -179,7 +179,7 @@ behavior and threat-model distinctions. ## Remove legacy engine selection Do not translate `engine="regex"`, `"smart"`, `"spacy"`, or `"gliner"` into -Core configuration. DataFog Core 0.2 owns detector composition and exposes +Core configuration. DataFog Core 0.3 owns detector composition and exposes `locale` as its scan setting. Detector provenance appears on each finding. Entity names are exact and case-sensitive. The built-in Core entities are: diff --git a/docs/guides/person-discovery.mdx b/docs/guides/person-discovery.mdx index a10bff1..d3657c9 100644 --- a/docs/guides/person-discovery.mdx +++ b/docs/guides/person-discovery.mdx @@ -5,8 +5,7 @@ icon: "user-shield" --- - Structured PERSON support is implemented and tested locally. Its package - release is pending. + Structured PERSON support requires DataFog Core 0.3.0 or newer. Structured scanning discovers person-name fields automatically and runs the diff --git a/docs/person-detection-plan.md b/docs/person-detection-plan.md index 6683cc7..47b9424 100644 --- a/docs/person-detection-plan.md +++ b/docs/person-detection-plan.md @@ -1,6 +1,6 @@ # PERSON detection with automatic field discovery -**Status: Core and binding implementation complete; release and downstream adoption pending.** +**Status: Core and binding implementation included in 0.3.0; downstream adoption pending.** The implemented contract is [ADR 002](adr/002-structured-person-discovery.md). Verification and measurement results are recorded in [structured performance](structured-performance.md). diff --git a/docs/privacy-operations-roadmap.md b/docs/privacy-operations-roadmap.md index 233222f..849384f 100644 --- a/docs/privacy-operations-roadmap.md +++ b/docs/privacy-operations-roadmap.md @@ -231,7 +231,7 @@ without a separately accepted host-managed key-custody boundary. ## Next workstream: PERSON detection with automatic field discovery -**Status: Core and bindings implemented; release and downstream adoption pending.** +**Status: Core and bindings included in 0.3.0; downstream adoption pending.** The [PERSON detection plan](person-detection-plan.md) extends Core with conservative automatic name-field discovery for structured JSON data, explicit @@ -244,7 +244,7 @@ completed privacy-operation slices or promise general prose name recognition. ## Slice 10: Finding-selection performance -**Status: merged; combined release pending.** +**Status: merged; included in 0.3.0.** Replace linear duplicate searches with indexed groups and repeated overlap scans with ordered interval selection. Preserve validation, filtering, @@ -253,12 +253,12 @@ compatibility fallback where mixed confidence prevents safe sorting. The [selection measurements](finding-selection-performance.md) document exact behavior comparisons, reproducible scaling benchmarks, and remaining performance -limits. Prepare one 0.3.0 release containing structured PERSON and this change -after review; individual feature PRs do not bump or publish package versions. +limits. The 0.3.0 release combines structured PERSON and this change with Slice 11; +individual feature PRs do not bump or publish package versions. ## Slice 11: Offset calculation and record conversion -**Status: implemented; review pending for inclusion in 0.3.0.** +**Status: merged; included in 0.3.0.** Reuse lazy text indexes for finding validation and Unicode range conversion, maintain running output positions, and remove per-record field copies from Node diff --git a/docs/reference/browser-wasm.mdx b/docs/reference/browser-wasm.mdx index ad88306..45b2541 100644 --- a/docs/reference/browser-wasm.mdx +++ b/docs/reference/browser-wasm.mdx @@ -4,11 +4,6 @@ description: "Browser WebAssembly initialization, stateless operations, and prov icon: "globe" --- - - The first `@datafog/wasm` npm release is pending. The API documented here is - implemented and tested in a real browser from the repository. - - ## Initialize the module Call `init` once before using another operation. diff --git a/docs/reference/node.mdx b/docs/reference/node.mdx index 795820b..295bbc9 100644 --- a/docs/reference/node.mdx +++ b/docs/reference/node.mdx @@ -4,11 +4,6 @@ description: "Node.js functions, TypeScript types, JavaScript ranges, and the as icon: "node-js" --- - - The first `@datafog/node` npm release is pending. The API documented here is - implemented and tested in the repository. - - ```javascript import { DataFogError, diff --git a/docs/releases/0-3-0.mdx b/docs/releases/0-3-0.mdx new file mode 100644 index 0000000..605bc46 --- /dev/null +++ b/docs/releases/0-3-0.mdx @@ -0,0 +1,62 @@ +--- +title: "0.3.0" +description: "Structured PERSON discovery and faster finding selection and text-position handling." +icon: "tag" +--- + +DataFog Core 0.3.0 adds structured PERSON discovery and improves performance +when processing many findings. Rust, Python, Node.js, and browser WASM share +the detection and stateless protection behavior. + +## Structured PERSON support + +- Discover explicit name fields such as `first_name`, `last_name`, and + `fullName` without a model or dictionary download. +- Supply concrete JSON Pointer mappings for fields that automatic discovery + leaves unresolved, such as `/customer/name`. +- Scan JSON string values with the original seven detectors and return each + finding with its field path. Protect the findings with the existing policies. +- Use structured pseudonymization, tokenization, and restoration through + Rust, Python, and Node provider integrations. Browser WASM retains its + existing restriction on provider-backed operations. + +PERSON detection uses field context. It does not recognize arbitrary names in +prose, and passing serialized JSON to `scan(text)` does not enable discovery. +See [Discover and protect person fields](/guides/person-discovery). + +## Performance + +- Index duplicate findings and resolve ordinary overlaps with ordered interval + selection. Built-in findings use an O(m log m) selection path, where m is the + finding count. +- Reuse lazy text indexes during validation and byte/code-point/UTF-16 range + conversion, maintain running output positions, and avoid copying a whole + field for every Node transformation record. +- Expose Rust's reusable `TextIndex` for converting multiple ranges from the + same string. See the [Rust reference](/reference/rust). + +On one local macOS ARM64 benchmark, selection of 4,096 disjoint findings fell +from 60.3 ms to 0.56 ms. Separately, the bookkeeping changes reduced a complete +Node structured scan-and-protect request with 1,024 findings in one Unicode +field from 242.3 ms to 3.2 ms. These measure different stages and baselines; +they are not a universal speedup guarantee. Short-field workloads changed +little, and one sparse scan-only case was about 10% slower. + +The [selection benchmark notes](https://github.com/DataFog/datafog-core/blob/main/docs/finding-selection-performance.md) +and [bookkeeping benchmark notes](https://github.com/DataFog/datafog-core/blob/main/docs/bookkeeping-performance.md) +include inputs, methodology, and limitations. + +## Compatibility and publishing + +Existing validation, offset semantics, transformation policies, and finding +preferences are preserved. Caller-supplied overlapping findings that mix +scored and unscored confidence can retain the original quadratic selection +algorithm to preserve its behavior. See [Findings and ranges](/concepts/findings-and-ranges). + +The Node and WASM release workflows publish to npm from GitHub Actions using +trusted publishing. Node builds cover macOS ARM64/x64, Linux GNU ARM64/x64, +and Windows x64. Node.js 24.x remains required. + +This release contains [PR #12](https://github.com/DataFog/datafog-core/pull/12), +[PR #13](https://github.com/DataFog/datafog-core/pull/13), and +[PR #14](https://github.com/DataFog/datafog-core/pull/14). diff --git a/scripts/check-release.py b/scripts/check-release.py new file mode 100644 index 0000000..ae8092b --- /dev/null +++ b/scripts/check-release.py @@ -0,0 +1,38 @@ +"""Reject mismatched package versions and release tags before publishing.""" + +import json +import os +from pathlib import Path +import sys +import tomllib + +root = Path(__file__).resolve().parent.parent +versions = {} +for relative in ( + "crates/core/Cargo.toml", + "bindings/python/Cargo.toml", + "bindings/node/Cargo.toml", + "bindings/wasm/Cargo.toml", + "bindings/python/pyproject.toml", +): + document = tomllib.loads((root / relative).read_text()) + versions[relative] = document.get("package", document.get("project"))["version"] +for binding in ("node", "wasm"): + for filename in ("package.json", "package-lock.json"): + relative = f"bindings/{binding}/{filename}" + document = json.loads((root / relative).read_text()) + versions[relative] = document["version"] + if filename == "package-lock.json": + versions[f"{relative}:root"] = document["packages"][""]["version"] +for package in tomllib.loads((root / "Cargo.lock").read_text())["package"]: + if package["name"] in {"datafog-core", "datafog-core-python", "datafog-node", "datafog-wasm"}: + versions[f"Cargo.lock:{package['name']}"] = package["version"] + +if len(set(versions.values())) != 1: + raise SystemExit(f"Release versions do not match: {versions}") +version = next(iter(versions.values())) +if os.environ.get("GITHUB_REF_TYPE") == "tag": + expected = f"{sys.argv[1]}-v{version}" + if os.environ.get("GITHUB_REF_NAME") != expected: + raise SystemExit(f"Expected release tag {expected}") +print(f"All package versions match {version}; release ref is valid.") diff --git a/scripts/test-node-package.mjs b/scripts/test-node-package.mjs index a37b226..382d508 100644 --- a/scripts/test-node-package.mjs +++ b/scripts/test-node-package.mjs @@ -550,9 +550,21 @@ void structuredManagerResult; } let tarball; +let platformTarball; try { - run("npm", ["run", "build"], nodePackage); + if (!process.argv.includes("--prebuilt")) { + run("npm", ["run", "build"], nodePackage); + } + const platformArgument = process.argv.indexOf("--platform-package"); + if (platformArgument !== -1) { + const platformDirectory = path.resolve(nodePackage, process.argv[platformArgument + 1]); + const packed = JSON.parse(execFileSync("npm", ["pack", "--json"], { + cwd: platformDirectory, + encoding: "utf8", + })); + platformTarball = path.join(platformDirectory, packed[0].filename); + } const packed = JSON.parse( execFileSync("npm", ["pack", "--json"], { @@ -587,6 +599,7 @@ try { "install", "--ignore-scripts", tarball, + ...(platformTarball ? [platformTarball] : []), `typescript@${nodePackageJson.devDependencies.typescript}`, ], temporaryDirectory, @@ -607,5 +620,8 @@ try { if (tarball) { rmSync(tarball, { force: true }); } + if (platformTarball) { + rmSync(platformTarball, { force: true }); + } rmSync(temporaryDirectory, { recursive: true, force: true }); }