From b71d494fb8c771cf419cad873f13a85367407da9 Mon Sep 17 00:00:00 2001 From: Sid Mohan <61345237+sidmohan0@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:50:51 -0700 Subject: [PATCH] fix(python): ship typing metadata and prepare 0.3.1 --- .github/workflows/python-binding.yml | 10 + Cargo.lock | 8 +- bindings/node/Cargo.toml | 2 +- bindings/node/package-lock.json | 4 +- bindings/node/package.json | 2 +- bindings/python/Cargo.toml | 2 +- bindings/python/pyproject.toml | 5 +- .../python/python/datafog_core/__init__.py | 57 +++ .../python/python/datafog_core/__init__.pyi | 381 ++++++++++++++++++ bindings/python/python/datafog_core/py.typed | 0 bindings/python/tests/requirements-typing.txt | 3 + bindings/python/tests/test_typing.py | 260 ++++++++++++ bindings/wasm/Cargo.toml | 2 +- bindings/wasm/package-lock.json | 4 +- bindings/wasm/package.json | 2 +- crates/core/Cargo.toml | 2 +- docs/development.mdx | 12 +- docs/docs.json | 2 +- docs/reference/python.mdx | 13 + docs/releases/0-3-1.mdx | 19 + 20 files changed, 768 insertions(+), 22 deletions(-) create mode 100644 bindings/python/python/datafog_core/__init__.py create mode 100644 bindings/python/python/datafog_core/__init__.pyi create mode 100644 bindings/python/python/datafog_core/py.typed create mode 100644 bindings/python/tests/requirements-typing.txt create mode 100644 bindings/python/tests/test_typing.py create mode 100644 docs/releases/0-3-1.mdx diff --git a/.github/workflows/python-binding.yml b/.github/workflows/python-binding.yml index 7e149ca..f76aed3 100644 --- a/.github/workflows/python-binding.yml +++ b/.github/workflows/python-binding.yml @@ -23,4 +23,14 @@ jobs: - run: | python -m venv .venv .venv/bin/python -m pip install target/wheels/*.whl + .venv/bin/python -m pip install -r bindings/python/tests/requirements-typing.txt .venv/bin/python bindings/python/tests/test_installed.py + .venv/bin/python bindings/python/tests/test_typing.py + - name: Verify typing survives a source distribution rebuild + if: matrix.python-version == '3.10' + run: | + maturin sdist --manifest-path bindings/python/Cargo.toml --out target/sdist + .venv/bin/python -m pip wheel --no-deps target/sdist/*.tar.gz --wheel-dir target/rebuilt-wheels + .venv/bin/python -m pip install --force-reinstall target/rebuilt-wheels/*.whl + .venv/bin/python bindings/python/tests/test_installed.py + .venv/bin/python bindings/python/tests/test_typing.py diff --git a/Cargo.lock b/Cargo.lock index d5fa492..07971d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -80,7 +80,7 @@ checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" [[package]] name = "datafog-core" -version = "0.3.0" +version = "0.3.1" dependencies = [ "base64", "futures", @@ -94,7 +94,7 @@ dependencies = [ [[package]] name = "datafog-core-python" -version = "0.3.0" +version = "0.3.1" dependencies = [ "datafog-core", "pyo3", @@ -104,7 +104,7 @@ dependencies = [ [[package]] name = "datafog-node" -version = "0.3.0" +version = "0.3.1" dependencies = [ "datafog-core", "napi", @@ -115,7 +115,7 @@ dependencies = [ [[package]] name = "datafog-wasm" -version = "0.3.0" +version = "0.3.1" dependencies = [ "datafog-core", "serde", diff --git a/bindings/node/Cargo.toml b/bindings/node/Cargo.toml index 19970ce..76cbc80 100644 --- a/bindings/node/Cargo.toml +++ b/bindings/node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-node" -version = "0.3.0" +version = "0.3.1" edition = "2024" rust-version = "1.88" description = "Node.js bindings for datafog-core" diff --git a/bindings/node/package-lock.json b/bindings/node/package-lock.json index 2f0cc09..f5ffd6d 100644 --- a/bindings/node/package-lock.json +++ b/bindings/node/package-lock.json @@ -1,12 +1,12 @@ { "name": "@datafog/node", - "version": "0.3.0", + "version": "0.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@datafog/node", - "version": "0.3.0", + "version": "0.3.1", "license": "MIT", "devDependencies": { "@napi-rs/cli": "3.8.6", diff --git a/bindings/node/package.json b/bindings/node/package.json index 75ca81b..718c0d4 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@datafog/node", - "version": "0.3.0", + "version": "0.3.1", "description": "Fast structured PII detection for Node.js, powered by Rust", "license": "MIT", "repository": { diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index b8e43bb..5ff13df 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-core-python" -version = "0.3.0" +version = "0.3.1" edition = "2024" rust-version = "1.88" description = "Python bindings for datafog-core" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index 7cdcdda..9d03027 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "datafog-core" -version = "0.3.0" +version = "0.3.1" description = "Fast structured PII detection implemented in Rust" readme = "../../README.md" license = "MIT" @@ -30,4 +30,5 @@ Repository = "https://github.com/DataFog/datafog-core" Issues = "https://github.com/DataFog/datafog-core/issues" [tool.maturin] -module-name = "datafog_core" +python-source = "python" +module-name = "datafog_core.datafog_core" diff --git a/bindings/python/python/datafog_core/__init__.py b/bindings/python/python/datafog_core/__init__.py new file mode 100644 index 0000000..c26a7b3 --- /dev/null +++ b/bindings/python/python/datafog_core/__init__.py @@ -0,0 +1,57 @@ +"""Fast structured PII detection implemented in Rust.""" + +from .datafog_core import ( + DataFogConfigurationError as DataFogConfigurationError, + DataFogFindingError as DataFogFindingError, + DataFogInternalError as DataFogInternalError, + DataFogKeyProviderError as DataFogKeyProviderError, + FieldMapping as FieldMapping, + Finding as Finding, + PrivacyManager as PrivacyManager, + Restoration as Restoration, + RestoreResult as RestoreResult, + StructuredFinding as StructuredFinding, + StructuredRestoration as StructuredRestoration, + StructuredRestoreResult as StructuredRestoreResult, + StructuredScanResult as StructuredScanResult, + StructuredTransformation as StructuredTransformation, + StructuredTransformResult as StructuredTransformResult, + TextRange as TextRange, + Transformation as Transformation, + TransformResult as TransformResult, + discover_fields as discover_fields, + scan as scan, + scan_and_transform as scan_and_transform, + scan_and_transform_structured as scan_and_transform_structured, + scan_structured as scan_structured, + transform as transform, + transform_structured as transform_structured, +) + +__all__ = [ + "DataFogConfigurationError", + "DataFogFindingError", + "DataFogInternalError", + "DataFogKeyProviderError", + "TextRange", + "Finding", + "Transformation", + "TransformResult", + "Restoration", + "RestoreResult", + "FieldMapping", + "StructuredFinding", + "StructuredScanResult", + "StructuredTransformation", + "StructuredTransformResult", + "StructuredRestoration", + "StructuredRestoreResult", + "PrivacyManager", + "scan", + "transform", + "scan_and_transform", + "discover_fields", + "scan_structured", + "transform_structured", + "scan_and_transform_structured", +] diff --git a/bindings/python/python/datafog_core/__init__.pyi b/bindings/python/python/datafog_core/__init__.pyi new file mode 100644 index 0000000..99b8f45 --- /dev/null +++ b/bindings/python/python/datafog_core/__init__.pyi @@ -0,0 +1,381 @@ +"""Types for the compiled API. Underscored helpers exist only for type checking.""" + +from collections.abc import Awaitable, Iterable, Sequence +from typing import Literal, NoReturn, Protocol, TypeAlias, TypedDict, final + +_JsonValue: TypeAlias = ( + None | bool | int | float | str | list["_JsonValue"] | dict[str, "_JsonValue"] +) +_JsonDocument: TypeAlias = list[_JsonValue] | dict[str, _JsonValue] + +class _ScanConfig(TypedDict, total=False): + locale: str + +class _StructuredScanConfig(_ScanConfig, total=False): + discover_person: bool + mappings: dict[str, Literal["PERSON"]] + exclude: list[str] | tuple[str, ...] + +class _Reveal(TypedDict): + direction: Literal["first", "last"] + count: int + +class _Redact(TypedDict): + strategy: Literal["redact"] + +class _Remove(TypedDict): + strategy: Literal["remove"] + +class _MaskRequired(TypedDict): + strategy: Literal["mask"] + +class _Mask(_MaskRequired, total=False): + character: str + reveal: _Reveal + +class _PseudonymizeRequired(TypedDict): + strategy: Literal["pseudonymize"] + key_ref: str + +class _Pseudonymize(_PseudonymizeRequired, total=False): + key_version: str + +class _Tokenize(TypedDict): + strategy: Literal["tokenize"] + token_ref: str + +_StrategyConfig: TypeAlias = _Redact | _Remove | _Mask | _Pseudonymize | _Tokenize + +class _RegexRequired(TypedDict): + pattern: str + +class _Regex(_RegexRequired, total=False): + case_sensitive: bool + +class _Allow(TypedDict, total=False): + exact: dict[str, list[str] | tuple[str, ...]] + regex: dict[str, list[_Regex] | tuple[_Regex, ...]] + +class _TransformRequired(TypedDict): + default: _StrategyConfig + +class _TransformationConfig(_TransformRequired, total=False): + entities: list[str] | tuple[str, ...] + overrides: dict[str, _StrategyConfig] + allow: _Allow + +class _ScanAndTransformRequired(TypedDict): + transform: _TransformationConfig + +class _ScanAndTransformConfig(_ScanAndTransformRequired, total=False): + scan: _ScanConfig + +class _StructuredScanAndTransformConfig(_ScanAndTransformRequired, total=False): + scan: _StructuredScanConfig + +class _PrivacyContext(TypedDict): + scope: str + +class _ResolvedKey(TypedDict): + key: bytes | Sequence[int] + resolved_version: str + +class _ResolvedKeyObject(Protocol): + @property + def key(self) -> bytes | Sequence[int]: ... + @property + def resolved_version(self) -> str: ... + +class _KeyProvider(Protocol): + def resolve_key( + self, key_ref: str, key_version: str | None, / + ) -> Awaitable[_ResolvedKey | _ResolvedKeyObject]: ... + +class _TokenizeItem(TypedDict): + id: str + exact_value: str + token_ref: str + +class _TokenizeResult(TypedDict): + id: str + payload: bytes | Sequence[int] + resolved_version: str + +class _TokenizeResultObject(Protocol): + @property + def id(self) -> str: ... + @property + def payload(self) -> bytes | Sequence[int]: ... + @property + def resolved_version(self) -> str: ... + +class _RestoreItem(TypedDict): + id: str + token_ref: str + resolved_version: str + payload: bytes + +class _RestoreResult(TypedDict): + id: str + value: str + +class _RestoreResultObject(Protocol): + @property + def id(self) -> str: ... + @property + def value(self) -> str: ... + +class _TokenProvider(Protocol): + def tokenize_batch( + self, scope: str, items: list[_TokenizeItem], / + ) -> Awaitable[Iterable[_TokenizeResult | _TokenizeResultObject]]: ... + def restore_batch( + self, scope: str, items: list[_RestoreItem], / + ) -> Awaitable[Iterable[_RestoreResult | _RestoreResultObject]]: ... + +class DataFogConfigurationError(ValueError): + code: str + reason: str | None + path: str | None + finding_index: int | None + +class DataFogFindingError(ValueError): + code: str + reason: str | None + path: str | None + finding_index: int | None + +class DataFogInternalError(RuntimeError): + code: str + reason: str | None + path: str | None + finding_index: int | None + +class DataFogKeyProviderError(RuntimeError): + code: str + reason: str | None + path: str | None + finding_index: int | None + +@final +class TextRange: + def __init__(self, start: int, end: int) -> None: ... + @property + def start(self) -> int: ... + @property + def end(self) -> int: ... + +@final +class Finding: + def __init__( + self, + entity_type: str, + matched_text: str, + byte_range: TextRange, + codepoint_range: TextRange, + detector_name: str, + confidence: float | None = None, + detector_version: str | None = None, + ) -> None: ... + @property + def entity_type(self) -> str: ... + @property + def matched_text(self) -> str: ... + @property + def byte_range(self) -> TextRange: ... + @property + def codepoint_range(self) -> TextRange: ... + @property + def confidence(self) -> float | None: ... + @property + def detector_name(self) -> str: ... + @property + def detector_version(self) -> str | None: ... + +# NoReturn prevents construction of native result-only classes on Python 3.10 +# without requiring typing_extensions.Never. +@final +class Transformation: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def entity_type(self) -> str: ... + @property + def source_byte_range(self) -> TextRange: ... + @property + def source_codepoint_range(self) -> TextRange: ... + @property + def confidence(self) -> float | None: ... + @property + def detector_name(self) -> str: ... + @property + def detector_version(self) -> str | None: ... + @property + def strategy(self) -> str: ... + @property + def replacement(self) -> str: ... + @property + def output_byte_range(self) -> TextRange: ... + @property + def output_codepoint_range(self) -> TextRange: ... + @property + def key_ref(self) -> str | None: ... + @property + def resolved_key_version(self) -> str | None: ... + @property + def token_ref(self) -> str | None: ... + @property + def resolved_token_version(self) -> str | None: ... + +@final +class TransformResult: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def text(self) -> str: ... + @property + def transformations(self) -> list[Transformation]: ... + +@final +class Restoration: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def source_byte_range(self) -> TextRange: ... + @property + def source_codepoint_range(self) -> TextRange: ... + @property + def output_byte_range(self) -> TextRange: ... + @property + def output_codepoint_range(self) -> TextRange: ... + @property + def token_ref(self) -> str: ... + @property + def resolved_token_version(self) -> str: ... + +@final +class RestoreResult: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def text(self) -> str: ... + @property + def restorations(self) -> list[Restoration]: ... + +@final +class FieldMapping: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def path(self) -> str: ... + @property + def entity_type(self) -> str: ... + @property + def source(self) -> str: ... + @property + def rule(self) -> str: ... + +@final +class StructuredFinding: + def __init__(self, path: str, finding: Finding) -> None: ... + @property + def path(self) -> str: ... + @property + def finding(self) -> Finding: ... + +@final +class StructuredScanResult: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def mappings(self) -> list[FieldMapping]: ... + @property + def findings(self) -> list[StructuredFinding]: ... + +@final +class StructuredTransformation: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def path(self) -> str: ... + @property + def transformation(self) -> Transformation: ... + +@final +class StructuredTransformResult: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def data(self) -> _JsonDocument: ... + @property + def transformations(self) -> list[StructuredTransformation]: ... + +@final +class StructuredRestoration: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def path(self) -> str: ... + @property + def restoration(self) -> Restoration: ... + +@final +class StructuredRestoreResult: + def __init__(self, _unconstructible: NoReturn) -> None: ... + @property + def data(self) -> _JsonDocument: ... + @property + def restorations(self) -> list[StructuredRestoration]: ... + +@final +class PrivacyManager: + def __init__( + self, + provider: _KeyProvider | None = None, + token_provider: _TokenProvider | None = None, + ) -> None: ... + def transform( + self, + text: str, + findings: Sequence[Finding], + config: _TransformationConfig, + context: _PrivacyContext | None = None, + ) -> Awaitable[TransformResult]: ... + def scan_and_transform( + self, + text: str, + config: _ScanAndTransformConfig, + context: _PrivacyContext | None = None, + ) -> Awaitable[TransformResult]: ... + def restore( + self, text: str, context: _PrivacyContext + ) -> Awaitable[RestoreResult]: ... + def transform_structured( + self, + data: _JsonDocument, + findings: Sequence[StructuredFinding], + config: _TransformationConfig, + context: _PrivacyContext | None = None, + ) -> Awaitable[StructuredTransformResult]: ... + def scan_and_transform_structured( + self, + data: _JsonDocument, + config: _StructuredScanAndTransformConfig, + context: _PrivacyContext | None = None, + ) -> Awaitable[StructuredTransformResult]: ... + def restore_structured( + self, data: _JsonDocument, context: _PrivacyContext + ) -> Awaitable[StructuredRestoreResult]: ... + +def scan(text: str, config: _ScanConfig | None = None) -> list[Finding]: ... +def transform( + text: str, findings: Sequence[Finding], config: _TransformationConfig +) -> TransformResult: ... +def scan_and_transform( + text: str, config: _ScanAndTransformConfig +) -> TransformResult: ... +def discover_fields( + data: _JsonDocument, config: _StructuredScanConfig | None = None +) -> list[FieldMapping]: ... +def scan_structured( + data: _JsonDocument, config: _StructuredScanConfig | None = None +) -> StructuredScanResult: ... +def transform_structured( + data: _JsonDocument, + findings: Sequence[StructuredFinding], + config: _TransformationConfig, +) -> StructuredTransformResult: ... +def scan_and_transform_structured( + data: _JsonDocument, config: _StructuredScanAndTransformConfig +) -> StructuredTransformResult: ... diff --git a/bindings/python/python/datafog_core/py.typed b/bindings/python/python/datafog_core/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/bindings/python/tests/requirements-typing.txt b/bindings/python/tests/requirements-typing.txt new file mode 100644 index 0000000..330cf96 --- /dev/null +++ b/bindings/python/tests/requirements-typing.txt @@ -0,0 +1,3 @@ +mypy==2.3.1 +pyright==1.1.411 +typing_extensions==4.16.0 diff --git a/bindings/python/tests/test_typing.py b/bindings/python/tests/test_typing.py new file mode 100644 index 0000000..d6842f5 --- /dev/null +++ b/bindings/python/tests/test_typing.py @@ -0,0 +1,260 @@ +"""Check typing metadata and downstream code against the installed wheel.""" + +import ast +import importlib.metadata +import inspect +import json +import subprocess +import sys +import tempfile +from pathlib import Path + +import datafog_core + +VALID = """ +from typing import TYPE_CHECKING +from typing_extensions import assert_type +from datafog_core import ( + DataFogConfigurationError, DataFogFindingError, DataFogInternalError, + DataFogKeyProviderError, TextRange, Finding, Transformation, TransformResult, + Restoration, RestoreResult, FieldMapping, StructuredFinding, + StructuredScanResult, StructuredTransformation, StructuredTransformResult, + StructuredRestoration, StructuredRestoreResult, PrivacyManager, + scan, transform, scan_and_transform, discover_fields, scan_structured, + transform_structured, scan_and_transform_structured, +) +if TYPE_CHECKING: + from datafog_core import ( + _TransformationConfig, _JsonDocument, _TokenizeItem, _TokenizeResult, + _RestoreItem, _RestoreResult, _ResolvedKey, + ) + +config: _TransformationConfig = { + "default": {"strategy": "mask", "character": "*", "reveal": {"direction": "last", "count": 4}}, + "entities": ["EMAIL"], + "overrides": { + "PHONE": {"strategy": "remove"}, + "PERSON": {"strategy": "pseudonymize", "key_ref": "names", "key_version": "1"}, + }, + "allow": {"exact": {"EMAIL": ["safe@example.com"]}, "regex": {"EMAIL": [{"pattern": "safe", "case_sensitive": False}]}}, +} +findings = scan("jane@example.com", {"locale": "en-US"}) +assert_type(findings, list[Finding]) +finding = findings[0] +assert_type(finding.entity_type, str) +assert_type(finding.matched_text, str) +assert_type(finding.byte_range, TextRange) +assert_type(finding.byte_range.start, int) +assert_type(finding.codepoint_range.end, int) +assert_type(finding.confidence, float | None) +assert_type(finding.detector_name, str) +assert_type(finding.detector_version, str | None) +manual = Finding("EMAIL", "jane@example.com", TextRange(0, 16), TextRange(0, 16), "custom", confidence=0.9) +assert_type(transform("jane@example.com", (manual,), config), TransformResult) +result = scan_and_transform("jane@example.com", {"scan": {"locale": "en-US"}, "transform": config}) +assert_type(result.text, str) +assert_type(result.transformations, list[Transformation]) +assert_type(result.transformations[0].source_byte_range, TextRange) +assert_type(result.transformations[0].output_codepoint_range, TextRange) +assert_type(result.transformations[0].resolved_key_version, str | None) +assert_type(result.transformations[0].resolved_token_version, str | None) +assert_type(scan("", None), list[Finding]) +data: _JsonDocument = {"name": "Jane", "users": [None, True, 1, 1.5, {"email": "jane@example.com"}]} +assert_type(discover_fields(data, {"discover_person": False, "mappings": {"/name": "PERSON"}, "exclude": ["/users"]}), list[FieldMapping]) +structured = scan_structured(data) +assert_type(structured, StructuredScanResult) +assert_type(structured.mappings, list[FieldMapping]) +assert_type(structured.mappings[0].rule, str) +assert_type(structured.findings, list[StructuredFinding]) +assert_type(structured.findings[0].finding, Finding) +assert_type(structured.findings[0].path, str) +located = StructuredFinding("/email", manual) +assert_type(transform_structured(data, [located], config), StructuredTransformResult) +structured_result = scan_and_transform_structured(data, {"transform": config}) +assert_type(structured_result.data, _JsonDocument) +assert_type(structured_result.transformations, list[StructuredTransformation]) +assert_type(structured_result.transformations[0].transformation, Transformation) + +class KeyProvider: + async def resolve_key(self, ref: str, version: str | None) -> _ResolvedKey: + return {"key": bytes(range(32)), "resolved_version": version or "1"} + +class AttributeKey: + key: bytes = bytes(range(32)) + resolved_version: str = "1" + +class AttributeKeyProvider: + async def resolve_key(self, ref: str, version: str | None) -> AttributeKey: + return AttributeKey() + +class TokenProvider: + async def tokenize_batch(self, scope: str, items: list[_TokenizeItem]) -> list[_TokenizeResult]: + return [{"id": item["id"], "payload": b"token", "resolved_version": "1"} for item in items] + async def restore_batch(self, scope: str, items: list[_RestoreItem]) -> list[_RestoreResult]: + return [{"id": item["id"], "value": "restored"} for item in items] + +PrivacyManager(AttributeKeyProvider()) +async def use_manager() -> None: + manager = PrivacyManager(provider=KeyProvider(), token_provider=TokenProvider()) + assert_type(await manager.transform("text", findings, config), TransformResult) + assert_type(await manager.scan_and_transform("text", {"transform": config}), TransformResult) + restored = await manager.restore("text", {"scope": "tenant"}) + assert_type(restored, RestoreResult) + assert_type(restored.restorations, list[Restoration]) + assert_type(restored.restorations[0].token_ref, str) + assert_type(restored.restorations[0].output_byte_range, TextRange) + assert_type(await manager.transform_structured(data, structured.findings, config), StructuredTransformResult) + assert_type(await manager.scan_and_transform_structured(data, {"transform": {"default": {"strategy": "tokenize", "token_ref": "names"}}}, {"scope": "tenant"}), StructuredTransformResult) + restored_data = await manager.restore_structured(data, {"scope": "tenant"}) + assert_type(restored_data, StructuredRestoreResult) + assert_type(restored_data.data, _JsonDocument) + assert_type(restored_data.restorations, list[StructuredRestoration]) + assert_type(restored_data.restorations[0].restoration, Restoration) + +for error in (DataFogConfigurationError(), DataFogFindingError(), DataFogInternalError(), DataFogKeyProviderError()): + assert_type(error.code, str) + assert_type(error.reason, str | None) + assert_type(error.path, str | None) + assert_type(error.finding_index, int | None) +""" + +# Each line must produce a diagnostic; a missing stub or Any cannot satisfy this. +INVALID = """ +from datafog_core import scan, transform, scan_structured, PrivacyManager, TransformResult +scan(123) # error +scan("text", {"locale": 123}) # error +scan("text", {"max_bytes": 100}) # error +scan("text")[0].entity_type = "OTHER" # error +scan("text")[0].byte_range.start = 1 # error +transform("text", ["not a finding"], {"default": {"strategy": "redact"}}) # error +transform("text", [], {"default": {"strategy": "invalid"}}) # error +transform("text", [], {"default": {"strategy": "pseudonymize"}}) # error +transform("text", [], {"default": {"strategy": "mask", "reveal": {"direction": "middle", "count": 4}}}) # error +scan_structured({"name": "Jane"}, {"mappings": {"/name": "EMAIL"}}) # error +scan_structured("not a document") # error +PrivacyManager(object()) # error +PrivacyManager().restore("text") # error +PrivacyManager().restore("text", {"scope": 12}) # error +TransformResult() # error +""" + + +def verify_metadata() -> None: + distribution = importlib.metadata.distribution("datafog-core") + files = {str(path) for path in distribution.files or []} + assert "datafog_core/py.typed" in files, files + assert "datafog_core/__init__.pyi" in files, files + stub = Path(distribution.locate_file("datafog_core/__init__.pyi")) + tree = ast.parse(stub.read_text()) + declarations = { + node.name: node + for node in tree.body + if isinstance(node, (ast.ClassDef, ast.FunctionDef)) + and not node.name.startswith("_") + } + native_module = importlib.import_module("datafog_core.datafog_core") + exports = set(native_module.__all__) + assert set(datafog_core.__all__) == exports + assert set(declarations) == exports, ( + set(declarations) - exports, + exports - set(declarations), + ) + for name, declaration in declarations.items(): + value = getattr(datafog_core, name) + if isinstance(declaration, ast.ClassDef): + # Every native read-only field needs a corresponding typed property. + properties = { + member.name + for member in declaration.body + if isinstance(member, ast.FunctionDef) + and any( + isinstance(d, ast.Name) and d.id == "property" + for d in member.decorator_list + ) + } + native = { + key + for key, member in vars(value).items() + if ( + inspect.isgetsetdescriptor(member) + or inspect.ismemberdescriptor(member) + ) + and not key.startswith("_") + } + assert properties == native, (name, properties, native) + + +def main() -> None: + verify_metadata() + with tempfile.TemporaryDirectory(prefix="datafog-typing-") as temporary: + root = Path(temporary) + (root / "valid.py").write_text(VALID) + (root / "invalid.py").write_text(INVALID) + (root / "pyrightconfig.json").write_text( + json.dumps( + { + "typeCheckingMode": "strict", + "pythonVersion": "3.10", + "reportPrivateUsage": False, + } + ) + ) + for checker in ("mypy", "pyright"): + command = [sys.executable, "-m", checker] + if checker == "mypy": + command += [ + "--strict", + "--python-version", + "3.10", + "--no-incremental", + "--show-column-numbers", + ] + else: + command += ["--pythonpath", sys.executable, "--outputjson"] + valid = subprocess.run( + command + ["valid.py"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + assert valid.returncode == 0, ( + f"{checker} valid usage failed:\n{valid.stdout}\n{valid.stderr}" + ) + invalid = subprocess.run( + command + ["invalid.py"], + cwd=root, + capture_output=True, + text=True, + check=False, + ) + assert invalid.returncode == 1, ( + f"{checker} expected type errors:\n{invalid.stdout}\n{invalid.stderr}" + ) + expected = { + number + for number, line in enumerate(INVALID.splitlines(), 1) + if "# error" in line + } + if checker == "pyright": + diagnostics = json.loads(invalid.stdout)["generalDiagnostics"] + actual = { + item["range"]["start"]["line"] + 1 + for item in diagnostics + if item["severity"] == "error" + } + else: + actual = { + int(line.split(":")[1]) + for line in invalid.stdout.splitlines() + if line.startswith("invalid.py:") and ": error:" in line + } + assert actual == expected, ( + f"{checker}: expected lines {expected}, got {actual}\n{invalid.stdout}" + ) + print(f"{checker}: valid usage passes; all invalid cases rejected.") + print("Installed typing metadata covers every runtime export and native property.") + + +if __name__ == "__main__": + main() diff --git a/bindings/wasm/Cargo.toml b/bindings/wasm/Cargo.toml index 359a132..37fda8d 100644 --- a/bindings/wasm/Cargo.toml +++ b/bindings/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-wasm" -version = "0.3.0" +version = "0.3.1" edition = "2024" rust-version = "1.88" description = "WebAssembly bindings for datafog-core" diff --git a/bindings/wasm/package-lock.json b/bindings/wasm/package-lock.json index 5d9d753..2fc86b9 100644 --- a/bindings/wasm/package-lock.json +++ b/bindings/wasm/package-lock.json @@ -1,12 +1,12 @@ { "name": "@datafog/wasm", - "version": "0.3.0", + "version": "0.3.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@datafog/wasm", - "version": "0.3.0", + "version": "0.3.1", "license": "MIT", "devDependencies": { "playwright": "1.62.1", diff --git a/bindings/wasm/package.json b/bindings/wasm/package.json index 18b45a5..fa0c98b 100644 --- a/bindings/wasm/package.json +++ b/bindings/wasm/package.json @@ -1,6 +1,6 @@ { "name": "@datafog/wasm", - "version": "0.3.0", + "version": "0.3.1", "description": "Fast structured PII detection for browsers, powered by Rust and WebAssembly", "license": "MIT", "repository": { diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index c42a3d2..14af6d3 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "datafog-core" -version = "0.3.0" +version = "0.3.1" edition = "2024" rust-version = "1.88" description = "Fast structured PII detection library" diff --git a/docs/development.mdx b/docs/development.mdx index ef6517f..9e53bee 100644 --- a/docs/development.mdx +++ b/docs/development.mdx @@ -44,6 +44,8 @@ python -m venv .venv .venv/bin/maturin build --manifest-path bindings/python/Cargo.toml --release .venv/bin/python -m pip install --force-reinstall target/wheels/*.whl .venv/bin/python bindings/python/tests/test_installed.py +.venv/bin/python -m pip install -r bindings/python/tests/requirements-typing.txt +.venv/bin/python bindings/python/tests/test_typing.py ``` ## Documentation preview @@ -80,14 +82,14 @@ or a release tag that does not match the package version. Run the quality gates and installed-package tests above on the release commit. Merge the release preparation PR, then create these tags on that same commit -for version 0.3.0: +for version 0.3.1: | Tag | Workflow | Destination | | --- | --- | --- | -| `rust-v0.3.0` | `publish-rust.yml` | crates.io | -| `python-v0.3.0` | `publish-python.yml` | PyPI | -| `node-v0.3.0` | `publish-node.yml` | npm: Node and five native platform packages | -| `wasm-v0.3.0` | `publish-wasm.yml` | npm: browser WASM | +| `rust-v0.3.1` | `publish-rust.yml` | crates.io | +| `python-v0.3.1` | `publish-python.yml` | PyPI | +| `node-v0.3.1` | `publish-node.yml` | npm: Node and five native platform packages | +| `wasm-v0.3.1` | `publish-wasm.yml` | npm: browser WASM | Each tag push starts its publishing workflow. Check all four runs and install the published versions before announcing the combined release. Registry diff --git a/docs/docs.json b/docs/docs.json index 6bc1c79..85f5abb 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -77,7 +77,7 @@ }, { "group": "Releases", - "pages": ["releases/0-3-0"] + "pages": ["releases/0-3-1", "releases/0-3-0"] }, { "group": "Contributing", diff --git a/docs/reference/python.mdx b/docs/reference/python.mdx index 7a59dfb..c8eceff 100644 --- a/docs/reference/python.mdx +++ b/docs/reference/python.mdx @@ -18,6 +18,19 @@ import datafog_core calls. +## Type checking + +Starting with 0.3.1, the wheel includes PEP 561 typing metadata and stubs for +all public functions, result objects, exceptions, and awaitable manager methods. +Pyright and mypy can infer finding and result types and check configuration +literals without a separate stub package. + +Configuration and provider helper types in the stub are prefixed with `_` and +exist only for type checking. If you use them in annotations, import them inside +`if TYPE_CHECKING:` and use postponed annotations (`from __future__ import +annotations`); they are not runtime exports. Runtime validation still enforces +value constraints such as supported locales, nonnegative ranges, and valid keys. + ## Synchronous functions ```python diff --git a/docs/releases/0-3-1.mdx b/docs/releases/0-3-1.mdx new file mode 100644 index 0000000..16b2a50 --- /dev/null +++ b/docs/releases/0-3-1.mdx @@ -0,0 +1,19 @@ +--- +title: "0.3.1" +description: "Python type information for the compiled DataFog Core API." +icon: "tag" +--- + +DataFog Core 0.3.1 adds Python typing support for the compiled extension, +addressing [issue #17](https://github.com/DataFog/datafog-core/issues/17). + +- Wheels include a complete API stub and the PEP 561 `py.typed` marker. +- Type checkers recognize imports, function signatures, immutable result + properties, configuration dictionaries, exceptions, and awaitable privacy + manager methods. +- Python CI checks valid and invalid downstream usage with Pyright and mypy, + including a wheel rebuilt from the source distribution. + +The Python package keeps its existing public imports. Detection, +transformation, and offset behavior are unchanged. All package versions advance +together under the repository's release policy.