Adopt locale-gated German regex defaults

Co-authored-by: Pranjal Parmar <pranjalrparmar@gmail.com>

Author: sidmohan0

README.md

@@ -78,10 +78,9 @@ print(datafog.sanitize("Card: 4111-1111-1111-1111", engine="regex"))
 
 ## German Structured PII
 
-German VAT IDs and German IBANs are detected by the default regex path because
-their country-code structure is specific enough for default-on screening.
-Broader German identifiers are available with explicit locale selection or
-explicit entity-type filtering.
+German structured PII is country-specific and opt-in. Use explicit locale
+selection or entity-type filtering when you want German VAT IDs, German IBANs,
+tax IDs, postal codes, passports, or residence permits.
 
 ```python
 import datafog
@@ -117,8 +116,8 @@ Use the engine that matches your accuracy and dependency constraints:
 
 - `regex`:
   - Fastest and always available.
-  - Best for structured entities: `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DE_VAT_ID`, `DE_IBAN`, `DATE`, `ZIP_CODE`.
-  - Use `locales=["de"]` for broader German structured IDs such as `DE_TAX_ID`, `DE_POSTAL_CODE`, and passport or residence permit numbers.
+  - Best for default structured entities: `EMAIL`, `PHONE`, `SSN`, `CREDIT_CARD`, `IP_ADDRESS`, `DATE`, `ZIP_CODE`.
+  - Use `locales=["de"]` for German structured IDs such as `DE_VAT_ID`, `DE_IBAN`, `DE_TAX_ID`, `DE_POSTAL_CODE`, and passport or residence permit numbers.
 - `spacy`:
   - Requires `pip install datafog[nlp]`.
   - Useful for unstructured entities like person and organization names.
@@ -190,7 +189,7 @@ datafog replace-text "john@example.com"
 # Hash detected entities
 datafog hash-text "john@example.com"
 
-# Enable broader German regex identifiers
+# Enable German regex identifiers
 datafog redact-text "Steuer-ID 12345678901" --locale de
 ```
 

datafog/core.py

@@ -221,7 +221,7 @@ def get_supported_entities(locales: List[str] | None = None) -> List[str]:
     Example:
         >>> entities = get_supported_entities()
         >>> print(entities)
-        ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DE_VAT_ID', 'DE_IBAN', 'DATE', 'ZIP_CODE']
+        ['EMAIL', 'PHONE', 'SSN', 'CREDIT_CARD', 'IP_ADDRESS', 'DATE', 'ZIP_CODE']
     """
     annotator = RegexAnnotator(locales=locales)
     legacy_map = {"DOB": "DATE", "ZIP": "ZIP_CODE"}

datafog/processing/text_processing/regex_annotator/regex_annotator.py

@@ -38,9 +38,8 @@ class RegexAnnotator:
         "DE_PASSPORT_NUMBER",
         "DE_RESIDENCE_PERMIT_NUMBER",
     ]
-    DEFAULT_LOCALIZED_LABELS = ["DE_VAT_ID", "DE_IBAN"]
     LABELS = BASE_LABELS + GERMAN_LABELS
-    DEFAULT_LABELS = BASE_LABELS + DEFAULT_LOCALIZED_LABELS
+    DEFAULT_LABELS = BASE_LABELS
     SUPPORTED_LOCALES = {"de", "de-de", "de_de"}
     LOCALE_LABELS = {
         "de": GERMAN_LABELS,
@@ -103,13 +102,18 @@ def __init__(
             # Supports dashed and no-dash formats.
             "SSN": re.compile(
                 r"""
-                (?<!\d)
                 (?:
+                    (?<!\d)
                     (?!000|666)\d{3}-(?!00)\d{2}-(?!0000)\d{4}
+                    (?!\d)
                     |
+                    (?<![A-Za-z0-9])
+                    (?<!DE)
+                    (?<!DE\s)
+                    (?<!DE-)
                     (?!000|666)\d{3}(?!00)\d{2}(?!0000)\d{4}
+                    (?![A-Za-z0-9])
                 )
-                (?!\d)
                 """,
                 re.IGNORECASE | re.MULTILINE | re.VERBOSE,
             ),

datafog/services/text_service.py

@@ -58,7 +58,7 @@ def __init__(
                 - "smart": Try RegexAnnotator → GLiNER → SpaCy cascade (requires nlp-advanced extra)
             gliner_model: GLiNER model name to use when engine is "gliner" or "smart"
             locales: Optional locale tags for regex detection. Use ["de"] to enable
-                broader German structured identifiers.
+                German structured identifiers.
 
         Raises:
             AssertionError: If an invalid engine type is provided

docs/cli.rst

@@ -22,8 +22,8 @@ commands. Install ``datafog[distributed]`` when using ``SparkService``.
 German locale support
 ---------------------
 
-German VAT IDs and German IBANs are detected by the default regex path. Broader
-German identifiers are opt-in through ``--locale de`` on the core text commands:
+German structured PII is opt-in through ``--locale de`` on the core text
+commands:
 
 .. code-block:: bash
 

docs/getting-started.rst

@@ -83,22 +83,21 @@ Agent-oriented helpers use the same lightweight text path:
 German Structured PII
 =====================
 
-The core regex engine includes German VAT IDs and German IBANs by default
-because they carry strong country-code structure:
+German structured PII is country-specific and opt-in, including German VAT IDs
+and German IBANs:
 
 .. code-block:: python
 
    import datafog
 
-   result = datafog.scan("USt-IdNr DE 123456789", engine="regex")
+   result = datafog.scan("USt-IdNr DE 123456789", engine="regex", locales=["de"])
    print([(entity.type, entity.text) for entity in result.entities])
 
-Broader German identifiers such as ``DE_TAX_ID``,
-``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
-``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER`` require explicit
-German locale selection or explicit ``entity_types`` filtering. This keeps
-ordinary ticket, SKU, order, and invoice IDs from becoming default-on false
-positives.
+German identifiers such as ``DE_VAT_ID``, ``DE_IBAN``, ``DE_TAX_ID``,
+``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``, ``DE_PASSPORT_NUMBER``, and
+``DE_RESIDENCE_PERMIT_NUMBER`` require explicit German locale selection or
+explicit ``entity_types`` filtering. This keeps ordinary ticket, SKU, order,
+and invoice IDs from becoming default-on false positives.
 
 .. code-block:: python
 

docs/python-sdk.rst

@@ -31,11 +31,11 @@ German locale coverage
 ----------------------
 
 DataFog 4.5 includes regex-only German structured PII support without adding
-dependencies. German VAT IDs and German IBANs are active in the default regex
-path. Broader German-only identifiers are opt-in because their raw shapes are
-common in ordinary product, ticket, invoice, and order data.
+dependencies. German-only identifiers are opt-in because their raw shapes are
+country-specific or common in ordinary product, ticket, invoice, and order
+data.
 
-Use ``locales=["de"]`` to enable the broader German set:
+Use ``locales=["de"]`` to enable the German set:
 
 .. code-block:: python
 
@@ -55,10 +55,9 @@ You can also request one German entity type directly:
        entity_types=["DE_TAX_ID"],
    )
 
-The opt-in German set currently covers ``DE_TAX_ID``,
-``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
-``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER``. The default set
-also covers ``DE_VAT_ID`` and ``DE_IBAN``.
+The opt-in German set currently covers ``DE_VAT_ID``, ``DE_IBAN``,
+``DE_TAX_ID``, ``DE_SOCIAL_SECURITY_NUMBER``, ``DE_POSTAL_CODE``,
+``DE_PASSPORT_NUMBER``, and ``DE_RESIDENCE_PERMIT_NUMBER``.
 
 Optional services
 -----------------

tests/corpus/structured_pii.json

@@ -735,8 +735,9 @@
     ]
   },
   {
-    "id": "de-vat-id-default",
+    "id": "de-vat-id-locale",
     "input": "USt-IdNr DE 123456789 ist gesetzt.",
+    "locales": ["de"],
     "expected_entities": [
       {
         "type": "DE_VAT_ID",
@@ -747,8 +748,9 @@
     ]
   },
   {
-    "id": "de-iban-default",
+    "id": "de-iban-locale",
     "input": "IBAN DE44 5001 0517 5407 3249 31 ist gueltig.",
+    "locales": ["de"],
     "expected_entities": [
       {
         "type": "DE_IBAN",

tests/test_de_pii_regex.py

@@ -24,12 +24,17 @@
         ),
     ],
 )
-def test_high_specificity_german_regex_default_cases(
+def test_german_regex_cases_require_german_locale_or_explicit_entity_type(
     label: str, text: str, expected: str
 ) -> None:
-    annotator = RegexAnnotator()
-    result = annotator.annotate(text)
-    assert expected in result[label]
+    default_result = RegexAnnotator().annotate(text)
+    assert expected not in default_result[label]
+
+    german_result = RegexAnnotator(locales=["de"]).annotate(text)
+    assert expected in german_result[label]
+
+    explicit_result = RegexAnnotator(enabled_labels=[label]).annotate(text)
+    assert expected in explicit_result[label]
 
 
 @pytest.mark.parametrize(
@@ -118,25 +123,39 @@ def test_redaction_and_service_locale_support() -> None:
     assert service_result["DE_PASSPORT_NUMBER"] == ["C12345678"]
 
 
-def test_german_vat_redaction_suppresses_inner_generic_ssn_match() -> None:
-    text = "USt-IdNr DE123456789 ist gesetzt."
-
+@pytest.mark.parametrize(
+    "text,vat_text",
+    [
+        ("USt-IdNr DE123456789 ist gesetzt.", "DE123456789"),
+        ("USt-IdNr DE 123456789 ist gesetzt.", "DE 123456789"),
+        ("USt-IdNr DE-123456789 ist gesetzt.", "DE-123456789"),
+    ],
+)
+def test_german_vat_redaction_suppresses_inner_generic_ssn_match(
+    text: str, vat_text: str
+) -> None:
     scan_result = scan(text, engine="regex")
-    assert [(entity.type, entity.text) for entity in scan_result.entities] == [
-        ("DE_VAT_ID", "DE123456789")
+    assert scan_result.entities == []
+
+    locale_scan_result = scan(text, engine="regex", locales=["de"])
+    assert [(entity.type, entity.text) for entity in locale_scan_result.entities] == [
+        ("DE_VAT_ID", vat_text)
     ]
 
-    redaction = scan_and_redact(text, engine="regex")
-    assert redaction.redacted_text == "USt-IdNr [DE_VAT_ID_1] ist gesetzt."
+    default_redaction = scan_and_redact(text, engine="regex")
+    assert default_redaction.redacted_text == text
+
+    redaction = scan_and_redact(text, engine="regex", locales=["de"])
+    assert redaction.redacted_text == text.replace(vat_text, "[DE_VAT_ID_1]")
 
 
 def test_top_level_helpers_and_supported_entities_respect_locale() -> None:
     default_entities = get_supported_entities()
-    assert "DE_VAT_ID" in default_entities
-    assert "DE_IBAN" in default_entities
-    assert "DE_TAX_ID" not in default_entities
+    assert all(not entity.startswith("DE_") for entity in default_entities)
 
     german_entities = get_supported_entities(locales=["de"])
+    assert "DE_VAT_ID" in german_entities
+    assert "DE_IBAN" in german_entities
     assert "DE_TAX_ID" in german_entities
     assert "DE_RESIDENCE_PERMIT_NUMBER" in german_entities
 

tests/test_detection_accuracy.py

@@ -280,9 +280,11 @@ def _canon_type(entity_type: str) -> str:
     return TYPE_ALIASES.get(raw, raw)
 
 
-def _extract_entities(text: str, engine: str) -> list[dict[str, Any]]:
+def _extract_entities(
+    text: str, engine: str, locales: list[str] | None = None
+) -> list[dict[str, Any]]:
     try:
-        result = scan(text=text, engine=engine)
+        result = scan(text=text, engine=engine, locales=locales)
     except (ImportError, EngineNotAvailable) as exc:
         pytest.skip(f"{engine} engine unavailable in this environment: {exc}")
 
@@ -347,7 +349,7 @@ def _assert_expected_found(
     case: dict[str, Any], engine: str, corpus_kind: str
 ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
     text = case["input"]
-    actual = _extract_entities(text, engine)
+    actual = _extract_entities(text, engine, locales=case.get("locales"))
     expected = _required_expected(case["expected_entities"], engine, corpus_kind)
 
     for exp in expected:
@@ -403,7 +405,9 @@ def _compute_metrics(
     for engine in engines:
         for corpus_kind, cases in corpora:
             for case in cases:
-                actual = _extract_entities(case["input"], engine)
+                actual = _extract_entities(
+                    case["input"], engine, locales=case.get("locales")
+                )
                 expected = _required_expected(
                     case["expected_entities"], engine, corpus_kind
                 )
@@ -490,7 +494,7 @@ def test_structured_pii_detection_slow(case: dict[str, Any], engine: str) -> Non
 @pytest.mark.parametrize("engine", FAST_ENGINES)
 def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None:
     _xfail_if_known_limitation(case, engine, "negative")
-    actual = _extract_entities(case["input"], engine)
+    actual = _extract_entities(case["input"], engine, locales=case.get("locales"))
     assert not actual, f"{case['id']} ({engine}) false positives: {actual}"
 
 
@@ -501,7 +505,7 @@ def test_negative_cases_fast(case: dict[str, Any], engine: str) -> None:
 @pytest.mark.parametrize("engine", SLOW_ENGINES)
 def test_negative_cases_slow(case: dict[str, Any], engine: str) -> None:
     _xfail_if_known_limitation(case, engine, "negative")
-    actual = _extract_entities(case["input"], engine)
+    actual = _extract_entities(case["input"], engine, locales=case.get("locales"))
     assert not actual, f"{case['id']} ({engine}) false positives: {actual}"