From 0b6b47b80e14d98d2a69e6392b351fc6d9dece8b Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 18:33:20 +0200 Subject: [PATCH 1/8] BOF: local JSONID registry From 4f81758d6bc062e2f057c2226a3e0a085628a48e Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 19:30:34 +0200 Subject: [PATCH 2/8] Update .gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 6071354..8e77aea 100644 --- a/.gitignore +++ b/.gitignore @@ -142,3 +142,7 @@ jsonid-integration-files/ # Secreta token.pypi jsonid_pronom.xml +*.csv +*.json +*.diff +token-pypi From 6ec6e4c2f2a9f33980097f9e2d87f82f4c666ad3 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 19:30:57 +0200 Subject: [PATCH 3/8] WIP: local registry --- src/jsonid/jsonid.py | 13 +++++++++++++ src/jsonid/local.py | 7 +++++++ 2 files changed, 20 insertions(+) create mode 100644 src/jsonid/local.py diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index 9d3379e..8f64dad 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -216,9 +216,18 @@ def main() -> None: ) parser.add_argument( "--registry", + "--local", help="path to a custom registry to lead into memory replacing the default", required=False, ) + parser.add_argument( + "--localonly", + "--lonly", + "--lonely", + help="if a local registry is specified, use this and this only", + required=False, + action="store_true", + ) # NB. consider output to stdout once the feature is more stable. parser.add_argument( "--pronom", @@ -285,6 +294,8 @@ def main() -> None: # Primary application functions. if args.registry: raise NotImplementedError("custom registry is not yet available") + if args.local_only: + raise NotImplementedError("todo...") if args.pronom: export.export_pronom() sys.exit() @@ -305,6 +316,8 @@ def main() -> None: logger.info("ok") sys.exit() if args.html: + if args.registry: + raise NotImplementedError("local registry output is not yet supported") helpers.html() sys.exit() if not strategy: diff --git a/src/jsonid/local.py b/src/jsonid/local.py new file mode 100644 index 0000000..ba78e0f --- /dev/null +++ b/src/jsonid/local.py @@ -0,0 +1,7 @@ +"""Functions supporting local registry use""" + + +def load_local_registry(): + """Load the local registry and return it as a data structure + to the caller. + """ From 05122dbfee505b10b71e9e843834683b82615fab Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 19:31:07 +0200 Subject: [PATCH 4/8] WIP: add local registry structure --- local/registry.toml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 local/registry.toml diff --git a/local/registry.toml b/local/registry.toml new file mode 100644 index 0000000..16fd85b --- /dev/null +++ b/local/registry.toml @@ -0,0 +1,19 @@ +[[entries]] + +name = "doctype1" +identifier = "local0001" + +[[doctype.markers]] + +key = "key1" +is = "value1" + +[[entries]] + +name = "doctype2" +identifier = "local0002" + +[[doctype.markers]] + +key = "key2" +is = "value2" From 5ef053ada8649beff3e9e919c7530abe88233b8a Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sat, 8 Aug 2026 22:40:39 +0200 Subject: [PATCH 5/8] WIP: add test harness --- src/jsonid/jsonid.py | 11 ++++++---- src/jsonid/local.py | 33 +++++++++++++++++++++++++++++- tests/test_local_registry.py | 39 ++++++++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 tests/test_local_registry.py diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index 8f64dad..9f62c2c 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -16,11 +16,12 @@ import helpers import lookup import registry + import local except ModuleNotFoundError: try: - from src.jsonid import export, file_processing, helpers, lookup, registry + from src.jsonid import export, file_processing, helpers, lookup, registry, local except ModuleNotFoundError: - from jsonid import export, file_processing, helpers, lookup, registry + from jsonid import export, file_processing, helpers, lookup, registry, local logger = None @@ -293,9 +294,11 @@ def main() -> None: # Primary application functions. if args.registry: - raise NotImplementedError("custom registry is not yet available") - if args.local_only: + local.load_and_parse_local_registry(path=args.registry) + if args.localonly: raise NotImplementedError("todo...") + + return if args.pronom: export.export_pronom() sys.exit() diff --git a/src/jsonid/local.py b/src/jsonid/local.py index ba78e0f..2c99ca5 100644 --- a/src/jsonid/local.py +++ b/src/jsonid/local.py @@ -1,7 +1,38 @@ """Functions supporting local registry use""" +import logging +import pathlib +import tomllib as toml -def load_local_registry(): +logger = logging.getLogger(__name__) + + +class LocalRegistryException(Exception): + """Exception to raise if something goes wrong with the local + registry. + """ + + +def load_and_parse_local_registry(path: str): + """Read the data use the data.""" + + registry = pathlib.Path(path) + if not registry.exists(): + raise LocalRegistryException("registry path not found") + + load_local_registry(registry) + + +def load_local_registry(registry: pathlib.Path): """Load the local registry and return it as a data structure to the caller. """ + + with registry.open() as data: + registry_data = data.read() + + reg = toml.loads(registry_data) + + logger.debug("local registry length: %d", len(reg["entries"])) + + assert False diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py new file mode 100644 index 0000000..72e2e68 --- /dev/null +++ b/tests/test_local_registry.py @@ -0,0 +1,39 @@ +"""Test functions associated with the local registry.""" + +from src.jsonid import local + +from typing import Final + +registry: Final[str] = """ +[[entries]] + +name = "doctype1" +identifier = "local0001" + +[[entries.markers]] + +key = "key1" +is = "value1" + +[[entries]] + +name = "doctype2" +identifier = "local0002" + +[[entries.markers]] + +key = "key2" +is = "value2" + +""" + +import io + + +def test_load_local(tmp_path): + """Ensure loading the local registry works as anticipated.""" + + a = tmp_path / "registry_path" + a.write_text(registry) + + local.load_local_registry(a) From 5e127976a5f3615c3b7691762c60865274c33c38 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sun, 9 Aug 2026 11:15:32 +0200 Subject: [PATCH 6/8] WIP: README and other pieces... --- README.md | 34 +++++++++++++++++++++++++++++++--- src/jsonid/jsonid.py | 5 +++-- src/jsonid/local.py | 3 +++ tests/test_local_registry.py | 9 +++++---- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1a23ed0..1e21bfa 100644 --- a/README.md +++ b/README.md @@ -405,9 +405,37 @@ within the `sops` object/value. ### Local rules -The plan is to allow local rules to be run alongside the global ruleset. I -expect this will be a bit further down the line when the ruleset and -metaddata is more stabilised. +You can define local rules in a local registry object. Local registries are +defined in TOML, and look as follows: + +```toml +[[entries]] + +name = "doctype1" +identifier = "local0001" + +[[entries.markers]] + +key = "key1" +is = "value1" + +[[entries]] + +name = "doctype2" +identifier = "local0002" + +[[entries.markers]] + +key = "key2" +is = "value2" +``` + +Markers follow the same pattern as the standard registry. + +#### Local only + +Use the `--localonly` flag to use _just_ your custom markers in your format +identification workflow. ## PRONOM diff --git a/src/jsonid/jsonid.py b/src/jsonid/jsonid.py index 9f62c2c..a148643 100644 --- a/src/jsonid/jsonid.py +++ b/src/jsonid/jsonid.py @@ -16,12 +16,13 @@ import helpers import lookup import registry + import local except ModuleNotFoundError: try: - from src.jsonid import export, file_processing, helpers, lookup, registry, local + from src.jsonid import export, file_processing, helpers, local, lookup, registry except ModuleNotFoundError: - from jsonid import export, file_processing, helpers, lookup, registry, local + from jsonid import export, file_processing, helpers, local, lookup, registry logger = None diff --git a/src/jsonid/local.py b/src/jsonid/local.py index 2c99ca5..17a8c17 100644 --- a/src/jsonid/local.py +++ b/src/jsonid/local.py @@ -35,4 +35,7 @@ def load_local_registry(registry: pathlib.Path): logger.debug("local registry length: %d", len(reg["entries"])) + for item in reg["entries"]: + print(item) + assert False diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 72e2e68..280c88d 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -1,10 +1,12 @@ """Test functions associated with the local registry.""" -from src.jsonid import local - from typing import Final -registry: Final[str] = """ +from src.jsonid import local + +registry: Final[ + str +] = """ [[entries]] name = "doctype1" @@ -27,7 +29,6 @@ """ -import io def test_load_local(tmp_path): From 3fa182da65a4cf7ce188b8f547a72e88adcd8225 Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Sun, 9 Aug 2026 11:28:19 +0200 Subject: [PATCH 7/8] WIP: look at testing optional entries NB. localref is an ID or URI local to the registry. --- README.md | 2 ++ local/registry.toml | 7 +++++-- tests/test_local_registry.py | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1e21bfa..837cc80 100644 --- a/README.md +++ b/README.md @@ -413,6 +413,7 @@ defined in TOML, and look as follows: name = "doctype1" identifier = "local0001" +localref = "http://example.com/repository/ID" [[entries.markers]] @@ -423,6 +424,7 @@ is = "value1" name = "doctype2" identifier = "local0002" +#localref = "http://example.com/repository/ID [[entries.markers]] diff --git a/local/registry.toml b/local/registry.toml index 16fd85b..aff854d 100644 --- a/local/registry.toml +++ b/local/registry.toml @@ -3,7 +3,10 @@ name = "doctype1" identifier = "local0001" -[[doctype.markers]] +# optional +localref = "http://example.com/repository/ID + +[[entries.markers]] key = "key1" is = "value1" @@ -13,7 +16,7 @@ is = "value1" name = "doctype2" identifier = "local0002" -[[doctype.markers]] +[[entries.markers]] key = "key2" is = "value2" diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 280c88d..03a02dd 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -30,7 +30,6 @@ """ - def test_load_local(tmp_path): """Ensure loading the local registry works as anticipated.""" From 77fef60db5cbb52fd3c8831327d0b12ac473e88f Mon Sep 17 00:00:00 2001 From: ross-spencer Date: Wed, 12 Aug 2026 00:23:14 +0200 Subject: [PATCH 8/8] WIP: slow progress but progress... --- README.md | 4 +- local/registry.toml | 9 +++-- src/jsonid/local.py | 75 +++++++++++++++++++++++++++++++++--- tests/test_local_registry.py | 5 +++ 4 files changed, 83 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 837cc80..cacd77d 100644 --- a/README.md +++ b/README.md @@ -413,7 +413,7 @@ defined in TOML, and look as follows: name = "doctype1" identifier = "local0001" -localref = "http://example.com/repository/ID" +localref = "http://example.com/doctype/spec/ID" [[entries.markers]] @@ -424,7 +424,7 @@ is = "value1" name = "doctype2" identifier = "local0002" -#localref = "http://example.com/repository/ID +localref = "http://example.com/doctype/spec/ID [[entries.markers]] diff --git a/local/registry.toml b/local/registry.toml index aff854d..a0c36eb 100644 --- a/local/registry.toml +++ b/local/registry.toml @@ -2,15 +2,18 @@ name = "doctype1" identifier = "local0001" - -# optional -localref = "http://example.com/repository/ID +localref = "http://example.com/repository/ID" [[entries.markers]] key = "key1" is = "value1" +[[entries.markers]] + +key = "key2" +is = "value2" + [[entries]] name = "doctype2" diff --git a/src/jsonid/local.py b/src/jsonid/local.py index 17a8c17..585e25b 100644 --- a/src/jsonid/local.py +++ b/src/jsonid/local.py @@ -4,6 +4,15 @@ import pathlib import tomllib as toml +try: + import registry_class + import registry_data +except ModuleNotFoundError: + try: + from src.jsonid import registry_class, registry_data + except ModuleNotFoundError: + from jsonid import registry_class, registry_data + logger = logging.getLogger(__name__) @@ -23,19 +32,75 @@ def load_and_parse_local_registry(path: str): load_local_registry(registry) +""" + registry_class.RegistryEntry( + identifier="jrid:0001", + name=[{"@en": "JavaScript Package Lock"}], + description=[{"@en": "describes an exact Node (NPM) module dependency tree"}], + markers=[ + {"KEY": "name", "EXISTS": None}, + {"KEY": "lockfileVersion", "EXISTS": None}, + {"KEY": "packages", "EXISTS": None}, + ], + ), +""" + + def load_local_registry(registry: pathlib.Path): """Load the local registry and return it as a data structure to the caller. """ with registry.open() as data: - registry_data = data.read() + local_registry_data = data.read() + + local_reg = toml.loads(local_registry_data) + + logger.debug("local registry length: %d", len(local_reg["entries"])) + + reg = registry_data.registry() + + for item in local_reg["entries"]: + + # TODO: cleanup, ensure keys are capitalized. + m = [] + for i in item["markers"]: + d = {} + for k, v in tuple(i.items()): + d.update({k.upper(): v}) + m.append(d) + + # TODO: variable naming. + a = registry_class.RegistryEntry( + identifier=item["identifier"], + name=[{"@en": "TODO"}], + description=[{"@en": "TODO"}], + markers=m, + ) - reg = toml.loads(registry_data) + print(a) + print(a.markers) + print("---") + # if local... + reg.append(a) - logger.debug("local registry length: %d", len(reg["entries"])) + # print(reg) - for item in reg["entries"]: - print(item) + """ + [[entries]] + + name = "doctype1" + identifier = "local0001" + localref = "http://example.com/repository/ID + + [[entries.markers]] + + key = "key1" + is = "value1" + + {'name': 'doctype1', 'identifier': 'local0001', 'markers': [{'key': 'key1', 'is': 'value1'}]} + + + """ assert False diff --git a/tests/test_local_registry.py b/tests/test_local_registry.py index 03a02dd..82559b2 100644 --- a/tests/test_local_registry.py +++ b/tests/test_local_registry.py @@ -17,6 +17,11 @@ key = "key1" is = "value1" +[[entries.markers]] + +key = "key2" +is = "value2" + [[entries]] name = "doctype2"