From b857b29c43ab5077d15b7a12996e7533e56c82f5 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 15 Jul 2026 17:02:05 +0200 Subject: [PATCH 1/6] fix(base): add isbn mapping, add missing resource types --- .../transform/xml_processing/rules/base.py | 54 +++++++++++++++++-- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 06d0d702..25f719d8 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -10,6 +10,8 @@ import datetime import logging import re +from idutils.normalizers import normalize_isbn +from isbnlib import NotValidISBNError from urllib.parse import ParseResult, urlparse from dateutil.parser import ParserError, parse @@ -341,6 +343,8 @@ def report_number(self, key, value): existing_ids.append(new_id) self["identifiers"] = existing_ids raise IgnoreKey("related_identifiers") + new_id["relation_type"] = {"id": "isvariantformof"} + new_id["resource_type"] = {"id": "publication-other"} if new_id in existing_ids: raise IgnoreKey("related_identifiers") return new_id @@ -363,6 +367,29 @@ def aleph_number(self, key, value): raise IgnoreKey("identifiers") +@model.over("related_identifiers", "(^020__)", override_tag=True) +@for_each_value +def isbn(self, key, value): + _isbn = StringValue(value.get("a", "")).parse() + _isbn_material = StringValue(value.get("u", "")).parse() + if _isbn: + try: + _isbn = normalize_isbn(_isbn) + + except NotValidISBNError as e: + raise UnexpectedValue("Not a valid ISBN.", field=key, value=value) + + new_id = { + "identifier": _isbn, + "scheme": "isbn", + "relation_type": {"id": "isversionof"}, + "resource_type": {"id": "publication-book"} + } + if new_id not in self.get("related_identifiers", []): + return new_id + raise IgnoreKey("related_identifiers") + + @model.over("identifiers", "^035__") @for_each_value def identifiers(self, key, value): @@ -441,6 +468,8 @@ def identifiers(self, key, value): rel_id not in self.get("identifiers", []) ): return rel_id + elif not scheme: + raise IgnoreKey("identifiers") else: if "HOLALE" in id_value: raise IgnoreKey("identifiers") @@ -903,21 +932,36 @@ def related_identifiers_787(self, key, value): raise IgnoreKey("related_identifiers") -@model.over("related_identifiers", "^775_") +@model.over("related_identifiers", "^775_", override_tag=True) @for_each_value def related_identifiers(self, key, value): """Translates related identifiers.""" - description = value.get("b") - year = value.get("c") recid = value.get("w") + volume_note = value.get("n") rel_ids = self.get("related_identifiers", []) + + if not recid: + raise IgnoreKey("related_identifiers") + + if volume_note: + if not re.search(r"v\.?\s*\d+", volume_note, re.IGNORECASE): + raise UnexpectedValue( + "Missing volume indication in 775__n", + field=key, + subfield="n", + value=value, + ) + resource_type = {"id": "publication-book"} + else: + resource_type = {"id": "publication-other"} + new_id = { "identifier": recid, "scheme": "cds", "relation_type": {"id": "references"}, - "resource_type": {"id": "other"}, + "resource_type": resource_type, } - if recid and new_id not in rel_ids: + if new_id not in rel_ids: return new_id raise IgnoreKey("related_identifiers") From 642704d70fd36633eeddac9148b9ba9fe2b9d130 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 15 Jul 2026 17:02:50 +0200 Subject: [PATCH 2/6] change(models): split models to add granular rules and queries --- .../rdm/records/transform/models/lep.py | 116 ++++++++++++++++++ .../rdm/records/transform/models/research.py | 2 +- setup.cfg | 3 +- 3 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 cds_migrator_kit/rdm/records/transform/models/lep.py diff --git a/cds_migrator_kit/rdm/records/transform/models/lep.py b/cds_migrator_kit/rdm/records/transform/models/lep.py new file mode 100644 index 00000000..ba7d7564 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/lep.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM CMS note model.""" +from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( + rdm_base_publication_model, +) +from cds_migrator_kit.rdm.records.transform.models.research import ResearchModel + + +class LEPResearchModel(ResearchModel): + """Translation model for research.""" + + __query__ = '980__:L3_Papers OR 980__:INTNOTEALEPHPRIV OR 980__:OPAL_Papers OR 980__:OPAL_Misc OR 980__:DELPHI_Misc OR 980__:DELPHI_Papers OR 980__:L3_Misc OR 693__.e:L3 OR 693__.e:DELPHI OR 693__.e:OPAL OR 693__.e:ALEPH OR 690C_.a:PUBLDELPHINOTE OR 690C_.a:PRIVDELPHINOTE OR 710__.g:"ALEPH Collaboration" OR 710__.g:"Aleph Collaboration" OR 980__.a:ALEPH_Papers OR 980__.a:ALEPHDRAFT OR 037__:CERN-ALEPH-PUB-* OR 037__:CERN-ALEPH-ARCH-DATA-* -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + + __ignore_keys__ = { + "594__a", # can be ignored for this collection + "775__p", # can be ignored for this collection - title of another volume + "775__c", # year of volume + "596__a", # multivolume tag + "300__x", # drop the physical description + "0248_a", + "0248_p", + "0248_q", + "0247_9", # provenance of the DOI + "030__a", # TODO coden designation to drop? + "035__h", # oai identifiers in 1215391 + "035__d", # oai identifiers in 1215391 + "035__t", # oai identifiers in 1215391 + "035__u", # oai identifiers in 1215391 + "035__m", # oai identifiers in 1215391 + "035__z", # oai identifiers in 1215391 + "336__a", # redundant field + "500__9", # provenance of the note + "520__9", # provenance of the description + "520__h", # provenance of the description + "852__c", # holdings will be taken separately + "852__h", + "037__c", # arxiv subject + "100__m", # email of contributor + "245__9", # title provenance + "270__m", # document contact email + "300__a", # number of pages + "340__a", # TODO ignore material? + "540__3", # TODO still ignore the material of the license? + "540__9", # TODO still ignore the material of the license? + "542__3", # TODO still ignore the material of the license? + "595__i", # TODO ?? + "695__e", # some inspire tag + "700__m", # email of contributor + "700__q", # TODO ignore? aliteration of the name, used for searching + "700__v", # TODO drop? + "773__x", # INSPIRE publication note + "773__t", # INSPIRE publication note + "773__0", # from SIS: can be ignored + "773__o", # from SIS: can be ignored + "773__z", # from SIS: can be ignored + "8564_8", # file id + "8564_s", # bibdoc id + "8564_x", # icon thumbnails sizes + "8564_y", # file description - done by files dump, sometimes these are used for open access calculation + "8564_w", # system field + "8564_z", # file comment, migrated via file metadata + "913__y", # citation + "913__v", # citation + "913__t", # citation + "913__a", # citation + "913__c", # citation + "916__y", # year, redundant value + "937__c", # last modified by + "937__s", # last modification date + "960__a", # base number + "961__c", # + "961__h", # + "961__l", # + "961__x", # + "964__a", # TODO: ignore? + "981__a", # duplicate record id + "999C50", + "999C52", # https://cds.cern.ch/record/2640188/export/hm?ln=en + "999C59", # https://cds.cern.ch/record/2284615/export/hm?ln=en + "999C5a", # https://cds.cern.ch/record/2678429/export/hm?ln=en + "999C5c", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5h", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5i", # https://cds.cern.ch/record/2284892/export/hm?ln=en + "999C5k", # https://cds.cern.ch/record/2671914/export/hm?ln=en + "999C5l", # https://cds.cern.ch/record/2283115/export/hm?ln=en + "999C5m", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5o", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5p", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5r", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5s", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5u", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5v", # https://cds.cern.ch/record/2283088/export/hm?ln=en + "999C5x", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C5y", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5z", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C6a", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6v", # https://cds.cern.ch/record/2284606/export/hm?ln=en + } + + _default_fields = { + "custom_fields": {}, + } + + +lep_research_model = LEPResearchModel( + bases=(rdm_base_publication_model,), + entry_point_group="cds_migrator_kit.migrator.rdm.rules.research", +) diff --git a/cds_migrator_kit/rdm/records/transform/models/research.py b/cds_migrator_kit/rdm/records/transform/models/research.py index e5ff48e9..821f5a35 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research.py +++ b/cds_migrator_kit/rdm/records/transform/models/research.py @@ -15,7 +15,7 @@ class ResearchModel(CdsOverdo): """Translation model for research.""" - __query__ = '980__:L3_Papers OR 980__:INTNOTEALEPHPRIV OR 980__:OPAL_Papers OR 980__:OPAL_Misc OR 980__:DELPHI_Misc OR 980__:DELPHI_Papers OR 980__:L3_Misc OR 693__.e:L3 OR 693__.e:DELPHI OR 693__.e:OPAL OR 693__.e:ALEPH OR 690C_.a:PUBLDELPHINOTE OR 690C_.a:PRIVDELPHINOTE OR 710__.g:"ALEPH Collaboration" OR 710__.g:"Aleph Collaboration" OR 980__.a:ALEPH_Papers OR 980__.a:ALEPHDRAFT OR 037__:CERN-ALEPH-PUB-* OR 037__:CERN-ALEPH-ARCH-DATA-* OR 980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + __query__ = '980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' __ignore_keys__ = { "0248_a", diff --git a/setup.cfg b/setup.cfg index 8ab5e8df..a3a8741b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,7 +66,8 @@ cds_migrator_kit.migrator.users.model = cds_migrator_kit.migrator.submitter.model = submitter = cds_migrator_kit.rdm.users.transform.xml_processing.models.submitter:submitter_model cds_migrator_kit.migrator.models = - lep_exp = cds_migrator_kit.rdm.records.transform.models.research:research_model + research = cds_migrator_kit.rdm.records.transform.models.research:research_model + lep_research = cds_migrator_kit.rdm.records.transform.models.lep:lep_research_model research_comm_model = cds_migrator_kit.rdm.records.transform.models.research_committee:research_comm_model fap = cds_migrator_kit.rdm.records.transform.models.fap:fap_model ssn = cds_migrator_kit.rdm.records.transform.models.summer_student_report:sspn_model From c4824ee72e80f4bc8514f858b989574940c84839 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 15 Jul 2026 17:03:52 +0200 Subject: [PATCH 3/6] add(rules): meeting rules (111 and 711), abbrevation field (691) --- .../xml_processing/rules/research.py | 72 ++++++ tests/cds-rdm/test_base_rules.py | 144 ++++++++++++ tests/cds-rdm/test_publications_rules.py | 222 ++++++++++++++++++ 3 files changed, 438 insertions(+) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index df1ae5c9..d300fbf2 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -1,4 +1,5 @@ import re +from datetime import datetime from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey @@ -187,6 +188,22 @@ def internal_notes(self, key, value): return internal_notes +@model.over("additional_descriptions", "^691__") +@for_each_value +@require(["a"]) +def abbreviation(self, key, value): + """Translates 691__a abbreviation into an additional description.""" + description_text = value.get("a") + if not description_text: + raise IgnoreKey("additional_descriptions") + return { + "description": f"Abbreviation: {description_text}", + "type": { + "id": "other", + }, + } + + @model.over("funding", "(^536__)") def funding(self, key, value): _custom_fields = self.get("custom_fields", {}) @@ -296,6 +313,61 @@ def journal(self, key, value): return _custom_fields +@model.over("custom_fields", "(^111__)|(^711__)", override_tag=True) +def meeting(self, key, value): + """Translates meeting name entries (111__, 711__) into meeting:meeting. + + 111__a -> title, 111__c -> place, 111__g -> acronym. 111__d -> dates; if + missing, fall back to 111__9 (parsed as YYYYMMDD), then to 111__f (a bare + year). 711__a -> title, but only if no meeting entry with that title + already exists (711 is an added entry, often duplicating the 111 + conference name). + """ + _custom_fields = self.get("custom_fields", {}) + meetings = _custom_fields.get("meeting:meeting", []) + + if key.startswith("111"): + title = StringValue(value.get("a", "")).parse() + place = StringValue(value.get("c", "")).parse() + date_recon = StringValue(value.get("9", "")).parse() + if date_recon: + try: + date_recon = datetime.strptime(date_recon, "%Y%m%d").strftime( + "%Y-%m-%d" + ) + except ValueError: + raise UnexpectedValue( + "Can't parse meeting date (111__9)", + field=key, + subfield="9", + value=value, + ) + date_year = StringValue(value.get("f", "")).parse() + dates = StringValue(value.get("d", "")).parse() or date_recon or date_year + acronym = StringValue(value.get("g", "")).parse() + + new_meeting = {} + if title: + new_meeting["title"] = title + if place: + new_meeting["place"] = place + if dates: + new_meeting["dates"] = dates + if acronym: + new_meeting["acronym"] = acronym + if new_meeting: + meetings.append(new_meeting) + else: + title = StringValue(value.get("a", "")).parse() + if title and not any(m.get("title") == title for m in meetings): + meetings.append({"title": title}) + + if meetings: + _custom_fields["meeting:meeting"] = meetings + self["custom_fields"] = _custom_fields + raise IgnoreKey("custom_fields") + + @model.over("_oa_license", "^540__", override=True) def oa_level_from_license(self, key, value): """Detect funding model; also runs base license logic for rights. diff --git a/tests/cds-rdm/test_base_rules.py b/tests/cds-rdm/test_base_rules.py index 5285fcd0..244a8fa7 100644 --- a/tests/cds-rdm/test_base_rules.py +++ b/tests/cds-rdm/test_base_rules.py @@ -13,10 +13,12 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import ( custom_fields_693, + isbn, normalize, note, recid, record_restriction, + related_identifiers, urls, ) @@ -287,6 +289,148 @@ def test_urls_custom_subfield(self): assert result[0]["identifier"] == "http://custom.com" +class TestIsbnRelatedIdentifiers: + """Test isbn function (020__ rule) from base.py.""" + + def test_isbn13_normalized_and_added(self): + """Test that a valid ISBN-13 is normalized and added to related_identifiers.""" + record = {} + result = isbn(record, "020__", {"a": "978-3-16-148410-0"}) + assert result == [ + { + "identifier": "978-3-16-148410-0", + "scheme": "isbn", + "relation_type": {"id": "isversionof"}, + "resource_type": {"id": "publication-book"}, + } + ] + + def test_isbn10_converted_to_isbn13(self): + """Test that a valid ISBN-10 is converted to its ISBN-13 form.""" + record = {} + result = isbn(record, "020__", {"a": "0-306-40615-2"}) + assert result == [ + { + "identifier": "978-0-306-40615-7", + "scheme": "isbn", + "relation_type": {"id": "isversionof"}, + "resource_type": {"id": "publication-book"}, + } + ] + + def test_isbn_missing_subfield_a_ignored(self): + """Test that 020__ without subfield 'a' is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + isbn(record, "020__", {}) + + def test_isbn_empty_subfield_a_ignored(self): + """Test that 020__a with an empty value is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + isbn(record, "020__", {"a": ""}) + + def test_isbn_no_existing_related_identifiers_does_not_raise(self): + """Regression test: isbn() must not fail on a record that has no + related_identifiers key yet (the common case for the first related + identifier found on a record).""" + record = {} + result = isbn(record, "020__", {"a": "978-3-16-148410-0"}) + assert result[0]["identifier"] == "978-3-16-148410-0" + + def test_isbn_no_duplicate(self): + """Test that an already-present ISBN is not added twice.""" + record = { + "related_identifiers": [ + { + "identifier": "978-3-16-148410-0", + "scheme": "isbn", + "relation_type": {"id": "isversionof"}, + "resource_type": {"id": "publication-book"}, + } + ] + } + with pytest.raises(IgnoreKey): + isbn(record, "020__", {"a": "978-3-16-148410-0"}) + + def test_isbn_malformed_value_does_not_raise(self): + """Documents current behavior: normalize_isbn() does not raise for + malformed input, so malformed ISBNs end up with an empty identifier + instead of triggering the UnexpectedValue branch.""" + record = {} + result = isbn(record, "020__", {"a": "123"}) + assert result[0]["identifier"] == "" + + +class TestRelatedIdentifiers775: + """Test related_identifiers function (775 field) from base.py.""" + + def test_775_no_volume_note_sets_publication_other(self): + """Test that missing 775__n results in resource_type publication-other.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456"}) + assert result == [ + { + "identifier": "123456", + "scheme": "cds", + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-other"}, + } + ] + + def test_775_volume_note_dot_notation_sets_publication_book(self): + """Test that 775__n with 'v.1' notation results in publication-book.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456", "n": "v.1"}) + assert result == [ + { + "identifier": "123456", + "scheme": "cds", + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-book"}, + } + ] + + def test_775_volume_note_no_dot_notation_sets_publication_book(self): + """Test that 775__n with 'v2' notation results in publication-book.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456", "n": "v2"}) + assert result[0]["resource_type"] == {"id": "publication-book"} + + def test_775_volume_note_with_space_sets_publication_book(self): + """Test that 775__n with 'v. 3' notation results in publication-book.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456", "n": "v. 3"}) + assert result[0]["resource_type"] == {"id": "publication-book"} + + def test_775_note_without_volume_indication_raises_error(self): + """Test that 775__n without volume indication raises UnexpectedValue.""" + record = {} + with pytest.raises(UnexpectedValue): + related_identifiers(record, "775__", {"w": "123456", "n": "some note"}) + + def test_775_no_recid_ignored(self): + """Test that missing 775__w is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + related_identifiers(record, "775__", {"n": "v.1"}) + + def test_775_no_duplicate(self): + """Test that duplicate related identifiers are not added twice.""" + record = { + "related_identifiers": [ + { + "identifier": "123456", + "scheme": "cds", + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-other"}, + } + ] + } + with pytest.raises(IgnoreKey): + related_identifiers(record, "775__", {"w": "123456"}) + + class TestNote: """Test note function from base.py.""" diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index 2a96fd26..78d307c9 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -12,12 +12,14 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.records.transform.xml_processing.rules.research import ( + abbreviation, deadline_date, imprint_info, internal_notes, isbn, issn, journal, + meeting, oa_level_from_license, udc, ) @@ -715,6 +717,226 @@ def test_deadline_date_invalid_raises_unexpected_value(self): deadline_date(record, "583__", {"c": "not-a-valid-date"}) +class TestAbbreviation: + """Test abbreviation function from research.py (691__a rule).""" + + def test_abbreviation_added_as_additional_description(self): + """Test that 691__a is translated into an additional description.""" + record = {} + result = abbreviation(record, "691__", {"a": "CMS"}) + assert result == [ + { + "description": "Abbreviation: CMS", + "type": {"id": "other"}, + } + ] + + def test_abbreviation_appends_to_existing_additional_descriptions(self): + """Test that 691__a is appended to an existing additional_descriptions list.""" + record = { + "additional_descriptions": [ + {"description": "Some other note", "type": {"id": "other"}} + ] + } + result = abbreviation(record, "691__", {"a": "CMS"}) + assert result == [ + { + "description": "Abbreviation: CMS", + "type": {"id": "other"}, + } + ] + + def test_abbreviation_missing_a_ignored(self): + """Test that 691__ without subfield 'a' is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + abbreviation(record, "691__", {}) + + def test_abbreviation_empty_a_ignored(self): + """Test that 691__a with an empty value is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + abbreviation(record, "691__", {"a": ""}) + + +class TestMeetingFrom111And711: + """Tests for meeting (111__/711__ rule) - builds meeting:meeting custom field.""" + + def test_111_full_fields_creates_meeting_entry(self): + """Test that 111__a/c/d/g populate title/place/dates/acronym.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + { + "a": "Lectures at Fermilab on SUSY", + "c": "Batavia, IL, USA", + "d": "10 Mar 2000", + "f": "2000", + "g": "batavia20000310", + }, + ) + assert record["custom_fields"]["meeting:meeting"] == [ + { + "title": "Lectures at Fermilab on SUSY", + "place": "Batavia, IL, USA", + "dates": "10 Mar 2000", + "acronym": "batavia20000310", + } + ] + + def test_111_falls_back_to_f_when_d_missing(self): + """Test that 111__f is used for dates when 111__d is absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "c": "Geneva", "f": "2000", "g": "ABC"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "2000" + + def test_111_d_takes_precedence_over_f(self): + """Test that 111__d is preferred over 111__f when both are present.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "d": "10 Mar 2000", "f": "2000"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "10 Mar 2000" + + def test_111_falls_back_to_9_when_d_missing(self): + """Test that 111__9 (YYYYMMDD) is used for dates when 111__d is absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "f": "2000", "9": "20000310"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "2000-03-10" + + def test_111_d_takes_precedence_over_9(self): + """Test that 111__d is preferred over 111__9 when both are present.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "d": "10 Mar 2000", "9": "20000310"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "10 Mar 2000" + + def test_111_9_takes_precedence_over_f(self): + """Test that 111__9 is preferred over 111__f (a bare year) when both + are present and 111__d is absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "f": "2000", "9": "20000310"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "2000-03-10" + + def test_111_invalid_9_raises_error(self): + """Test that an unparseable 111__9 raises UnexpectedValue.""" + record = {} + with pytest.raises(UnexpectedValue): + meeting(record, "111__", {"a": "Some Meeting", "9": "not-a-date"}) + + def test_111_without_dates_omits_dates_key(self): + """Test that no 'dates' key is set when both 111__d and 111__f are absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "111__", {"a": "Some Meeting"}) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert "dates" not in meeting_entry + + def test_111_without_title_omits_title_key(self): + """Test that a 111 field without subfield 'a' creates an entry with + no 'title' key rather than dropping the other subfields.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "111__", {"c": "Geneva"}) + assert record["custom_fields"]["meeting:meeting"] == [{"place": "Geneva"}] + + def test_711_adds_new_title_when_no_matching_meeting_exists(self): + """Test that 711__a creates a new meeting entry when its title is new.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "711__", {"a": "Joint Experimental Theoretical Seminar"}) + assert record["custom_fields"]["meeting:meeting"] == [ + {"title": "Joint Experimental Theoretical Seminar"} + ] + + def test_711_skips_duplicate_title(self): + """Test that 711__a is not added again when a meeting with the same + title already exists (e.g. duplicating the 111 conference name).""" + record = { + "custom_fields": { + "meeting:meeting": [{"title": "SUSY at DELPHI", "place": "Geneva"}] + } + } + with pytest.raises(IgnoreKey): + meeting(record, "711__", {"a": "SUSY at DELPHI"}) + assert record["custom_fields"]["meeting:meeting"] == [ + {"title": "SUSY at DELPHI", "place": "Geneva"} + ] + + def test_111_then_two_711_only_adds_distinct_titles(self): + """Reproduces record 436657: 111 sets the full title, and only the + 711 entry with a distinct title is appended as a new meeting.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + { + "a": "Lectures at Fermilab on SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy", + "c": "Batavia, IL, USA", + "d": "10 Mar 2000", + "g": "batavia20000310", + }, + ) + with pytest.raises(IgnoreKey): + meeting( + record, + "711__", + {"a": "SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy"}, + ) + with pytest.raises(IgnoreKey): + meeting( + record, + "711__", + {"a": "Joint Experimental Theoretical Physics Seminar"}, + ) + meetings = record["custom_fields"]["meeting:meeting"] + assert len(meetings) == 3 + titles = [m["title"] for m in meetings] + assert ( + "Lectures at Fermilab on SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy" + in titles + ) + assert "SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy" in titles + assert "Joint Experimental Theoretical Physics Seminar" in titles + + def test_711_without_title_no_meeting_added(self): + """Test that a 711 field without subfield 'a' produces no meeting entry.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "711__", {}) + assert record["custom_fields"].get("meeting:meeting", []) == [] + + class TestLicenseAndFundingFrom540: """Tests for oa_level_from_license (540__ rule).""" From a77df0c149878767cc2b5e263b5c190658db918d Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 15 Jul 2026 17:04:30 +0200 Subject: [PATCH 4/6] add(transform): use meeting title for missing conference proceedings title --- .../rdm/records/transform/transform.py | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index b03eaa65..a21b9eca 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -413,9 +413,22 @@ def _resource_type(entry): try: return entry["resource_type"] except KeyError: - raise MissingRequiredField( - message="resource_type", field="980" - ) + raise MissingRequiredField(message="resource_type", field="980") + + def _title(entry, resource_type): + title = entry.get("title") + if title: + return title + # 245 (title) is sometimes absent on conference proceedings + # records; fall back to the conference name (111__a) stored on + # the first meeting entry. + if resource_type.get("id") == "publication-conferenceproceeding": + meetings = entry.get("custom_fields", {}).get("meeting:meeting", []) + for meeting_entry in meetings: + meeting_title = meeting_entry.get("title") + if meeting_title: + return meeting_title + return title def _publication_date(entry, dump_record): pub_date = entry.get("publication_date") @@ -485,10 +498,11 @@ def subjects(json_entry): _subjects = subjects(json_entry) table_of_contents(json_entry) + _resource_type_value = _resource_type(json_entry) metadata = { "creators": creators(json_entry), - "title": json_entry.get("title"), - "resource_type": _resource_type(json_entry), + "title": _title(json_entry, _resource_type_value), + "resource_type": _resource_type_value, "description": json_entry.get("description"), "publication_date": _publication_date(json_entry, record_dump), "contributors": creators(json_entry, key="contributors"), From 514484b5e3f0cc9d2c3402eaf5f6ab39b2596385 Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 15 Jul 2026 17:04:56 +0200 Subject: [PATCH 5/6] fix(contributors): drop jacow identifier from orcid field --- .../transform/xml_processing/quality/contributors.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cds_migrator_kit/transform/xml_processing/quality/contributors.py b/cds_migrator_kit/transform/xml_processing/quality/contributors.py index e58c9218..f67f246e 100644 --- a/cds_migrator_kit/transform/xml_processing/quality/contributors.py +++ b/cds_migrator_kit/transform/xml_processing/quality/contributors.py @@ -6,7 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM contributors migration module.""" - +import idutils import re from dojson.utils import force_list @@ -164,9 +164,13 @@ def extract_json_contributor_ids(info, orcid_subfield="k"): author_orcid = info.get(orcid_subfield) if author_orcid: author_orcid = author_orcid.replace("ORCID:", "") - new_id = {"identifier": author_orcid, "scheme": "orcid"} - if new_id not in ids: - ids.append(new_id) + if not author_orcid.lower().startswith("jacow-"): + if idutils.is_orcid(author_orcid): + new_id = {"identifier": author_orcid, "scheme": "orcid"} + if new_id not in ids: + ids.append(new_id) + else: + raise UnexpectedValue(message="Author has invalid orcid", value=author_orcid, stage="transform") inspire = info.get("i", "") if inspire and inspire.startswith("INSPIRE-"): From 7641f0d36e589d7e341bd0ced7961c861a99bf3b Mon Sep 17 00:00:00 2001 From: Karolina Przerwa Date: Wed, 15 Jul 2026 17:46:05 +0200 Subject: [PATCH 6/6] fix(tests): fix isbns rule source --- .../transform/xml_processing/rules/base.py | 27 +------ .../xml_processing/rules/research.py | 36 ++++----- tests/cds-rdm/test_base_rules.py | 74 ------------------- tests/cds-rdm/test_publications_rules.py | 23 ++++++ 4 files changed, 37 insertions(+), 123 deletions(-) diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 25f719d8..1629a4b1 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -10,8 +10,6 @@ import datetime import logging import re -from idutils.normalizers import normalize_isbn -from isbnlib import NotValidISBNError from urllib.parse import ParseResult, urlparse from dateutil.parser import ParserError, parse @@ -30,9 +28,9 @@ IDENTIFIERS_SCHEMES_TO_DROP, KEYWORD_SCHEMES_TO_DROP, PID_SCHEMES_TO_STORE_IN_IDENTIFIERS, + PID_SCHEMES_TO_STORE_IN_RELATED_IDENTIFIERS, RECOGNISED_KEYWORD_SCHEMES, udc_pattern, - PID_SCHEMES_TO_STORE_IN_RELATED_IDENTIFIERS, ) from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model as model, @@ -367,29 +365,6 @@ def aleph_number(self, key, value): raise IgnoreKey("identifiers") -@model.over("related_identifiers", "(^020__)", override_tag=True) -@for_each_value -def isbn(self, key, value): - _isbn = StringValue(value.get("a", "")).parse() - _isbn_material = StringValue(value.get("u", "")).parse() - if _isbn: - try: - _isbn = normalize_isbn(_isbn) - - except NotValidISBNError as e: - raise UnexpectedValue("Not a valid ISBN.", field=key, value=value) - - new_id = { - "identifier": _isbn, - "scheme": "isbn", - "relation_type": {"id": "isversionof"}, - "resource_type": {"id": "publication-book"} - } - if new_id not in self.get("related_identifiers", []): - return new_id - raise IgnoreKey("related_identifiers") - - @model.over("identifiers", "^035__") @for_each_value def identifiers(self, key, value): diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index d300fbf2..d6b0f0dc 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -45,7 +45,7 @@ def _sub(v, code): return val[0] if val else "" -@model.over("isbns", "^020__") +@model.over("isbns", "^020__", override_tag=True) def isbn(self, key, value): _custom_fields = self.get("custom_fields", {}) _isbn = StringValue(value.get("a", "")).parse() @@ -57,34 +57,24 @@ def isbn(self, key, value): except NotValidISBNError as e: raise UnexpectedValue("Not a valid ISBN.", field=key, value=value) is_cern_isbn = _isbn.startswith("978-92-9083") - thesis_fields = _custom_fields.get("imprint:imprint", {}) - thesis_fields["isbn"] = _isbn - _custom_fields["imprint:imprint"] = thesis_fields - - if is_cern_isbn: - # TODO, should we have ISBN as internal? - destination = "related_identifiers" - new_id = { - "identifier": _isbn, - "scheme": "isbn", - "relation_type": {"id": "isvariantformof"}, - "resource_type": {"id": "publication-book"}, - } - else: - destination = "related_identifiers" - new_id = { - "identifier": _isbn, - "scheme": "isbn", - "relation_type": {"id": "isvariantformof"}, - "resource_type": {"id": "publication-book"}, - } + imprint_fields = _custom_fields.get("imprint:imprint", {}) + imprint_fields["isbn"] = _isbn + _custom_fields["imprint:imprint"] = imprint_fields + + destination = "related_identifiers" + new_id = { + "identifier": _isbn, + "scheme": "isbn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-book"}, + } ids = self.get(destination, []) if new_id not in ids: ids.append(new_id) self[destination] = ids self["custom_fields"] = _custom_fields - raise IgnoreKey("custom_fields") + raise IgnoreKey("isbns") @model.over("related_identifiers", "(^022__)") diff --git a/tests/cds-rdm/test_base_rules.py b/tests/cds-rdm/test_base_rules.py index 244a8fa7..91e14492 100644 --- a/tests/cds-rdm/test_base_rules.py +++ b/tests/cds-rdm/test_base_rules.py @@ -13,7 +13,6 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.records.transform.xml_processing.rules.base import ( custom_fields_693, - isbn, normalize, note, recid, @@ -289,79 +288,6 @@ def test_urls_custom_subfield(self): assert result[0]["identifier"] == "http://custom.com" -class TestIsbnRelatedIdentifiers: - """Test isbn function (020__ rule) from base.py.""" - - def test_isbn13_normalized_and_added(self): - """Test that a valid ISBN-13 is normalized and added to related_identifiers.""" - record = {} - result = isbn(record, "020__", {"a": "978-3-16-148410-0"}) - assert result == [ - { - "identifier": "978-3-16-148410-0", - "scheme": "isbn", - "relation_type": {"id": "isversionof"}, - "resource_type": {"id": "publication-book"}, - } - ] - - def test_isbn10_converted_to_isbn13(self): - """Test that a valid ISBN-10 is converted to its ISBN-13 form.""" - record = {} - result = isbn(record, "020__", {"a": "0-306-40615-2"}) - assert result == [ - { - "identifier": "978-0-306-40615-7", - "scheme": "isbn", - "relation_type": {"id": "isversionof"}, - "resource_type": {"id": "publication-book"}, - } - ] - - def test_isbn_missing_subfield_a_ignored(self): - """Test that 020__ without subfield 'a' is ignored.""" - record = {} - with pytest.raises(IgnoreKey): - isbn(record, "020__", {}) - - def test_isbn_empty_subfield_a_ignored(self): - """Test that 020__a with an empty value is ignored.""" - record = {} - with pytest.raises(IgnoreKey): - isbn(record, "020__", {"a": ""}) - - def test_isbn_no_existing_related_identifiers_does_not_raise(self): - """Regression test: isbn() must not fail on a record that has no - related_identifiers key yet (the common case for the first related - identifier found on a record).""" - record = {} - result = isbn(record, "020__", {"a": "978-3-16-148410-0"}) - assert result[0]["identifier"] == "978-3-16-148410-0" - - def test_isbn_no_duplicate(self): - """Test that an already-present ISBN is not added twice.""" - record = { - "related_identifiers": [ - { - "identifier": "978-3-16-148410-0", - "scheme": "isbn", - "relation_type": {"id": "isversionof"}, - "resource_type": {"id": "publication-book"}, - } - ] - } - with pytest.raises(IgnoreKey): - isbn(record, "020__", {"a": "978-3-16-148410-0"}) - - def test_isbn_malformed_value_does_not_raise(self): - """Documents current behavior: normalize_isbn() does not raise for - malformed input, so malformed ISBNs end up with an empty identifier - instead of triggering the UnexpectedValue branch.""" - record = {} - result = isbn(record, "020__", {"a": "123"}) - assert result[0]["identifier"] == "" - - class TestRelatedIdentifiers775: """Test related_identifiers function (775 field) from base.py.""" diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index 78d307c9..eff4b89e 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -106,6 +106,29 @@ def test_isbn_relation_type(self): assert isbn_rel["relation_type"]["id"] == "isvariantformof" assert isbn_rel["resource_type"]["id"] == "publication-book" + def test_isbn_no_duplicate(self): + """Test that an already-present ISBN is not added twice.""" + record = { + "related_identifiers": [ + { + "identifier": "978-3-16-148410-0", + "scheme": "isbn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-book"}, + } + ] + } + with pytest.raises(IgnoreKey): + isbn(record, "020__", {"a": "978-3-16-148410-0"}) + assert record["related_identifiers"] == [ + { + "identifier": "978-3-16-148410-0", + "scheme": "isbn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-book"}, + } + ] + class TestIssn: """Test issn function from publications.py."""