diff --git a/cds_migrator_kit/rdm/records/transform/models/lep.py b/cds_migrator_kit/rdm/records/transform/models/lep.py new file mode 100644 index 00000000..ba7d7564 --- /dev/null +++ b/cds_migrator_kit/rdm/records/transform/models/lep.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2025 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""CDS-RDM CMS note model.""" +from cds_migrator_kit.rdm.records.transform.models.base_publication_record import ( + rdm_base_publication_model, +) +from cds_migrator_kit.rdm.records.transform.models.research import ResearchModel + + +class LEPResearchModel(ResearchModel): + """Translation model for research.""" + + __query__ = '980__:L3_Papers OR 980__:INTNOTEALEPHPRIV OR 980__:OPAL_Papers OR 980__:OPAL_Misc OR 980__:DELPHI_Misc OR 980__:DELPHI_Papers OR 980__:L3_Misc OR 693__.e:L3 OR 693__.e:DELPHI OR 693__.e:OPAL OR 693__.e:ALEPH OR 690C_.a:PUBLDELPHINOTE OR 690C_.a:PRIVDELPHINOTE OR 710__.g:"ALEPH Collaboration" OR 710__.g:"Aleph Collaboration" OR 980__.a:ALEPH_Papers OR 980__.a:ALEPHDRAFT OR 037__:CERN-ALEPH-PUB-* OR 037__:CERN-ALEPH-ARCH-DATA-* -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + + __ignore_keys__ = { + "594__a", # can be ignored for this collection + "775__p", # can be ignored for this collection - title of another volume + "775__c", # year of volume + "596__a", # multivolume tag + "300__x", # drop the physical description + "0248_a", + "0248_p", + "0248_q", + "0247_9", # provenance of the DOI + "030__a", # TODO coden designation to drop? + "035__h", # oai identifiers in 1215391 + "035__d", # oai identifiers in 1215391 + "035__t", # oai identifiers in 1215391 + "035__u", # oai identifiers in 1215391 + "035__m", # oai identifiers in 1215391 + "035__z", # oai identifiers in 1215391 + "336__a", # redundant field + "500__9", # provenance of the note + "520__9", # provenance of the description + "520__h", # provenance of the description + "852__c", # holdings will be taken separately + "852__h", + "037__c", # arxiv subject + "100__m", # email of contributor + "245__9", # title provenance + "270__m", # document contact email + "300__a", # number of pages + "340__a", # TODO ignore material? + "540__3", # TODO still ignore the material of the license? + "540__9", # TODO still ignore the material of the license? + "542__3", # TODO still ignore the material of the license? + "595__i", # TODO ?? + "695__e", # some inspire tag + "700__m", # email of contributor + "700__q", # TODO ignore? aliteration of the name, used for searching + "700__v", # TODO drop? + "773__x", # INSPIRE publication note + "773__t", # INSPIRE publication note + "773__0", # from SIS: can be ignored + "773__o", # from SIS: can be ignored + "773__z", # from SIS: can be ignored + "8564_8", # file id + "8564_s", # bibdoc id + "8564_x", # icon thumbnails sizes + "8564_y", # file description - done by files dump, sometimes these are used for open access calculation + "8564_w", # system field + "8564_z", # file comment, migrated via file metadata + "913__y", # citation + "913__v", # citation + "913__t", # citation + "913__a", # citation + "913__c", # citation + "916__y", # year, redundant value + "937__c", # last modified by + "937__s", # last modification date + "960__a", # base number + "961__c", # + "961__h", # + "961__l", # + "961__x", # + "964__a", # TODO: ignore? + "981__a", # duplicate record id + "999C50", + "999C52", # https://cds.cern.ch/record/2640188/export/hm?ln=en + "999C59", # https://cds.cern.ch/record/2284615/export/hm?ln=en + "999C5a", # https://cds.cern.ch/record/2678429/export/hm?ln=en + "999C5c", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5h", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5i", # https://cds.cern.ch/record/2284892/export/hm?ln=en + "999C5k", # https://cds.cern.ch/record/2671914/export/hm?ln=en + "999C5l", # https://cds.cern.ch/record/2283115/export/hm?ln=en + "999C5m", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5o", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5p", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5r", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5s", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5u", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5v", # https://cds.cern.ch/record/2283088/export/hm?ln=en + "999C5x", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C5y", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C5z", # https://cds.cern.ch/record/2710809/export/hm?ln=en + "999C6a", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6t", # https://cds.cern.ch/record/2284606/export/hm?ln=en + "999C6v", # https://cds.cern.ch/record/2284606/export/hm?ln=en + } + + _default_fields = { + "custom_fields": {}, + } + + +lep_research_model = LEPResearchModel( + bases=(rdm_base_publication_model,), + entry_point_group="cds_migrator_kit.migrator.rdm.rules.research", +) diff --git a/cds_migrator_kit/rdm/records/transform/models/research.py b/cds_migrator_kit/rdm/records/transform/models/research.py index e5ff48e9..821f5a35 100644 --- a/cds_migrator_kit/rdm/records/transform/models/research.py +++ b/cds_migrator_kit/rdm/records/transform/models/research.py @@ -15,7 +15,7 @@ class ResearchModel(CdsOverdo): """Translation model for research.""" - __query__ = '980__:L3_Papers OR 980__:INTNOTEALEPHPRIV OR 980__:OPAL_Papers OR 980__:OPAL_Misc OR 980__:DELPHI_Misc OR 980__:DELPHI_Papers OR 980__:L3_Misc OR 693__.e:L3 OR 693__.e:DELPHI OR 693__.e:OPAL OR 693__.e:ALEPH OR 690C_.a:PUBLDELPHINOTE OR 690C_.a:PRIVDELPHINOTE OR 710__.g:"ALEPH Collaboration" OR 710__.g:"Aleph Collaboration" OR 980__.a:ALEPH_Papers OR 980__.a:ALEPHDRAFT OR 037__:CERN-ALEPH-PUB-* OR 037__:CERN-ALEPH-ARCH-DATA-* OR 980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' + __query__ = '980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES) OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM' __ignore_keys__ = { "0248_a", diff --git a/cds_migrator_kit/rdm/records/transform/transform.py b/cds_migrator_kit/rdm/records/transform/transform.py index b03eaa65..a21b9eca 100644 --- a/cds_migrator_kit/rdm/records/transform/transform.py +++ b/cds_migrator_kit/rdm/records/transform/transform.py @@ -413,9 +413,22 @@ def _resource_type(entry): try: return entry["resource_type"] except KeyError: - raise MissingRequiredField( - message="resource_type", field="980" - ) + raise MissingRequiredField(message="resource_type", field="980") + + def _title(entry, resource_type): + title = entry.get("title") + if title: + return title + # 245 (title) is sometimes absent on conference proceedings + # records; fall back to the conference name (111__a) stored on + # the first meeting entry. + if resource_type.get("id") == "publication-conferenceproceeding": + meetings = entry.get("custom_fields", {}).get("meeting:meeting", []) + for meeting_entry in meetings: + meeting_title = meeting_entry.get("title") + if meeting_title: + return meeting_title + return title def _publication_date(entry, dump_record): pub_date = entry.get("publication_date") @@ -485,10 +498,11 @@ def subjects(json_entry): _subjects = subjects(json_entry) table_of_contents(json_entry) + _resource_type_value = _resource_type(json_entry) metadata = { "creators": creators(json_entry), - "title": json_entry.get("title"), - "resource_type": _resource_type(json_entry), + "title": _title(json_entry, _resource_type_value), + "resource_type": _resource_type_value, "description": json_entry.get("description"), "publication_date": _publication_date(json_entry, record_dump), "contributors": creators(json_entry, key="contributors"), diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py index 06d0d702..1629a4b1 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py @@ -28,9 +28,9 @@ IDENTIFIERS_SCHEMES_TO_DROP, KEYWORD_SCHEMES_TO_DROP, PID_SCHEMES_TO_STORE_IN_IDENTIFIERS, + PID_SCHEMES_TO_STORE_IN_RELATED_IDENTIFIERS, RECOGNISED_KEYWORD_SCHEMES, udc_pattern, - PID_SCHEMES_TO_STORE_IN_RELATED_IDENTIFIERS, ) from cds_migrator_kit.rdm.records.transform.models.base_record import ( rdm_base_record_model as model, @@ -341,6 +341,8 @@ def report_number(self, key, value): existing_ids.append(new_id) self["identifiers"] = existing_ids raise IgnoreKey("related_identifiers") + new_id["relation_type"] = {"id": "isvariantformof"} + new_id["resource_type"] = {"id": "publication-other"} if new_id in existing_ids: raise IgnoreKey("related_identifiers") return new_id @@ -441,6 +443,8 @@ def identifiers(self, key, value): rel_id not in self.get("identifiers", []) ): return rel_id + elif not scheme: + raise IgnoreKey("identifiers") else: if "HOLALE" in id_value: raise IgnoreKey("identifiers") @@ -903,21 +907,36 @@ def related_identifiers_787(self, key, value): raise IgnoreKey("related_identifiers") -@model.over("related_identifiers", "^775_") +@model.over("related_identifiers", "^775_", override_tag=True) @for_each_value def related_identifiers(self, key, value): """Translates related identifiers.""" - description = value.get("b") - year = value.get("c") recid = value.get("w") + volume_note = value.get("n") rel_ids = self.get("related_identifiers", []) + + if not recid: + raise IgnoreKey("related_identifiers") + + if volume_note: + if not re.search(r"v\.?\s*\d+", volume_note, re.IGNORECASE): + raise UnexpectedValue( + "Missing volume indication in 775__n", + field=key, + subfield="n", + value=value, + ) + resource_type = {"id": "publication-book"} + else: + resource_type = {"id": "publication-other"} + new_id = { "identifier": recid, "scheme": "cds", "relation_type": {"id": "references"}, - "resource_type": {"id": "other"}, + "resource_type": resource_type, } - if recid and new_id not in rel_ids: + if new_id not in rel_ids: return new_id raise IgnoreKey("related_identifiers") diff --git a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py index df1ae5c9..d6b0f0dc 100644 --- a/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py +++ b/cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py @@ -1,4 +1,5 @@ import re +from datetime import datetime from dateutil.parser import ParserError, parse from dojson.errors import IgnoreKey @@ -44,7 +45,7 @@ def _sub(v, code): return val[0] if val else "" -@model.over("isbns", "^020__") +@model.over("isbns", "^020__", override_tag=True) def isbn(self, key, value): _custom_fields = self.get("custom_fields", {}) _isbn = StringValue(value.get("a", "")).parse() @@ -56,34 +57,24 @@ def isbn(self, key, value): except NotValidISBNError as e: raise UnexpectedValue("Not a valid ISBN.", field=key, value=value) is_cern_isbn = _isbn.startswith("978-92-9083") - thesis_fields = _custom_fields.get("imprint:imprint", {}) - thesis_fields["isbn"] = _isbn - _custom_fields["imprint:imprint"] = thesis_fields - - if is_cern_isbn: - # TODO, should we have ISBN as internal? - destination = "related_identifiers" - new_id = { - "identifier": _isbn, - "scheme": "isbn", - "relation_type": {"id": "isvariantformof"}, - "resource_type": {"id": "publication-book"}, - } - else: - destination = "related_identifiers" - new_id = { - "identifier": _isbn, - "scheme": "isbn", - "relation_type": {"id": "isvariantformof"}, - "resource_type": {"id": "publication-book"}, - } + imprint_fields = _custom_fields.get("imprint:imprint", {}) + imprint_fields["isbn"] = _isbn + _custom_fields["imprint:imprint"] = imprint_fields + + destination = "related_identifiers" + new_id = { + "identifier": _isbn, + "scheme": "isbn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-book"}, + } ids = self.get(destination, []) if new_id not in ids: ids.append(new_id) self[destination] = ids self["custom_fields"] = _custom_fields - raise IgnoreKey("custom_fields") + raise IgnoreKey("isbns") @model.over("related_identifiers", "(^022__)") @@ -187,6 +178,22 @@ def internal_notes(self, key, value): return internal_notes +@model.over("additional_descriptions", "^691__") +@for_each_value +@require(["a"]) +def abbreviation(self, key, value): + """Translates 691__a abbreviation into an additional description.""" + description_text = value.get("a") + if not description_text: + raise IgnoreKey("additional_descriptions") + return { + "description": f"Abbreviation: {description_text}", + "type": { + "id": "other", + }, + } + + @model.over("funding", "(^536__)") def funding(self, key, value): _custom_fields = self.get("custom_fields", {}) @@ -296,6 +303,61 @@ def journal(self, key, value): return _custom_fields +@model.over("custom_fields", "(^111__)|(^711__)", override_tag=True) +def meeting(self, key, value): + """Translates meeting name entries (111__, 711__) into meeting:meeting. + + 111__a -> title, 111__c -> place, 111__g -> acronym. 111__d -> dates; if + missing, fall back to 111__9 (parsed as YYYYMMDD), then to 111__f (a bare + year). 711__a -> title, but only if no meeting entry with that title + already exists (711 is an added entry, often duplicating the 111 + conference name). + """ + _custom_fields = self.get("custom_fields", {}) + meetings = _custom_fields.get("meeting:meeting", []) + + if key.startswith("111"): + title = StringValue(value.get("a", "")).parse() + place = StringValue(value.get("c", "")).parse() + date_recon = StringValue(value.get("9", "")).parse() + if date_recon: + try: + date_recon = datetime.strptime(date_recon, "%Y%m%d").strftime( + "%Y-%m-%d" + ) + except ValueError: + raise UnexpectedValue( + "Can't parse meeting date (111__9)", + field=key, + subfield="9", + value=value, + ) + date_year = StringValue(value.get("f", "")).parse() + dates = StringValue(value.get("d", "")).parse() or date_recon or date_year + acronym = StringValue(value.get("g", "")).parse() + + new_meeting = {} + if title: + new_meeting["title"] = title + if place: + new_meeting["place"] = place + if dates: + new_meeting["dates"] = dates + if acronym: + new_meeting["acronym"] = acronym + if new_meeting: + meetings.append(new_meeting) + else: + title = StringValue(value.get("a", "")).parse() + if title and not any(m.get("title") == title for m in meetings): + meetings.append({"title": title}) + + if meetings: + _custom_fields["meeting:meeting"] = meetings + self["custom_fields"] = _custom_fields + raise IgnoreKey("custom_fields") + + @model.over("_oa_license", "^540__", override=True) def oa_level_from_license(self, key, value): """Detect funding model; also runs base license logic for rights. diff --git a/cds_migrator_kit/transform/xml_processing/quality/contributors.py b/cds_migrator_kit/transform/xml_processing/quality/contributors.py index e58c9218..f67f246e 100644 --- a/cds_migrator_kit/transform/xml_processing/quality/contributors.py +++ b/cds_migrator_kit/transform/xml_processing/quality/contributors.py @@ -6,7 +6,7 @@ # the terms of the MIT License; see LICENSE file for more details. """CDS-RDM contributors migration module.""" - +import idutils import re from dojson.utils import force_list @@ -164,9 +164,13 @@ def extract_json_contributor_ids(info, orcid_subfield="k"): author_orcid = info.get(orcid_subfield) if author_orcid: author_orcid = author_orcid.replace("ORCID:", "") - new_id = {"identifier": author_orcid, "scheme": "orcid"} - if new_id not in ids: - ids.append(new_id) + if not author_orcid.lower().startswith("jacow-"): + if idutils.is_orcid(author_orcid): + new_id = {"identifier": author_orcid, "scheme": "orcid"} + if new_id not in ids: + ids.append(new_id) + else: + raise UnexpectedValue(message="Author has invalid orcid", value=author_orcid, stage="transform") inspire = info.get("i", "") if inspire and inspire.startswith("INSPIRE-"): diff --git a/setup.cfg b/setup.cfg index 8ab5e8df..a3a8741b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,7 +66,8 @@ cds_migrator_kit.migrator.users.model = cds_migrator_kit.migrator.submitter.model = submitter = cds_migrator_kit.rdm.users.transform.xml_processing.models.submitter:submitter_model cds_migrator_kit.migrator.models = - lep_exp = cds_migrator_kit.rdm.records.transform.models.research:research_model + research = cds_migrator_kit.rdm.records.transform.models.research:research_model + lep_research = cds_migrator_kit.rdm.records.transform.models.lep:lep_research_model research_comm_model = cds_migrator_kit.rdm.records.transform.models.research_committee:research_comm_model fap = cds_migrator_kit.rdm.records.transform.models.fap:fap_model ssn = cds_migrator_kit.rdm.records.transform.models.summer_student_report:sspn_model diff --git a/tests/cds-rdm/test_base_rules.py b/tests/cds-rdm/test_base_rules.py index 5285fcd0..91e14492 100644 --- a/tests/cds-rdm/test_base_rules.py +++ b/tests/cds-rdm/test_base_rules.py @@ -17,6 +17,7 @@ note, recid, record_restriction, + related_identifiers, urls, ) @@ -287,6 +288,75 @@ def test_urls_custom_subfield(self): assert result[0]["identifier"] == "http://custom.com" +class TestRelatedIdentifiers775: + """Test related_identifiers function (775 field) from base.py.""" + + def test_775_no_volume_note_sets_publication_other(self): + """Test that missing 775__n results in resource_type publication-other.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456"}) + assert result == [ + { + "identifier": "123456", + "scheme": "cds", + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-other"}, + } + ] + + def test_775_volume_note_dot_notation_sets_publication_book(self): + """Test that 775__n with 'v.1' notation results in publication-book.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456", "n": "v.1"}) + assert result == [ + { + "identifier": "123456", + "scheme": "cds", + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-book"}, + } + ] + + def test_775_volume_note_no_dot_notation_sets_publication_book(self): + """Test that 775__n with 'v2' notation results in publication-book.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456", "n": "v2"}) + assert result[0]["resource_type"] == {"id": "publication-book"} + + def test_775_volume_note_with_space_sets_publication_book(self): + """Test that 775__n with 'v. 3' notation results in publication-book.""" + record = {} + result = related_identifiers(record, "775__", {"w": "123456", "n": "v. 3"}) + assert result[0]["resource_type"] == {"id": "publication-book"} + + def test_775_note_without_volume_indication_raises_error(self): + """Test that 775__n without volume indication raises UnexpectedValue.""" + record = {} + with pytest.raises(UnexpectedValue): + related_identifiers(record, "775__", {"w": "123456", "n": "some note"}) + + def test_775_no_recid_ignored(self): + """Test that missing 775__w is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + related_identifiers(record, "775__", {"n": "v.1"}) + + def test_775_no_duplicate(self): + """Test that duplicate related identifiers are not added twice.""" + record = { + "related_identifiers": [ + { + "identifier": "123456", + "scheme": "cds", + "relation_type": {"id": "references"}, + "resource_type": {"id": "publication-other"}, + } + ] + } + with pytest.raises(IgnoreKey): + related_identifiers(record, "775__", {"w": "123456"}) + + class TestNote: """Test note function from base.py.""" diff --git a/tests/cds-rdm/test_publications_rules.py b/tests/cds-rdm/test_publications_rules.py index 2a96fd26..eff4b89e 100644 --- a/tests/cds-rdm/test_publications_rules.py +++ b/tests/cds-rdm/test_publications_rules.py @@ -12,12 +12,14 @@ from cds_migrator_kit.errors import UnexpectedValue from cds_migrator_kit.rdm.records.transform.xml_processing.rules.research import ( + abbreviation, deadline_date, imprint_info, internal_notes, isbn, issn, journal, + meeting, oa_level_from_license, udc, ) @@ -104,6 +106,29 @@ def test_isbn_relation_type(self): assert isbn_rel["relation_type"]["id"] == "isvariantformof" assert isbn_rel["resource_type"]["id"] == "publication-book" + def test_isbn_no_duplicate(self): + """Test that an already-present ISBN is not added twice.""" + record = { + "related_identifiers": [ + { + "identifier": "978-3-16-148410-0", + "scheme": "isbn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-book"}, + } + ] + } + with pytest.raises(IgnoreKey): + isbn(record, "020__", {"a": "978-3-16-148410-0"}) + assert record["related_identifiers"] == [ + { + "identifier": "978-3-16-148410-0", + "scheme": "isbn", + "relation_type": {"id": "isvariantformof"}, + "resource_type": {"id": "publication-book"}, + } + ] + class TestIssn: """Test issn function from publications.py.""" @@ -715,6 +740,226 @@ def test_deadline_date_invalid_raises_unexpected_value(self): deadline_date(record, "583__", {"c": "not-a-valid-date"}) +class TestAbbreviation: + """Test abbreviation function from research.py (691__a rule).""" + + def test_abbreviation_added_as_additional_description(self): + """Test that 691__a is translated into an additional description.""" + record = {} + result = abbreviation(record, "691__", {"a": "CMS"}) + assert result == [ + { + "description": "Abbreviation: CMS", + "type": {"id": "other"}, + } + ] + + def test_abbreviation_appends_to_existing_additional_descriptions(self): + """Test that 691__a is appended to an existing additional_descriptions list.""" + record = { + "additional_descriptions": [ + {"description": "Some other note", "type": {"id": "other"}} + ] + } + result = abbreviation(record, "691__", {"a": "CMS"}) + assert result == [ + { + "description": "Abbreviation: CMS", + "type": {"id": "other"}, + } + ] + + def test_abbreviation_missing_a_ignored(self): + """Test that 691__ without subfield 'a' is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + abbreviation(record, "691__", {}) + + def test_abbreviation_empty_a_ignored(self): + """Test that 691__a with an empty value is ignored.""" + record = {} + with pytest.raises(IgnoreKey): + abbreviation(record, "691__", {"a": ""}) + + +class TestMeetingFrom111And711: + """Tests for meeting (111__/711__ rule) - builds meeting:meeting custom field.""" + + def test_111_full_fields_creates_meeting_entry(self): + """Test that 111__a/c/d/g populate title/place/dates/acronym.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + { + "a": "Lectures at Fermilab on SUSY", + "c": "Batavia, IL, USA", + "d": "10 Mar 2000", + "f": "2000", + "g": "batavia20000310", + }, + ) + assert record["custom_fields"]["meeting:meeting"] == [ + { + "title": "Lectures at Fermilab on SUSY", + "place": "Batavia, IL, USA", + "dates": "10 Mar 2000", + "acronym": "batavia20000310", + } + ] + + def test_111_falls_back_to_f_when_d_missing(self): + """Test that 111__f is used for dates when 111__d is absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "c": "Geneva", "f": "2000", "g": "ABC"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "2000" + + def test_111_d_takes_precedence_over_f(self): + """Test that 111__d is preferred over 111__f when both are present.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "d": "10 Mar 2000", "f": "2000"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "10 Mar 2000" + + def test_111_falls_back_to_9_when_d_missing(self): + """Test that 111__9 (YYYYMMDD) is used for dates when 111__d is absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "f": "2000", "9": "20000310"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "2000-03-10" + + def test_111_d_takes_precedence_over_9(self): + """Test that 111__d is preferred over 111__9 when both are present.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "d": "10 Mar 2000", "9": "20000310"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "10 Mar 2000" + + def test_111_9_takes_precedence_over_f(self): + """Test that 111__9 is preferred over 111__f (a bare year) when both + are present and 111__d is absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + {"a": "Some Meeting", "f": "2000", "9": "20000310"}, + ) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert meeting_entry["dates"] == "2000-03-10" + + def test_111_invalid_9_raises_error(self): + """Test that an unparseable 111__9 raises UnexpectedValue.""" + record = {} + with pytest.raises(UnexpectedValue): + meeting(record, "111__", {"a": "Some Meeting", "9": "not-a-date"}) + + def test_111_without_dates_omits_dates_key(self): + """Test that no 'dates' key is set when both 111__d and 111__f are absent.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "111__", {"a": "Some Meeting"}) + meeting_entry = record["custom_fields"]["meeting:meeting"][0] + assert "dates" not in meeting_entry + + def test_111_without_title_omits_title_key(self): + """Test that a 111 field without subfield 'a' creates an entry with + no 'title' key rather than dropping the other subfields.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "111__", {"c": "Geneva"}) + assert record["custom_fields"]["meeting:meeting"] == [{"place": "Geneva"}] + + def test_711_adds_new_title_when_no_matching_meeting_exists(self): + """Test that 711__a creates a new meeting entry when its title is new.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "711__", {"a": "Joint Experimental Theoretical Seminar"}) + assert record["custom_fields"]["meeting:meeting"] == [ + {"title": "Joint Experimental Theoretical Seminar"} + ] + + def test_711_skips_duplicate_title(self): + """Test that 711__a is not added again when a meeting with the same + title already exists (e.g. duplicating the 111 conference name).""" + record = { + "custom_fields": { + "meeting:meeting": [{"title": "SUSY at DELPHI", "place": "Geneva"}] + } + } + with pytest.raises(IgnoreKey): + meeting(record, "711__", {"a": "SUSY at DELPHI"}) + assert record["custom_fields"]["meeting:meeting"] == [ + {"title": "SUSY at DELPHI", "place": "Geneva"} + ] + + def test_111_then_two_711_only_adds_distinct_titles(self): + """Reproduces record 436657: 111 sets the full title, and only the + 711 entry with a distinct title is appended as a new meeting.""" + record = {} + with pytest.raises(IgnoreKey): + meeting( + record, + "111__", + { + "a": "Lectures at Fermilab on SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy", + "c": "Batavia, IL, USA", + "d": "10 Mar 2000", + "g": "batavia20000310", + }, + ) + with pytest.raises(IgnoreKey): + meeting( + record, + "711__", + {"a": "SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy"}, + ) + with pytest.raises(IgnoreKey): + meeting( + record, + "711__", + {"a": "Joint Experimental Theoretical Physics Seminar"}, + ) + meetings = record["custom_fields"]["meeting:meeting"] + assert len(meetings) == 3 + titles = [m["title"] for m in meetings] + assert ( + "Lectures at Fermilab on SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy" + in titles + ) + assert "SUSY at DELPHI, LEP200 and LEP-2000 and LEP-Legacy" in titles + assert "Joint Experimental Theoretical Physics Seminar" in titles + + def test_711_without_title_no_meeting_added(self): + """Test that a 711 field without subfield 'a' produces no meeting entry.""" + record = {} + with pytest.raises(IgnoreKey): + meeting(record, "711__", {}) + assert record["custom_fields"].get("meeting:meeting", []) == [] + + class TestLicenseAndFundingFrom540: """Tests for oa_level_from_license (540__ rule)."""