Lep fixes 2

cds_migrator_kit/rdm/records/transform/models/lep.py

@@ -0,0 +1,116 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2025 CERN.
+#
+# CDS-RDM is free software; you can redistribute it and/or modify it under
+# the terms of the MIT License; see LICENSE file for more details.
+
+"""CDS-RDM CMS note model."""
+from cds_migrator_kit.rdm.records.transform.models.base_publication_record import (
+    rdm_base_publication_model,
+)
+from cds_migrator_kit.rdm.records.transform.models.research import ResearchModel
+
+
+class LEPResearchModel(ResearchModel):
+    """Translation model for research."""
+
+    __query__ = '980__:L3_Papers OR 980__:INTNOTEALEPHPRIV OR 980__:OPAL_Papers OR 980__:OPAL_Misc OR 980__:DELPHI_Misc OR 980__:DELPHI_Papers OR 980__:L3_Misc OR 693__.e:L3 OR 693__.e:DELPHI OR 693__.e:OPAL OR 693__.e:ALEPH OR 690C_.a:PUBLDELPHINOTE OR 690C_.a:PRIVDELPHINOTE OR 710__.g:"ALEPH Collaboration" OR 710__.g:"Aleph Collaboration" OR 980__.a:ALEPH_Papers OR 980__.a:ALEPHDRAFT OR 037__:CERN-ALEPH-PUB-* OR 037__:CERN-ALEPH-ARCH-DATA-* -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'
+
+    __ignore_keys__ = {
+        "594__a", # can be ignored for this collection
+        "775__p", # can be ignored for this collection - title of another volume
+        "775__c", # year of volume
+        "596__a", # multivolume tag
+        "300__x", # drop the physical description
+        "0248_a",
+        "0248_p",
+        "0248_q",
+        "0247_9",  # provenance of the DOI
+        "030__a",  # TODO coden designation to drop?
+        "035__h",  # oai identifiers in 1215391
+        "035__d",  # oai identifiers in 1215391
+        "035__t",  # oai identifiers in 1215391
+        "035__u",  # oai identifiers in 1215391
+        "035__m",  # oai identifiers in 1215391
+        "035__z",  # oai identifiers in 1215391
+        "336__a",  # redundant field
+        "500__9",  # provenance of the note
+        "520__9",  # provenance of the description
+        "520__h",  # provenance of the description
+        "852__c",  # holdings will be taken separately
+        "852__h",
+        "037__c",  # arxiv subject
+        "100__m",  # email of contributor
+        "245__9",  # title provenance
+        "270__m",  # document contact email
+        "300__a",  # number of pages
+        "340__a",  # TODO ignore material?
+        "540__3",  # TODO still ignore the material of the license?
+        "540__9",  # TODO still ignore the material of the license?
+        "542__3",  # TODO still ignore the material of the license?
+        "595__i",  # TODO ??
+        "695__e",  # some inspire tag
+        "700__m",  # email of contributor
+        "700__q",  # TODO ignore? aliteration of the name, used for searching
+        "700__v",  # TODO drop?
+        "773__x",  # INSPIRE publication note
+        "773__t",  # INSPIRE publication note
+        "773__0",  # from SIS: can be ignored
+        "773__o",  # from SIS: can be ignored
+        "773__z",  # from SIS: can be ignored
+        "8564_8",  # file id
+        "8564_s",  # bibdoc id
+        "8564_x",  # icon thumbnails sizes
+        "8564_y",  # file description - done by files dump, sometimes these are used for open access calculation
+        "8564_w",  # system field
+        "8564_z",  # file comment, migrated via file metadata
+        "913__y",  # citation
+        "913__v",  # citation
+        "913__t",  # citation
+        "913__a",  # citation
+        "913__c",  # citation
+        "916__y",  # year, redundant value
+        "937__c",  # last modified by
+        "937__s",  # last modification date
+        "960__a",  # base number
+        "961__c",  #
+        "961__h",  #
+        "961__l",  #
+        "961__x",  #
+        "964__a",  # TODO: ignore?
+        "981__a",  # duplicate record id
+        "999C50",
+        "999C52",  # https://cds.cern.ch/record/2640188/export/hm?ln=en
+        "999C59",  # https://cds.cern.ch/record/2284615/export/hm?ln=en
+        "999C5a",  # https://cds.cern.ch/record/2678429/export/hm?ln=en
+        "999C5c",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5h",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5i",  # https://cds.cern.ch/record/2284892/export/hm?ln=en
+        "999C5k",  # https://cds.cern.ch/record/2671914/export/hm?ln=en
+        "999C5l",  # https://cds.cern.ch/record/2283115/export/hm?ln=en
+        "999C5m",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5o",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5p",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5r",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5s",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5t",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5u",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5v",  # https://cds.cern.ch/record/2283088/export/hm?ln=en
+        "999C5x",  # https://cds.cern.ch/record/2710809/export/hm?ln=en
+        "999C5y",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C5z",  # https://cds.cern.ch/record/2710809/export/hm?ln=en
+        "999C6a",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C6t",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+        "999C6v",  # https://cds.cern.ch/record/2284606/export/hm?ln=en
+    }
+
+    _default_fields = {
+        "custom_fields": {},
+    }
+
+
+lep_research_model = LEPResearchModel(
+    bases=(rdm_base_publication_model,),
+    entry_point_group="cds_migrator_kit.migrator.rdm.rules.research",
+)

cds_migrator_kit/rdm/records/transform/models/research.py

@@ -15,7 +15,7 @@
 class ResearchModel(CdsOverdo):
     """Translation model for research."""
 
-    __query__ = '980__:L3_Papers OR 980__:INTNOTEALEPHPRIV OR 980__:OPAL_Papers OR 980__:OPAL_Misc OR 980__:DELPHI_Misc OR 980__:DELPHI_Papers OR 980__:L3_Misc OR 693__.e:L3 OR 693__.e:DELPHI OR 693__.e:OPAL OR 693__.e:ALEPH OR 690C_.a:PUBLDELPHINOTE OR 690C_.a:PRIVDELPHINOTE OR 710__.g:"ALEPH Collaboration" OR 710__.g:"Aleph Collaboration" OR 980__.a:ALEPH_Papers OR 980__.a:ALEPHDRAFT OR 037__:CERN-ALEPH-PUB-* OR 037__:CERN-ALEPH-ARCH-DATA-* OR 980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES)  OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'
+    __query__ = '980__:LCD-Notes OR 980__:LCD-NOTES OR 693__.e:"DAMPE RE29" OR 037__:DIRAC-NOTE* OR 037__:DIRAC-Note* OR 037__:DIRAC-CONF* OR 037__:DIRAC-DOC* OR 037__:DIRAC-PUB* OR 693__:UA2 OR 693__:UA4 OR 693:__UA5 OR 693__:UA8 OR 980__:ANTARESCERNTALK OR (980__.a:"POSTER" AND 693__.e:ANTARES)  OR 980__:INTNOTEHARPCDPPUBL OR 980__:PRIVIMXGAM OR 980__:PRIVANTARES -980__:THESIS -037__:CERN-STUDENTS-Note-* -980__:DELETED -980__.c:MIGRATED -980__.a:DUMMY -690C_.a:SCICOM'
 
     __ignore_keys__ = {
         "0248_a",

cds_migrator_kit/rdm/records/transform/transform.py

@@ -413,9 +413,22 @@ def _resource_type(entry):
             try:
                 return entry["resource_type"]
             except KeyError:
-                raise MissingRequiredField(
-                    message="resource_type", field="980"
-                )
+                raise MissingRequiredField(message="resource_type", field="980")
+
+        def _title(entry, resource_type):
+            title = entry.get("title")
+            if title:
+                return title
+            # 245 (title) is sometimes absent on conference proceedings
+            # records; fall back to the conference name (111__a) stored on
+            # the first meeting entry.
+            if resource_type.get("id") == "publication-conferenceproceeding":
+                meetings = entry.get("custom_fields", {}).get("meeting:meeting", [])
+                for meeting_entry in meetings:
+                    meeting_title = meeting_entry.get("title")
+                    if meeting_title:
+                        return meeting_title
+            return title
 
         def _publication_date(entry, dump_record):
             pub_date = entry.get("publication_date")
@@ -485,10 +498,11 @@ def subjects(json_entry):
         _subjects = subjects(json_entry)
         table_of_contents(json_entry)
 
+        _resource_type_value = _resource_type(json_entry)
         metadata = {
             "creators": creators(json_entry),
-            "title": json_entry.get("title"),
-            "resource_type": _resource_type(json_entry),
+            "title": _title(json_entry, _resource_type_value),
+            "resource_type": _resource_type_value,
             "description": json_entry.get("description"),
             "publication_date": _publication_date(json_entry, record_dump),
             "contributors": creators(json_entry, key="contributors"),

cds_migrator_kit/rdm/records/transform/xml_processing/rules/base.py

@@ -28,9 +28,9 @@
     IDENTIFIERS_SCHEMES_TO_DROP,
     KEYWORD_SCHEMES_TO_DROP,
     PID_SCHEMES_TO_STORE_IN_IDENTIFIERS,
+    PID_SCHEMES_TO_STORE_IN_RELATED_IDENTIFIERS,
     RECOGNISED_KEYWORD_SCHEMES,
     udc_pattern,
-    PID_SCHEMES_TO_STORE_IN_RELATED_IDENTIFIERS,
 )
 from cds_migrator_kit.rdm.records.transform.models.base_record import (
     rdm_base_record_model as model,
@@ -341,6 +341,8 @@ def report_number(self, key, value):
             existing_ids.append(new_id)
             self["identifiers"] = existing_ids
             raise IgnoreKey("related_identifiers")
+    new_id["relation_type"] = {"id": "isvariantformof"}
+    new_id["resource_type"] = {"id": "publication-other"}
     if new_id in existing_ids:
         raise IgnoreKey("related_identifiers")
     return new_id
@@ -441,6 +443,8 @@ def identifiers(self, key, value):
             rel_id not in self.get("identifiers", [])
         ):
             return rel_id
+        elif not scheme:
+            raise IgnoreKey("identifiers")
         else:
             if "HOLALE" in id_value:
                 raise IgnoreKey("identifiers")
@@ -903,21 +907,36 @@ def related_identifiers_787(self, key, value):
     raise IgnoreKey("related_identifiers")
 
 
-@model.over("related_identifiers", "^775_")
+@model.over("related_identifiers", "^775_", override_tag=True)
 @for_each_value
 def related_identifiers(self, key, value):
     """Translates related identifiers."""
-    description = value.get("b")
-    year = value.get("c")
     recid = value.get("w")
+    volume_note = value.get("n")
     rel_ids = self.get("related_identifiers", [])
+
+    if not recid:
+        raise IgnoreKey("related_identifiers")
+
+    if volume_note:
+        if not re.search(r"v\.?\s*\d+", volume_note, re.IGNORECASE):
+            raise UnexpectedValue(
+                "Missing volume indication in 775__n",
+                field=key,
+                subfield="n",
+                value=value,
+            )
+        resource_type = {"id": "publication-book"}
+    else:
+        resource_type = {"id": "publication-other"}
+
     new_id = {
         "identifier": recid,
         "scheme": "cds",
         "relation_type": {"id": "references"},
-        "resource_type": {"id": "other"},
+        "resource_type": resource_type,
     }
-    if recid and new_id not in rel_ids:
+    if new_id not in rel_ids:
         return new_id
     raise IgnoreKey("related_identifiers")
 

cds_migrator_kit/rdm/records/transform/xml_processing/rules/research.py

@@ -1,4 +1,5 @@
 import re
+from datetime import datetime
 
 from dateutil.parser import ParserError, parse
 from dojson.errors import IgnoreKey
@@ -44,7 +45,7 @@ def _sub(v, code):
     return val[0] if val else ""
 
 
-@model.over("isbns", "^020__")
+@model.over("isbns", "^020__", override_tag=True)
 def isbn(self, key, value):
     _custom_fields = self.get("custom_fields", {})
     _isbn = StringValue(value.get("a", "")).parse()
@@ -56,34 +57,24 @@ def isbn(self, key, value):
         except NotValidISBNError as e:
             raise UnexpectedValue("Not a valid ISBN.", field=key, value=value)
         is_cern_isbn = _isbn.startswith("978-92-9083")
-        thesis_fields = _custom_fields.get("imprint:imprint", {})
-        thesis_fields["isbn"] = _isbn
-        _custom_fields["imprint:imprint"] = thesis_fields
-
-        if is_cern_isbn:
-            # TODO, should we have ISBN as internal?
-            destination = "related_identifiers"
-            new_id = {
-                "identifier": _isbn,
-                "scheme": "isbn",
-                "relation_type": {"id": "isvariantformof"},
-                "resource_type": {"id": "publication-book"},
-            }
-        else:
-            destination = "related_identifiers"
-            new_id = {
-                "identifier": _isbn,
-                "scheme": "isbn",
-                "relation_type": {"id": "isvariantformof"},
-                "resource_type": {"id": "publication-book"},
-            }
+        imprint_fields = _custom_fields.get("imprint:imprint", {})
+        imprint_fields["isbn"] = _isbn
+        _custom_fields["imprint:imprint"] = imprint_fields
+
+        destination = "related_identifiers"
+        new_id = {
+            "identifier": _isbn,
+            "scheme": "isbn",
+            "relation_type": {"id": "isvariantformof"},
+            "resource_type": {"id": "publication-book"},
+        }
         ids = self.get(destination, [])
 
         if new_id not in ids:
             ids.append(new_id)
         self[destination] = ids
     self["custom_fields"] = _custom_fields
-    raise IgnoreKey("custom_fields")
+    raise IgnoreKey("isbns")
 
 
 @model.over("related_identifiers", "(^022__)")
@@ -187,6 +178,22 @@ def internal_notes(self, key, value):
     return internal_notes
 
 
+@model.over("additional_descriptions", "^691__")
+@for_each_value
+@require(["a"])
+def abbreviation(self, key, value):
+    """Translates 691__a abbreviation into an additional description."""
+    description_text = value.get("a")
+    if not description_text:
+        raise IgnoreKey("additional_descriptions")
+    return {
+        "description": f"Abbreviation: {description_text}",
+        "type": {
+            "id": "other",
+        },
+    }
+
+
 @model.over("funding", "(^536__)")
 def funding(self, key, value):
     _custom_fields = self.get("custom_fields", {})
@@ -296,6 +303,61 @@ def journal(self, key, value):
     return _custom_fields
 
 
+@model.over("custom_fields", "(^111__)|(^711__)", override_tag=True)
+def meeting(self, key, value):
+    """Translates meeting name entries (111__, 711__) into meeting:meeting.
+
+    111__a -> title, 111__c -> place, 111__g -> acronym. 111__d -> dates; if
+    missing, fall back to 111__9 (parsed as YYYYMMDD), then to 111__f (a bare
+    year). 711__a -> title, but only if no meeting entry with that title
+    already exists (711 is an added entry, often duplicating the 111
+    conference name).
+    """
+    _custom_fields = self.get("custom_fields", {})
+    meetings = _custom_fields.get("meeting:meeting", [])
+
+    if key.startswith("111"):
+        title = StringValue(value.get("a", "")).parse()
+        place = StringValue(value.get("c", "")).parse()
+        date_recon = StringValue(value.get("9", "")).parse()
+        if date_recon:
+            try:
+                date_recon = datetime.strptime(date_recon, "%Y%m%d").strftime(
+                    "%Y-%m-%d"
+                )
+            except ValueError:
+                raise UnexpectedValue(
+                    "Can't parse meeting date (111__9)",
+                    field=key,
+                    subfield="9",
+                    value=value,
+                )
+        date_year = StringValue(value.get("f", "")).parse()
+        dates = StringValue(value.get("d", "")).parse() or date_recon or date_year
+        acronym = StringValue(value.get("g", "")).parse()
+
+        new_meeting = {}
+        if title:
+            new_meeting["title"] = title
+        if place:
+            new_meeting["place"] = place
+        if dates:
+            new_meeting["dates"] = dates
+        if acronym:
+            new_meeting["acronym"] = acronym
+        if new_meeting:
+            meetings.append(new_meeting)
+    else:
+        title = StringValue(value.get("a", "")).parse()
+        if title and not any(m.get("title") == title for m in meetings):
+            meetings.append({"title": title})
+
+    if meetings:
+        _custom_fields["meeting:meeting"] = meetings
+    self["custom_fields"] = _custom_fields
+    raise IgnoreKey("custom_fields")
+
+
 @model.over("_oa_license", "^540__", override=True)
 def oa_level_from_license(self, key, value):
     """Detect funding model; also runs base license logic for rights.