From 97c5438827b6feb09b3fe67d7fef91680accc684 Mon Sep 17 00:00:00 2001 From: zaihuaji Date: Wed, 26 Aug 2026 10:44:43 -0500 Subject: [PATCH] pg_dbi.py: drop the retired UCAR People DB and use dssdb.dssgrp only; bump version to 3.0.11 The UCAR People DB/API no longer exists, so ucar_user_info() and its pgperson/pgusername helpers can never succeed. dssdb.dssgrp is now the only source of GDEX specialist information; a user missing from dssgrp gets a minimal dssdb.user record filled with UNKNOWN placeholders. Closed dssdb.user records are no longer reopened. Co-Authored-By: Claude Opus 4.6 --- pyproject.toml | 2 +- src/rda_python_common/pg_dbi.py | 167 ++++++++------------------------ 2 files changed, 41 insertions(+), 128 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e5c8660..9eb8f4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_common" -version = "3.0.10" +version = "3.0.11" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] diff --git a/src/rda_python_common/pg_dbi.py b/src/rda_python_common/pg_dbi.py index f344e15..e7a3c01 100644 --- a/src/rda_python_common/pg_dbi.py +++ b/src/rda_python_common/pg_dbi.py @@ -85,7 +85,7 @@ class PgDBI(PgLOG): curtran (int): Transaction counter: 0 = idle, >0 = inside a transaction. NMISSES (list): Cached list of scientist IDs (userno) not found in the DB. LMISSES (list): Cached list of login names not found in the DB. - USRWARN (int): Set to 1 once the 'run filluser' reminder has been logged. + USRWARN (int): Set to 1 once the incomplete-user reminder has been logged. TABLES (dict): Cache of table-field default info keyed by table name. SEQUENCES (dict): Cache of sequence field names keyed by table name. SPECIALIST (dict): Cache of specialist records keyed by dataset ID. @@ -1702,19 +1702,17 @@ def pgcheck(self, tablename, logact = 0): return (self.SUCCESS if ret else self.FAILURE) def dssgrp_user_info(self, userno, logname = None): - """Build a partial dssdb.user record from dssdb.dssgrp. + """Build a dssdb.user record from dssdb.dssgrp. - Fallback for when the UCAR People API is unreachable, so a usable user record - can still be added for a DECS group member. Fields absent from dssgrp are set - to the values implied by DECS membership; the record is marked stat_flag 'M' - so that 'filluser -i' completes it once the People API is available again. + dssdb.dssgrp is the only source of specialist information; fields absent from + it are set to the values implied by DECS group membership. Args: userno (int): Scientist number; pass 0 to look up by logname instead. logname (str | None): UCAR login name; used when userno is 0. Returns: - dict | None: Partial mapping of column_name → value suitable for + dict | None: Mapping of column_name → value suitable for pgadd('dssdb.user'), or None when not in dssgrp. """ cond = "userno = {}".format(userno) if userno else "logname = '{}'".format(logname) @@ -1726,15 +1724,16 @@ def dssgrp_user_info(self, userno, logname = None): pgrec['org_name'] = 'NCAR' pgrec['country'] = 'UNITED.STATES' pgrec['email'] = pgrec['ucaremail'] = email - pgrec['stat_flag'] = 'M' + pgrec['stat_flag'] = 'A' return pgrec def add_missed_user(self, userno, logname = None): """Add a dssdb.user record for a user missing from the table. - Tries the UCAR People API first, then dssgrp, and finally falls back to a bare - stub. A reminder to run filluser is logged once per process whenever the added - record is incomplete (stat_flag 'M'). + Builds the record from dssdb.dssgrp for a DECS group member. For anyone not + in dssgrp, adds a record filled with whatever can be derived from the login + name, using 'UNKNOWN' for the fields that cannot be filled in; such a record + is marked stat_flag 'M' and logs a reminder once per process. Args: userno (int): Scientist number; pass 0 to add by logname instead. @@ -1743,46 +1742,41 @@ def add_missed_user(self, userno, logname = None): Returns: int: The new user.uid, or 0 if the insert fails. """ - pgrec = self.ucar_user_info(userno, logname) - if not pgrec: pgrec = self.dssgrp_user_info(userno, logname) + pgrec = self.dssgrp_user_info(userno, logname) if not pgrec: - pgrec = {'userno': userno} if userno else {'logname': logname} - pgrec['stat_flag'] = 'M' - if pgrec['stat_flag'] == 'M': self.incomplete_user_warning() + pgrec = {'stat_flag': 'M', 'lstname': "UNKNOWN", 'fstname': "UNKNOWN"} + if userno: pgrec['userno'] = userno + if logname: + pgrec['logname'] = logname + pgrec['email'] = pgrec['ucaremail'] = logname + '@ucar.edu' + pgrec['org_type'] = 'NCAR' # a UCAR login name outside of the DECS group + pgrec['org_name'] = 'NCAR' + pgrec['country'] = 'UNITED.STATES' + else: + pgrec['org_type'] = "OTHER" + pgrec['org_name'] = pgrec['country'] = "UNKNOWN" + self.incomplete_user_warning(userno, logname) return self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID)) - def incomplete_user_warning(self): - """Log a one-time reminder to run filluser for incomplete dssdb.user records.""" - if self.USRWARN: return - self.USRWARN = 1 - self.pglog("Incomplete record in dssdb.user; run 'filluser -i' later to complete it", self.LGWNEM) - - def reopen_user_record(self, pgrec): - """Reactivate a closed dssdb.user record and return its uid. - - Clears until_date and marks the record incomplete (stat_flag 'M') so that - 'filluser -i' refreshes it. Records that are not closed are returned as is. - Only call this for a present-day lookup; reopening on a historical lookup - would wrongly reactivate a user who has since left. + def incomplete_user_warning(self, userno, logname = None): + """Log a one-time reminder that an incomplete dssdb.user record was added. Args: - pgrec (dict): Record holding at least 'uid' and 'until_date'. - - Returns: - int: The uid of the record. + userno (int): Scientist number of the added user; 0 if unknown. + logname (str | None): UCAR login name of the added user; None if unknown. """ - if pgrec['until_date']: - record = {'until_date': None, 'stat_flag': 'M'} - self.pgupdt("dssdb.user", record, "uid = {}".format(pgrec['uid']), self.PGDBI['ERRLOG']) - self.pglog("Closed user.uid = {} is reopened".format(pgrec['uid']), self.LGWNEM) - return pgrec['uid'] + if self.USRWARN: return + self.USRWARN = 1 + who = logname if logname else userno + self.pglog("{}: Not in dssdb.dssgrp; incomplete record added to dssdb.user".format(who), self.LGWNEM) def check_user_uid(self, userno, date = None): """Return the user.uid for a scientist ID, adding a record if missing. Looks up the active user record for userno on the given date. If not found, - logs a warning, attempts a date-range-independent lookup, and finally fetches - UCAR person info to insert a new user record. + logs a warning and retries without the date range, returning the uid of a + closed record as is; a closed record is never reactivated. Only when no + record exists at all is a new one added from dssdb.dssgrp. Args: userno (int | str): UCAR scientist number. @@ -1794,7 +1788,6 @@ def check_user_uid(self, userno, date = None): """ if not userno: return 0 if type(userno) is str: userno = int(userno) - istoday = (date is None) if date is None: datecond = "until_date IS NULL" date = 'today' @@ -1806,8 +1799,8 @@ def check_user_uid(self, userno, date = None): self.pglog("{}: Scientist ID NOT on file for {}".format(userno, date), self.LGWNEM) self.NMISSES.append(userno) # check again if a user is on file with different date range - pgrec = self.pgget("dssdb.user", "uid, until_date", "userno = {}".format(userno), self.PGDBI['ERRLOG']) - if pgrec: return self.reopen_user_record(pgrec) if istoday else pgrec['uid'] + pgrec = self.pgget("dssdb.user", "uid", "userno = {}".format(userno), self.PGDBI['ERRLOG']) + if pgrec: return pgrec['uid'] uid = self.add_missed_user(userno) if uid: self.pglog("{}: Scientist ID Added as user.uid = {}".format(userno, uid), self.LGWNEM) return uid @@ -1815,7 +1808,8 @@ def check_user_uid(self, userno, date = None): def get_user_uid(self, logname, date = None): """Return the user.uid for a UCAR login name, adding a record if missing. - Similar to check_user_uid() but looks up by logname instead of userno. + Similar to check_user_uid() but looks up by logname instead of userno; a + closed record is likewise returned as is and never reactivated. Args: logname (str): UCAR login name. @@ -1825,7 +1819,6 @@ def get_user_uid(self, logname, date = None): int: user.uid on success, 0 if logname is falsy or insert fails. """ if not logname: return 0 - istoday = (not date) if not date: date = 'today' datecond = "until_date IS NULL" @@ -1837,92 +1830,12 @@ def get_user_uid(self, logname, date = None): self.pglog("{}: UCAR Login Name NOT on file for {}".format(logname, date), self.LGWNEM) self.LMISSES.append(logname) # check again if a user is on file with different date range - pgrec = self.pgget("dssdb.user", "uid, until_date", "logname = '{}'".format(logname), self.PGDBI['ERRLOG']) - if pgrec: return self.reopen_user_record(pgrec) if istoday else pgrec['uid'] + pgrec = self.pgget("dssdb.user", "uid", "logname = '{}'".format(logname), self.PGDBI['ERRLOG']) + if pgrec: return pgrec['uid'] uid = self.add_missed_user(0, logname) if uid: self.pglog("{}: UCAR Login Name Added as user.uid = {}".format(logname, uid), self.LGWNEM) return uid - def ucar_user_info(self, userno, logname = None): - """Fetch UCAR person info for a scientist ID or login name via pgperson/pgusername. - - Runs the pgperson command-line tool and parses its key<=>value output. - Maps UCAR API fields to database column names, normalises country code, - organisation type, and employment dates. - - Args: - userno (int): Scientist number; pass 0 to look up by logname instead. - logname (str | None): UCAR login name; used when userno is 0. - - Returns: - dict | None: Mapping of column_name → value suitable for pgadd('dssdb.user'), - or None when pgperson returns no output. - """ - matches = { - 'upid': "upid", - 'uid': "userno", - 'username': "logname", - 'lastName': "lstname", - 'firstName': "fstname", - 'active': "stat_flag", - 'internalOrg': "division", - 'externalOrg': "org_name", - 'country': "country", - 'forwardEmail': "email", - 'email': "ucaremail", - 'phone': "phoneno" - } - # pgperson/pgusername are installed by rda_python_metrics; skip quietly when the - # commands are not on this host so that callers can fall back to dssgrp - if not self.valid_command("pgperson"): return None - buf = self.pgsystem("pgperson " + ("-uid {}".format(userno) if userno else "-username {}".format(logname)), self.LOGWRN, 20) - if not buf: return None - pgrec = {} - for line in buf.split('\n'): - ms = re.match(r'^(.+)<=>(.*)$', line) - if ms: - (key, val) = ms.groups() - if key in matches: - if key == 'upid' and 'upid' in pgrec: break # get one record only - pgrec[matches[key]] = val - if not pgrec: return None - # the People API omits keys it has no value for, so read them defensively - if userno: - pgrec['userno'] = userno - elif pgrec.get('userno'): - pgrec['userno'] = userno = int(pgrec['userno']) - if pgrec.get('upid'): pgrec['upid'] = int(pgrec['upid']) - if pgrec.get('stat_flag'): pgrec['stat_flag'] = 'A' if pgrec['stat_flag'] == "True" else 'C' - email = pgrec.get('email') - if email and re.search(r'(@|\.)ucar\.edu$', email, re.I): - email = pgrec['email'] = pgrec.get('ucaremail') - pgrec['org_name'] = 'NCAR' - country = pgrec.get('country') - if email or country: pgrec['country'] = self.set_country_code(email, country) - val = "NCAR" if pgrec.get('division') else None - pgrec['org_type'] = self.get_org_type(val, email) - logname = pgrec.get('logname') - if not logname or not self.valid_command("pgusername"): return pgrec - buf = self.pgsystem("pgusername {}".format(logname), self.LOGWRN, 20) - if not buf: return pgrec - for line in buf.split('\n'): - ms = re.match(r'^(.+)<=>(.*)$', line) - if ms: - (key, val) = ms.groups() - if key == 'startDate': - m = re.match(r'^(\d+-\d+-\d+)\s', val) - if m: - pgrec['start_date'] = m.group(1) - else: - pgrec['start_date'] = val - if key == 'endDate': - m = re.match(r'^(\d+-\d+-\d+)\s', val) - if m: - pgrec['until_date'] = m.group(1) - else: - pgrec['until_date'] = val - return pgrec - def set_country_code(self, email, country = None): """Normalise a country name or derive it from an email domain.