Skip to content

pg_dbi.py: drop the retired UCAR People DB and use dssdb.dssgrp only;… #119

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zaihuaji merged 1 commit into from
Aug 26, 2026
Merged

pg_dbi.py: drop the retired UCAR People DB and use dssdb.dssgrp only;… #119

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

[project]
name = "rda_python_common"
version = "3.0.10"
version = "3.0.11"
Comment on lines 5 to +7
authors = [
{ name="Zaihua Ji", email="zji@ucar.edu" },
]
Expand Down
167 changes: 40 additions & 127 deletions src/rda_python_common/pg_dbi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class PgDBI(PgLOG):
curtran (int): Transaction counter: 0 = idle, >0 = inside a transaction.
NMISSES (list): Cached list of scientist IDs (userno) not found in the DB.
LMISSES (list): Cached list of login names not found in the DB.
USRWARN (int): Set to 1 once the 'run filluser' reminder has been logged.
USRWARN (int): Set to 1 once the incomplete-user reminder has been logged.
TABLES (dict): Cache of table-field default info keyed by table name.
SEQUENCES (dict): Cache of sequence field names keyed by table name.
SPECIALIST (dict): Cache of specialist records keyed by dataset ID.
Expand Down Expand Up @@ -1702,19 +1702,17 @@ def pgcheck(self, tablename, logact = 0):
return (self.SUCCESS if ret else self.FAILURE)

def dssgrp_user_info(self, userno, logname = None):
"""Build a partial dssdb.user record from dssdb.dssgrp.
"""Build a dssdb.user record from dssdb.dssgrp.

Fallback for when the UCAR People API is unreachable, so a usable user record
can still be added for a DECS group member. Fields absent from dssgrp are set
to the values implied by DECS membership; the record is marked stat_flag 'M'
so that 'filluser -i' completes it once the People API is available again.
dssdb.dssgrp is the only source of specialist information; fields absent from
it are set to the values implied by DECS group membership.

Args:
userno (int): Scientist number; pass 0 to look up by logname instead.
logname (str | None): UCAR login name; used when userno is 0.

Returns:
dict | None: Partial mapping of column_name → value suitable for
dict | None: Mapping of column_name → value suitable for
pgadd('dssdb.user'), or None when not in dssgrp.
"""
cond = "userno = {}".format(userno) if userno else "logname = '{}'".format(logname)
Expand All @@ -1726,15 +1724,16 @@ def dssgrp_user_info(self, userno, logname = None):
pgrec['org_name'] = 'NCAR'
pgrec['country'] = 'UNITED.STATES'
pgrec['email'] = pgrec['ucaremail'] = email
pgrec['stat_flag'] = 'M'
pgrec['stat_flag'] = 'A'
return pgrec

def add_missed_user(self, userno, logname = None):
"""Add a dssdb.user record for a user missing from the table.

Tries the UCAR People API first, then dssgrp, and finally falls back to a bare
stub. A reminder to run filluser is logged once per process whenever the added
record is incomplete (stat_flag 'M').
Builds the record from dssdb.dssgrp for a DECS group member. For anyone not
in dssgrp, adds a record filled with whatever can be derived from the login
name, using 'UNKNOWN' for the fields that cannot be filled in; such a record
is marked stat_flag 'M' and logs a reminder once per process.

Args:
userno (int): Scientist number; pass 0 to add by logname instead.
Expand All @@ -1743,46 +1742,41 @@ def add_missed_user(self, userno, logname = None):
Returns:
int: The new user.uid, or 0 if the insert fails.
"""
pgrec = self.ucar_user_info(userno, logname)
if not pgrec: pgrec = self.dssgrp_user_info(userno, logname)
pgrec = self.dssgrp_user_info(userno, logname)
if not pgrec:
pgrec = {'userno': userno} if userno else {'logname': logname}
pgrec['stat_flag'] = 'M'
if pgrec['stat_flag'] == 'M': self.incomplete_user_warning()
pgrec = {'stat_flag': 'M', 'lstname': "UNKNOWN", 'fstname': "UNKNOWN"}
if userno: pgrec['userno'] = userno
if logname:
pgrec['logname'] = logname
pgrec['email'] = pgrec['ucaremail'] = logname + '@ucar.edu'
pgrec['org_type'] = 'NCAR' # a UCAR login name outside of the DECS group
pgrec['org_name'] = 'NCAR'
pgrec['country'] = 'UNITED.STATES'
else:
pgrec['org_type'] = "OTHER"
pgrec['org_name'] = pgrec['country'] = "UNKNOWN"
self.incomplete_user_warning(userno, logname)
return self.pgadd("dssdb.user", pgrec, (self.PGDBI['EXITLG']|self.AUTOID))

def incomplete_user_warning(self):
"""Log a one-time reminder to run filluser for incomplete dssdb.user records."""
if self.USRWARN: return
self.USRWARN = 1
self.pglog("Incomplete record in dssdb.user; run 'filluser -i' later to complete it", self.LGWNEM)

def reopen_user_record(self, pgrec):
"""Reactivate a closed dssdb.user record and return its uid.

Clears until_date and marks the record incomplete (stat_flag 'M') so that
'filluser -i' refreshes it. Records that are not closed are returned as is.
Only call this for a present-day lookup; reopening on a historical lookup
would wrongly reactivate a user who has since left.
def incomplete_user_warning(self, userno, logname = None):
"""Log a one-time reminder that an incomplete dssdb.user record was added.

Args:
pgrec (dict): Record holding at least 'uid' and 'until_date'.

Returns:
int: The uid of the record.
userno (int): Scientist number of the added user; 0 if unknown.
logname (str | None): UCAR login name of the added user; None if unknown.
"""
if pgrec['until_date']:
record = {'until_date': None, 'stat_flag': 'M'}
self.pgupdt("dssdb.user", record, "uid = {}".format(pgrec['uid']), self.PGDBI['ERRLOG'])
self.pglog("Closed user.uid = {} is reopened".format(pgrec['uid']), self.LGWNEM)
return pgrec['uid']
if self.USRWARN: return
self.USRWARN = 1
who = logname if logname else userno
self.pglog("{}: Not in dssdb.dssgrp; incomplete record added to dssdb.user".format(who), self.LGWNEM)
Comment on lines +1768 to +1771

def check_user_uid(self, userno, date = None):
"""Return the user.uid for a scientist ID, adding a record if missing.

Looks up the active user record for userno on the given date. If not found,
logs a warning, attempts a date-range-independent lookup, and finally fetches
UCAR person info to insert a new user record.
logs a warning and retries without the date range, returning the uid of a
closed record as is; a closed record is never reactivated. Only when no
record exists at all is a new one added from dssdb.dssgrp.
Comment on lines 1776 to +1779

Args:
userno (int | str): UCAR scientist number.
Expand All @@ -1794,7 +1788,6 @@ def check_user_uid(self, userno, date = None):
"""
if not userno: return 0
if type(userno) is str: userno = int(userno)
istoday = (date is None)
if date is None:
datecond = "until_date IS NULL"
date = 'today'
Expand All @@ -1806,16 +1799,17 @@ def check_user_uid(self, userno, date = None):
self.pglog("{}: Scientist ID NOT on file for {}".format(userno, date), self.LGWNEM)
self.NMISSES.append(userno)
# check again if a user is on file with different date range
pgrec = self.pgget("dssdb.user", "uid, until_date", "userno = {}".format(userno), self.PGDBI['ERRLOG'])
if pgrec: return self.reopen_user_record(pgrec) if istoday else pgrec['uid']
pgrec = self.pgget("dssdb.user", "uid", "userno = {}".format(userno), self.PGDBI['ERRLOG'])
if pgrec: return pgrec['uid']
uid = self.add_missed_user(userno)
if uid: self.pglog("{}: Scientist ID Added as user.uid = {}".format(userno, uid), self.LGWNEM)
return uid

def get_user_uid(self, logname, date = None):
"""Return the user.uid for a UCAR login name, adding a record if missing.

Similar to check_user_uid() but looks up by logname instead of userno.
Similar to check_user_uid() but looks up by logname instead of userno; a
closed record is likewise returned as is and never reactivated.

Args:
logname (str): UCAR login name.
Expand All @@ -1825,7 +1819,6 @@ def get_user_uid(self, logname, date = None):
int: user.uid on success, 0 if logname is falsy or insert fails.
"""
if not logname: return 0
istoday = (not date)
if not date:
date = 'today'
datecond = "until_date IS NULL"
Expand All @@ -1837,92 +1830,12 @@ def get_user_uid(self, logname, date = None):
self.pglog("{}: UCAR Login Name NOT on file for {}".format(logname, date), self.LGWNEM)
self.LMISSES.append(logname)
# check again if a user is on file with different date range
pgrec = self.pgget("dssdb.user", "uid, until_date", "logname = '{}'".format(logname), self.PGDBI['ERRLOG'])
if pgrec: return self.reopen_user_record(pgrec) if istoday else pgrec['uid']
pgrec = self.pgget("dssdb.user", "uid", "logname = '{}'".format(logname), self.PGDBI['ERRLOG'])
if pgrec: return pgrec['uid']
uid = self.add_missed_user(0, logname)
if uid: self.pglog("{}: UCAR Login Name Added as user.uid = {}".format(logname, uid), self.LGWNEM)
return uid

def ucar_user_info(self, userno, logname = None):
"""Fetch UCAR person info for a scientist ID or login name via pgperson/pgusername.

Runs the pgperson command-line tool and parses its key<=>value output.
Maps UCAR API fields to database column names, normalises country code,
organisation type, and employment dates.

Args:
userno (int): Scientist number; pass 0 to look up by logname instead.
logname (str | None): UCAR login name; used when userno is 0.

Returns:
dict | None: Mapping of column_name → value suitable for pgadd('dssdb.user'),
or None when pgperson returns no output.
"""
matches = {
'upid': "upid",
'uid': "userno",
'username': "logname",
'lastName': "lstname",
'firstName': "fstname",
'active': "stat_flag",
'internalOrg': "division",
'externalOrg': "org_name",
'country': "country",
'forwardEmail': "email",
'email': "ucaremail",
'phone': "phoneno"
}
# pgperson/pgusername are installed by rda_python_metrics; skip quietly when the
# commands are not on this host so that callers can fall back to dssgrp
if not self.valid_command("pgperson"): return None
buf = self.pgsystem("pgperson " + ("-uid {}".format(userno) if userno else "-username {}".format(logname)), self.LOGWRN, 20)
if not buf: return None
pgrec = {}
for line in buf.split('\n'):
ms = re.match(r'^(.+)<=>(.*)$', line)
if ms:
(key, val) = ms.groups()
if key in matches:
if key == 'upid' and 'upid' in pgrec: break # get one record only
pgrec[matches[key]] = val
if not pgrec: return None
# the People API omits keys it has no value for, so read them defensively
if userno:
pgrec['userno'] = userno
elif pgrec.get('userno'):
pgrec['userno'] = userno = int(pgrec['userno'])
if pgrec.get('upid'): pgrec['upid'] = int(pgrec['upid'])
if pgrec.get('stat_flag'): pgrec['stat_flag'] = 'A' if pgrec['stat_flag'] == "True" else 'C'
email = pgrec.get('email')
if email and re.search(r'(@|\.)ucar\.edu$', email, re.I):
email = pgrec['email'] = pgrec.get('ucaremail')
pgrec['org_name'] = 'NCAR'
country = pgrec.get('country')
if email or country: pgrec['country'] = self.set_country_code(email, country)
val = "NCAR" if pgrec.get('division') else None
pgrec['org_type'] = self.get_org_type(val, email)
logname = pgrec.get('logname')
if not logname or not self.valid_command("pgusername"): return pgrec
buf = self.pgsystem("pgusername {}".format(logname), self.LOGWRN, 20)
if not buf: return pgrec
for line in buf.split('\n'):
ms = re.match(r'^(.+)<=>(.*)$', line)
if ms:
(key, val) = ms.groups()
if key == 'startDate':
m = re.match(r'^(\d+-\d+-\d+)\s', val)
if m:
pgrec['start_date'] = m.group(1)
else:
pgrec['start_date'] = val
if key == 'endDate':
m = re.match(r'^(\d+-\d+-\d+)\s', val)
if m:
pgrec['until_date'] = m.group(1)
else:
pgrec['until_date'] = val
return pgrec

def set_country_code(self, email, country = None):
"""Normalise a country name or derive it from an email domain.

Expand Down
Loading