From 47965fa9480712dce1dfbde1f9d8c1efa41540d5 Mon Sep 17 00:00:00 2001 From: zaihuaji Date: Thu, 10 Sep 2026 10:52:48 -0500 Subject: [PATCH] pg_dbi.py: read db passwords from the environment first; bump version to 3.0.15 Containers on CIRRUS get their db passwords injected as environment variables by a Kubernetes secret, so look them up there before the .pgpass file and OpenBao, and cache a password found by either of those back into the environment so that the child processes on local and PBS batch hosts inherit it instead of looking it up again. Name the secret from the db server node and the login name, so that the same name serves the OpenBao path (kv/gdex/) and the environment variable (_). Prefer the node of the host being connected to, so that a node holding its own passwords, such as pgdb02, is served its own secrets, and fall back to the node mapped for the db name, such as pgdb01 for pgdb02, for a node that shares them without holding secrets of its own. Co-Authored-By: Claude Opus 4.6 --- README.md | 2 +- pyproject.toml | 2 +- src/rda_python_common/__init__.py | 2 +- src/rda_python_common/pg_dbi.py | 134 +++++++++++++++++++++++++----- 4 files changed, 114 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 91c52aa..cefd3e6 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN) python -c "import rda_python_common; print(rda_python_common.__version__)" ``` -You should see the installed version (currently `3.0.14`). If the import +You should see the installed version (currently `3.0.15`). If the import fails, double-check that the active Python environment is the one where you ran `pip install`. diff --git a/pyproject.toml b/pyproject.toml index 8cf8a03..4143e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_common" -version = "3.0.14" +version = "3.0.15" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] diff --git a/src/rda_python_common/__init__.py b/src/rda_python_common/__init__.py index 3877cdd..74cdbfa 100644 --- a/src/rda_python_common/__init__.py +++ b/src/rda_python_common/__init__.py @@ -22,7 +22,7 @@ from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit -__version__ = "3.0.14" +__version__ = "3.0.15" __all__ = [ "PgLOG", diff --git a/src/rda_python_common/pg_dbi.py b/src/rda_python_common/pg_dbi.py index e7a3c01..941c0b1 100644 --- a/src/rda_python_common/pg_dbi.py +++ b/src/rda_python_common/pg_dbi.py @@ -76,7 +76,7 @@ class PgDBI(PgLOG): using psycopg (v3) when available, falling back to psycopg2. Supports single and batch INSERT, SELECT, UPDATE, and DELETE operations, transaction management, schema introspection, user lookups, usage tracking, and - credential retrieval from .pgpass or OpenBao. + credential retrieval from the environment, .pgpass or OpenBao. Inherits all logging and utility helpers from PgLOG. @@ -96,7 +96,10 @@ class PgDBI(PgLOG): set from psycopg/psycopg2 depending on which driver is in use. DBPORTS (dict): Mapping of database names to non-default TCP port numbers. DBPASS (dict): Credentials loaded from .pgpass, keyed by (host, port, db, user). - DBBAOS (dict): Credentials loaded from OpenBao, keyed by database name. + DBBAOS (dict): Credentials loaded from OpenBao, keyed by db server node. + DBNODES (dict): Mapping of database names to their db server nodes, used when the + node cannot be taken from the host being connected to. + DBSKEYS (dict): Mapping of login names to secret key names ('pass' by default). DBNAMES (dict): Mapping of schema names to their parent database names. DBSOCKS (dict): Mapping of database names to Unix socket paths. VIEWHOMES (dict): Mapping of hostnames to home directories for the view host. @@ -139,6 +142,19 @@ def __init__(self): self.DBPORTS = {'default': 0} self.DBPASS = {} self.DBBAOS = {} + # hard coded db server nodes for dbnames, for hosts not named for their nodes; a node + # names the OpenBao path (kv/gdex/) and the password environment variable + # (_) the same secrets are passed in through + self.DBNODES = { + 'ivaddb': 'pgdb03', + 'ispddb': 'pgdb03', + 'default': 'pgdb01', + } + # hard coded secret key names for db login names, defaulting to pass + self.DBSKEYS = { + 'postgres': 'password', + 'metadata': 'metapass', + } # hard coded db names for given schema names self.DBNAMES = { 'ivaddb': 'ivaddb', @@ -2617,19 +2633,91 @@ def pgname(self, str, sign = None): def get_pgpass_password(self): """Return the database password for the current connection settings. - Checks PGDBI['PWNAME'] first, then tries the .pgpass file (get_pgpassword()), - and finally falls back to OpenBao (get_baopassword()). + Checks PGDBI['PWNAME'] first, then the environment (get_envpassword(), as set + for containers on CIRRUS), then the .pgpass file (get_pgpassword()), and + finally falls back to OpenBao (get_baopassword()). A password found on local + or batch hosts is cached in the environment (set_envpassword()) so that the + child processes inherit it instead of looking it up again. Returns: str | None: Password string, or None when no credential is found. """ if self.PGDBI['PWNAME']: return self.PGDBI['PWNAME'] + pwname = self.get_envpassword() + if pwname: return pwname pwname = self.get_pgpassword() if not pwname: pwname = self.get_baopassword() - if not pwname: - self.pglog("Unable to find password for {} in .pgpass or OpenBao".format(self.PGDBI['DBNAME']), self.PGDBI['ERRLOG']) + if pwname: + self.set_envpassword(pwname) + else: + self.pglog("Unable to find password for {} in environment, .pgpass or OpenBao".format(self.PGDBI['DBNAME']), self.PGDBI['ERRLOG']) return pwname + def get_dbnodes(self): + """Return the db server nodes holding the secrets of the current connection. + + The node named by DBSHOST comes first, such as pgdb02 for the view-only host, + so that a node with its own passwords is served its own secrets. The DBNODES + node of the current DBNAME follows it as a fallback, such as pgdb01 for pgdb02, + for a node that shares the passwords of the default one without holding secrets + of its own. + + Returns: + list: Node names, such as ['pgdb02', 'pgdb01'], the first one preferred. + """ + dbnodes = [] + ms = re.match(r'^(pgdb\d+)$', self.PGDBI['DBSHOST']) if self.PGDBI['DBSHOST'] else None + if ms: dbnodes.append(ms.group(1)) + dbnode = self.DBNODES.get(self.PGDBI['DBNAME'], self.DBNODES['default']) + if dbnode not in dbnodes: dbnodes.append(dbnode) + + return dbnodes + + def get_secret_name(self, dbnode): + """Build the name of the secret holding the password of the current connection. + + The key comes from DBSKEYS for the current LNNAME, defaulting to 'pass'. + The pair names the secret in OpenBao (kv/gdex/ key ) and the + environment variable (_) the same secret is passed in through + on CIRRUS. + + Args: + dbnode (str): Db server node holding the secret, such as 'pgdb01'. + + Returns: + tuple: (node, key) naming the db server node and the secret key. + """ + lnname = self.PGDBI['LNNAME'] + + return (dbnode, self.DBSKEYS.get(lnname, lnname + 'pass')) + + def get_envpassword(self): + """Look up the password in the environment, as set for containers on CIRRUS. + + The variable name is the uppercased '_' secret name, for example + PGDB01_DSSDBPASS for login name dssdb and PGDB03_IVADDBPASS for ivaddb. The + fallback nodes of get_dbnodes() are tried in turn. + + Returns: + str | None: Password string, or None when no variable is set. + """ + for dbnode in self.get_dbnodes(): + pwname = os.environ.get("{}_{}".format(*self.get_secret_name(dbnode)).upper()) + if pwname: return pwname + + return None + + def set_envpassword(self, pwname): + """Cache a password in the environment under the name get_envpassword() reads. + + Called for a password found in .pgpass or OpenBao, on local and PBS batch hosts, + so that the child processes of the current one inherit it and skip the lookup. + + Args: + pwname (str): Password to pass on to the child processes. + """ + os.environ["{}_{}".format(*self.get_secret_name(self.get_dbnodes()[0])).upper()] = pwname + def get_pgpassword(self): """Look up the password in the cached .pgpass credentials. @@ -2646,16 +2734,20 @@ def get_pgpassword(self): return pwname def get_baopassword(self): - """Look up the password from OpenBao for the current database and login name. + """Look up the password from OpenBao for the current db server node and login name. - Loads OpenBao secrets for PGDBI['DBNAME'] on first call (or when not cached). + Loads the OpenBao secrets of a node on first call (or when not cached), trying + the fallback nodes of get_dbnodes() in turn. Returns: str | None: Password string, or None when not found in OpenBao. """ - dbname = self.PGDBI['DBNAME'] - if dbname not in self.DBBAOS: self.read_openbao() - return self.DBBAOS[dbname].get(self.PGDBI['LNNAME']) + for dbnode in self.get_dbnodes(): + if dbnode not in self.DBBAOS: self.read_openbao(dbnode) + pwname = self.DBBAOS[dbnode].get(self.PGDBI['LNNAME']) + if pwname: return pwname + + return None def read_pgpass(self): """Read the .pgpass file and populate DBPASS with credentials. @@ -2675,21 +2767,17 @@ def read_pgpass(self): except Exception: pass - def read_openbao(self): - """Read OpenBao secrets and populate DBBAOS with credentials for DBNAME. + def read_openbao(self, dbnode): + """Read OpenBao secrets and populate DBBAOS with the credentials of a db server node. Uses the hvac client to fetch key-value secrets from the configured BAOURL. Parses keys matching 'pass' patterns to extract database usernames and passwords. + + Args: + dbnode (str): Db server node naming the secret path, such as 'pgdb01'. """ - dbname = self.PGDBI['DBNAME'] - self.DBBAOS[dbname] = {} - url = 'https://bao.k8s.ucar.edu/' - baopath = { - 'ivaddb': 'gdex/pgdb03', - 'ispddb': 'gdex/pgdb03', - 'default': 'gdex/pgdb01' - } - dbpath = baopath[dbname] if dbname in baopath else baopath['default'] + self.DBBAOS[dbnode] = {} + dbpath = 'gdex/' + dbnode client = hvac.Client(url=self.PGDBI.get('BAOURL')) client.token = self.PGLOG.get('BAOTOKEN') try: @@ -2711,4 +2799,4 @@ def read_openbao(self): baoname = 'metadata' if pre == 'meta' else pre elif suf == 'word': baoname = 'postgres' - if baoname: self.DBBAOS[dbname][baoname] = baos[key] + if baoname: self.DBBAOS[dbnode][baoname] = baos[key]