From 495afb70c062617a62031210bf0ba512548f6078 Mon Sep 17 00:00:00 2001 From: zaihuaji Date: Thu, 17 Sep 2026 08:59:07 -0500 Subject: [PATCH] pg_file.py: download object store directory keys; bump version to 3.0.16 A zarr store on the object store is a set of objects sharing a key prefix, not a single object, so 'isd_s3_cli go -k ' fails with a 404 on HeadObject. object_copy_local() now hands such a key to the new object_copy_local_directory(), which lists the prefix and downloads every member into its place under the target directory. The single object download is factored out into object_get_local(), which takes the expected size as an argument so the member sizes returned by object_glob() can be reused; rechecking each member key on its own would misread chunk names such as 'time/1' as directories, since listing is a prefix match that also returns 'time/10'. Co-Authored-By: Claude Opus 4.6 --- README.md | 2 +- pyproject.toml | 2 +- src/rda_python_common/__init__.py | 2 +- src/rda_python_common/pg_file.py | 63 ++++++++++++++++++++++++++++++- 4 files changed, 64 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cefd3e6..edce822 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,7 @@ PgLOG.pglog("hello", PgLOG.LOGWRN) python -c "import rda_python_common; print(rda_python_common.__version__)" ``` -You should see the installed version (currently `3.0.15`). If the import +You should see the installed version (currently `3.0.16`). If the import fails, double-check that the active Python environment is the one where you ran `pip install`. diff --git a/pyproject.toml b/pyproject.toml index 4143e1e..34e23b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "rda_python_common" -version = "3.0.15" +version = "3.0.16" authors = [ { name="Zaihua Ji", email="zji@ucar.edu" }, ] diff --git a/src/rda_python_common/__init__.py b/src/rda_python_common/__init__.py index 74cdbfa..e406c96 100644 --- a/src/rda_python_common/__init__.py +++ b/src/rda_python_common/__init__.py @@ -22,7 +22,7 @@ from . import PgLOG, PgUtil, PgDBI, PgFile, PgLock, PgCMD, PgSIG, PgOPT, PgSplit -__version__ = "3.0.15" +__version__ = "3.0.16" __all__ = [ "PgLOG", diff --git a/src/rda_python_common/pg_file.py b/src/rda_python_common/pg_file.py index b467557..d96eded 100644 --- a/src/rda_python_common/pg_file.py +++ b/src/rda_python_common/pg_file.py @@ -766,7 +766,9 @@ def object_copy_local(self, tofile, fromfile, bucket = None, logact = 0): """Download a file from the object store to the local filesystem. Changes to the target directory, downloads using isd_s3_cli, verifies size, - sets permissions, and renames if needed. Retries once on failure. + sets permissions, and renames if needed. Retries once on failure. A key that + is a prefix of multiple objects, such as a zarr store, is downloaded as a + directory by object_copy_local_directory(). Args: tofile (str): Destination local file path. @@ -783,6 +785,28 @@ def object_copy_local(self, tofile, fromfile, bucket = None, logact = 0): if not finfo: if finfo != None: return ret return self.lmsg(fromfile, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], tofile), logact) + if not finfo['isfile']: return self.object_copy_local_directory(tofile, fromfile, bucket, logact) + return self.object_get_local(tofile, fromfile, finfo['data_size'], bucket, logact) + + # Download a single object of a known size to a local file + # tofile - target file name + # fromfile - source object key name + # fsize - size of the object, to verify the download against + # bucket - bucket name on Object store + def object_get_local(self, tofile, fromfile, fsize, bucket, logact = 0): + """Download one object key to a local file and verify its size. + + Args: + tofile (str): Destination local file path. + fromfile (str): Object key (source path in the bucket). + fsize (int): Expected size of the object. + bucket (str): Source bucket. + logact (int): Logging action flags; default 0. + + Returns: + int: self.SUCCESS on success, self.FAILURE on error. + """ + ret = self.FAILURE ocmd = self.OBJCTCMD cmd = "{} go -k {} -b {}".format(ocmd, fromfile, bucket) fromname = op.basename(fromfile) @@ -797,7 +821,7 @@ def object_copy_local(self, tofile, fromfile, bucket = None, logact = 0): buf = self.pgsystem(cmd, logact, self.CMDBTH) info = self.check_local_file(fromname, 143, logact|self.PFSIZE) # 1+2+4+8+128 if info: - if info['data_size'] == finfo['data_size']: + if info['data_size'] == fsize: self.set_local_mode(fromfile, info['isfile'], 0, info['mode'], info['logname'], logact) if toname == fromname or self.move_local_file(toname, fromname, logact): ret = self.SUCCESS @@ -811,6 +835,41 @@ def object_copy_local(self, tofile, fromfile, bucket = None, logact = 0): self.change_local_directory(odir, logact) return ret + # Copy an object directory, a key prefix holding multiple objects, to local + # todir - target local directory name + # fromdir - source object key prefix + # bucket - bucket name on Object store + def object_copy_local_directory(self, todir, fromdir, bucket = None, logact = 0): + """Download every object under a key prefix into a local directory. + + The object store has no directories; a zarr store and the like is a set of + objects sharing a key prefix. Each object is downloaded individually and + keeps its position relative to the prefix under todir. + + Args: + todir (str): Destination local directory path. + fromdir (str): Object key prefix (source path in the bucket). + bucket (str | None): Source bucket; defaults to PGLOG['OBJCTBKT']. + logact (int): Logging action flags; default 0. + + Returns: + int: self.SUCCESS on success, self.FAILURE on error. + """ + if not bucket: bucket = self.PGLOG['OBJCTBKT'] + ms = re.match(r'^(.+)/$', fromdir) + if ms: fromdir = ms.group(1) + flist = self.object_glob(fromdir, bucket, 0, logact) + if flist == self.FAILURE: return self.FAILURE + prefix = fromdir + '/' + keys = [key for key in flist if key.startswith(prefix)] + if not keys: + return self.lmsg(fromdir, "{}-{} to copy to {}".format(self.OHOST, self.PGLOG['MISSFILE'], todir), logact) + plen = len(prefix) + for key in keys: + tofile = "{}/{}".format(todir, key[plen:]) + if not self.object_get_local(tofile, key, flist[key]['data_size'], bucket, logact): return self.FAILURE + return self.SUCCESS + # Copy a remote file to object # tofile - target object file name # fromfile - source remote file name