From b40b524370abb5e667315e82363488c873cbefaa Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 10 Sep 2026 18:28:31 +0000 Subject: [PATCH 1/2] Register +ndr/+format/+blosc as MATLAB-only in the bridge NDR-matlab now owns a Blosc container codec at +ndr/+format/+blosc, backed by numcodecs.Blosc in a private subprocess venv (sonpipe pattern; ndr.util.blosc.setup manages the venv, MATLAB's pyenv is never touched). There is no Python mirror and none is planned: Python callers should use numcodecs.Blosc directly, and any MATLAB wrapper on top would just be a rename over the same subprocess call. The new bridge yaml records the four MATLAB functions (encode, decode, isBlosc, header) with status: matlab_only so the mirror inventory stays honest. Co-Authored-By: Claude Opus 4.7 Claude-Session: https://claude.ai/code/session_01N67xH9BejMzZp4GNAq8m7w --- src/ndr/format/blosc/__init__.py | 18 +++++ .../blosc/ndr_matlab_python_bridge.yaml | 81 +++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 src/ndr/format/blosc/__init__.py create mode 100644 src/ndr/format/blosc/ndr_matlab_python_bridge.yaml diff --git a/src/ndr/format/blosc/__init__.py b/src/ndr/format/blosc/__init__.py new file mode 100644 index 0000000..1dc77cb --- /dev/null +++ b/src/ndr/format/blosc/__init__.py @@ -0,0 +1,18 @@ +"""Blosc container codec -- MATLAB-side surface only. + +Mirrors ``+ndr/+format/+blosc/`` in NDR-matlab. The MATLAB package +exists so callers there can encode/decode Blosc v1 containers without +a system ``zstd`` binary; under the hood it calls ``numcodecs.Blosc`` +in a private subprocess venv. + +There is no Python mirror. Python callers should use ``numcodecs`` +directly: + + from numcodecs import Blosc + codec = Blosc(cname='zstd', clevel=5, shuffle=Blosc.SHUFFLE) + container = codec.encode(numpy_buffer) + raw = codec.decode(container) + +See ``ndr_matlab_python_bridge.yaml`` in this directory for the +per-function contract. +""" diff --git a/src/ndr/format/blosc/ndr_matlab_python_bridge.yaml b/src/ndr/format/blosc/ndr_matlab_python_bridge.yaml new file mode 100644 index 0000000..8dc4b49 --- /dev/null +++ b/src/ndr/format/blosc/ndr_matlab_python_bridge.yaml @@ -0,0 +1,81 @@ +# ndr_matlab_python_bridge.yaml -- src/ndr/format/blosc/ +# The Primary Contract for the +ndr/+format/+blosc MATLAB package. + +project_metadata: + bridge_version: "1.1" + naming_policy: "Strict MATLAB Mirror" + indexing_policy: "Semantic Parity (1-based for user concepts, 0-based for internal data)" + +# ========================================================================= +# +ndr/+format/+blosc -- Blosc v1 container codec (encode + decode) +# (VH-Lab/NDR-matlab claude/lightsheet-zarr-ndi-viewer-djp5vk). +# +# The MATLAB package is deliberately a thin surface over `numcodecs.Blosc` +# invoked as a subprocess against a private venv NDR-matlab sets up on +# first use (see +ndr/+util/+blosc). It exists so that MATLAB callers +# (chiefly the lightsheet document builder in NDI-matlab) do not need a +# system `zstd` binary and do not need to touch MATLAB's `pyenv`, which +# is process-global and would fight the customer's Python setup. +# +# THE PYTHON SIDE HAS NO MIRROR AND IS NOT PLANNED. Python callers that +# need to write or read a Blosc container should use `numcodecs.Blosc` +# directly: +# +# from numcodecs import Blosc +# codec = Blosc(cname='zstd', clevel=5, shuffle=Blosc.SHUFFLE) +# container_bytes = codec.encode(numpy_buffer) # bytes-out +# raw_bytes = codec.decode(container_bytes) # bytes-in +# +# The MATLAB `encode` / `decode` calls are byte-identical to the +# corresponding numcodecs Blosc.encode / Blosc.decode calls, because +# they ARE those calls (over a subprocess). A wrapper here would only +# add a rename. +# ========================================================================= +functions: + + - name: encode + matlab_path: "+ndr/+format/+blosc/encode.m" + matlab_last_sync_hash: "9a87c0a" + status: matlab_only + decision_log: >- + Delegates to numcodecs.Blosc.encode via a subprocess. The MATLAB + argument list mirrors the numcodecs constructor: cname (default + 'zstd'), clevel (default 5), shuffle (0/1/2, default 1), + blocksize (default 0 = auto), typesize (element size in bytes, + inferred from a typed array's class or supplied explicitly for + raw uint8). Python callers use numcodecs.Blosc().encode(buf) + directly; no port planned. + + - name: decode + matlab_path: "+ndr/+format/+blosc/decode.m" + matlab_last_sync_hash: "9a87c0a" + status: matlab_only + decision_log: >- + Delegates to numcodecs.Blosc.decode via a subprocess. Reads + typesize out of the container header (numcodecs handles it), so + the MATLAB signature is just `decode(container)`. Python callers + use numcodecs.Blosc().decode(bytes) directly; no port planned. + + - name: isBlosc + matlab_path: "+ndr/+format/+blosc/isBlosc.m" + matlab_last_sync_hash: "9a87c0a" + status: matlab_only + decision_log: >- + Informed header sniff: checks the first byte is a supported Blosc + version (1 or 2) and that the cbytes field in the header equals + the buffer length. Blosc v1 has no fixed magic number so this is + a probe, not a proof. Python callers usually decide from + .zarray.compressor.id and would inline the same check with + numpy.frombuffer; no port planned. + + - name: header + matlab_path: "+ndr/+format/+blosc/header.m" + matlab_last_sync_hash: "9a87c0a" + status: matlab_only + decision_log: >- + Metadata-only header parser: returns version, versionlz, flags, + shuffle bits, memcpy bit, typesize, nbytes, blocksize, cbytes. + Codec identity is NOT reported because the Blosc flag encoding of + it has drifted across versions and .zarray.compressor.cname is + the source of truth. Python callers should read the header with + struct.unpack when they need it; no port planned. From 4be27f02e8e89d9c228445213bff1b4c74ff1733 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 16 Sep 2026 14:04:09 +0000 Subject: [PATCH 2/2] Bring omezarr/smartspim ports into parity with NDR-matlab main Follow-up to VH-Lab/NDR-matlab#139 (merged there): the probe/reduce helpers and the omezarr/smartspim reader classes were already ported; this commit adds the pytest coverage that was missing on the Python side and bumps the four bridge entries that had drifted. Tests mirror the MATLAB unit tests function-for-function: - tests/test_omezarr_probe.py mirrors TestOMEZarrProbe.m: ok / n_pyramids / pyramid_names / axes_order / level0_* on the shared programmatic fixture, plus the not-ok paths (missing dir, empty dir). - tests/test_omezarr_reduce.py mirrors TestOMEZarrReduce.m: mean/max, dtype (max preserves, mean promotes to float64), per-axis factor vector, trailing partial block kept, arity + reduction validation, factor-1 pass-through. - tests/test_omezarr_reader.py mirrors TestOMEZarrReader.m: pinned / unpinned / unknown / sidecar resolveepoch, all metadata methods, single-frame / multi-frame / Level>1 pixel reads against ground truth, and mean-vs-max routing proof at level 2. - tests/test_smartspim_reader.py mirrors TestSmartSPIMReader.m: pinned / unpinned-channel / unpinned-tile / sidecar resolveepoch, all metadata methods, single-frame / multi-frame / cross-channel reads against the synthetic fixture, out-of-range error. Programmatic fixtures mirror ndr.test.format.{omezarr,smartspim} .makeExampleFixture.m: dual-pyramid Zarr v2 store with a shared level-0 array (uncompressed C-order chunks are enough since the read path is zarr.open); a synthetic SmartSPIM tree with real TIFF slices whose pixels encode (channelIndex, z) so cross-channel routing has a byte-for-byte proof. Bridge yaml updates (all drift picked up by test_matlab_bridge_completeness.py): - +ndr/+format/+blosc/encode.m 9a87c0a -> aa89463 - +ndr/+format/+blosc/decode.m 9a87c0a -> aa89463 - +ndr/+format/+omezarr/readArray.m a847431 -> 8bea2d2 - +ndr/+format/+omezarr/private/ decompressBlosc.m a847431 -> 5a44303 Every one is MATLAB-side blosc plumbing (batch API, persistent server, MEX preference, hand-rolled codec dispatch replaced with one-line delegation). Python has always used numcodecs.Blosc through zarr directly, so all four are hash-bump-only per the decision_log notes. +ndr/+util/+blosc/* MEX bootstrapping stays deliberately unported (numcodecs.Blosc on the Python side; see the matlab_only entries in src/ndr/format/blosc/ndr_matlab_python_bridge.yaml). Co-Authored-By: Claude Opus 4.7 Claude-Session: https://claude.ai/code/session_01UdLHfApBhntCz64BatnwY5 --- .../blosc/ndr_matlab_python_bridge.yaml | 17 +- src/ndr/format/omezarr/__init__.py | 4 + .../omezarr/ndr_matlab_python_bridge.yaml | 18 +- tests/_omezarr_fixture.py | 223 ++++++++++++++++++ tests/_smartspim_fixture.py | 217 +++++++++++++++++ tests/test_omezarr_probe.py | 61 +++++ tests/test_omezarr_reader.py | 190 +++++++++++++++ tests/test_omezarr_reduce.py | 78 ++++++ tests/test_smartspim_reader.py | 162 +++++++++++++ 9 files changed, 966 insertions(+), 4 deletions(-) create mode 100644 tests/_omezarr_fixture.py create mode 100644 tests/_smartspim_fixture.py create mode 100644 tests/test_omezarr_probe.py create mode 100644 tests/test_omezarr_reader.py create mode 100644 tests/test_omezarr_reduce.py create mode 100644 tests/test_smartspim_reader.py diff --git a/src/ndr/format/blosc/ndr_matlab_python_bridge.yaml b/src/ndr/format/blosc/ndr_matlab_python_bridge.yaml index 8dc4b49..28963c1 100644 --- a/src/ndr/format/blosc/ndr_matlab_python_bridge.yaml +++ b/src/ndr/format/blosc/ndr_matlab_python_bridge.yaml @@ -35,7 +35,7 @@ functions: - name: encode matlab_path: "+ndr/+format/+blosc/encode.m" - matlab_last_sync_hash: "9a87c0a" + matlab_last_sync_hash: "aa89463" status: matlab_only decision_log: >- Delegates to numcodecs.Blosc.encode via a subprocess. The MATLAB @@ -46,9 +46,16 @@ functions: raw uint8). Python callers use numcodecs.Blosc().encode(buf) directly; no port planned. + Hash bumped 2026-09-16 across four MATLAB-only commits + (f9cb0a2 args-string cleanup, b5ae537 clevel cap at 9, + 8bea2d2 batch API + persistent server plumbing, aa89463 blosc + MEX preference). Every one is MATLAB-side subprocess or MEX + plumbing; Python uses numcodecs.Blosc directly, so there is + nothing to port. + - name: decode matlab_path: "+ndr/+format/+blosc/decode.m" - matlab_last_sync_hash: "9a87c0a" + matlab_last_sync_hash: "aa89463" status: matlab_only decision_log: >- Delegates to numcodecs.Blosc.decode via a subprocess. Reads @@ -56,6 +63,12 @@ functions: the MATLAB signature is just `decode(container)`. Python callers use numcodecs.Blosc().decode(bytes) directly; no port planned. + Hash bumped 2026-09-16 across two MATLAB-only commits + (8bea2d2 batch API + persistent server plumbing, + aa89463 blosc MEX preference). Both are MATLAB-side subprocess + or MEX plumbing; Python uses numcodecs.Blosc directly, so there + is nothing to port. + - name: isBlosc matlab_path: "+ndr/+format/+blosc/isBlosc.m" matlab_last_sync_hash: "9a87c0a" diff --git a/src/ndr/format/omezarr/__init__.py b/src/ndr/format/omezarr/__init__.py index b673ee8..c3988d7 100644 --- a/src/ndr/format/omezarr/__init__.py +++ b/src/ndr/format/omezarr/__init__.py @@ -5,14 +5,18 @@ from ndr.format.omezarr.isOMEZarr import isOMEZarr from ndr.format.omezarr.listPyramids import listPyramids +from ndr.format.omezarr.probe import probe from ndr.format.omezarr.readArray import readArray from ndr.format.omezarr.readAttrs import readAttrs +from ndr.format.omezarr.reduce import reduce from ndr.format.omezarr.resolveArrayPath import resolveArrayPath __all__ = [ "isOMEZarr", "listPyramids", + "probe", "readArray", "readAttrs", + "reduce", "resolveArrayPath", ] diff --git a/src/ndr/format/omezarr/ndr_matlab_python_bridge.yaml b/src/ndr/format/omezarr/ndr_matlab_python_bridge.yaml index 8fc6866..5a25133 100644 --- a/src/ndr/format/omezarr/ndr_matlab_python_bridge.yaml +++ b/src/ndr/format/omezarr/ndr_matlab_python_bridge.yaml @@ -56,7 +56,7 @@ functions: - name: readArray matlab_path: "+ndr/+format/+omezarr/readArray.m" status: regular_port - matlab_last_sync_hash: "a847431" + matlab_last_sync_hash: "8bea2d2" python_path: "ndr/format/omezarr/readArray.py" decision_log: >- Reads pixels for one region of one pyramid level and returns an n-D @@ -80,6 +80,13 @@ functions: (v2, C-order, Blosc/Zstd or uncompressed, fixed-length numeric dtypes). A store the MATLAB reader accepts is also accepted here. + Hash bumped 2026-09-16 to include 8bea2d2, which restructured + the MATLAB reader into three passes (plan, batched fread, + batched blosc.decodeMany) to amortize Python-subprocess spawn + cost. Python's zarr.open() already reads and decodes chunks in + one call with no subprocess round-trip, so there is no MATLAB + change to port here. + - name: probe matlab_path: "+ndr/+format/+omezarr/probe.m" status: regular_port @@ -112,7 +119,7 @@ functions: - name: decompressBlosc matlab_path: "+ndr/+format/+omezarr/private/decompressBlosc.m" - matlab_last_sync_hash: "a847431" + matlab_last_sync_hash: "5a44303" status: ported_differently python_path: "ndr/format/omezarr/readArray.py" decision_log: >- @@ -122,6 +129,13 @@ functions: `numcodecs.Blosc`, so readArray never needs to see the container bytes. Blosc-compressed stores read correctly in Python. + Hash bumped 2026-09-16 to include 5a44303, which replaces the + MATLAB hand-rolled codec-per-block loop with a one-line + delegation to ndr.format.blosc.decode (which itself routes + through numcodecs.Blosc in a MATLAB-owned venv). Python has + always been on numcodecs.Blosc directly, so there is nothing + new to port. + - name: decompressZstd matlab_path: "+ndr/+format/+omezarr/private/decompressZstd.m" matlab_last_sync_hash: "248f7f2" diff --git a/tests/_omezarr_fixture.py b/tests/_omezarr_fixture.py new file mode 100644 index 0000000..426ef23 --- /dev/null +++ b/tests/_omezarr_fixture.py @@ -0,0 +1,223 @@ +"""Programmatic OME-Zarr fixture used by the omezarr tests. + +Python mirror of ``+ndr/+test/+format/+omezarr/makeExampleFixture.m``. + +Builds a dual-pyramid store that matches the lab layout in +``ferret-lightsheet/formats/OurUsualZarrFormat.md``: one shared level-0 +array plus separate ``mean`` and ``max`` downsample chains. Without +``with_chunks`` only metadata files are written; with ``with_chunks`` a +seeded uint16 volume is written through the ``zarr`` library so +chunks exist on disk and their ground-truth downsamples come back for +comparison. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import numpy as np + + +@dataclass +class GroundTruth: + level0: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.uint16)) + mean_level1: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.uint16)) + max_level1: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.uint16)) + mean_level2: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.uint16)) + max_level2: np.ndarray = field(default_factory=lambda: np.empty(0, dtype=np.uint16)) + shape0: tuple[int, ...] = () + chunk_shape0: tuple[int, ...] = () + dtype: str = " None: + path.write_text(json.dumps(obj)) + + +def _write_zgroup(dir_path: Path) -> None: + _write_json(dir_path / ".zgroup", {"zarr_format": 2}) + + +def _write_zarray( + array_dir: Path, shape: tuple[int, ...], chunks: tuple[int, ...], dtype: str +) -> None: + array_dir.mkdir(parents=True, exist_ok=True) + meta = { + "zarr_format": 2, + "shape": list(shape), + "chunks": list(chunks), + "dtype": dtype, + "compressor": None, + "fill_value": 0, + "order": "C", + "filters": None, + "dimension_separator": ".", + } + _write_json(array_dir / ".zarray", meta) + + +def _make_dataset(path_str: str, scale: list[int]) -> dict[str, Any]: + return { + "path": path_str, + "coordinateTransformations": [{"type": "scale", "scale": scale}], + } + + +def _downsample_block(vol: np.ndarray, reducer: str) -> np.ndarray: + """Mirror ``downsampleBlock`` from the MATLAB fixture. + + Downsample by 2 on the last three axes (c, z, y, x volumes: keep C, + reduce Z/Y/X). Pad odd axes with edge values so 2x coarsening is + well-defined on odd extents. + """ + sz = list(vol.shape) + out_shape = (sz[0], -(-sz[1] // 2), -(-sz[2] // 2), -(-sz[3] // 2)) + padded = vol + if sz[1] % 2 == 1: + padded = np.concatenate([padded, padded[:, -1:, :, :]], axis=1) + if sz[2] % 2 == 1: + padded = np.concatenate([padded, padded[:, :, -1:, :]], axis=2) + if sz[3] % 2 == 1: + padded = np.concatenate([padded, padded[:, :, :, -1:]], axis=3) + ps = padded.shape + p = padded.astype(np.float64) + reshaped = p.reshape(ps[0], ps[1] // 2, 2, ps[2] // 2, 2, ps[3] // 2, 2) + if reducer == "mean": + agg = reshaped.mean(axis=(2, 4, 6)) + else: + agg = reshaped.max(axis=(2, 4, 6)) + out = np.rint(agg).astype(np.uint16) + return out[: out_shape[0], : out_shape[1], : out_shape[2], : out_shape[3]] + + +def _write_chunks(array_dir: Path, vol: np.ndarray, chunks: tuple[int, ...]) -> None: + """Write ``vol`` under ``array_dir`` as uncompressed C-order chunk files. + + Uses the same ``...`` layout the MATLAB fixture writes. + Uncompressed chunks are enough here because ``zarr.open`` reads them + via the same code path as any other codec, and skipping Blosc keeps + the test dependency-free (no zstd binary, no numcodecs.Blosc). + """ + sz = vol.shape + n_chunks = tuple(-(-sz[d] // chunks[d]) for d in range(len(sz))) + dtype = vol.dtype + for c in range(n_chunks[0]): + for z in range(n_chunks[1]): + for y in range(n_chunks[2]): + for x in range(n_chunks[3]): + chunk = np.zeros(chunks, dtype=dtype) + src_start = np.array([c, z, y, x]) * np.array(chunks) + src_end = np.minimum(src_start + np.array(chunks), np.array(sz)) + dst_end = src_end - src_start + chunk[: dst_end[0], : dst_end[1], : dst_end[2], : dst_end[3]] = vol[ + src_start[0] : src_end[0], + src_start[1] : src_end[1], + src_start[2] : src_end[2], + src_start[3] : src_end[3], + ] + key = f"{c}.{z}.{y}.{x}" + (array_dir / key).write_bytes(chunk.tobytes(order="C")) + + +def make_example_fixture( + parent_dir: str | Path, + *, + with_chunks: bool = False, + seed: int = 20260906, +) -> tuple[Path, GroundTruth]: + """Write a dual-pyramid OME-Zarr store and return its root and ground truth.""" + parent = Path(parent_dir) + parent.mkdir(parents=True, exist_ok=True) + fixture_dir = parent / "example.zarr" + if fixture_dir.exists(): + import shutil + + shutil.rmtree(fixture_dir) + fixture_dir.mkdir(parents=True) + + dtype = "_Em__Ch"`` channel directory per channel. Each +channel folder holds ``xml_import.xml``, ``xml_merging.xml``, and one +tile directory per stack; each tile directory holds a small stack of +zero-padded 2D TIFF z-slices with pixel_value = 100*channelIndex + z. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import numpy as np + +_NUM_SLICES = 4 +_HEIGHT = 8 +_WIDTH = 10 +_VOXEL_V = 4 +_VOXEL_H = 4 +_VOXEL_D = 1 + +_CHANNELS = [ + { + "name": "Ex_561_Em_561F_Ch2", + "laser": 561, + "filter": "561F", + "chNum": 2, + "left": 45, + "right": 45, + }, + { + "name": "Ex_640_Em_640F_Ch3", + "laser": 640, + "filter": "640F", + "chNum": 3, + "left": 40, + "right": 40, + }, +] + +_STACKS_IMPORT = [ + {"row": 0, "col": 0, "absV": 0, "absH": 0, "absD": 0, "dirName": "100000/100000_200000"}, + {"row": 1, "col": 0, "absV": 100, "absH": 0, "absD": 0, "dirName": "100000/100000_300000"}, +] +_STACKS_MERGING = [ + {"row": 0, "col": 0, "absV": 2, "absH": 1, "absD": 0, "dirName": "100000/100000_200000"}, + {"row": 1, "col": 0, "absV": 98, "absH": 3, "absD": 0, "dirName": "100000/100000_300000"}, +] + + +def _stitcher_xml(stacks: list[dict[str, Any]]) -> str: + lines = [''] + lines.append('') + lines.append('') + lines.append(' ') + lines.append(' ') + lines.append(' ') + lines.append(f' ') + lines.append(' ') + lines.append(' ') + lines.append( + f' ' + ) + lines.append(" ") + for s in stacks: + lines.append( + f' ' + ) + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append(" ") + lines.append("") + return "\n".join(lines) + + +def _extract_y(second: str, x_str: str) -> str: + prefix = x_str + "_" + if second.startswith(prefix): + return second[len(prefix) :] + parts = second.split("_") + return parts[-1] + + +def _metadata(channels: list[dict[str, Any]], stacks: list[dict[str, Any]]) -> dict[str, Any]: + laser_power = [ + {"wavelength": c["laser"], "left___": c["left"], "right___": c["right"]} for c in channels + ] + tiles = [] + for c in channels: + for s in stacks: + parts = s["dirName"].split("/") + x_str = parts[0] + y_str = _extract_y(parts[1], x_str) + tiles.append( + { + "X": x_str, + "Y": y_str, + "Z": "-1000", + "Laser": str(c["laser"]), + "Side": str(0 if s["row"] % 2 == 0 else 1), + "Exposure": "2", + "Acquire": "1", + "Filter": c["filter"], + "FilterChannel": str(c["chNum"]), + "NumImages": str(_NUM_SLICES), + } + ) + sample_metadata = { + "acquisition_ID": "test_acquisition_001", + "objective": "LCT 1.625x", + "um_per_pix": 4, + "z_step_um": 4, + "horizontal_resolution": _WIDTH, + "vertical_resolution": _HEIGHT, + "z_range": -100.0, + "scanning": "FAST", + "destripe": "256/0", + "destripe_status": "done", + "laser_power": laser_power, + } + return {"sample_metadata": sample_metadata, "tiles": tiles} + + +def _sequence(channels: list[dict[str, Any]]) -> dict[str, Any]: + laser_power = [ + {"wavelength": c["laser"], "left___": c["left"], "right___": c["right"]} for c in channels + ] + imaging_steps = [{"laser": c["laser"], "filter": c["filter"]} for c in channels] + z_section = { + "step_size__um_": 4, + "top__um_": -100.0, + "bottom__um_": 100.0, + "default_step_size": True, + } + tile_boundary = { + "top__pix_": 10.0, + "left__pix_": 5.0, + "bottom__pix_": 90.0, + "right__pix_": 95.0, + "left_right_split__pix_": 50.0, + } + focus_points = [ + { + "objective": "LCT 1.625x", + "wavelength": 561, + "z_position": [-50, 50], + "focus_position": [-10, -12], + } + ] + step = { + "GUID": "TEST-GUID-0001", + "name": "test_acquisition", + "disabled": False, + "destripe": True, + "z": z_section, + "imaging_steps": imaging_steps, + "laser_power": laser_power, + "tile_boundary": tile_boundary, + "focus_points": focus_points, + } + return {"objective": "LCT 1.625x", "immersion": "1.52+", "steps": [step]} + + +def make_example_fixture(parent_dir: str | Path) -> Path: + """Write a synthetic SmartSPIM directory and return its root path. + + Pixel values encode ``(channelIndex, z)`` so tests can verify a read + hits the right slice: ``pixel_value(channelIndex, z) = 100 * + channelIndex + z``, where ``channelIndex`` is 1-based (matching MATLAB). + """ + try: + import tifffile + except ImportError as err: # pragma: no cover + raise ImportError("tifffile is required for the smartspim fixture") from err + + parent = Path(parent_dir) + parent.mkdir(parents=True, exist_ok=True) + fixture_dir = parent / "BOTTOM" + if fixture_dir.exists(): + import shutil + + shutil.rmtree(fixture_dir) + fixture_dir.mkdir(parents=True) + + import_xml = _stitcher_xml(_STACKS_IMPORT) + merging_xml = _stitcher_xml(_STACKS_MERGING) + + for ci, c in enumerate(_CHANNELS, start=1): + ch_dir = fixture_dir / c["name"] + ch_dir.mkdir() + (ch_dir / "xml_import.xml").write_text(import_xml) + (ch_dir / "xml_merging.xml").write_text(merging_xml) + for s in _STACKS_IMPORT: + tile_dir = ch_dir + for part in [p for p in s["dirName"].split("/") if p]: + tile_dir = tile_dir / part + tile_dir.mkdir(parents=True, exist_ok=True) + for z in range(_NUM_SLICES): + img = np.full((_HEIGHT, _WIDTH), 100 * ci + z, dtype=np.uint16) + tifffile.imwrite(tile_dir / f"{z:06d}.tiff", img) + + (fixture_dir / "metadata.json").write_text(json.dumps(_metadata(_CHANNELS, _STACKS_IMPORT))) + (fixture_dir / "sequence.json").write_text(json.dumps(_sequence(_CHANNELS))) + + return fixture_dir diff --git a/tests/test_omezarr_probe.py b/tests/test_omezarr_probe.py new file mode 100644 index 0000000..d5cab1b --- /dev/null +++ b/tests/test_omezarr_probe.py @@ -0,0 +1,61 @@ +"""Tests for :func:`ndr.format.omezarr.probe`. + +Python mirror of ``tools/tests/+ndr/+unittest/+format/+omezarr/TestOMEZarrProbe.m``. +Exercises the cheap ``ok/nPyramids/pyramidNames/axesOrder/level0*`` fields +against the shared programmatic OME-Zarr fixture, plus the not-ok paths +(missing dir, empty dir). +""" + +from __future__ import annotations + +import pytest + +from ndr.format.omezarr.probe import probe +from tests._omezarr_fixture import make_example_fixture + + +@pytest.fixture +def fixture_dir(tmp_path): + return make_example_fixture(tmp_path, with_chunks=False)[0] + + +def test_ok_on_fixture(fixture_dir): + info = probe(str(fixture_dir)) + assert info.ok is True + + +def test_not_ok_on_missing_dir(tmp_path): + info = probe(str(tmp_path / "does-not-exist")) + assert info.ok is False + assert info.n_pyramids == 0 + assert info.pyramid_names == [] + + +def test_not_ok_on_empty_directory(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + info = probe(str(empty)) + assert info.ok is False + + +def test_pyramid_count_and_names(fixture_dir): + info = probe(str(fixture_dir)) + assert info.n_pyramids == 2 + assert "mean" in info.pyramid_names + assert "max" in info.pyramid_names + + +def test_first_pyramid_metadata(fixture_dir): + info = probe(str(fixture_dir)) + assert info.axes_order == "czyx" + assert len(info.axes_units) == 4 + assert len(info.level0_shape) == 4 + assert len(info.level0_chunks) == 4 + assert len(info.level0_scale) == 4 + assert info.dtype != "" + + +def test_n_levels_per_pyramid(fixture_dir): + info = probe(str(fixture_dir)) + assert len(info.n_levels_per_pyramid) == 2 + assert all(n >= 1 for n in info.n_levels_per_pyramid) diff --git a/tests/test_omezarr_reader.py b/tests/test_omezarr_reader.py new file mode 100644 index 0000000..5220b34 --- /dev/null +++ b/tests/test_omezarr_reader.py @@ -0,0 +1,190 @@ +"""Tests for :class:`ndr.reader.omezarr.ndr_reader_omezarr`. + +Python mirror of +``tools/tests/+ndr/+unittest/+reader/+omezarr/TestOMEZarrReader.m``. + +Metadata coverage: resolveepoch (pinned / unpinned / unknown / +with-sidecars), numframes, framesize, dimensionorder, datatype, +epochclock, t0_t1, frametimes, getchannelsepoch. + +Pixel-read coverage: single frame matches ground truth, multi-frame +in requested order, ``Level`` selects a lower-resolution level, and +mean vs max return different pixels at level 2 (the pyramid pinning +actually routes). +""" + +from __future__ import annotations + +import numpy as np +import pytest + +pytest.importorskip("zarr") + +from ndr.reader.omezarr import ndr_reader_omezarr # noqa: E402 +from ndr.time.clocktype import ClockType # noqa: E402 +from tests._omezarr_fixture import make_example_fixture # noqa: E402 + + +@pytest.fixture +def fixture(tmp_path): + return make_example_fixture(tmp_path, with_chunks=True) + + +def _z_slice(vol: np.ndarray, z_index: int) -> np.ndarray: + """Extract one z-plane from a (c, z, y, x) volume as (y, x, c).""" + plane_czyx = vol[:, z_index - 1, :, :] + plane_yxc = np.transpose(plane_czyx, (1, 2, 0)) + return plane_yxc.reshape(plane_yxc.shape[0], plane_yxc.shape[1], plane_yxc.shape[2]) + + +class TestResolveEpoch: + def test_pinned_pyramid(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + info = r.resolveepoch([str(fixture_dir), "mean"]) + assert info["zarrPath"] == str(fixture_dir) + assert info["pyramidName"] == "mean" + assert info["axisIndex"]["c"] == 1 + assert info["axisIndex"]["z"] == 2 + assert info["axisIndex"]["y"] == 3 + assert info["axisIndex"]["x"] == 4 + + def test_unpinned_pyramid_errors(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + with pytest.raises(ValueError, match="does not pin a pyramid"): + r.resolveepoch(str(fixture_dir)) + with pytest.raises(ValueError, match="does not pin a pyramid"): + r.resolveepoch([str(fixture_dir)]) + + def test_unknown_pyramid_errors(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + with pytest.raises(ValueError, match='"median"'): + r.resolveepoch([str(fixture_dir), "median"]) + + def test_sidecars_ignored(self, fixture, tmp_path): + fixture_dir, _ = fixture + sidecar = tmp_path / "sidecar.json" + sidecar.write_text('{"note":"acquisition sidecar"}') + r = ndr_reader_omezarr() + info = r.resolveepoch([[str(fixture_dir), "max"], str(sidecar)]) + assert info["pyramidName"] == "max" + assert info["zarrPath"] == str(fixture_dir) + + +class TestMetadata: + def test_numframes(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + assert r.numframes([str(fixture_dir), "mean"], 1) == 8 + + def test_framesize(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + assert r.framesize([str(fixture_dir), "mean"], 1) == [8, 10, 1, 1, 8] + + def test_dimensionorder(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + assert r.dimensionorder([str(fixture_dir), "mean"], 1) == "YXCZT" + + def test_datatype(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + assert r.datatype([str(fixture_dir), "mean"], 1) == "uint16" + + def test_frametimes_all_nan(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + t = r.frametimes([str(fixture_dir), "mean"], 1, [1, 5, 8]) + assert len(t) == 3 + assert np.all(np.isnan(t)) + + def test_epochclock_is_no_time(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + ec = r.epochclock([str(fixture_dir), "mean"], 1) + assert isinstance(ec, list) + assert isinstance(ec[0], ClockType) + assert ec[0].type == "no_time" + + def test_t0_t1_is_nan(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + t0t1 = r.t0_t1([str(fixture_dir), "mean"], 1) + assert len(t0t1) == 1 + assert np.all(np.isnan(t0t1[0])) + + def test_getchannelsepoch(self, fixture): + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + ch = r.getchannelsepoch([str(fixture_dir), "mean"], 1) + assert len(ch) == 1 + assert ch[0]["name"] == "image1" + assert ch[0]["type"] == "image" + + +class TestReadFrames: + def test_single_frame_matches_ground_truth(self, fixture): + fixture_dir, gt = fixture + r = ndr_reader_omezarr() + frame_idx = 3 + frames = r.readframes([str(fixture_dir), "mean"], 1, [frame_idx]) + expected = _z_slice(gt.level0, frame_idx) + assert frames.shape == (8, 10, 1, 1, 1) + assert frames.dtype == np.uint16 + np.testing.assert_array_equal(frames[:, :, :, 0, 0], expected) + + def test_multiple_frames_in_order(self, fixture): + fixture_dir, gt = fixture + r = ndr_reader_omezarr() + frame_inds = [1, 4, 7] + frames = r.readframes([str(fixture_dir), "mean"], 1, frame_inds) + assert frames.shape == (8, 10, 1, 1, 3) + for k, f in enumerate(frame_inds): + expected = _z_slice(gt.level0, f) + np.testing.assert_array_equal(frames[:, :, :, 0, k], expected) + + def test_level_option_selects_lower_resolution(self, fixture): + # Level 2 in the reader is pyramid.levels[1], which points at + # 'mean/1' -- the 2x downsample (fixture gt.mean_level1 shape + # [1 4 4 5]). Level 1 is the shared root. + fixture_dir, gt = fixture + r = ndr_reader_omezarr() + frames = r.readframes([str(fixture_dir), "mean"], 1, [1], Level=2) + expected = _z_slice(gt.mean_level1, 1) + assert frames.shape == (4, 5, 1, 1, 1) + np.testing.assert_array_equal(frames[:, :, :, 0, 0], expected) + + def test_mean_and_max_differ_at_level2(self, fixture): + # The whole point of pinning a pyramid: mean and max return + # different pixels at the same coordinates. + fixture_dir, gt = fixture + assert not np.array_equal(gt.mean_level1, gt.max_level1) + + r = ndr_reader_omezarr() + mean_frame = r.readframes([str(fixture_dir), "mean"], 1, [1], Level=2) + max_frame = r.readframes([str(fixture_dir), "max"], 1, [1], Level=2) + np.testing.assert_array_equal(mean_frame[:, :, :, 0, 0], _z_slice(gt.mean_level1, 1)) + np.testing.assert_array_equal(max_frame[:, :, :, 0, 0], _z_slice(gt.max_level1, 1)) + assert not np.array_equal(mean_frame, max_frame) + + def test_all_frames_default(self, fixture): + # An empty frameind reads all frames. + fixture_dir, _ = fixture + r = ndr_reader_omezarr() + frames = r.readframes([str(fixture_dir), "mean"], 1, []) + assert frames.shape == (8, 10, 1, 1, 8) + + +class TestZarrClassMapping: + def test_maps_common_dtypes(self): + assert ndr_reader_omezarr.zarrClass("i4") == "int32" + assert ndr_reader_omezarr.zarrClass("|u1") == "uint8" + assert ndr_reader_omezarr.zarrClass("f8") == "double" + + def test_rejects_unsupported(self): + with pytest.raises(ValueError): + ndr_reader_omezarr.zarrClass(" 2x2 output. Mean case. + a = np.arange(1, 25).reshape(4, 6) + r = reduce(a, [2, 3], "mean") + assert r.shape == (2, 2) + + +def test_wrong_factor_arity_errors(): + a = np.zeros((3, 4)) + with pytest.raises(ValueError): + reduce(a, [2, 2, 2], "mean") + + +def test_factor_must_be_positive_integer(): + a = np.zeros((4, 4)) + with pytest.raises(ValueError): + reduce(a, 0, "mean") + + +def test_reduction_must_be_member(): + a = np.zeros((4, 4)) + with pytest.raises(ValueError): + reduce(a, 2, "median") + + +def test_factor_one_passes_through(): + # Factor 1 on every axis should reproduce the input (mean returns + # double, max preserves dtype). + a = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.uint16) + r = reduce(a, 1, "max") + np.testing.assert_array_equal(r, a) + assert r.dtype == np.uint16 diff --git a/tests/test_smartspim_reader.py b/tests/test_smartspim_reader.py new file mode 100644 index 0000000..82140fc --- /dev/null +++ b/tests/test_smartspim_reader.py @@ -0,0 +1,162 @@ +"""Tests for :class:`ndr.reader.smartspim.ndr_reader_smartspim`. + +Python mirror of +``tools/tests/+ndr/+unittest/+reader/+smartspim/TestSmartSPIMReader.m``. + +Metadata coverage: resolveepoch (pinned / unpinned channel / unpinned +tile / with-sidecars), numframes, framesize, dimensionorder, datatype, +epochclock, t0_t1, frametimes, getchannelsepoch. + +Pixel-read coverage: single-frame read against the synthetic fixture, +multi-frame read in the requested order, different-channel epoch returns +different pixels (pinning actually routes), out-of-range frame indices +error. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +tifffile = pytest.importorskip("tifffile") + +from ndr.reader.smartspim import ndr_reader_smartspim # noqa: E402 +from ndr.time.clocktype import ClockType # noqa: E402 +from tests._smartspim_fixture import make_example_fixture # noqa: E402 + +_CH561 = "Ex_561_Em_561F_Ch2" +_CH640 = "Ex_640_Em_640F_Ch3" +_TILE1 = "100000/100000_200000" +_TILE2 = "100000/100000_300000" + + +@pytest.fixture +def fixture_dir(tmp_path): + return make_example_fixture(tmp_path) + + +class TestResolveEpoch: + def test_pinned_tile(self, fixture_dir): + r = ndr_reader_smartspim() + info = r.resolveepoch([str(fixture_dir), _CH561, _TILE1]) + assert info["rootDir"] == str(fixture_dir) + assert info["channelName"] == _CH561 + assert info["tileId"] == _TILE1 + assert info["tile"]["numSlices"] == 4 + assert info["tile"]["height"] == 8 + assert info["tile"]["width"] == 10 + + def test_unpinned_channel_errors(self, fixture_dir): + r = ndr_reader_smartspim() + with pytest.raises(ValueError, match="does not pin a channel"): + r.resolveepoch(str(fixture_dir)) + with pytest.raises(ValueError, match="does not pin a channel"): + r.resolveepoch([str(fixture_dir)]) + + def test_unpinned_tile_errors(self, fixture_dir): + r = ndr_reader_smartspim() + with pytest.raises(ValueError, match="does not pin a tile"): + r.resolveepoch([str(fixture_dir), _CH561]) + + def test_missing_root_errors(self, fixture_dir, tmp_path): + r = ndr_reader_smartspim() + with pytest.raises((NotADirectoryError, FileNotFoundError)): + r.resolveepoch([str(tmp_path / "does-not-exist"), _CH561, "x/y"]) + + def test_sidecars_ignored(self, fixture_dir, tmp_path): + sidecar = tmp_path / "sidecar.json" + sidecar.write_text('{"note":"acquisition sidecar"}') + r = ndr_reader_smartspim() + info = r.resolveepoch([[str(fixture_dir), _CH640, _TILE2], str(sidecar)]) + assert info["channelName"] == _CH640 + assert info["tileId"] == _TILE2 + + +class TestMetadata: + def _es(self, fixture_dir): + return [str(fixture_dir), _CH561, _TILE1] + + def test_numframes(self, fixture_dir): + r = ndr_reader_smartspim() + assert r.numframes(self._es(fixture_dir), 1) == 4 + + def test_framesize(self, fixture_dir): + r = ndr_reader_smartspim() + assert r.framesize(self._es(fixture_dir), 1) == [8, 10, 1, 1, 4] + + def test_dimensionorder(self, fixture_dir): + r = ndr_reader_smartspim() + assert r.dimensionorder(self._es(fixture_dir), 1) == "YXCZT" + + def test_datatype(self, fixture_dir): + r = ndr_reader_smartspim() + assert r.datatype(self._es(fixture_dir), 1) == "uint16" + + def test_frametimes_all_nan(self, fixture_dir): + r = ndr_reader_smartspim() + t = r.frametimes(self._es(fixture_dir), 1, [1, 3]) + assert len(t) == 2 + assert np.all(np.isnan(t)) + + def test_epochclock_is_no_time(self, fixture_dir): + r = ndr_reader_smartspim() + ec = r.epochclock(self._es(fixture_dir), 1) + assert isinstance(ec, list) + assert isinstance(ec[0], ClockType) + assert ec[0].type == "no_time" + + def test_t0_t1_is_nan(self, fixture_dir): + r = ndr_reader_smartspim() + t0t1 = r.t0_t1(self._es(fixture_dir), 1) + assert len(t0t1) == 1 + assert np.all(np.isnan(t0t1[0])) + + def test_getchannelsepoch(self, fixture_dir): + r = ndr_reader_smartspim() + ch = r.getchannelsepoch(self._es(fixture_dir), 1) + assert len(ch) == 1 + assert ch[0]["name"] == "image1" + assert ch[0]["type"] == "image" + + +class TestReadFrames: + def test_single_frame_matches_fixture(self, fixture_dir): + # Fixture rule: pixel_value(channelIndex=1, z=0) = 100 + r = ndr_reader_smartspim() + frames = r.readframes([str(fixture_dir), _CH561, _TILE1], 1, [1]) + assert frames.shape == (8, 10, 1, 1, 1) + assert frames.dtype == np.uint16 + assert frames[0, 0, 0, 0, 0] == np.uint16(100) + + def test_multiple_frames_in_order(self, fixture_dir): + r = ndr_reader_smartspim() + frame_inds = [1, 3, 4] + frames = r.readframes([str(fixture_dir), _CH561, _TILE1], 1, frame_inds) + assert frames.shape == (8, 10, 1, 1, 3) + # Fixture: pixel = 100 * 1 + (frameIdx - 1) + expected = np.array([100, 102, 103], dtype=np.uint16) + actual = np.array( + [ + frames[0, 0, 0, 0, 0], + frames[0, 0, 0, 0, 1], + frames[0, 0, 0, 0, 2], + ], + dtype=np.uint16, + ) + np.testing.assert_array_equal(actual, expected) + + def test_different_channel_different_pixels(self, fixture_dir): + # Pins actually route: Ex_640 has channelIndex=2, so pixel(z=0)=200. + r = ndr_reader_smartspim() + frames = r.readframes([str(fixture_dir), _CH640, _TILE1], 1, [1]) + assert frames[0, 0, 0, 0, 0] == np.uint16(200) + + def test_read_all_frames_default(self, fixture_dir): + r = ndr_reader_smartspim() + frames = r.readframes([str(fixture_dir), _CH561, _TILE1], 1, []) + assert frames.shape == (8, 10, 1, 1, 4) + + def test_out_of_range_errors(self, fixture_dir): + r = ndr_reader_smartspim() + with pytest.raises((IndexError, ValueError)): + r.readframes([str(fixture_dir), _CH561, _TILE1], 1, [1, 99])