diff --git a/.github/workflows/cli-tests.yml b/.github/workflows/cli-tests.yml new file mode 100644 index 0000000..c57d739 --- /dev/null +++ b/.github/workflows/cli-tests.yml @@ -0,0 +1,59 @@ +name: CLI tests + +# Build ("compile") and test the Python command-line tool across Linux, +# Windows, and macOS Apple Silicon (macos-14, running natively on arm64). The +# tests use a fake sonpy shim, so they do not require CED's proprietary +# binaries. + +on: + push: + branches: [ "main" ] + paths-ignore: + - '**.md' + pull_request: + branches: [ "main" ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: ${{ matrix.os }} / py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest # Linux + python-version: '3.11' + - os: windows-latest # Windows + python-version: '3.11' + - os: macos-14 # macOS Apple Silicon + python-version: '3.11' + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Upgrade build tooling + run: python -m pip install --upgrade pip build + + - name: Compile / build the package + run: | + python -m compileall src + python -m build + + - name: Install package (without CED sonpy) and test deps + run: | + python -m pip install . --no-deps + python -m pip install numpy pytest + + - name: Run CLI tests + run: python -m pytest -q diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000..8be2a23 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,62 @@ +name: Integration tests (real sonpy) + +# Exercises the REAL CED sonpy binding end-to-end against the checked-in +# example file (example/spike2data.smrx), on every platform. +# +# CED ships sonpy wheels for Python 3.14 on manylinux, macOS and Windows. +# Note: CED's macOS wheel is labelled universal2 but actually contains an +# x86_64-only binary (no native arm64). On Apple Silicon (macos-14) we +# therefore run every Python command through `arch -x86_64`, which executes the +# x86_64 slice of the universal2 python.org interpreter under Rosetta 2 so the +# x86_64 sonpy loads. Other platforms run their native x86_64 Python. (The +# fake-sonpy suites in cli-tests.yml run natively and cover more Python +# versions.) + +on: + push: + branches: [ "main" ] + paths-ignore: + - '**.md' + pull_request: + branches: [ "main" ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + integration: + name: sonpy / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest # Linux (manylinux wheel) + py: python + - os: windows-latest # Windows + py: python + - os: macos-14 # macOS Apple Silicon -> x86_64 under Rosetta + py: arch -x86_64 python + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python 3.14 (x86_64) + uses: actions/setup-python@v5 + with: + python-version: '3.14' + architecture: x64 + + - name: Install sonpipe with the real CED sonpy + run: | + ${{ matrix.py }} -m pip install --upgrade pip + ${{ matrix.py }} -m pip install .[test] + + - name: Verify sonpy is importable and exposes SonFile + run: ${{ matrix.py }} -c "from sonpipe.sonfile import load_sonpy; m = load_sonpy(); print('sonpy module:', m.__name__, '-> SonFile:', hasattr(m, 'SonFile'))" + + - name: Run integration tests against example/spike2data.smrx + run: ${{ matrix.py }} -m pytest -m integration -rs -v diff --git a/.github/workflows/matlab-tests.yml b/.github/workflows/matlab-tests.yml new file mode 100644 index 0000000..4bf07f3 --- /dev/null +++ b/.github/workflows/matlab-tests.yml @@ -0,0 +1,72 @@ +name: MATLAB tests + +# matbox-style CI (mirroring VH-Lab/NDR-matlab) for the +sonpipe MATLAB +# wrappers. Runs the matlab.unittest suite across Linux, Windows, and macOS +# Apple Silicon (macos-14). The MATLAB tests drive a fake sonpipe CLI +# (Python + numpy), so CED's proprietary sonpy binaries are not required. +# +# Licensing: matlab-actions provides MathWorks licensing for free on PUBLIC +# repositories, so no token or secret is needed. (If this repo is kept private, +# MATLAB in CI requires a batch licensing token supplied via the +# MLM_LICENSE_TOKEN environment variable.) + +on: + push: + branches: [ "main" ] + paths-ignore: + - '**.md' + pull_request: + branches: [ "main" ] + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: MATLAB / ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest # Linux + - windows-latest # Windows + - macos-14 # macOS Apple Silicon + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python (for the fake sonpipe CLI) + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install numpy + run: python -m pip install --upgrade pip numpy + + - name: Start virtual display server + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y xvfb + Xvfb :99 & + echo "DISPLAY=:99" >> $GITHUB_ENV + + - name: Set up MATLAB + uses: matlab-actions/setup-matlab@v3 + with: + release: latest + cache: true + + - name: Run MATLAB unit tests + uses: matlab-actions/run-command@v3 + with: + command: | + addpath(fullfile(pwd,'matlab')); + addpath(fullfile(pwd,'test')); + results = runtests('sonpipe.unittest', 'OutputDetail', 'Detailed'); + disp(table(results)); + assert(all(~[results.Failed]), 'Some sonpipe MATLAB tests failed'); diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58ae0c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,21 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ +build/ +dist/ +.pytest_cache/ +.venv/ +venv/ +*.so +*.pyd + +# MATLAB +*.p +*.asv +*.mex* + +# OS / editor +.DS_Store +*.swp diff --git a/README.md b/README.md index d44b546..8ec1924 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,299 @@ # sonpipe -A wrapper for reading 64-bit smrx files via command line tools (requires sonpy) + +A lightweight command-line **bridge** for reading Cambridge Electronic Design +(CED) Spike2 data files. `sonpipe` extracts data from proprietary Spike2 files +with CED's [`sonpy`](https://pypi.org/project/sonpy/) library and streams it as +**raw binary bytes** to standard output, so a host environment such as MATLAB +can ingest it in chunks — quickly, predictably, and cross-platform. + +It supports **both** Spike2 file formats transparently: + +* **32-bit `.smr`** (legacy "son32") +* **64-bit `.smrx`** ("son64") + +The command-line tools imitate the reading functions of the +[`ndr.format.ced`](https://github.com/VH-Lab/NDR-matlab) package in NDR-matlab, +and a companion MATLAB package (`+sonpipe`) provides drop-in analogues of those +functions that call the CLI for you. + +--- + +## Why a CLI bridge? + +1. **Interpreter isolation.** Python runs in its own process, invoked by a + system call. It never shares MATLAB's memory space, so there are no version + locks, environment conflicts, or interpreter crashes inside your workspace. + +2. **Licensing via pip.** CED's `sonpy` is proprietary. sonpipe does **not** + bundle or redistribute it; instead `pip install sonpipe` declares `sonpy` as + a dependency, so pip fetches the official, authorized CED binaries directly. + (On Apple Silicon, CED's current wheel is x86_64-only — see the Apple Silicon + note under Installation.) + +3. **No text-parsing overhead.** Waveforms and event times are written as raw + little-endian binary, not JSON/CSV text. MATLAB captures the byte stream and + reinterprets it directly (`typecast` / `fread`) with no number→text→number + round-trips. + +4. **Controlled chunking.** The host drives ingestion, requesting blocks by + sample index (`--start`/`--count`) or time window (`--t0`/`--t1`), keeping + memory usage low and predictable even for multi-gigabyte recordings. + +--- + +## Installation + +### Recommended: the install script + +The install scripts set up sonpipe in an **isolated virtual environment** and +put the `sonpipe` command on your PATH, so it never collides with other Python +packages. From a checkout of this repo: + +```bash +# Linux / macOS +./install.sh +``` +```powershell +# Windows (PowerShell) +./install.ps1 -AddToPath +``` + +On Linux/macOS this creates a venv at `~/.local/share/sonpipe/venv` and links +the command at `~/.local/bin/sonpipe`. On Windows the venv lives under +`%LOCALAPPDATA%\sonpipe`. Both print the exact `sonpipe.executable(...)` line to +paste into MATLAB. Run `./install.sh --help` for options (custom prefix, Python, +installing from PyPI, etc.). + +> **Python version.** CED ships `sonpy` wheels for **Python 3.14 on Linux and +> macOS** (and Python 3.9–3.14 on Windows). The installer prefers a +> `python3.14` interpreter; if `sonpy` cannot be imported it tells you to +> install 3.14 and re-run with `--python "$(command -v python3.14)"`. +> +> **Apple Silicon.** CED's current macOS `sonpy` wheel contains an x86_64-only +> binary (despite its `universal2` label), so on Apple Silicon Macs you need an +> **x86_64 Python running under Rosetta 2** until CED ships a native arm64 +> build. Everything else (the sonpipe CLI itself, the MATLAB package, and the +> fake-sonpy test suites) runs natively on arm64. + +### Manual: pip + +```bash +pip install sonpipe # once published to PyPI +pip install . # from a checkout +sonpipe --version +``` + +This also installs `sonpy` (from CED, via PyPI) and `numpy`. For an isolated, +PATH-managed command you can alternatively use `pipx install sonpipe`. + +> **Note on the CED license.** `sonpy` is proprietary CED software. It is +> fetched by pip at install time and is intentionally not included in this +> repository. + +--- + +## Command-line usage + +`sonpipe` has four sub-commands. Metadata commands emit JSON; `read` emits raw +binary (except for markers, which are JSON). + +| Sub-command | NDR-matlab analogue | +| ------------------------ | -------------------------------------------- | +| `sonpipe header` | `ndr.format.ced.read_SOMSMR_header` | +| `sonpipe sampleinterval` | `ndr.format.ced.read_SOMSMR_sampleinterval` | +| `sonpipe read` | `ndr.format.ced.read_SOMSMR_datafile` | +| `sonpipe channels` | (convenience listing) | + +### header — file and channel metadata (JSON) + +```bash +sonpipe header recording.smrx --pretty +``` + +Channels are reported by their **Spike2 channel number** (1-based). The `kind` +field is the CED data-type code (see table below), which matches both `sonpy`'s +`DataType` enum and NDR-matlab's `channelinfo.kind`. + +### sampleinterval — timing for one channel (JSON) + +```bash +sonpipe sampleinterval recording.smrx -c 21 +# {"channel":21,"sampleinterval":4e-05,"samplerate":25000.0,"total_samples":...,"total_time":...} +``` + +### read — stream channel data + +Waveform, by sample block (raw little-endian `double`, scaled to real units): + +```bash +sonpipe read recording.smrx -c 21 --start 0 --count 500000 > block0.bin +``` + +Waveform, by time window; raw unscaled 16-bit ADC values: + +```bash +sonpipe read recording.smrx -c 21 --t0 0 --t1 10 --raw > adc.bin +``` + +Event channel (event times in seconds, as `double`): + +```bash +sonpipe read recording.smrx -c 24 > events.bin +``` + +Marker / TextMark channel (JSON — times, code bytes, optional text): + +```bash +sonpipe read recording.smrx -c 30 +``` + +Useful `read` flags: `--raw` (int16 ADC values), `--dtype` +{double,single,int16,int32,int64}, `--endian` {little,big,native}, `--json` +(debugging), `--no-size-warning`. + +> **Large reads.** Because the pipe can be slow for very large transfers, +> sonpipe prints a warning to stderr when a read exceeds **50 MB**. Read in +> smaller blocks, or pass `--no-size-warning` to silence it. + +### channels — quick tab-separated listing + +```bash +sonpipe channels recording.smrx +# 21 Adc analog_in 25000.0000 Hz Vm +``` + +### Channel kinds + +| kind | name | ndr_type | read as | +| ----- | -------- | --------- | ----------------------------- | +| 1 | Adc | analog_in | binary waveform (int16→real) | +| 2/3/4 | Event\* | event | binary event times (double) | +| 5 | Marker | mark | JSON (time + code bytes) | +| 6 | AdcMark | mark | JSON (WaveMark) | +| 7 | RealMark | mark | JSON | +| 8 | TextMark | text | JSON (time + text) | +| 9 | RealWave | analog_in | binary waveform (float) | + +--- + +## MATLAB usage + +The `matlab/+sonpipe` package provides drop-in analogues of the +`ndr.format.ced.*` functions, backed by the CLI. + +**Setup** + +1. `pip install sonpipe` +2. Add the folder that *contains* `+sonpipe` to the MATLAB path: + ```matlab + addpath('/path/to/sonpipe/matlab') + ``` +3. If the `sonpipe` command is not on the system PATH, tell MATLAB where it is: + ```matlab + sonpipe.executable('/full/path/to/sonpipe') % or 'python3 -m sonpipe' + ``` + +**Example** + +```matlab +f = '/data/recording.smrx'; % or a legacy .smr file + +h = sonpipe.read_SOMSMR_header(f); % file + channel header +sr = 1 / sonpipe.read_SOMSMR_sampleinterval(f, h, 21); + +% Read waveform channel 21 from t = 0 to t = 100 s +[data, total_samples, total_time, ~, t] = ... + sonpipe.read_SOMSMR_datafile(f, h, 21, 0, 100); + +plot(t, data); xlabel('Time (s)'); ylabel(h.channelinfo(1).units); +``` + +Because these mirror `ndr.format.ced.*`, existing code often ports by swapping +the package prefix (`ndr.format.ced` → `sonpipe`). + +Helpers: `sonpipe.channels(f)` (NDR-style channel struct array), +`sonpipe.channelinfo(h, n)` (one channel's header entry), `sonpipe.executable` +(locate/set the CLI command). + +--- + +## Development, testing, and CI + +**Python (CLI) tests** use a fake `sonpy` shim, so they run anywhere: + +```bash +pip install -e . --no-deps +pip install numpy pytest +pytest -q +``` + +**MATLAB tests** live in `test/+sonpipe/+unittest` and drive a fake CLI +(`fakecli.py`, which runs the real sonpipe code against synthetic data), so +they too need no CED binaries — only Python + numpy: + +```matlab +addpath('matlab'); addpath('test'); +results = runtests('sonpipe.unittest'); +``` + +### Where do the test files come from? + +The default suites use **no real Spike2 files**. `tests/fakesonpy.py` is a +pure-Python stand-in for CED's `sonpy` that serves synthetic in-memory channels +(a ramp waveform, a sine `RealWave`, events, markers, text markers); the MATLAB +tests drive `fakecli.py`, which runs the real sonpipe code against that same +fake. This keeps the suites deterministic and lets them run everywhere — +including Linux, where CED's `sonpy` does not install cleanly. + +To additionally validate the **real** `sonpy` binding, there is an integration +suite that runs the actual CED sonpy end-to-end (both the `SmrxFile` wrapper and +the CLI as a subprocess) against the checked-in `example/spike2data.smrx`. It is +skipped unless `sonpy` is importable; point `SONPIPE_TEST_FILE` at another file +to use your own: + +```bash +pip install sonpy # real CED sonpy (Python 3.14 on Linux/macOS) +pytest -m integration # uses example/spike2data.smrx by default +``` + +In CI, the integration suite runs on **Linux, Windows, and macOS Apple Silicon** +using Python 3.14. On the macOS runner it exercises the x86_64 `sonpy` under +Rosetta 2 (`arch -x86_64`), since CED ships no native arm64 build. + +### Continuous integration + +Continuous integration (see `.github/workflows/`) mirrors the matbox style used +by NDR-matlab and runs on **Linux, Windows, and macOS Apple Silicon**. The +macOS runner covers both architectures: the CLI and MATLAB suites run natively +on arm64, while the real-`sonpy` suite runs x86_64 under Rosetta 2: + +* **CLI tests** build the Python package (`compileall` + `python -m build`) and + run the pytest suite. +* **MATLAB tests** run the `matlab.unittest` suite (which drives the CLI via a + fake). matlab-actions provides MathWorks licensing for free on public repos; + no secret is needed. + +The workflows trigger on pushes to `main`, pull requests targeting `main`, and +manual dispatch. + +--- + +## Repository layout + +``` +sonpipe/ +├── src/sonpipe/ # Python package (CLI + sonpy wrapper) +├── tests/ # Python tests (fake sonpy + real-sonpy integration) +├── matlab/+sonpipe/ # MATLAB client package (imitates ndr.format.ced.*) +├── test/+sonpipe/+unittest/# MATLAB unit tests + fake CLI +├── example/spike2data.smrx # a real 64-bit Spike2 file used by integration tests +└── .github/workflows/ # cross-platform CI (CLI, MATLAB, real-sonpy) +``` + +--- + +## License + +sonpipe is released under the MIT License (see `LICENSE`). It depends on, but +does not include, CED's proprietary `sonpy` library, which carries its own +license. diff --git a/example/spike2data.smrx b/example/spike2data.smrx new file mode 100644 index 0000000..f80ebe3 Binary files /dev/null and b/example/spike2data.smrx differ diff --git a/install.ps1 b/install.ps1 new file mode 100644 index 0000000..aac319b --- /dev/null +++ b/install.ps1 @@ -0,0 +1,123 @@ +<# +.SYNOPSIS + sonpipe installer for Windows. + +.DESCRIPTION + Installs the sonpipe command-line tool (and CED's sonpy) into an isolated + Python virtual environment under %LOCALAPPDATA%\sonpipe, verifies the + install, and prints the line you need to wire sonpipe into MATLAB. + Optionally adds the venv's Scripts directory to your user PATH. + + On Windows, CED ships sonpy wheels for Python 3.9 - 3.14, so any of those + interpreters will work. + +.PARAMETER Venv + Virtual environment location (default: %LOCALAPPDATA%\sonpipe\venv). + +.PARAMETER Python + Python launcher/interpreter to use (default: "py -3", falling back to + "python"). + +.PARAMETER Source + Path or pip spec to install (default: the repo if run inside it, else + "sonpipe" from PyPI). + +.PARAMETER Pypi + Force installing "sonpipe" from PyPI. + +.PARAMETER AddToPath + Persistently add the venv Scripts directory to your user PATH. + +.EXAMPLE + ./install.ps1 -AddToPath +#> + +[CmdletBinding()] +param( + [string]$Venv, + [string]$Python, + [string]$Source, + [switch]$Pypi, + [switch]$AddToPath +) + +$ErrorActionPreference = 'Stop' + +function Info($m) { Write-Host "[sonpipe] $m" -ForegroundColor Cyan } +function Warn($m) { Write-Host "[sonpipe] WARNING: $m" -ForegroundColor Yellow } +function Fail($m) { Write-Host "[sonpipe] ERROR: $m" -ForegroundColor Red; exit 1 } + +if (-not $Venv) { $Venv = Join-Path $env:LOCALAPPDATA 'sonpipe\venv' } + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path + +# Choose the Python command. +$pyCmd = $null +if ($Python) { + $pyCmd = $Python +} elseif (Get-Command py -ErrorAction SilentlyContinue) { + $pyCmd = 'py -3' +} elseif (Get-Command python -ErrorAction SilentlyContinue) { + $pyCmd = 'python' +} else { + Fail "No Python found. Install Python 3 (https://www.python.org/downloads/) and retry." +} +Info "Using Python command: $pyCmd" + +# Decide what to install. +if (-not $Source) { + if ($Pypi) { + $Source = 'sonpipe' + } elseif (Test-Path (Join-Path $ScriptDir 'pyproject.toml')) { + $Source = $ScriptDir + } else { + $Source = 'sonpipe' + } +} +Info "Installing from: $Source" + +# Create the venv. +Info "Creating virtual environment: $Venv" +$venvParent = Split-Path -Parent $Venv +if (-not (Test-Path $venvParent)) { New-Item -ItemType Directory -Force -Path $venvParent | Out-Null } +Invoke-Expression "$pyCmd -m venv `"$Venv`"" + +$PY = Join-Path $Venv 'Scripts\python.exe' +if (-not (Test-Path $PY)) { Fail "venv python not found at $PY" } + +Info "Upgrading pip" +& $PY -m pip install --upgrade pip | Out-Null + +Info "Installing sonpipe (also fetches CED's sonpy from PyPI)" +& $PY -m pip install $Source + +# Verify. +Info "Verifying sonpipe CLI ..." +& $PY -m sonpipe --version +if ($LASTEXITCODE -ne 0) { Fail "sonpipe did not install correctly." } + +Info "Verifying CED sonpy import ..." +& $PY -c "import sonpy; print('sonpy', sonpy.__version__)" +$sonpyOk = ($LASTEXITCODE -eq 0) +if (-not $sonpyOk) { + Warn "sonpy could not be imported -- no sonpy wheel for this Python. Try a" + Warn "different Python 3.x (CED ships Windows wheels for 3.9 - 3.14)." +} + +$Cmd = Join-Path $Venv 'Scripts\sonpipe.exe' +$ScriptsDir = Join-Path $Venv 'Scripts' + +if ($AddToPath) { + $userPath = [Environment]::GetEnvironmentVariable('Path', 'User') + if ($userPath -notlike "*$ScriptsDir*") { + [Environment]::SetEnvironmentVariable('Path', "$userPath;$ScriptsDir", 'User') + Info "Added $ScriptsDir to your user PATH (restart your shell to pick it up)." + } +} + +Write-Host "" +Info "Done." +Write-Host " Command: `"$Cmd`" --help" +Write-Host " MATLAB: add the repo's 'matlab' folder to your path, then run:" +Write-Host " sonpipe.executable('$Cmd')" +if (-not $sonpyOk) { exit 1 } diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..3aa720f --- /dev/null +++ b/install.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# sonpipe installer for Linux and macOS. +# +# Installs the sonpipe command-line tool (and CED's sonpy) into an isolated +# Python virtual environment, and symlinks the `sonpipe` command into your +# ~/.local/bin so it is on your PATH. Then verifies the install and prints the +# line you need to wire sonpipe into MATLAB. +# +# venv: ~/.local/share/sonpipe/venv (isolated; holds sonpy + numpy) +# command: ~/.local/bin/sonpipe (symlink onto your PATH) +# +# Usage: +# ./install.sh [options] +# +# Options: +# --prefix DIR Base install prefix (default: ~/.local); the venv goes in +# DIR/share/sonpipe/venv and the command in DIR/bin. +# --venv DIR Override the virtual environment location. +# --bin-dir DIR Override where the `sonpipe` command is linked. +# --python PY Python interpreter to build the venv with (default: auto, +# prefers python3.14 for sonpy wheel availability). +# --pypi Install from PyPI (default when not run inside the repo). +# --source PATH Install from this path or pip spec (default: the repo). +# --no-symlink Do not create the ~/.local/bin symlink. +# -h, --help Show this help. +# +# Note: CED ships sonpy Linux/macOS wheels only for Python 3.14, so this script +# prefers python3.14. If sonpy fails to import afterwards, install Python 3.14 +# and re-run with: --python "$(command -v python3.14)". + +set -euo pipefail + +PREFIX="$HOME/.local" +VENV_DIR="" +BIN_DIR="" +PYTHON="" +SOURCE="" +SYMLINK=1 + +log() { printf '\033[1;34m[sonpipe]\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m[sonpipe] WARNING:\033[0m %s\n' "$*" >&2; } +err() { printf '\033[1;31m[sonpipe] ERROR:\033[0m %s\n' "$*" >&2; exit 1; } + +while [ $# -gt 0 ]; do + case "$1" in + --prefix) PREFIX="$2"; shift 2 ;; + --venv) VENV_DIR="$2"; shift 2 ;; + --bin-dir) BIN_DIR="$2"; shift 2 ;; + --python) PYTHON="$2"; shift 2 ;; + --source) SOURCE="$2"; shift 2 ;; + --pypi) SOURCE="sonpipe"; shift ;; + --no-symlink) SYMLINK=0; shift ;; + -h|--help) sed -n '2,38p' "$0" | sed 's/^#\{0,1\} \{0,1\}//'; exit 0 ;; + *) err "unknown option: $1 (use --help)" ;; + esac +done + +VENV_DIR="${VENV_DIR:-$PREFIX/share/sonpipe/venv}" +BIN_DIR="${BIN_DIR:-$PREFIX/bin}" + +# Resolve the directory this script lives in (following symlinks). +SOURCE_PATH="${BASH_SOURCE[0]}" +while [ -h "$SOURCE_PATH" ]; do + DIR="$(cd -P "$(dirname "$SOURCE_PATH")" >/dev/null 2>&1 && pwd)" + SOURCE_PATH="$(readlink "$SOURCE_PATH")" + [[ "$SOURCE_PATH" != /* ]] && SOURCE_PATH="$DIR/$SOURCE_PATH" +done +SCRIPT_DIR="$(cd -P "$(dirname "$SOURCE_PATH")" >/dev/null 2>&1 && pwd)" + +# Pick a Python interpreter (prefer 3.14 for sonpy wheel availability). +if [ -z "$PYTHON" ]; then + for c in python3.14 python3 python; do + if command -v "$c" >/dev/null 2>&1; then PYTHON="$c"; break; fi + done +fi +[ -n "$PYTHON" ] || err "no Python interpreter found; install Python 3.14 and retry." +command -v "$PYTHON" >/dev/null 2>&1 || err "python not found: $PYTHON" + +PYVER="$("$PYTHON" -c 'import sys; print("%d.%d" % sys.version_info[:2])')" +log "Using Python $PYVER ($("$PYTHON" -c 'import sys; print(sys.executable)'))" +if [ "$PYVER" != "3.14" ]; then + warn "CED sonpy ships Linux/macOS wheels only for Python 3.14; you are on $PYVER." + warn "If sonpy fails to import below, re-run with: --python \"\$(command -v python3.14)\"" +fi + +# Decide what to install. +if [ -z "$SOURCE" ]; then + if [ -f "$SCRIPT_DIR/pyproject.toml" ]; then SOURCE="$SCRIPT_DIR"; else SOURCE="sonpipe"; fi +fi +log "Installing from: $SOURCE" + +log "Creating virtual environment: $VENV_DIR" +mkdir -p "$(dirname "$VENV_DIR")" +"$PYTHON" -m venv "$VENV_DIR" +PY="$VENV_DIR/bin/python" + +log "Upgrading pip" +"$PY" -m pip install --upgrade pip >/dev/null + +log "Installing sonpipe (also fetches CED's sonpy from PyPI)" +"$PY" -m pip install "$SOURCE" + +# Verify. +log "Verifying sonpipe CLI ..." +"$PY" -m sonpipe --version || err "sonpipe did not install correctly." + +log "Verifying CED sonpy import ..." +SONPY_OK=1 +if ! "$PY" -c "import sonpy; print('sonpy', sonpy.__version__)"; then + SONPY_OK=0 + warn "sonpy could not be imported -- most likely no sonpy wheel exists for" + warn "Python $PYVER on this platform. Install Python 3.14 and re-run:" + warn " ./install.sh --python \"\$(command -v python3.14)\"" +fi + +# Symlink the command onto the PATH. +CMD="$VENV_DIR/bin/sonpipe" +if [ "$SYMLINK" -eq 1 ]; then + mkdir -p "$BIN_DIR" + ln -sf "$CMD" "$BIN_DIR/sonpipe" + CMD="$BIN_DIR/sonpipe" + case ":$PATH:" in + *":$BIN_DIR:"*) : ;; + *) warn "$BIN_DIR is not on your PATH. Add it, e.g.:" + warn " echo 'export PATH=\"$BIN_DIR:\$PATH\"' >> ~/.profile" ;; + esac +fi + +echo +log "Done." +echo " Command: $CMD --help" +echo " MATLAB: add the repo's 'matlab' folder to your path, then run:" +echo " sonpipe.executable('$CMD')" +[ "$SONPY_OK" -eq 1 ] || exit 1 diff --git a/matlab/+sonpipe/Contents.m b/matlab/+sonpipe/Contents.m new file mode 100644 index 0000000..cdb6839 --- /dev/null +++ b/matlab/+sonpipe/Contents.m @@ -0,0 +1,42 @@ +% SONPIPE MATLAB client for the sonpipe command-line bridge. +% +% The +sonpipe package reads Cambridge Electronic Design (CED) Spike2 data +% files by calling the standalone "sonpipe" command-line tool, which in turn +% uses CED's proprietary sonpy library. Python runs in its own isolated +% process; nothing is loaded into the MATLAB interpreter, so there are no +% version locks or interpreter crashes. +% +% Both file formats are supported transparently, because sonpy reads both: +% * 32-bit .smr (legacy Spike2 "son32" files) +% * 64-bit .smrx ("son64" files) +% +% These functions are drop-in analogues of the ndr.format.ced.* functions in +% NDR-matlab, so code written against those functions works with only a +% change of package prefix (ndr.format.ced -> sonpipe). +% +% Reading functions: +% read_SOMSMR_header - read file + channel header information +% read_SOMSMR_sampleinterval - sample interval / rate for one channel +% read_SOMSMR_datafile - read waveform, event or marker data +% +% Helpers: +% channels - list channels as an NDR-style struct array +% channelinfo - look up one channel's header entry +% executable - locate / set / query the sonpipe CLI command +% +% Setup: +% 1. Install the CLI: pip install sonpipe +% 2. Add this folder's PARENT (the one containing +sonpipe) to the MATLAB +% path: addpath('/path/to/sonpipe/matlab') +% 3. (optional) If "sonpipe" is not on the system PATH, point MATLAB at it: +% sonpipe.executable('/full/path/to/sonpipe') +% or set the SONPIPE environment variable. +% +% Example: +% f = '/data/example.smrx'; +% h = sonpipe.read_SOMSMR_header(f); +% sr = 1/sonpipe.read_SOMSMR_sampleinterval(f, h, 21); +% [d,~,~,~,t] = sonpipe.read_SOMSMR_datafile(f, h, 21, 0, 100); +% plot(t, d); +% +% See also NDR (https://github.com/VH-Lab/NDR-matlab). diff --git a/matlab/+sonpipe/channelinfo.m b/matlab/+sonpipe/channelinfo.m new file mode 100644 index 0000000..c59fddb --- /dev/null +++ b/matlab/+sonpipe/channelinfo.m @@ -0,0 +1,22 @@ +function ci = channelinfo(header, channel_number) +% SONPIPE.CHANNELINFO - Look up one channel's entry in a sonpipe header +% +% CI = sonpipe.channelinfo(HEADER, CHANNEL_NUMBER) +% +% Returns the element of HEADER.channelinfo whose 'number' field equals the +% 1-based Spike2 CHANNEL_NUMBER. HEADER is the struct returned by +% sonpipe.read_SOMSMR_header. Errors if the channel is not present. +% +% See also sonpipe.read_SOMSMR_header + + if isempty(header) || ~isfield(header, 'channelinfo') || isempty(header.channelinfo) + error('sonpipe:noChannels', 'The header contains no channels.'); + end + numbers = [header.channelinfo.number]; + idx = find(numbers == channel_number, 1); + if isempty(idx) + error('sonpipe:noChannel', ... + 'Channel number %d is not recorded in this file.', channel_number); + end + ci = header.channelinfo(idx); +end diff --git a/matlab/+sonpipe/channels.m b/matlab/+sonpipe/channels.m new file mode 100644 index 0000000..f519e29 --- /dev/null +++ b/matlab/+sonpipe/channels.m @@ -0,0 +1,52 @@ +function channels = channels(filename, header) +% SONPIPE.CHANNELS - List channels in a CED SMR/SMRX file (NDR-style struct) +% +% CHANNELS = sonpipe.channels(FILENAME) +% CHANNELS = sonpipe.channels(FILENAME, HEADER) +% +% Returns a struct array describing every recorded (non-Off) channel, in the +% style of ndr.reader.ced_smr/getchannelsepoch. If HEADER is omitted it is +% read from the file. +% +% Each element has fields: +% name - NDR-style channel name (e.g. 'ai1' for analog_in ch 1) +% type - NDR channel type ('analog_in','event','mark','text') +% number - Spike2 channel number (1-based) +% kind - CED data-type code +% time_channel - the channel number (used as its time reference) +% +% See also sonpipe.read_SOMSMR_header + + if nargin < 2 || isempty(header) + header = sonpipe.read_SOMSMR_header(filename); + end + + channels = struct('name', {}, 'type', {}, 'number', {}, ... + 'kind', {}, 'time_channel', {}); + + for k = 1:numel(header.channelinfo) + c = header.channelinfo(k); + entry.name = [prefix(c.ndr_type) int2str(c.number)]; + entry.type = c.ndr_type; + entry.number = c.number; + entry.kind = c.kind; + entry.time_channel = c.number; + channels(end+1) = entry; %#ok + end +end + +function p = prefix(ndrtype) +% Short NDR name prefix per channel type (mirrors ndr.reader.base.mfdaq_prefix). + switch ndrtype + case 'analog_in' + p = 'ai'; + case 'event' + p = 'e'; + case 'mark' + p = 'mk'; + case 'text' + p = 'text'; + otherwise + p = 'ch'; + end +end diff --git a/matlab/+sonpipe/executable.m b/matlab/+sonpipe/executable.m new file mode 100644 index 0000000..649e798 --- /dev/null +++ b/matlab/+sonpipe/executable.m @@ -0,0 +1,62 @@ +function exe = executable(newvalue) +% SONPIPE.EXECUTABLE - Locate, set, or query the sonpipe command-line tool +% +% EXE = sonpipe.executable() +% +% Returns a command string that invokes the sonpipe CLI (for example +% 'sonpipe' or 'python3 -m sonpipe'). The result is cached for the MATLAB +% session after the first successful lookup. +% +% sonpipe.executable(NEWVALUE) +% +% Sets (and caches) the command string to NEWVALUE. Use this if the CLI is +% installed somewhere that is not on the system PATH, e.g. +% sonpipe.executable('/opt/venv/bin/sonpipe') +% sonpipe.executable('python3 -m sonpipe') +% +% Lookup order when no value is cached: +% 1. the SONPIPE environment variable, if set +% 2. 'sonpipe' on the system PATH +% 3. 'python3 -m sonpipe' +% 4. 'python -m sonpipe' +% +% Each candidate is verified by running " --version". +% +% See also sonpipe.read_SOMSMR_header + + persistent CACHED + + if nargin >= 1 + CACHED = newvalue; + exe = CACHED; + return; + end + + if ~isempty(CACHED) + exe = CACHED; + return; + end + + candidates = {}; + envval = getenv('SONPIPE'); + if ~isempty(envval) + candidates{end+1} = envval; + end + candidates{end+1} = 'sonpipe'; + candidates{end+1} = 'python3 -m sonpipe'; + candidates{end+1} = 'python -m sonpipe'; + + for i = 1:numel(candidates) + [status, ~] = sonpipe.runcmd([candidates{i} ' --version']); + if status == 0 + CACHED = candidates{i}; + exe = CACHED; + return; + end + end + + error('sonpipe:executableNotFound', ... + ['Could not locate the sonpipe CLI. Install it with ' ... + '''pip install sonpipe'', or point MATLAB at it with ' ... + 'sonpipe.executable(PATH) or the SONPIPE environment variable.']); +end diff --git a/matlab/+sonpipe/private/invoke_binary.m b/matlab/+sonpipe/private/invoke_binary.m new file mode 100644 index 0000000..39d40c5 --- /dev/null +++ b/matlab/+sonpipe/private/invoke_binary.m @@ -0,0 +1,44 @@ +function data = invoke_binary(args, precision) +% INVOKE_BINARY - Run the sonpipe CLI and capture its raw binary output +% +% DATA = invoke_binary(ARGS, PRECISION) +% +% Runs the sonpipe CLI with argument string ARGS, redirecting its (binary) +% stdout to a temporary file, then reads that file back as a column vector of +% the given PRECISION (e.g. 'double', 'int16', 'single'). sonpipe writes +% little-endian bytes, which are read back accordingly, mirroring the +% "capture the pipe and typecast" pattern with an on-disk buffer (MATLAB's +% system() mangles binary captured directly to a char array). +% +% Informational messages from the CLI go to stderr and are not captured here, +% so the temporary file contains pure sample bytes. +% +% This is a private helper for the +sonpipe package. + + exe = sonpipe.executable(); + tmp = [tempname() '.bin']; + cleaner = onCleanup(@() deletefile(tmp)); + + cmd = sprintf('%s %s > "%s"', exe, args, tmp); + [status, msg] = sonpipe.runcmd(cmd); + if status ~= 0 + error('sonpipe:cliError', ... + 'sonpipe failed (status %d) for command:\n %s\n%s', ... + status, cmd, msg); + end + + fid = fopen(tmp, 'r', 'l'); % little-endian + if fid < 0 + error('sonpipe:tmpRead', ... + 'Could not read sonpipe output file: %s', tmp); + end + closer = onCleanup(@() fclose(fid)); + data = fread(fid, Inf, ['*' precision]); + data = data(:); +end + +function deletefile(fname) + if exist(fname, 'file') + delete(fname); + end +end diff --git a/matlab/+sonpipe/private/invoke_text.m b/matlab/+sonpipe/private/invoke_text.m new file mode 100644 index 0000000..47238ce --- /dev/null +++ b/matlab/+sonpipe/private/invoke_text.m @@ -0,0 +1,20 @@ +function txt = invoke_text(args) +% INVOKE_TEXT - Run the sonpipe CLI and capture its text (JSON) output +% +% TXT = invoke_text(ARGS) +% +% ARGS is the argument string passed to the sonpipe CLI (everything after the +% executable name). Returns the captured stdout as a char row vector. Raises +% an error if the CLI exits with a nonzero status. +% +% This is a private helper for the +sonpipe package. + + exe = sonpipe.executable(); + cmd = [exe ' ' args]; + [status, txt] = sonpipe.runcmd(cmd); + if status ~= 0 + error('sonpipe:cliError', ... + 'sonpipe failed (status %d) for command:\n %s\n%s', ... + status, cmd, txt); + end +end diff --git a/matlab/+sonpipe/read_SOMSMR_datafile.m b/matlab/+sonpipe/read_SOMSMR_datafile.m new file mode 100644 index 0000000..134f77a --- /dev/null +++ b/matlab/+sonpipe/read_SOMSMR_datafile.m @@ -0,0 +1,132 @@ +function [data, total_samples, total_time, blockinfo, time] = read_SOMSMR_datafile(filename, header, channel_number, t0, t1) +% SONPIPE.READ_SOMSMR_DATAFILE - Read samples from a CED SMR/SMRX file +% +% [DATA, TOTAL_SAMPLES, TOTAL_TIME, BLOCKINFO, TIME] = ... +% sonpipe.read_SOMSMR_datafile(FILENAME, HEADER, CHANNEL_NUMBER, T0, T1) +% +% Reads data for CHANNEL_NUMBER (1-based Spike2 channel number) from the CED +% Spike2 file FILENAME, between times T0 and T1 (seconds). If T0 is negative +% it is clamped to 0; if T1 is Inf the read runs to the end of the recording. +% HEADER may be [] (it will be read from the file). +% +% Works with both 32-bit .smr and 64-bit .smrx files. This is a drop-in +% analogue of ndr.format.ced.read_SOMSMR_datafile, backed by the sonpipe CLI. +% +% The behaviour depends on the channel kind: +% * Waveform (Adc/RealWave): DATA is a column of scaled real-unit samples; +% TIME is the matching column of sample times (seconds). Data is streamed +% from the CLI as raw little-endian doubles for speed. +% * Event (kinds 2/3/4): DATA and TIME are equal columns of event times (s). +% * Marker (kinds 5/6/7/8): DATA holds the marker codes (or, for TextMark, +% a char matrix of the text); TIME holds the marker times (seconds). +% +% Outputs: +% DATA - the channel data (see above) +% TOTAL_SAMPLES - estimated total waveform samples ([] for non-waveforms) +% TOTAL_TIME - estimated recording duration in seconds +% BLOCKINFO - always [] here; sonpy abstracts away SON disk blocks. The +% output is retained for signature compatibility. +% TIME - times (seconds) matching DATA +% +% Note: only one channel can be read per call, matching the NDR function. +% +% See also sonpipe.read_SOMSMR_header, sonpipe.read_SOMSMR_sampleinterval + + data = []; + total_samples = []; + total_time = []; + blockinfo = []; + time = []; + + if isempty(header) + header = sonpipe.read_SOMSMR_header(filename); + end + if numel(channel_number) > 1 + error('sonpipe:singleChannel', ... + 'Only one channel may be read per call; CHANNEL_NUMBER must be scalar.'); + end + + ci = sonpipe.channelinfo(header, channel_number); + kind = ci.kind; + total_time = ci.max_time; + + if any(kind == [1 9]) % ---- waveform ---- + sr = ci.samplerate; + total_samples = ci.num_samples; + if t0 < 0 || isinf(t0) + t0 = 0; + end + s0 = max(0, floor(t0 * sr)); + if isinf(t1) + s1 = total_samples - 1; + else + s1 = floor(t1 * sr); + end + count = s1 - s0 + 1; + if count <= 0 + return; + end + args = sprintf('read "%s" -c %d --start %d --count %d', ... + filename, channel_number, s0, count); + data = invoke_binary(args, 'double'); + n = numel(data); + time = (s0 + (0:n-1)') / sr; + + elseif any(kind == [2 3 4]) % ---- event ---- + args = [sprintf('read "%s" -c %d', filename, channel_number) timewindow(t0, t1)]; + data = invoke_binary(args, 'double'); + time = data; + + else % ---- marker / wavemark / realmark / textmark (5,6,7,8) ---- + args = [sprintf('read "%s" -c %d', filename, channel_number) timewindow(t0, t1)]; + txt = invoke_text(args); + r = jsondecode(txt); + [time, data] = markers_to_output(r); + end +end + +function s = timewindow(t0, t1) +% Build the --t0/--t1 argument fragment, omitting defaults (t0<=0 -> start of +% file, t1=Inf -> end of file). + s = ''; + if isfinite(t0) && t0 > 0 + s = [s sprintf(' --t0 %.12g', t0)]; + end + if isfinite(t1) + s = [s sprintf(' --t1 %.12g', t1)]; + end +end + +function [time, data] = markers_to_output(r) + time = []; + data = []; + if ~isfield(r, 'markers') || isempty(r.markers) + return; + end + m = r.markers; + if iscell(m) % ragged; normalise to struct array + tmp = m{1}; + for i = 2:numel(m) + tmp(i) = m{i}; + end + m = tmp; + end + n = numel(m); + time = zeros(n, 1); + for i = 1:n + time(i) = m(i).time; + end + if isfield(m, 'text') + txt = cell(n, 1); + for i = 1:n + txt{i} = m(i).text; + end + data = char(txt); + else + ncode = numel(m(1).code); + data = zeros(n, ncode); + for i = 1:n + data(i, :) = m(i).code(:)'; + end + end +end diff --git a/matlab/+sonpipe/read_SOMSMR_header.m b/matlab/+sonpipe/read_SOMSMR_header.m new file mode 100644 index 0000000..d11f217 --- /dev/null +++ b/matlab/+sonpipe/read_SOMSMR_header.m @@ -0,0 +1,73 @@ +function header = read_SOMSMR_header(filename) +% SONPIPE.READ_SOMSMR_HEADER - Read header information from a CED SMR/SMRX file +% +% HEADER = sonpipe.read_SOMSMR_header(FILENAME) +% +% Returns a structure HEADER describing the CED Spike2 file FILENAME. Works +% with both 32-bit .smr and 64-bit .smrx files (sonpy opens either). +% +% This is a drop-in analogue of ndr.format.ced.read_SOMSMR_header, but reads +% the file through the sonpipe command-line tool (sonpy) rather than sigTOOL. +% +% HEADER contains two substructures: +% -------------------------------------------------------------------- +% fileinfo | Information about the file (timebase, duration, #channels) +% channelinfo | Struct array, one entry per recorded (non-Off) channel +% +% HEADER.fileinfo fields include: +% timebase - seconds per clock tick +% max_channels - number of channel slots +% max_time_ticks - duration of the recording in ticks +% max_time - duration of the recording in seconds +% dTimeBase, usPerTime, maxFTime - aliases kept for compatibility with the +% classic SON/sigTOOL header (dTimeBase*maxFTime*usPerTime +% equals max_time). +% +% HEADER.channelinfo(k) fields include: +% number - Spike2 channel number (1-based) +% kind - CED data-type code (1=Adc, 2/3/4=Event, 5=Marker, +% 6=AdcMark/WaveMark, 7=RealMark, 8=TextMark, 9=RealWave) +% kind_name - human-readable kind +% ndr_type - NDR channel type ('analog_in','event','mark','text') +% title, units - channel title and units +% sampleinterval - seconds between samples (NaN for non-waveform channels) +% samplerate - 1/sampleinterval (NaN for non-waveform channels) +% scale, offset - Adc-to-real-units conversion (waveform channels) +% num_samples - estimated total waveform samples +% +% See also sonpipe.read_SOMSMR_datafile, sonpipe.read_SOMSMR_sampleinterval + + txt = invoke_text(sprintf('header "%s"', filename)); + raw = jsondecode(txt); + + header.fileinfo = raw.fileinfo; + % Compatibility aliases so callers written against the classic SON header + % keep working (see ndr.reader.ced_smr/t0_t1 and event block timing). + header.fileinfo.usPerTime = 1; + header.fileinfo.dTimeBase = raw.fileinfo.timebase; + header.fileinfo.maxFTime = raw.fileinfo.max_time_ticks; + + if isfield(raw, 'channelinfo') + header.channelinfo = raw.channelinfo; + else + header.channelinfo = struct([]); + end + + % jsondecode may return a cell array if the channel structs were ragged; the + % CLI emits uniform fields precisely so this stays a struct array, but guard + % just in case. + if iscell(header.channelinfo) + header.channelinfo = cell2struct_uniform(header.channelinfo); + end +end + +function s = cell2struct_uniform(c) + if isempty(c) + s = struct([]); + return; + end + s = c{1}; + for i = 2:numel(c) + s(i) = c{i}; + end +end diff --git a/matlab/+sonpipe/read_SOMSMR_sampleinterval.m b/matlab/+sonpipe/read_SOMSMR_sampleinterval.m new file mode 100644 index 0000000..9f999ce --- /dev/null +++ b/matlab/+sonpipe/read_SOMSMR_sampleinterval.m @@ -0,0 +1,33 @@ +function [sampleinterval, total_samples, total_time] = read_SOMSMR_sampleinterval(filename, header, channel_number) +% SONPIPE.READ_SOMSMR_SAMPLEINTERVAL - Sample interval for a CED SMR/SMRX channel +% +% [SAMPLEINTERVAL, TOTAL_SAMPLES, TOTAL_TIME] = ... +% sonpipe.read_SOMSMR_sampleinterval(FILENAME, HEADER, CHANNEL_NUMBER) +% +% Returns the sample interval (in seconds) for CHANNEL_NUMBER of the CED +% Spike2 file FILENAME. CHANNEL_NUMBER is the 1-based Spike2 channel number. +% HEADER is accepted for signature compatibility with +% ndr.format.ced.read_SOMSMR_sampleinterval and may be [] (it is not needed). +% +% For non-waveform channels (events, markers) SAMPLEINTERVAL is NaN. +% +% Outputs: +% SAMPLEINTERVAL - seconds between samples (NaN if not a waveform) +% TOTAL_SAMPLES - estimated number of samples in the channel +% TOTAL_TIME - estimated duration of the recording, in seconds +% +% See also sonpipe.read_SOMSMR_header, sonpipe.read_SOMSMR_datafile + + txt = invoke_text(sprintf('sampleinterval "%s" -c %d', filename, channel_number)); + r = jsondecode(txt); + + sampleinterval = emptytonan(r.sampleinterval); + total_samples = emptytonan(r.total_samples); + total_time = emptytonan(r.total_time); +end + +function v = emptytonan(v) + if isempty(v) + v = NaN; + end +end diff --git a/matlab/+sonpipe/runcmd.m b/matlab/+sonpipe/runcmd.m new file mode 100644 index 0000000..ce55a3d --- /dev/null +++ b/matlab/+sonpipe/runcmd.m @@ -0,0 +1,37 @@ +function [status, out] = runcmd(cmd) +% SONPIPE.RUNCMD - Run a system command with a Python-safe environment +% +% [STATUS, OUT] = sonpipe.runcmd(CMD) +% +% Wrapper around SYSTEM that temporarily clears LD_LIBRARY_PATH and +% DYLD_LIBRARY_PATH before running CMD, then restores them. MATLAB overrides +% these variables with paths to its own bundled shared libraries; if a child +% Python process inherits them it may load MATLAB's (often older) libraries +% instead of its own and fail to start. Clearing them for the duration of the +% call lets the system/Conda/venv Python run normally. +% +% On Windows this is a plain passthrough to SYSTEM. +% +% See also sonpipe.executable + + if ispc + [status, out] = system(cmd); + return; + end + + names = {'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH'}; + saved = cell(size(names)); + for i = 1:numel(names) + saved{i} = getenv(names{i}); + setenv(names{i}, ''); + end + restorer = onCleanup(@() restoreEnv(names, saved)); %#ok + + [status, out] = system(cmd); +end + +function restoreEnv(names, saved) + for i = 1:numel(names) + setenv(names{i}, saved{i}); + end +end diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..77e3015 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "sonpipe" +version = "0.1.0" +description = "A command-line bridge that streams data from 64-bit CED Spike2 .smrx/.smr files (via sonpy) as raw binary for fast ingestion by MATLAB and other tools." +readme = "README.md" +requires-python = ">=3.8" +license = { file = "LICENSE" } +authors = [{ name = "VH-Lab" }] +keywords = ["spike2", "ced", "smrx", "smr", "sonpy", "electrophysiology", "matlab", "ndr"] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", +] + +# NOTE: sonpy is a proprietary Cambridge Electronic Design (CED) package. +# We declare it as a dependency so that `pip install sonpipe` legally and +# automatically fetches the official, authorized sonpy binaries from PyPI/CED. +# We deliberately do NOT vendor or redistribute sonpy in this repository, in +# order to comply with CED's license. +dependencies = [ + "numpy>=1.17", + "sonpy>=1.7", +] + +[project.optional-dependencies] +test = ["pytest>=6.0"] + +[project.urls] +Homepage = "https://github.com/VH-Lab/sonpipe" +Source = "https://github.com/VH-Lab/sonpipe" +"sonpy (CED)" = "https://pypi.org/project/sonpy/" + +[project.scripts] +sonpipe = "sonpipe.cli:main" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +markers = [ + "integration: exercises the real CED sonpy against a real Spike2 file (requires sonpy installed and the SONPIPE_TEST_FILE environment variable); skipped otherwise", +] diff --git a/src/sonpipe/__init__.py b/src/sonpipe/__init__.py new file mode 100644 index 0000000..6598161 --- /dev/null +++ b/src/sonpipe/__init__.py @@ -0,0 +1,63 @@ +"""sonpipe - a CLI bridge for reading CED Spike2 .smrx / .smr files via sonpy. + +sonpipe extracts data from proprietary 64-bit ``.smrx`` (and legacy 32-bit +``.smr``) files produced by the Cambridge Electronic Design (CED) Spike2 +acquisition system and streams it, as raw binary bytes, to standard output. +This lets a host environment such as MATLAB drive data ingestion in chunks and +reinterpret the byte stream directly (e.g. with ``typecast``) without any +text-parsing overhead. + +The proprietary reader, :mod:`sonpy`, is provided by CED and is fetched +automatically by ``pip install sonpipe``. It is intentionally *not* vendored +in this repository, to comply with CED's license. + +The command-line tools mirror the reading functions found in the NDR-matlab +``ndr.format.ced`` package: + +=========================== ============================================== +sonpipe sub-command NDR-matlab equivalent +=========================== ============================================== +``sonpipe header`` ``ndr.format.ced.read_SOMSMR_header`` +``sonpipe sampleinterval`` ``ndr.format.ced.read_SOMSMR_sampleinterval`` +``sonpipe read`` ``ndr.format.ced.read_SOMSMR_datafile`` +=========================== ============================================== +""" + +from .channels import ( + ADC, + ADC_MARK, + EVENT_BOTH, + EVENT_FALL, + EVENT_RISE, + KIND_NAMES, + MARKER, + OFF, + REAL_MARK, + REAL_WAVE, + TEXT_MARK, + kind_name, + ndr_type, +) +from .errors import SonpipeError +from .sonfile import SmrxFile + +__version__ = "0.1.0" + +__all__ = [ + "SmrxFile", + "SonpipeError", + "kind_name", + "ndr_type", + "KIND_NAMES", + "OFF", + "ADC", + "EVENT_FALL", + "EVENT_RISE", + "EVENT_BOTH", + "MARKER", + "ADC_MARK", + "REAL_MARK", + "TEXT_MARK", + "REAL_WAVE", + "__version__", +] diff --git a/src/sonpipe/__main__.py b/src/sonpipe/__main__.py new file mode 100644 index 0000000..84ac55f --- /dev/null +++ b/src/sonpipe/__main__.py @@ -0,0 +1,8 @@ +"""Enable ``python -m sonpipe``.""" + +import sys + +from .cli import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/sonpipe/channels.py b/src/sonpipe/channels.py new file mode 100644 index 0000000..f4d4916 --- /dev/null +++ b/src/sonpipe/channels.py @@ -0,0 +1,75 @@ +"""CED Spike2 channel-type codes and their mappings. + +The integer codes below are shared by three worlds and, conveniently, agree: + +* the CED ``sonpy`` ``DataType`` enum (``int(sonpy.lib.DataType.Adc) == 1``), +* the legacy sigTOOL/SON library ``kind`` numbers used by NDR-matlab's + ``ndr.format.ced`` functions, and +* the values sonpipe emits in its ``header`` output. + +Because they agree, code that consumes sonpipe's ``kind`` field behaves exactly +like the existing NDR-matlab reader, which switches on the same numbers. +""" + +# --- DataType / kind codes ------------------------------------------------- +OFF = 0 # channel slot is unused +ADC = 1 # integer waveform (analog, regularly sampled) +EVENT_FALL = 2 # event on a positive-to-negative transition +EVENT_RISE = 3 # event on a negative-to-positive transition +EVENT_BOTH = 4 # event on either transition ("level") +MARKER = 5 # generic marker (4 code bytes per event) +ADC_MARK = 6 # WaveMark: a marker with an attached short waveform (spike) +REAL_MARK = 7 # marker with attached real-valued data +TEXT_MARK = 8 # marker with attached text +REAL_WAVE = 9 # single-precision floating-point waveform + +KIND_NAMES = { + OFF: "Off", + ADC: "Adc", + EVENT_FALL: "EventFall", + EVENT_RISE: "EventRise", + EVENT_BOTH: "EventBoth", + MARKER: "Marker", + ADC_MARK: "AdcMark", + REAL_MARK: "RealMark", + TEXT_MARK: "TextMark", + REAL_WAVE: "RealWave", +} + +# Groupings by how the data is read. +WAVEFORM_KINDS = frozenset({ADC, REAL_WAVE}) +EVENT_KINDS = frozenset({EVENT_FALL, EVENT_RISE, EVENT_BOTH}) +MARKER_KINDS = frozenset({MARKER, ADC_MARK, REAL_MARK, TEXT_MARK}) + + +def kind_name(kind): + """Return the human-readable name for a channel ``kind`` code.""" + return KIND_NAMES.get(int(kind), "Unknown") + + +def ndr_type(kind): + """Map a CED ``kind`` code to an NDR reader channel type. + + Mirrors ``ndr.reader.ced_smr.cedsmrheader2readerchanneltype`` so the names + line up with the rest of the NDR ecosystem: + + ================ =============== + kind ndr_type + ================ =============== + Adc, RealWave ``analog_in`` + Event* ``event`` + Marker, AdcMark, ``mark`` + RealMark + TextMark ``text`` + ================ =============== + """ + kind = int(kind) + if kind in WAVEFORM_KINDS: + return "analog_in" + if kind in EVENT_KINDS: + return "event" + if kind in (MARKER, ADC_MARK, REAL_MARK): + return "mark" + if kind == TEXT_MARK: + return "text" + return "unknown" diff --git a/src/sonpipe/cli.py b/src/sonpipe/cli.py new file mode 100644 index 0000000..c7dbb6b --- /dev/null +++ b/src/sonpipe/cli.py @@ -0,0 +1,322 @@ +"""Command-line interface for sonpipe. + +Sub-commands +------------ +``header`` Print JSON metadata for the file and all channels. +``sampleinterval`` Print JSON sample-interval info for one channel. +``read`` Stream channel data. Waveforms and events are written as + raw little-endian binary by default (for fast ingestion by + MATLAB's ``typecast``); markers are written as JSON. + +Design notes +------------ +Metadata commands (``header``, ``sampleinterval``) emit JSON on stdout because +they are called rarely and their volume is tiny. The performance-critical +``read`` command emits *raw bytes only* on stdout so a host such as MATLAB can +capture the pipe and reinterpret it with ``typecast`` -- no number-to-text and +back conversion. Anything informational is written to stderr so it never +pollutes the binary stream. +""" + +import argparse +import json +import math +import sys + +import numpy as np + +from . import __version__, channels +from .errors import SonpipeError +from .sonfile import SmrxFile + +_DTYPES = { + "double": " WARN_BYTES: + sys.stderr.write( + "sonpipe: warning: reading ~{:.1f} MB of {} (> {} MB). The " + "command-line bridge can be slow for large reads; consider reading " + "in smaller blocks with --start/--count or --t0/--t1 (suppress this " + "warning with --no-size-warning).\n".format( + nbytes / 1048576.0, label, WARN_BYTES // 1048576 + ) + ) + + +def _open(path): + return SmrxFile(path) + + +def _dump_json(obj, pretty): + if pretty: + json.dump(obj, sys.stdout, indent=2, sort_keys=False) + else: + json.dump(obj, sys.stdout, separators=(",", ":")) + sys.stdout.write("\n") + + +# -------------------------------------------------------------------------- +# sub-command handlers +# -------------------------------------------------------------------------- + +def cmd_header(args): + smrx = _open(args.file) + out = { + "fileinfo": smrx.file_info(), + "channelinfo": smrx.all_channel_info(), + } + _dump_json(out, args.pretty) + return 0 + + +def cmd_sampleinterval(args): + smrx = _open(args.file) + info = smrx.channel_info(args.channel) + if info is None: + raise SonpipeError("Channel {} is not recorded in the file.".format(args.channel)) + sample_interval = info.get("sampleinterval") + out = { + "channel": args.channel, + "kind": info["kind"], + "kind_name": info["kind_name"], + "sampleinterval": sample_interval, + "samplerate": info.get("samplerate"), + "total_samples": info.get("num_samples"), + "total_time": info.get("max_time"), + } + _dump_json(out, args.pretty) + return 0 + + +def _write_binary(arr, dtype, endian): + np_dtype = np.dtype(_DTYPES[dtype]) + if endian == "big": + np_dtype = np_dtype.newbyteorder(">") + elif endian == "native": + np_dtype = np_dtype.newbyteorder("=") + # else little (the explicit default already encoded in _DTYPES) + buf = np.ascontiguousarray(arr.astype(np_dtype, copy=False)) + sys.stdout.buffer.write(buf.tobytes()) + sys.stdout.buffer.flush() + return buf.size + + +def cmd_read(args): + smrx = _open(args.file) + info = smrx.channel_info(args.channel) + if info is None: + raise SonpipeError("Channel {} is not recorded in the file.".format(args.channel)) + kind = info["kind"] + + if kind in channels.WAVEFORM_KINDS: + return _read_waveform(smrx, args, info) + if kind in channels.EVENT_KINDS: + return _read_events(smrx, args, info) + if kind in channels.MARKER_KINDS: + return _read_markers(smrx, args, info) + raise SonpipeError("Unsupported channel kind {} ({}).".format(kind, info["kind_name"])) + + +def _estimate_wave_samples(args, info): + """Best-effort estimate of how many samples a waveform read will return.""" + if args.count is not None: + return max(0, args.count) + samplerate = info.get("samplerate") + num_samples = info.get("num_samples") or 0 + start = args.start or 0 + if args.t0 is not None or args.t1 is not None: + if not samplerate: + return None + t0 = args.t0 or 0.0 + if args.t1 is None or math.isinf(args.t1): + return max(0, num_samples) + return max(0, int((args.t1 - t0) * samplerate)) + return max(0, num_samples - start) + + +def _read_waveform(smrx, args, info): + scaled = not args.raw + dtype = args.dtype if args.dtype is not None else ("int16" if args.raw else "double") + + if not args.json: + est_samples = _estimate_wave_samples(args, info) + if est_samples is not None: + _warn_size(est_samples * _itemsize(dtype), args, "waveform samples") + + data = smrx.read_waveform( + args.channel, + start=args.start, + count=args.count, + t0=args.t0, + t1=args.t1, + scaled=scaled, + ) + if args.json: + _dump_json({ + "channel": args.channel, + "kind": info["kind"], + "samplerate": info.get("samplerate"), + "count": int(data.size), + "data": data.tolist(), + }, args.pretty) + return 0 + + n = _write_binary(data, dtype, args.endian) + sys.stderr.write("sonpipe: wrote {} samples ({}) for channel {}\n".format( + n, dtype, args.channel)) + return 0 + + +def _read_events(smrx, args, info): + times = smrx.read_events(args.channel, t0=args.t0, t1=args.t1) + dtype = args.dtype or "double" + _warn_size(times.size * _itemsize(dtype), args, "event times") + if args.json: + _dump_json({ + "channel": args.channel, + "kind": info["kind"], + "count": int(times.size), + "times": times.tolist(), + }, args.pretty) + return 0 + n = _write_binary(times, dtype, args.endian) + sys.stderr.write("sonpipe: wrote {} event times ({}) for channel {}\n".format( + n, dtype, args.channel)) + return 0 + + +def _read_markers(smrx, args, info): + markers = smrx.read_markers(args.channel, t0=args.t0, t1=args.t1) + # Rough size estimate: time + 4 code bytes + optional text per marker. + _warn_size(len(markers) * 48, args, "markers") + # Markers are heterogeneous (times + code bytes + optional text); JSON only. + _dump_json({ + "channel": args.channel, + "kind": info["kind"], + "kind_name": info["kind_name"], + "count": len(markers), + "markers": markers, + }, args.pretty) + return 0 + + +def cmd_channels(args): + """Convenience listing: one line per channel (human/script friendly).""" + smrx = _open(args.file) + for info in smrx.all_channel_info(): + sr = info.get("samplerate") + sr_str = "{:.4f} Hz".format(sr) if sr else "-" + sys.stdout.write("{number}\t{kind_name}\t{ndr_type}\t{sr}\t{title}\n".format( + number=info["number"], + kind_name=info["kind_name"], + ndr_type=info["ndr_type"], + sr=sr_str, + title=info.get("title", ""), + )) + return 0 + + +# -------------------------------------------------------------------------- +# argument parsing +# -------------------------------------------------------------------------- + +def build_parser(): + parser = argparse.ArgumentParser( + prog="sonpipe", + description=( + "Stream data from CED Spike2 .smrx/.smr files as raw binary " + "(via sonpy) for fast ingestion by MATLAB and other tools." + ), + ) + parser.add_argument("--version", action="version", + version="sonpipe {}".format(__version__)) + sub = parser.add_subparsers(dest="command", required=True) + + p_header = sub.add_parser( + "header", help="print JSON metadata for the file and every channel") + p_header.add_argument("file", help="path to a .smrx or .smr file") + p_header.add_argument("--pretty", action="store_true", + help="pretty-print the JSON output") + p_header.set_defaults(func=cmd_header) + + p_si = sub.add_parser( + "sampleinterval", + help="print JSON sample-interval / sample-rate info for one channel") + p_si.add_argument("file", help="path to a .smrx or .smr file") + p_si.add_argument("-c", "--channel", type=int, required=True, + help="Spike2 channel number (1-based)") + p_si.add_argument("--pretty", action="store_true") + p_si.set_defaults(func=cmd_sampleinterval) + + p_read = sub.add_parser( + "read", + help="stream channel data (raw binary for waveforms/events, JSON for markers)") + p_read.add_argument("file", help="path to a .smrx or .smr file") + p_read.add_argument("-c", "--channel", type=int, required=True, + help="Spike2 channel number (1-based)") + # Sample-based chunking (waveforms). + p_read.add_argument("--start", type=int, default=None, + help="first sample index (0-based) for waveform reads") + p_read.add_argument("--count", type=int, default=None, + help="number of samples to read for waveform reads") + # Time-based windowing (all channel types). + p_read.add_argument("--t0", type=float, default=None, + help="start time in seconds (default: start of file)") + p_read.add_argument("--t1", type=float, default=None, + help="end time in seconds (default: end of file)") + p_read.add_argument("--raw", action="store_true", + help="for Adc waveforms, emit raw int16 ADC values " + "instead of scaled real units") + p_read.add_argument("--dtype", choices=sorted(_DTYPES), default=None, + help="binary output dtype (default: double for scaled " + "waveforms/events, int16 for --raw)") + p_read.add_argument("--endian", choices=("little", "big", "native"), + default="little", + help="byte order of the binary output (default: little)") + p_read.add_argument("--json", action="store_true", + help="emit JSON instead of raw binary (debugging)") + p_read.add_argument("--no-size-warning", action="store_true", + help="suppress the >50 MB slow-read warning") + p_read.add_argument("--pretty", action="store_true") + p_read.set_defaults(func=cmd_read) + + p_ch = sub.add_parser( + "channels", help="print a tab-separated one-line-per-channel listing") + p_ch.add_argument("file", help="path to a .smrx or .smr file") + p_ch.set_defaults(func=cmd_channels) + + return parser + + +def main(argv=None): + parser = build_parser() + args = parser.parse_args(argv) + try: + return args.func(args) + except SonpipeError as exc: + sys.stderr.write("sonpipe: error: {}\n".format(exc)) + return 2 + except BrokenPipeError: # pragma: no cover - consumer closed the pipe early + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/src/sonpipe/errors.py b/src/sonpipe/errors.py new file mode 100644 index 0000000..421533b --- /dev/null +++ b/src/sonpipe/errors.py @@ -0,0 +1,11 @@ +"""Exception types for sonpipe.""" + + +class SonpipeError(Exception): + """Base class for all sonpipe errors. + + Raised for user-facing problems such as a missing ``sonpy`` install, a file + that cannot be opened, or a request for a channel that is not present. The + CLI catches this exception and prints a clean message to stderr instead of a + traceback. + """ diff --git a/src/sonpipe/sonfile.py b/src/sonpipe/sonfile.py new file mode 100644 index 0000000..967c14e --- /dev/null +++ b/src/sonpipe/sonfile.py @@ -0,0 +1,510 @@ +"""A thin, testable wrapper around a CED ``sonpy`` ``SonFile``. + +This module isolates every call into ``sonpy`` so that: + +* the rest of sonpipe deals in plain integers, numpy arrays and dicts, and +* tests can inject a fake ``sonpy`` implementation (see ``tests/fakesonpy.py``) + and run on machines where the proprietary CED binaries are not available. + +Channel numbering +----------------- +``sonpy`` addresses channels with a 0-based index, but Spike2 (and NDR-matlab) +refer to channels by their 1-based "channel number" as shown in the Spike2 +Sampling Configuration. sonpipe speaks the Spike2 convention on its public +surface: a channel *number* ``n`` maps to the sonpy *index* ``n - 1``. +""" + +import math +import os + +import numpy as np + +from . import channels +from .errors import SonpipeError + + +def _resolve_son_module(root, importer): + """Return the sonpy module object that actually exposes ``SonFile``. + + CED packages sonpy inconsistently across platforms and versions: + + * some wheels expose the API at the top level (``sonpy.__init__`` does + ``from .sonpy import *``), so ``sonpy.SonFile`` works; + * some wheels ship an empty ``__init__`` and the API lives in the compiled + submodule ``sonpy.sonpy``; + * older references used a ``sonpy.lib`` submodule. + + ``root`` is the imported top-level ``sonpy`` package and ``importer`` is a + callable like ``importlib.import_module``. Returns the module exposing + ``SonFile``, or ``None`` if none of the known layouts do. + """ + if hasattr(root, "SonFile"): + return root + for sub in ("lib", "sonpy"): + mod = getattr(root, sub, None) + if mod is None: + try: + mod = importer("sonpy." + sub) + except Exception: + mod = None + if mod is not None and hasattr(mod, "SonFile"): + return mod + return None + + +def load_sonpy(): + """Import and return the sonpy module that exposes ``SonFile``. + + ``sonpy`` is a proprietary CED package and is not vendored with sonpipe. + """ + import importlib + + try: + root = importlib.import_module("sonpy") + except Exception as exc: # pragma: no cover - exercised only without sonpy + raise SonpipeError( + "The 'sonpy' package (Cambridge Electronic Design) is required but " + "could not be imported.\n" + "Install it with: pip install sonpy\n" + "sonpy is proprietary CED software and is fetched from PyPI on " + "install; it is intentionally not bundled with sonpipe.\n" + f"(import error: {exc})" + ) + + module = _resolve_son_module(root, importlib.import_module) + if module is None: + raise SonpipeError( + "sonpy is installed but no module exposing 'SonFile' was found " + "(checked sonpy, sonpy.lib, sonpy.sonpy). This is an unexpected " + "sonpy version/layout." + ) + return module + + +def _call(method, index): + """Call a sonpy accessor that may want the channel positionally or as ``chan=``.""" + try: + return method(index) + except TypeError: + return method(chan=index) + + +class SmrxFile: + """Read metadata and data from a single ``.smrx`` / ``.smr`` file.""" + + def __init__(self, path, sonlib=None): + if sonlib is None: + sonlib = load_sonpy() + self._son = sonlib + self.path = os.fspath(path) + if not os.path.exists(self.path): + raise SonpipeError("File not found: {}".format(self.path)) + + # sonpy opens in read-only mode when the second argument is True. + self.f = sonlib.SonFile(self.path, True) + self._check_open_error() + + self.timebase = float(self.f.GetTimeBase()) + self.max_channels = int(self.f.MaxChannels()) + + # -- open / error handling --------------------------------------------- + + def _check_open_error(self): + """Raise if sonpy reports the file could not be opened.""" + getter = getattr(self.f, "GetOpenError", None) + if getter is None: + return + try: + err = getter() + except Exception: + return + # sonpy returns 0 (or an enum whose int() is 0) on success. + try: + code = int(err) + except (TypeError, ValueError): + code = 0 if err is None else 1 + if code != 0: + raise SonpipeError( + "sonpy could not open '{}' (open error code {}). " + "Is it a valid Spike2 .smrx/.smr file?".format(self.path, code) + ) + + # -- channel discovery ------------------------------------------------- + + def index_for_number(self, number): + """Convert a Spike2 1-based channel *number* to a sonpy 0-based *index*.""" + index = int(number) - 1 + if index < 0 or index >= self.max_channels: + raise SonpipeError( + "Channel number {} is out of range (file has {} channel " + "slots).".format(number, self.max_channels) + ) + return index + + def kind(self, index): + """Return the integer channel-type ``kind`` for a sonpy channel *index*.""" + return int(self.f.ChannelType(index)) + + def channel_numbers(self): + """Return the Spike2 channel numbers of every non-Off channel.""" + numbers = [] + for index in range(self.max_channels): + if self.kind(index) != channels.OFF: + numbers.append(index + 1) + return numbers + + def _text(self, method_name, index): + method = getattr(self.f, method_name, None) + if method is None: + return "" + try: + value = _call(method, index) + except Exception: + return "" + if value is None: + return "" + return str(value).strip() + + def _num(self, method_name, index, default=None): + method = getattr(self.f, method_name, None) + if method is None: + return default + try: + return float(_call(method, index)) + except Exception: + return default + + def channel_info(self, number): + """Return a metadata dict for the given Spike2 channel *number*. + + Returns ``None`` if the channel slot is Off (unused). + """ + index = self.index_for_number(number) + kind = self.kind(index) + if kind == channels.OFF: + return None + + max_ticks = self._num("ChannelMaxTime", index, default=0.0) + max_ticks = 0 if max_ticks is None else int(max_ticks) + + info = { + "number": number, + "index": index, + "kind": kind, + "kind_name": channels.kind_name(kind), + "ndr_type": channels.ndr_type(kind), + "title": self._text("GetChannelTitle", index), + "units": self._text("GetChannelUnits", index), + "comment": self._text("GetChannelComment", index), + "max_time_ticks": max_ticks, + "max_time": max_ticks * self.timebase, + # The following keys are always present (null for non-waveform + # channels) so that consumers such as MATLAB's jsondecode receive a + # uniform struct array rather than a ragged cell array. + "sampleinterval": None, + "samplerate": None, + "divide": None, + "ideal_rate": None, + "scale": None, + "offset": None, + "num_samples": None, + } + + if kind in channels.WAVEFORM_KINDS: + divide = int(self.f.ChannelDivide(index)) + info["divide"] = divide + sample_interval = divide * self.timebase + info["sampleinterval"] = sample_interval + info["samplerate"] = (1.0 / sample_interval) if sample_interval > 0 else None + info["ideal_rate"] = self._num("GetIdealRate", index) + info["scale"] = self._num("GetChannelScale", index, default=1.0) + info["offset"] = self._num("GetChannelOffset", index, default=0.0) + info["num_samples"] = (max_ticks // divide) if divide > 0 else 0 + + return info + + def all_channel_info(self): + """Return metadata dicts for every non-Off channel, ordered by number.""" + out = [] + for number in self.channel_numbers(): + info = self.channel_info(number) + if info is not None: + out.append(info) + return out + + def file_info(self): + """Return file-level metadata (timebase, duration, channel count).""" + max_time_ticks = self._file_max_ticks() + info = { + "path": self.path, + "timebase": self.timebase, + "max_channels": self.max_channels, + "max_time_ticks": max_time_ticks, + "max_time": max_time_ticks * self.timebase, + } + for name, key in (("GetFileVersion", "version"), ("AppID", "app_id")): + method = getattr(self.f, name, None) + if method is not None: + try: + info[key] = method() + except Exception: + pass + return info + + def _file_max_ticks(self): + getter = getattr(self.f, "GetMaxTime", None) + if getter is not None: + try: + return int(getter()) + except Exception: + pass + # Fall back to the largest per-channel max time. + longest = 0 + for index in range(self.max_channels): + if self.kind(index) == channels.OFF: + continue + ticks = self._num("ChannelMaxTime", index, default=0.0) or 0.0 + longest = max(longest, int(ticks)) + return longest + + # -- time <-> tick <-> sample helpers ---------------------------------- + + def seconds_to_ticks(self, seconds): + """Round a time in seconds to the nearest integer clock tick.""" + return int(round(seconds / self.timebase)) + + def _wave_tick_range(self, index, start, count, t0, t1): + """Resolve a read request into ``(tfrom_ticks, tupto_ticks, nmax)``. + + Either sample-based (``start``/``count``) or time-based (``t0``/``t1``) + arguments may be given. Sample-based reads assume the waveform's first + sample sits at tick 0, which is the common Spike2 case. + """ + divide = int(self.f.ChannelDivide(index)) + if divide <= 0: + divide = 1 + max_ticks = int(self._num("ChannelMaxTime", index, default=0.0) or 0) + + if start is not None or count is not None: + start = 0 if start is None else int(start) + if start < 0: + start = 0 + tfrom = start * divide + if count is None: + tupto = max_ticks + 1 + nmax = max(0, (max_ticks - tfrom) // divide + 1) + else: + count = int(count) + tupto = (start + count) * divide + nmax = count + else: + # Time-based. + tfrom = 0 if (t0 is None or t0 < 0) else self.seconds_to_ticks(t0) + if t1 is None or math.isinf(t1): + tupto = max_ticks + 1 + else: + tupto = self.seconds_to_ticks(t1) + 1 + span = max(0, tupto - tfrom) + nmax = span // divide + 2 + + if tupto > max_ticks + 1: + tupto = max_ticks + 1 + return tfrom, tupto, int(nmax) + + def _event_tick_range(self, t0, t1): + max_ticks = self._file_max_ticks() + tfrom = 0 if (t0 is None or t0 < 0) else self.seconds_to_ticks(t0) + if t1 is None or math.isinf(t1): + tupto = max_ticks + 1 + else: + tupto = self.seconds_to_ticks(t1) + 1 + return int(tfrom), int(tupto) + + # -- reads ------------------------------------------------------------- + + def read_waveform(self, number, start=None, count=None, t0=None, t1=None, + scaled=True): + """Read waveform samples for a channel and return a numpy array. + + For ``Adc`` channels the raw 16-bit integers are converted to real + units with ``value = adc * scale / 6553.6 + offset`` when ``scaled`` is + true; otherwise the raw ``int16`` values are returned. ``RealWave`` + channels are already in real units. + """ + index = self.index_for_number(number) + kind = self.kind(index) + if kind not in channels.WAVEFORM_KINDS: + raise SonpipeError( + "Channel {} is {} (kind {}), not a waveform channel.".format( + number, channels.kind_name(kind), kind + ) + ) + tfrom, tupto, nmax = self._wave_tick_range(index, start, count, t0, t1) + if nmax <= 0: + return np.zeros(0, dtype=np.float64 if scaled else _wave_raw_dtype(kind)) + + if kind == channels.ADC: + raw = np.asarray(self.f.ReadInts(index, nmax, tfrom, tupto)) + if scaled: + scale = self._num("GetChannelScale", index, default=1.0) + offset = self._num("GetChannelOffset", index, default=0.0) + return raw.astype(np.float64) * (scale / 6553.6) + offset + return raw.astype(np.int16) + else: # REAL_WAVE + raw = np.asarray(self.f.ReadFloats(index, nmax, tfrom, tupto)) + return raw.astype(np.float64 if scaled else np.float32) + + def read_events(self, number, t0=None, t1=None, chunk=1_000_000): + """Read event times (seconds) for an event or marker channel. + + Reads in ticks and converts to seconds using the file time base. The + read is chunked so channels with more than ``chunk`` events are not + truncated. + """ + index = self.index_for_number(number) + kind = self.kind(index) + tfrom, tupto = self._event_tick_range(t0, t1) + + reader = self._event_reader_for(kind) + pieces = [] + cursor = tfrom + while cursor < tupto: + ticks = np.asarray(reader(index, chunk, cursor, tupto), dtype=np.int64) + if ticks.size == 0: + break + pieces.append(ticks) + if ticks.size < chunk: + break + cursor = int(ticks[-1]) + 1 + if not pieces: + return np.zeros(0, dtype=np.float64) + return np.concatenate(pieces).astype(np.float64) * self.timebase + + def _event_reader_for(self, kind): + """Return a callable ``(index, nmax, tfrom, tupto) -> tick array``. + + For pure event channels this is ``ReadEvents``. Marker channels do not + have a plain-event reader, so we pull markers and take their ticks. + """ + if kind in channels.EVENT_KINDS: + return lambda i, n, a, b: self.f.ReadEvents(i, n, a, b) + + marker_method = self._marker_method(kind) + + def read_marker_ticks(i, n, a, b): + markers = marker_method(i, n, a, b) + return _marker_ticks(markers) + + return read_marker_ticks + + def _marker_method(self, kind): + name = { + channels.MARKER: "ReadMarkers", + channels.ADC_MARK: "ReadWaveMarks", + channels.REAL_MARK: "ReadRealMarks", + channels.TEXT_MARK: "ReadTextMarks", + }.get(kind) + method = getattr(self.f, name, None) if name else None + if method is None: + # Fall back to generic ReadMarkers if the specific reader is absent. + method = getattr(self.f, "ReadMarkers", None) + if method is None: + raise SonpipeError( + "sonpy provides no marker reader for kind {}.".format(kind) + ) + return method + + def read_markers(self, number, t0=None, t1=None, chunk=1_000_000): + """Read markers for a marker channel as a list of dicts. + + Each entry has ``time`` (seconds) and ``code`` (list of up to four + marker code bytes). Text markers additionally carry ``text``. + """ + index = self.index_for_number(number) + kind = self.kind(index) + if kind not in channels.MARKER_KINDS: + raise SonpipeError( + "Channel {} is {} (kind {}), not a marker channel.".format( + number, channels.kind_name(kind), kind + ) + ) + tfrom, tupto = self._event_tick_range(t0, t1) + marker_method = self._marker_method(kind) + + out = [] + cursor = tfrom + while cursor < tupto: + markers = marker_method(index, chunk, cursor, tupto) + markers = list(markers) if markers is not None else [] + if not markers: + break + for m in markers: + out.append(_marker_to_dict(m, self.timebase)) + if len(markers) < chunk: + break + last_tick = _marker_tick(markers[-1]) + if last_tick is None: + break + cursor = int(last_tick) + 1 + return out + + +def _wave_raw_dtype(kind): + return np.int16 if kind == channels.ADC else np.float32 + + +def _marker_tick(marker): + for attr in ("Tick", "tick", "Time", "time"): + if hasattr(marker, attr): + try: + return int(getattr(marker, attr)) + except Exception: + pass + return None + + +def _marker_ticks(markers): + if markers is None: + return np.zeros(0, dtype=np.int64) + ticks = [_marker_tick(m) for m in markers] + ticks = [t for t in ticks if t is not None] + return np.asarray(ticks, dtype=np.int64) + + +def _marker_code(marker): + for attr in ("Code", "code"): + if hasattr(marker, attr): + value = getattr(marker, attr) + try: + return [int(c) for c in value] + except TypeError: + try: + return [int(value)] + except Exception: + return [] + # Some sonpy versions expose Code1..Code4. + codes = [] + for i in range(1, 5): + attr = "Code{}".format(i) + if hasattr(marker, attr): + try: + codes.append(int(getattr(marker, attr))) + except Exception: + pass + return codes + + +def _marker_to_dict(marker, timebase): + tick = _marker_tick(marker) + entry = { + "tick": tick, + "time": (tick * timebase) if tick is not None else None, + "code": _marker_code(marker), + } + for attr in ("Text", "text"): + if hasattr(marker, attr): + text = getattr(marker, attr) + if text is not None: + entry["text"] = str(text) + break + return entry diff --git a/test/+sonpipe/+unittest/DatafileTest.m b/test/+sonpipe/+unittest/DatafileTest.m new file mode 100644 index 0000000..bd1907f --- /dev/null +++ b/test/+sonpipe/+unittest/DatafileTest.m @@ -0,0 +1,66 @@ +classdef DatafileTest < sonpipe.unittest.TestCase +% sonpipe.unittest.DatafileTest - Tests for sonpipe.read_SOMSMR_datafile + + methods (Test) + function testWaveformFullRead(tc) + [d, ts, tt, bi, t] = sonpipe.read_SOMSMR_datafile(tc.File, [], 1, 0, Inf); + tc.verifyEqual(numel(d), 1000); + tc.verifyEqual(size(d, 2), 1); % column vector + tc.verifyEqual(numel(t), 1000); + tc.verifyEqual(t(1), 0, 'AbsTol', 1e-12); + % First raw sample is -1000; scaled = -1000*scale/6553.6 + offset. + tc.verifyEqual(d(1), -1000 * (2 / 6553.6) + 0.5, 'RelTol', 1e-9); + tc.verifyEmpty(bi); % blockinfo not modelled + tc.verifyEqual(ts, 1000); % total_samples + tc.verifyEqual(tt, 0.1, 'RelTol', 1e-6); % total_time + end + + function testWaveformTimeWindow(tc) + [d, ~, ~, ~, t] = sonpipe.read_SOMSMR_datafile(tc.File, [], 1, 0, 0.005); + tc.verifyGreaterThanOrEqual(numel(d), 50); + tc.verifyLessThanOrEqual(numel(d), 52); + tc.verifyEqual(numel(t), numel(d)); + tc.verifyEqual(t(1), 0, 'AbsTol', 1e-12); + end + + function testRealWaveChannel(tc) + d = sonpipe.read_SOMSMR_datafile(tc.File, [], 3, 0, Inf); + tc.verifyEqual(numel(d), 1000); + tc.verifyEqual(size(d, 2), 1); + end + + function testEventChannel(tc) + [d, ts, ~, ~, t] = sonpipe.read_SOMSMR_datafile(tc.File, [], 4, 0, Inf); + tc.verifyEqual(d, t); % events: data == time + tc.verifyEqual(numel(d), 1000); + tc.verifyEqual(d(1), 0, 'AbsTol', 1e-12); + tc.verifyEqual(d(2), 1e-4, 'RelTol', 1e-6); + tc.verifyEmpty(ts); % no sample count for events + end + + function testEventTimeWindow(tc) + [d, ~, ~, ~, ~] = sonpipe.read_SOMSMR_datafile(tc.File, [], 4, 0, 0.001); + tc.verifyTrue(all(d <= 0.001 + 1e-12)); + end + + function testTextMarkerChannel(tc) + [d, ~, ~, ~, t] = sonpipe.read_SOMSMR_datafile(tc.File, [], 6, 0, Inf); + tc.verifyTrue(ischar(d)); + tc.verifyEqual(size(d, 1), numel(t)); + tc.verifyEqual(strtrim(d(1, :)), 'note0'); + tc.verifyEqual(t(1), 0, 'AbsTol', 1e-12); + end + + function testMarkerChannel(tc) + [d, ~, ~, ~, t] = sonpipe.read_SOMSMR_datafile(tc.File, [], 5, 0, Inf); + tc.verifyEqual(size(d, 1), numel(t)); + tc.verifyGreaterThanOrEqual(size(d, 2), 1); % marker code columns + end + + function testMultiChannelErrors(tc) + tc.verifyError(... + @() sonpipe.read_SOMSMR_datafile(tc.File, [], [1 3], 0, Inf), ... + 'sonpipe:singleChannel'); + end + end +end diff --git a/test/+sonpipe/+unittest/HeaderTest.m b/test/+sonpipe/+unittest/HeaderTest.m new file mode 100644 index 0000000..9d7f9a6 --- /dev/null +++ b/test/+sonpipe/+unittest/HeaderTest.m @@ -0,0 +1,40 @@ +classdef HeaderTest < sonpipe.unittest.TestCase +% sonpipe.unittest.HeaderTest - Tests for sonpipe.read_SOMSMR_header / channels + + methods (Test) + function testFileInfo(tc) + h = sonpipe.read_SOMSMR_header(tc.File); + tc.verifyEqual(h.fileinfo.timebase, 1e-6, 'RelTol', 1e-9); + tc.verifyEqual(h.fileinfo.maxFTime, 100000); + tc.verifyEqual(h.fileinfo.dTimeBase, 1e-6, 'RelTol', 1e-9); + tc.verifyEqual(h.fileinfo.usPerTime, 1); + end + + function testChannelNumbersSkipOff(tc) + h = sonpipe.read_SOMSMR_header(tc.File); + % Spike2 slot #2 is Off and must be omitted. + tc.verifyEqual([h.channelinfo.number], [1 3 4 5 6]); + end + + function testAdcChannelInfo(tc) + h = sonpipe.read_SOMSMR_header(tc.File); + ci = sonpipe.channelinfo(h, 1); + tc.verifyEqual(ci.kind, 1); + tc.verifyEqual(ci.ndr_type, 'analog_in'); + tc.verifyEqual(ci.samplerate, 10000, 'RelTol', 1e-6); + tc.verifyEqual(ci.num_samples, 1000); + end + + function testChannelsHelperNames(tc) + chans = sonpipe.channels(tc.File); + tc.verifyEqual({chans.name}, {'ai1', 'ai3', 'e4', 'mk5', 'text6'}); + tc.verifyEqual({chans.type}, ... + {'analog_in', 'analog_in', 'event', 'mark', 'text'}); + end + + function testUnknownChannelErrors(tc) + h = sonpipe.read_SOMSMR_header(tc.File); + tc.verifyError(@() sonpipe.channelinfo(h, 99), 'sonpipe:noChannel'); + end + end +end diff --git a/test/+sonpipe/+unittest/SampleIntervalTest.m b/test/+sonpipe/+unittest/SampleIntervalTest.m new file mode 100644 index 0000000..a38f641 --- /dev/null +++ b/test/+sonpipe/+unittest/SampleIntervalTest.m @@ -0,0 +1,26 @@ +classdef SampleIntervalTest < sonpipe.unittest.TestCase +% sonpipe.unittest.SampleIntervalTest - Tests for read_SOMSMR_sampleinterval + + methods (Test) + function testWaveformInterval(tc) + [si, ns, tt] = sonpipe.read_SOMSMR_sampleinterval(tc.File, [], 1); + tc.verifyEqual(si, 1e-4, 'RelTol', 1e-6); + tc.verifyEqual(1/si, 10000, 'RelTol', 1e-6); + tc.verifyEqual(ns, 1000); + tc.verifyEqual(tt, 0.1, 'RelTol', 1e-6); + end + + function testEventIntervalIsNaN(tc) + si = sonpipe.read_SOMSMR_sampleinterval(tc.File, [], 4); + tc.verifyTrue(isnan(si)); + end + + function testHeaderArgumentIgnored(tc) + % Passing a header should give the same answer as passing []. + h = sonpipe.read_SOMSMR_header(tc.File); + si1 = sonpipe.read_SOMSMR_sampleinterval(tc.File, h, 1); + si2 = sonpipe.read_SOMSMR_sampleinterval(tc.File, [], 1); + tc.verifyEqual(si1, si2); + end + end +end diff --git a/test/+sonpipe/+unittest/TestCase.m b/test/+sonpipe/+unittest/TestCase.m new file mode 100644 index 0000000..fa2cd71 --- /dev/null +++ b/test/+sonpipe/+unittest/TestCase.m @@ -0,0 +1,59 @@ +classdef TestCase < matlab.unittest.TestCase +% sonpipe.unittest.TestCase - Base fixture for the +sonpipe MATLAB unit tests +% +% Points sonpipe.executable at a fake sonpipe CLI (fakecli.py) that runs the +% real sonpipe Python code against a synthetic in-memory file, so the MATLAB +% wrappers can be tested end-to-end without CED's proprietary sonpy binaries. +% +% Tests are skipped (filtered) if a working Python 3 with numpy is not found. + + properties + File % path to a temporary (synthetic) data file + end + + methods (TestClassSetup) + function configureFakeCli(tc) + here = fileparts(mfilename('fullpath')); + fakecli = fullfile(here, 'fakecli.py'); + + py = sonpipe.unittest.TestCase.findPython(); + tc.assumeFalse(isempty(py), ... + 'Python 3 not found on PATH; skipping +sonpipe MATLAB tests.'); + + cmd = sprintf('%s "%s"', py, fakecli); + [status, ~] = sonpipe.runcmd([cmd ' --version']); + tc.assumeEqual(status, 0, ... + ['The sonpipe fake CLI could not run (is numpy installed for ' ... + 'this Python?); skipping +sonpipe MATLAB tests.']); + + sonpipe.executable(cmd); + + tc.File = [tempname() '.smrx']; + fid = fopen(tc.File, 'w'); + fwrite(fid, 0, 'uint8'); + fclose(fid); + end + end + + methods (TestClassTeardown) + function cleanupFile(tc) + if ~isempty(tc.File) && exist(tc.File, 'file') + delete(tc.File); + end + end + end + + methods (Static) + function py = findPython() + py = ''; + candidates = {'python3', 'python'}; + for i = 1:numel(candidates) + [status, ~] = sonpipe.runcmd([candidates{i} ' --version']); + if status == 0 + py = candidates{i}; + return; + end + end + end + end +end diff --git a/test/+sonpipe/+unittest/fakecli.py b/test/+sonpipe/+unittest/fakecli.py new file mode 100644 index 0000000..e38c1ae --- /dev/null +++ b/test/+sonpipe/+unittest/fakecli.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""A stand-in for the ``sonpipe`` command-line tool, for MATLAB unit tests. + +It runs the *real* sonpipe CLI code, but injects the test fake sonpy module in +place of CED's proprietary ``sonpy``. That way the MATLAB ``+sonpipe`` wrappers +can be exercised end-to-end (argument construction, JSON decoding, binary +typecast, channel dispatch, time-vector math) on any machine, without the CED +binaries and without a Python/MATLAB in-process bridge. + +The file served is synthetic: contents are ignored and the channel layout comes +from ``tests/fakesonpy.py`` (Spike2 #1 Adc, #3 RealWave, #4 EventFall, +#5 Marker, #6 TextMark). +""" + +import os +import sys + +_HERE = os.path.dirname(os.path.abspath(__file__)) +# .../repo/test/+sonpipe/+unittest/fakecli.py -> repo is three levels up. +_REPO = os.path.abspath(os.path.join(_HERE, os.pardir, os.pardir, os.pardir)) + +sys.path.insert(0, os.path.join(_REPO, "src")) # the sonpipe package +sys.path.insert(0, os.path.join(_REPO, "tests")) # fakesonpy + +import fakesonpy # noqa: E402 +from sonpipe import cli, sonfile # noqa: E402 + +# Force the CLI to use the fake reader instead of importing CED's sonpy. +sonfile.load_sonpy = lambda: fakesonpy + + +if __name__ == "__main__": + sys.exit(cli.main()) diff --git a/test/+sonpipe/+unittest/run_all.m b/test/+sonpipe/+unittest/run_all.m new file mode 100644 index 0000000..f8a556a --- /dev/null +++ b/test/+sonpipe/+unittest/run_all.m @@ -0,0 +1,29 @@ +function results = run_all(varargin) +% SONPIPE.UNITTEST.RUN_ALL - Run every +sonpipe MATLAB unit test +% +% RESULTS = sonpipe.unittest.run_all() +% +% Adds the repository's 'matlab' (source) and 'test' folders to the MATLAB +% path and runs all tests in the sonpipe.unittest namespace. Any extra +% arguments are forwarded to RUNTESTS (e.g. 'OutputDetail','Detailed'). +% +% Requires a working Python 3 with numpy so the fake sonpipe CLI can run; +% individual tests are skipped if it is unavailable. +% +% Example: +% results = sonpipe.unittest.run_all(); +% assert(all(~[results.Failed])); + + here = fileparts(mfilename('fullpath')); + repo = fileparts(fileparts(fileparts(here))); % .../repo + + addpath(fullfile(repo, 'matlab')); + addpath(fullfile(repo, 'test')); + + results = runtests('sonpipe.unittest', varargin{:}); + + if nargout == 0 + disp(results); + clear results; + end +end diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e35914a --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,28 @@ +"""Shared pytest fixtures: wire sonpipe up to the fake sonpy.""" + +import pytest + +import fakesonpy +from sonpipe import sonfile + + +@pytest.fixture(autouse=True) +def use_fake_sonpy(request, monkeypatch): + """Make ``SmrxFile`` (and therefore the CLI) use the fake sonpy module. + + Integration tests (marked ``integration``) are exempt: they run against the + real CED sonpy and a real Spike2 file. + """ + if request.node.get_closest_marker("integration"): + yield + return + monkeypatch.setattr(sonfile, "load_sonpy", lambda: fakesonpy) + yield + + +@pytest.fixture +def smrx_path(tmp_path): + """A real (but empty) file on disk; content is served by the fake sonpy.""" + p = tmp_path / "example.smrx" + p.write_bytes(b"") # SmrxFile only checks that the path exists + return str(p) diff --git a/tests/fakesonpy.py b/tests/fakesonpy.py new file mode 100644 index 0000000..5f98357 --- /dev/null +++ b/tests/fakesonpy.py @@ -0,0 +1,160 @@ +"""A minimal fake of the CED ``sonpy.lib`` module for testing. + +It reproduces the parts of the ``sonpy`` API that :class:`sonpipe.SmrxFile` +relies on, so the package can be exercised on machines without CED's +proprietary binaries. It is *not* a real SON file reader -- it serves synthetic +in-memory data whose shape matches what real sonpy returns. + +The channel layout of the default fake file (mirrors a small Spike2 file): + +=============== ===== ========================================= +sonpy index kind contents +=============== ===== ========================================= +0 (Spike2 #1) Adc ramp waveform 0,1,2,... (int16), 10 kHz +1 (Spike2 #2) Off unused slot +2 (Spike2 #3) RealWave sine-ish float waveform, 10 kHz +3 (Spike2 #4) EventFall event ticks every 100 ticks +4 (Spike2 #5) Marker markers every 250 ticks +5 (Spike2 #6) TextMark text markers +=============== ===== ========================================= +""" + +import numpy as np + + +class DataType: + """Integer-valued stand-in for sonpy's ``DataType`` enum.""" + + Off = 0 + Adc = 1 + EventFall = 2 + EventRise = 3 + EventBoth = 4 + Marker = 5 + AdcMark = 6 + RealMark = 7 + TextMark = 8 + RealWave = 9 + + +class _Marker: + def __init__(self, tick, code=(0, 0, 0, 0), text=None): + self.Tick = int(tick) + self.Code = list(code) + if text is not None: + self.Text = text + + +# The synthetic file: per-channel description. +TIMEBASE = 1e-6 # 1 microsecond per tick +DIVIDE = 100 # 100 ticks per sample -> 10 kHz +N_SAMPLES = 1000 # waveform length in samples +MAX_TICKS = N_SAMPLES * DIVIDE + +_ADC_SCALE = 2.0 # real = adc * scale/6553.6 + offset +_ADC_OFFSET = 0.5 + + +class SonFile: + """Fake ``SonFile`` serving the synthetic layout described above.""" + + def __init__(self, path, read_only=True): + self.path = path + self.read_only = read_only + self._open_error = 0 + # Waveform sample data. + self._adc = (np.arange(N_SAMPLES) % 2000 - 1000).astype(np.int16) + t = np.arange(N_SAMPLES) / (1.0 / (DIVIDE * TIMEBASE)) + self._realwave = np.sin(2 * np.pi * 5 * t).astype(np.float32) + # Event / marker tick tables. + self._event_ticks = np.arange(0, MAX_TICKS, 100, dtype=np.int64) + self._marker_ticks = np.arange(0, MAX_TICKS, 250, dtype=np.int64) + self._text_ticks = np.arange(0, MAX_TICKS, 400, dtype=np.int64) + + # -- file-level --------------------------------------------------------- + + def GetOpenError(self): + return self._open_error + + def GetTimeBase(self): + return TIMEBASE + + def MaxChannels(self): + return 6 + + def GetMaxTime(self): + return MAX_TICKS + + def GetFileVersion(self): + return 9 + + # -- per-channel metadata ---------------------------------------------- + + def ChannelType(self, chan): + return { + 0: DataType.Adc, + 1: DataType.Off, + 2: DataType.RealWave, + 3: DataType.EventFall, + 4: DataType.Marker, + 5: DataType.TextMark, + }[chan] + + def ChannelDivide(self, chan): + if chan in (0, 2): + return DIVIDE + return 0 + + def ChannelMaxTime(self, chan): + return MAX_TICKS + + def GetIdealRate(self, chan): + return 1.0 / (DIVIDE * TIMEBASE) + + def GetChannelScale(self, chan): + return _ADC_SCALE + + def GetChannelOffset(self, chan): + return _ADC_OFFSET + + def GetChannelTitle(self, chan): + return {0: "Ramp", 2: "Sine", 3: "Trig", 4: "Mark", 5: "Notes"}.get(chan, "") + + def GetChannelUnits(self, chan): + return {0: "V", 2: "mV"}.get(chan, "") + + def GetChannelComment(self, chan): + return "" + + # -- reads -------------------------------------------------------------- + + def _wave_slice(self, data, nMax, tFrom, tUpto, divide): + # Sample i sits at tick i*divide; select those in [tFrom, tUpto). + first = max(0, (tFrom + divide - 1) // divide) if tFrom > 0 else 0 + last = min(len(data), (tUpto + divide - 1) // divide) + sel = data[first:last] + if nMax is not None and nMax >= 0: + sel = sel[:nMax] + return sel + + def ReadInts(self, chan, nMax, tFrom, tUpto): + return self._wave_slice(self._adc, nMax, tFrom, tUpto, DIVIDE) + + def ReadFloats(self, chan, nMax, tFrom, tUpto): + return self._wave_slice(self._realwave, nMax, tFrom, tUpto, DIVIDE) + + def ReadEvents(self, chan, nMax, tFrom, tUpto): + ticks = self._event_ticks + sel = ticks[(ticks >= tFrom) & (ticks < tUpto)] + return sel[:nMax] + + def ReadMarkers(self, chan, nMax, tFrom, tUpto): + ticks = self._marker_ticks + sel = ticks[(ticks >= tFrom) & (ticks < tUpto)][:nMax] + return [_Marker(t, code=(int(t) % 256, 0, 0, 0)) for t in sel] + + def ReadTextMarks(self, chan, nMax, tFrom, tUpto): + ticks = self._text_ticks + sel = ticks[(ticks >= tFrom) & (ticks < tUpto)][:nMax] + return [_Marker(t, code=(1, 0, 0, 0), text="note{}".format(int(t))) + for t in sel] diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..260dbf0 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,143 @@ +"""End-to-end tests of the sonpipe CLI (with the fake sonpy).""" + +import json + +import numpy as np +import pytest + +import fakesonpy +from sonpipe import cli + + +def run(capsysbinary, argv): + """Run the CLI, returning (exit_code, stdout_bytes, stderr_text).""" + code = cli.main(argv) + captured = capsysbinary.readouterr() + return code, captured.out, captured.err.decode() if isinstance(captured.err, bytes) else captured.err + + +def test_header_json(capsysbinary, smrx_path): + code, out, _ = run(capsysbinary, ["header", smrx_path]) + assert code == 0 + doc = json.loads(out.decode()) + assert doc["fileinfo"]["timebase"] == pytest.approx(fakesonpy.TIMEBASE) + numbers = [c["number"] for c in doc["channelinfo"]] + assert numbers == [1, 3, 4, 5, 6] # Off slot #2 excluded + adc = doc["channelinfo"][0] + assert adc["kind"] == 1 + assert adc["kind_name"] == "Adc" + assert adc["ndr_type"] == "analog_in" + + +def test_sampleinterval_json(capsysbinary, smrx_path): + code, out, _ = run(capsysbinary, ["sampleinterval", smrx_path, "-c", "1"]) + assert code == 0 + doc = json.loads(out.decode()) + assert doc["channel"] == 1 + assert doc["samplerate"] == pytest.approx(10000.0) + assert doc["total_samples"] == fakesonpy.N_SAMPLES + + +def test_sampleinterval_off_channel_errors(capsysbinary, smrx_path): + code, out, err = run(capsysbinary, ["sampleinterval", smrx_path, "-c", "2"]) + assert code == 2 + assert "not recorded" in err + + +def test_read_waveform_binary_double(capsysbinary, smrx_path): + code, out, err = run(capsysbinary, ["read", smrx_path, "-c", "1"]) + assert code == 0 + data = np.frombuffer(out, dtype=" 0 + assert "text" in doc["markers"][0] + + +def test_large_read_warns(capsysbinary, smrx_path, monkeypatch): + # Lower the threshold so the modest fake file trips it. + from sonpipe import cli as climod + monkeypatch.setattr(climod, "WARN_BYTES", 100) + code, out, err = run(capsysbinary, ["read", smrx_path, "-c", "1"]) + assert code == 0 + assert "warning" in err.lower() + assert "MB" in err + + +def test_large_read_warning_suppressed(capsysbinary, smrx_path, monkeypatch): + from sonpipe import cli as climod + monkeypatch.setattr(climod, "WARN_BYTES", 100) + code, out, err = run(capsysbinary, [ + "read", smrx_path, "-c", "1", "--no-size-warning"]) + assert code == 0 + assert "warning" not in err.lower() + + +def test_channels_listing(capsysbinary, smrx_path): + code, out, _ = run(capsysbinary, ["channels", smrx_path]) + assert code == 0 + lines = out.decode().strip().splitlines() + assert len(lines) == 5 # 5 non-Off channels + assert lines[0].startswith("1\tAdc\tanalog_in") diff --git a/tests/test_integration.py b/tests/test_integration.py new file mode 100644 index 0000000..1ba46a5 --- /dev/null +++ b/tests/test_integration.py @@ -0,0 +1,145 @@ +"""Integration tests against the *real* CED sonpy and a real Spike2 file. + +Unlike the rest of the suite (which uses a synthetic fake sonpy), these exercise +the actual sonpy binding end-to-end -- both the :class:`SmrxFile` wrapper and the +CLI (as a subprocess, so the raw-binary pipe is exercised too). + +They run when BOTH of the following hold: + +* the ``sonpy`` package (proprietary CED) is importable, and +* a real Spike2 file is available -- by default the repository's + ``example/spike2data.smrx``, or whatever ``SONPIPE_TEST_FILE`` points at. + +Otherwise they are skipped. In CI they run on a Python for which CED ships a +sonpy wheel (Python 3.14 covers Linux, Windows, and macOS Intel/Apple Silicon). +""" + +import json +import os +import subprocess +import sys + +import numpy as np +import pytest + +from sonpipe import channels +from sonpipe.sonfile import SmrxFile + +pytestmark = pytest.mark.integration + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_REPO = os.path.dirname(_HERE) +_DEFAULT_FILE = os.path.join(_REPO, "example", "spike2data.smrx") +_FILE = os.environ.get("SONPIPE_TEST_FILE") or ( + _DEFAULT_FILE if os.path.exists(_DEFAULT_FILE) else None +) + +try: + import sonpy # noqa: F401 + _HAVE_SONPY = True +except Exception: + _HAVE_SONPY = False + + +@pytest.fixture +def real_file(): + if not _FILE: + pytest.skip("no real Spike2 file (set SONPIPE_TEST_FILE or add example/spike2data.smrx)") + if not os.path.exists(_FILE): + pytest.skip("test file does not exist: {}".format(_FILE)) + if not _HAVE_SONPY: + pytest.skip("CED sonpy is not installed (pip install sonpy)") + return _FILE + + +def _first_waveform(smrx): + for info in smrx.all_channel_info(): + if info["kind"] in channels.WAVEFORM_KINDS: + return info + return None + + +# -- SmrxFile wrapper -------------------------------------------------------- + +def test_header_has_valid_channels(real_file): + smrx = SmrxFile(real_file) + infos = smrx.all_channel_info() + assert len(infos) > 0 + for info in infos: + assert info["kind"] in channels.KIND_NAMES + assert info["number"] >= 1 + assert info["ndr_type"] in ("analog_in", "event", "mark", "text") + + +def test_file_info(real_file): + smrx = SmrxFile(real_file) + fi = smrx.file_info() + assert fi["timebase"] > 0 + assert fi["max_time_ticks"] >= 0 + + +def test_waveform_read(real_file): + smrx = SmrxFile(real_file) + wf = _first_waveform(smrx) + if wf is None: + pytest.skip("no waveform channels in the test file") + ch = wf["number"] + + assert wf["sampleinterval"] > 0 + assert abs(wf["samplerate"] * wf["sampleinterval"] - 1.0) < 1e-6 + + full = smrx.read_waveform(ch) + assert full.dtype == np.float64 + assert full.size > 0 + assert np.all(np.isfinite(full)) + + # A count-limited read from the same origin is a prefix of a longer read. + n = min(100, full.size) + head = smrx.read_waveform(ch, start=0, count=n) + assert head.size == n + np.testing.assert_allclose(head, full[:n]) + + # Raw (unscaled) reads come back as int16 for Adc channels. + if wf["kind"] == channels.ADC: + raw = smrx.read_waveform(ch, start=0, count=n, scaled=False) + assert raw.dtype == np.int16 + assert raw.size == n + + +def test_event_or_marker_read(real_file): + smrx = SmrxFile(real_file) + for info in smrx.all_channel_info(): + if info["kind"] in channels.EVENT_KINDS: + times = smrx.read_events(info["number"]) + assert times.dtype == np.float64 + assert np.all(np.diff(times) >= 0) # non-decreasing + return + if info["kind"] in channels.MARKER_KINDS: + markers = smrx.read_markers(info["number"]) + for m in markers: + assert "time" in m + return + pytest.skip("no event or marker channels in the test file") + + +# -- CLI end-to-end (subprocess, exercises the raw-binary pipe) --------------- + +def _cli(*args): + return subprocess.run( + [sys.executable, "-m", "sonpipe", *args], + check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ).stdout + + +def test_cli_header_and_binary_read(real_file): + doc = json.loads(_cli("header", real_file)) + assert "fileinfo" in doc and "channelinfo" in doc + waves = [c for c in doc["channelinfo"] if c["kind"] in (channels.ADC, channels.REAL_WAVE)] + if not waves: + pytest.skip("no waveform channels in the test file") + ch = str(waves[0]["number"]) + + raw = _cli("read", real_file, "-c", ch, "--start", "0", "--count", "100") + data = np.frombuffer(raw, dtype=" sonpy index 0. + assert smrx.index_for_number(1) == 0 + assert smrx.index_for_number(6) == 5 + with pytest.raises(SonpipeError): + smrx.index_for_number(999) + + +def test_adc_channel_info(smrx_path): + smrx = open_file(smrx_path) + info = smrx.channel_info(1) # Spike2 #1 -> Adc + assert info["kind"] == channels.ADC + assert info["ndr_type"] == "analog_in" + assert info["title"] == "Ramp" + assert info["units"] == "V" + assert info["sampleinterval"] == pytest.approx(fakesonpy.DIVIDE * fakesonpy.TIMEBASE) + assert info["samplerate"] == pytest.approx(10000.0) + assert info["num_samples"] == fakesonpy.N_SAMPLES + + +def test_off_channel_info_is_none(smrx_path): + smrx = open_file(smrx_path) + assert smrx.channel_info(2) is None # Spike2 #2 is Off + + +def test_event_channel_info_has_nan_interval(smrx_path): + smrx = open_file(smrx_path) + info = smrx.channel_info(4) # EventFall + assert info["kind"] == channels.EVENT_FALL + assert info["sampleinterval"] is None + assert info["samplerate"] is None + + +def test_file_info(smrx_path): + smrx = open_file(smrx_path) + fi = smrx.file_info() + assert fi["timebase"] == pytest.approx(fakesonpy.TIMEBASE) + assert fi["max_time_ticks"] == fakesonpy.MAX_TICKS + assert fi["max_time"] == pytest.approx(fakesonpy.MAX_TICKS * fakesonpy.TIMEBASE) + + +# -- waveform reads --------------------------------------------------------- + +def test_read_full_adc_scaled(smrx_path): + smrx = open_file(smrx_path) + data = smrx.read_waveform(1) + assert data.dtype == np.float64 + assert data.size == fakesonpy.N_SAMPLES + # First raw sample is -1000; scaled = -1000*scale/6553.6 + offset. + expected0 = -1000 * (2.0 / 6553.6) + 0.5 + assert data[0] == pytest.approx(expected0) + + +def test_read_adc_raw_is_int16(smrx_path): + smrx = open_file(smrx_path) + data = smrx.read_waveform(1, scaled=False) + assert data.dtype == np.int16 + assert data[0] == -1000 + + +def test_read_adc_chunk_start_count(smrx_path): + smrx = open_file(smrx_path) + full = smrx.read_waveform(1, scaled=False) + chunk = smrx.read_waveform(1, start=100, count=50, scaled=False) + assert chunk.size == 50 + np.testing.assert_array_equal(chunk, full[100:150]) + + +def test_read_adc_chunks_tile_full(smrx_path): + smrx = open_file(smrx_path) + full = smrx.read_waveform(1, scaled=False) + pieces = [] + for start in range(0, fakesonpy.N_SAMPLES, 250): + pieces.append(smrx.read_waveform(1, start=start, count=250, scaled=False)) + joined = np.concatenate(pieces) + np.testing.assert_array_equal(joined, full) + + +def test_read_realwave(smrx_path): + smrx = open_file(smrx_path) + data = smrx.read_waveform(3) # RealWave + assert data.dtype == np.float64 + assert data.size == fakesonpy.N_SAMPLES + + +def test_read_waveform_time_window(smrx_path): + smrx = open_file(smrx_path) + # 0 to 0.005 s at 10 kHz -> ~50 samples. + data = smrx.read_waveform(1, t0=0.0, t1=0.005, scaled=False) + assert 49 <= data.size <= 52 + + +def test_read_waveform_on_event_channel_errors(smrx_path): + smrx = open_file(smrx_path) + with pytest.raises(SonpipeError): + smrx.read_waveform(4) + + +# -- event / marker reads --------------------------------------------------- + +def test_read_events_returns_seconds(smrx_path): + smrx = open_file(smrx_path) + times = smrx.read_events(4) + assert times.dtype == np.float64 + # Events every 100 ticks at 1 us/tick -> every 1e-4 s. + assert times[0] == pytest.approx(0.0) + assert times[1] == pytest.approx(1e-4) + + +def test_read_events_time_window(smrx_path): + smrx = open_file(smrx_path) + times = smrx.read_events(4, t0=0.0, t1=0.001) + assert np.all(times <= 0.001 + 1e-12) + + +def test_read_events_chunked_no_truncation(smrx_path): + smrx = open_file(smrx_path) + full = smrx.read_events(4) + chunked = smrx.read_events(4, chunk=7) # tiny chunk forces the read loop + np.testing.assert_allclose(full, chunked) + + +def test_read_markers(smrx_path): + smrx = open_file(smrx_path) + markers = smrx.read_markers(5) # Spike2 #5 -> generic Marker + assert len(markers) > 0 + first = markers[0] + assert first["time"] == pytest.approx(0.0) + assert first["code"] # marker code bytes present + + +def test_read_text_markers(smrx_path): + smrx = open_file(smrx_path) + markers = smrx.read_markers(6) # Spike2 #6 -> TextMark + assert len(markers) > 0 + assert "text" in markers[0] + assert markers[0]["text"].startswith("note") + + +def test_read_markers_on_waveform_errors(smrx_path): + smrx = open_file(smrx_path) + with pytest.raises(SonpipeError): + smrx.read_markers(1)