warpedatom · warpedatom · Jul 27, 2026 · Jul 27, 2026 · sourcery-ai · Jul 27, 2026
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,44 @@
 All notable changes to OffsetScan are documented in this file.
 The project follows semantic versioning.
 
+## [Unreleased]
+
+### Planned
+
+- `offsetscan rich` — PE Rich-header parsing and rich hash (build-toolchain fingerprint for
+  clustering / attribution), the fast-follow to 0.3.0's TLSH similarity clustering.
+
+## [0.3.0] - 2026-07-27
+
+### Added
+
+- **`offsetscan cluster` — TLSH similarity hashing and corpus clustering.** Computes a
+  canonical TLSH (`T1...`) locality-sensitive digest per file and groups files whose pairwise
+  TLSH distance is within `--threshold` (default 70; lower = stricter, 0 = identical) into
+  single-linkage clusters. Emits one record per file — `File`/`FileSize`/`SHA256`/`Tlsh`/
+  `ClusterId`/`Warnings` — where `ClusterId` is shared across near-duplicates and null for
+  singletons. This is the first OffsetScan capability with no OffsetInspect equivalent: it is
+  inherently corpus-scale, answering "which of these thousands of samples are variants of one
+  another?" (repacks, malware families, near-duplicates) — variants that a different SHA-256
+  hides but TLSH still groups. Verified on real PEs: a modified copy (different SHA-256) still
+  clusters with its original, while an unrelated system DLL does not.
+  - Files too small/uniform for TLSH, or unreadable, are reported with a warning and left
+    unclustered rather than aborting the run.
+  - Reading and hashing are parallelised (rayon); cluster numbering is deterministic (ordered
+    by each cluster's smallest member path), independent of input order or thread scheduling.
+  - Digests carry the `T1` version prefix, so they are directly comparable with other canonical
+    TLSH tooling.
+- Depends on the pure-Rust `tlsh2` crate (`diff` feature) — no C toolchain or `libclang`
+  required, unlike the optional `yara-scan` feature — so `cargo install offsetscan` keeps
+  working with no system prerequisites.
+
+### Notes
+
+- `cluster` is a whole-corpus operation: unlike the streaming subcommands it needs every file
+  before assigning cluster ids. `--ndjson` still emits one record per line; `--csv` is not
+  supported for it (the `Warnings` field is a list). It is a new OffsetScan-only output and
+  does not participate in the OffsetInspect JSON parity contract.
+
 ## [0.2.3] - 2026-07-21
 
 ### Documentation

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,9 +1,9 @@
 [package]
 name = "offsetscan"
-version = "0.2.3"
+version = "0.3.0"
 edition = "2021"
 authors = ["DreadHost Research"]
-description = "Standalone native corpus-scale engine for PE parsing, entropy, string extraction, imphash, and IOC panels — JSON schema-compatible with OffsetInspect's static-triage output (Get-OffsetPEInfo/Get-OffsetEntropy/Get-OffsetString/Get-OffsetIOC)."
+description = "Standalone native corpus-scale engine for PE parsing, entropy, string extraction, imphash, and IOC panels (JSON schema-compatible with OffsetInspect's Get-OffsetPEInfo/Get-OffsetEntropy/Get-OffsetString/Get-OffsetIOC), plus TLSH similarity clustering for grouping related samples across a corpus."
 license = "MIT"
 repository = "https://github.com/warpedatom/OffsetScan"
 readme = "README.md"
@@ -35,6 +35,10 @@ glob = "0.3"
 csv = "1"
 # Timestamps in ISO 8601 to match ThreatScanResult's UTC scan timestamp field
 chrono = { version = "0.4", features = ["serde"] }
+# TLSH locality-sensitive hashing for the `cluster` subcommand. Pure Rust (no C toolchain
+# or libclang, unlike the optional yara-scan feature), so `cargo install offsetscan` needs
+# no system prerequisites. The `diff` feature provides pairwise distance for clustering.
+tlsh2 = { version = "1.1", features = ["diff"] }
 
 [dependencies.yara]
 version = "0.32"

diff --git a/README.md b/README.md
@@ -49,6 +49,10 @@ identical panels for the same file — **including the imphash**. The
 serialized field names (`MD5`/`SHA1`/`SHA256`/`IsPE`/`IsPE32Plus`) are locked
 by unit tests so the interchange contract can't silently drift.
 
+One subcommand deliberately sits **outside** this contract: `offsetscan cluster`
+(TLSH similarity clustering) has no OffsetInspect equivalent — it is inherently
+corpus-scale, so it lives only here. See [Similarity clustering](#similarity-clustering).
+
 ### Verified parity
 
 Both engines produce the same IOC panel for a file — captured at runtime against the same
@@ -141,6 +145,10 @@ offsetscan ioc ./samples --recurse --ndjson
 
 # Flat CSV for spreadsheets/SIEM (ioc only; columns match the JSON field names):
 offsetscan ioc ./samples --recurse --csv > iocs.csv
+
+# Similarity clustering — group variants/repacks/near-duplicates across a corpus:
+offsetscan cluster ./samples --recurse
+offsetscan cluster ./samples --recurse --threshold 40   # stricter (fewer, tighter clusters)
 ```
 
 By default all commands emit a pretty-printed JSON array to stdout, matching
@@ -163,6 +171,41 @@ $results | Export-OffsetThreatReport -Path ./engagement.md -IocJsonPath ./ioc.js
 Because the JSON shape matches `Get-OffsetIOC` field-for-field, any consumer of
 that shape accepts OffsetScan's output as a drop-in, faster-at-scale alternative.
 
+## Similarity clustering
+
+`offsetscan cluster` computes a canonical [TLSH](https://github.com/trendmicro/tlsh)
+locality-sensitive digest for each file and groups files whose pairwise TLSH distance is
+within `--threshold` (default 70; lower = stricter, `0` = identical) into single-linkage
+clusters. It answers the question single-file inspection can't: *which of these thousands of
+samples are variants of one another?* — repacks, malware families, near-duplicates. A modified
+copy has a different SHA-256, so exact-hash dedup misses it; TLSH still places it beside its
+original.
+
+One record per file:
+
+```json
+[
+  { "File": "a.bin", "FileSize": 836232, "SHA256": "26410f49…",
+    "Tlsh": "T187056B2E…", "ClusterId": 1, "Warnings": [] },
+  { "File": "b_modified.bin", "FileSize": 836232, "SHA256": "06b26733…",
+    "Tlsh": "T187056A2E…", "ClusterId": 1, "Warnings": [] },
+  { "File": "unrelated.bin", "FileSize": 1881576, "SHA256": "6fcd8218…",
+    "Tlsh": "T1B8956C5D…", "ClusterId": null, "Warnings": [] }
+]
+```
+
+`ClusterId` is shared across near-duplicates and `null` for a singleton. Files too small or too
-`ClusterId` is shared across near-duplicates and `null` for a singleton. Files too small or too
+`ClusterId` is shared across near-duplicates and is `null` for a singleton. Files too small or too
-`ClusterId` is shared across near-duplicates and `null` for a singleton. Files too small or too
+`ClusterId` is shared across near-duplicates and is `null` for a singleton. Files too small or too
+uniform for TLSH (and unreadable files) come back with `Tlsh: null` and a `Warnings` entry
+rather than failing the run. Digests carry the `T1` version prefix, so they compare directly
+with other canonical TLSH tooling. Clustering is deterministic — cluster numbering is ordered by
+each cluster's smallest member path, so output doesn't depend on input order or thread timing.
+
+Notes: `cluster` is a whole-corpus operation (every file is hashed before ids are assigned), so
+unlike the other subcommands it isn't streamed as files finish; `--ndjson` still emits one record
+per line, and `--csv` isn't supported for it. Pairwise comparison is `O(n²)` — sized for the
+few-thousand-file corpora this targets. It depends only on the pure-Rust `tlsh2` crate, so
+`cargo install offsetscan` needs no system prerequisites (unlike the `yara-scan` feature).
+
 ## What's intentionally NOT here
 
 - AMSI / Microsoft Defender scanning — stays in OffsetInspect (Windows-only,