diff --git a/CHANGELOG.md b/CHANGELOG.md index ab8232b..2e6ce20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,44 @@ All notable changes to OffsetScan are documented in this file. The project follows semantic versioning. +## [Unreleased] + +### Planned + +- `offsetscan rich` — PE Rich-header parsing and rich hash (build-toolchain fingerprint for + clustering / attribution), the fast-follow to 0.3.0's TLSH similarity clustering. + +## [0.3.0] - 2026-07-27 + +### Added + +- **`offsetscan cluster` — TLSH similarity hashing and corpus clustering.** Computes a + canonical TLSH (`T1...`) locality-sensitive digest per file and groups files whose pairwise + TLSH distance is within `--threshold` (default 70; lower = stricter, 0 = identical) into + single-linkage clusters. Emits one record per file — `File`/`FileSize`/`SHA256`/`Tlsh`/ + `ClusterId`/`Warnings` — where `ClusterId` is shared across near-duplicates and null for + singletons. This is the first OffsetScan capability with no OffsetInspect equivalent: it is + inherently corpus-scale, answering "which of these thousands of samples are variants of one + another?" (repacks, malware families, near-duplicates) — variants that a different SHA-256 + hides but TLSH still groups. Verified on real PEs: a modified copy (different SHA-256) still + clusters with its original, while an unrelated system DLL does not. + - Files too small/uniform for TLSH, or unreadable, are reported with a warning and left + unclustered rather than aborting the run. + - Reading and hashing are parallelised (rayon); cluster numbering is deterministic (ordered + by each cluster's smallest member path), independent of input order or thread scheduling. + - Digests carry the `T1` version prefix, so they are directly comparable with other canonical + TLSH tooling. +- Depends on the pure-Rust `tlsh2` crate (`diff` feature) — no C toolchain or `libclang` + required, unlike the optional `yara-scan` feature — so `cargo install offsetscan` keeps + working with no system prerequisites. + +### Notes + +- `cluster` is a whole-corpus operation: unlike the streaming subcommands it needs every file + before assigning cluster ids. `--ndjson` still emits one record per line; `--csv` is not + supported for it (the `Warnings` field is a list). It is a new OffsetScan-only output and + does not participate in the OffsetInspect JSON parity contract. + ## [0.2.3] - 2026-07-21 ### Documentation diff --git a/Cargo.lock b/Cargo.lock index 3d76258..569d468 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -488,7 +488,7 @@ dependencies = [ [[package]] name = "offsetscan" -version = "0.2.3" +version = "0.3.0" dependencies = [ "chrono", "clap", @@ -501,6 +501,7 @@ dependencies = [ "serde_json", "sha1", "sha2", + "tlsh2", "walkdir", "yara", ] @@ -763,6 +764,12 @@ dependencies = [ "syn", ] +[[package]] +name = "tlsh2" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7184dfffa0d05e10284df327c6cb52f0d62c5aa6c9a8b9236fa151d4361882f1" + [[package]] name = "typenum" version = "1.20.1" diff --git a/Cargo.toml b/Cargo.toml index a3117cc..1c9b11a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "offsetscan" -version = "0.2.3" +version = "0.3.0" edition = "2021" authors = ["DreadHost Research"] -description = "Standalone native corpus-scale engine for PE parsing, entropy, string extraction, imphash, and IOC panels — JSON schema-compatible with OffsetInspect's static-triage output (Get-OffsetPEInfo/Get-OffsetEntropy/Get-OffsetString/Get-OffsetIOC)." +description = "Standalone native corpus-scale engine for PE parsing, entropy, string extraction, imphash, and IOC panels (JSON schema-compatible with OffsetInspect's Get-OffsetPEInfo/Get-OffsetEntropy/Get-OffsetString/Get-OffsetIOC), plus TLSH similarity clustering for grouping related samples across a corpus." license = "MIT" repository = "https://github.com/warpedatom/OffsetScan" readme = "README.md" @@ -35,6 +35,10 @@ glob = "0.3" csv = "1" # Timestamps in ISO 8601 to match ThreatScanResult's UTC scan timestamp field chrono = { version = "0.4", features = ["serde"] } +# TLSH locality-sensitive hashing for the `cluster` subcommand. Pure Rust (no C toolchain +# or libclang, unlike the optional yara-scan feature), so `cargo install offsetscan` needs +# no system prerequisites. The `diff` feature provides pairwise distance for clustering. +tlsh2 = { version = "1.1", features = ["diff"] } [dependencies.yara] version = "0.32" diff --git a/README.md b/README.md index ac6983f..225c3e9 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,10 @@ identical panels for the same file — **including the imphash**. The serialized field names (`MD5`/`SHA1`/`SHA256`/`IsPE`/`IsPE32Plus`) are locked by unit tests so the interchange contract can't silently drift. +One subcommand deliberately sits **outside** this contract: `offsetscan cluster` +(TLSH similarity clustering) has no OffsetInspect equivalent — it is inherently +corpus-scale, so it lives only here. See [Similarity clustering](#similarity-clustering). + ### Verified parity Both engines produce the same IOC panel for a file — captured at runtime against the same @@ -141,6 +145,10 @@ offsetscan ioc ./samples --recurse --ndjson # Flat CSV for spreadsheets/SIEM (ioc only; columns match the JSON field names): offsetscan ioc ./samples --recurse --csv > iocs.csv + +# Similarity clustering — group variants/repacks/near-duplicates across a corpus: +offsetscan cluster ./samples --recurse +offsetscan cluster ./samples --recurse --threshold 40 # stricter (fewer, tighter clusters) ``` By default all commands emit a pretty-printed JSON array to stdout, matching @@ -163,6 +171,41 @@ $results | Export-OffsetThreatReport -Path ./engagement.md -IocJsonPath ./ioc.js Because the JSON shape matches `Get-OffsetIOC` field-for-field, any consumer of that shape accepts OffsetScan's output as a drop-in, faster-at-scale alternative. +## Similarity clustering + +`offsetscan cluster` computes a canonical [TLSH](https://github.com/trendmicro/tlsh) +locality-sensitive digest for each file and groups files whose pairwise TLSH distance is +within `--threshold` (default 70; lower = stricter, `0` = identical) into single-linkage +clusters. It answers the question single-file inspection can't: *which of these thousands of +samples are variants of one another?* — repacks, malware families, near-duplicates. A modified +copy has a different SHA-256, so exact-hash dedup misses it; TLSH still places it beside its +original. + +One record per file: + +```json +[ + { "File": "a.bin", "FileSize": 836232, "SHA256": "26410f49…", + "Tlsh": "T187056B2E…", "ClusterId": 1, "Warnings": [] }, + { "File": "b_modified.bin", "FileSize": 836232, "SHA256": "06b26733…", + "Tlsh": "T187056A2E…", "ClusterId": 1, "Warnings": [] }, + { "File": "unrelated.bin", "FileSize": 1881576, "SHA256": "6fcd8218…", + "Tlsh": "T1B8956C5D…", "ClusterId": null, "Warnings": [] } +] +``` + +`ClusterId` is shared across near-duplicates and `null` for a singleton. Files too small or too +uniform for TLSH (and unreadable files) come back with `Tlsh: null` and a `Warnings` entry +rather than failing the run. Digests carry the `T1` version prefix, so they compare directly +with other canonical TLSH tooling. Clustering is deterministic — cluster numbering is ordered by +each cluster's smallest member path, so output doesn't depend on input order or thread timing. + +Notes: `cluster` is a whole-corpus operation (every file is hashed before ids are assigned), so +unlike the other subcommands it isn't streamed as files finish; `--ndjson` still emits one record +per line, and `--csv` isn't supported for it. Pairwise comparison is `O(n²)` — sized for the +few-thousand-file corpora this targets. It depends only on the pure-Rust `tlsh2` crate, so +`cargo install offsetscan` needs no system prerequisites (unlike the `yara-scan` feature). + ## What's intentionally NOT here - AMSI / Microsoft Defender scanning — stays in OffsetInspect (Windows-only, diff --git a/src/cluster.rs b/src/cluster.rs new file mode 100644 index 0000000..c65bd1c --- /dev/null +++ b/src/cluster.rs @@ -0,0 +1,342 @@ +//! `cluster` — TLSH similarity hashing and single-linkage clustering for corpus-scale triage. +//! +//! Computes a canonical TLSH (`T1...`) locality-sensitive digest per file and groups files +//! whose pairwise TLSH distance is within a threshold into single-linkage clusters. This is +//! the first OffsetScan capability with no OffsetInspect equivalent: it is inherently +//! corpus-scale, answering "which of these thousands of samples are variants of one another?" +//! (repacks, malware families, near-duplicates) — a question a single-file inspector cannot. +//! +//! It is OffsetScan-only output and does NOT participate in the OffsetInspect JSON parity +//! contract (OffsetInspect emits no TLSH object), so it lives outside `schema.rs`. + +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::path::PathBuf; +use tlsh2::TlshDefaultBuilder; + +const TLSH_NULL_WARNING: &str = + "input too small or too uniform for TLSH (needs ~50 bytes of varied data)"; + +/// One file's similarity record. +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct ClusterRecord { + pub file: String, + pub file_size: u64, + #[serde(rename = "SHA256")] + pub sha256: String, + /// Canonical TLSH digest (with the `T1` version prefix), or null when the file is too + /// small / too uniform for TLSH to produce one. + pub tlsh: Option, + /// 1-based cluster id shared by files within `threshold` TLSH distance of each other + /// (single-linkage). null when the file has no near neighbour (a singleton) or no digest. + pub cluster_id: Option, + pub warnings: Vec, +} + +/// Read files in parallel, then cluster. Read errors become a per-file warning rather than +/// aborting the run — a corpus scan should survive one unreadable sample. +pub fn run(files: Vec, threshold: i32) -> Vec { + let reads: Vec<(String, Result, String>)> = files + .par_iter() + .map(|f| { + ( + f.to_string_lossy().to_string(), + std::fs::read(f).map_err(|e| e.to_string()), + ) + }) + .collect(); + cluster_reads(reads, threshold) +} + +/// Core, I/O-free so it is directly unit-testable: given each file's name and bytes (or a +/// read error), compute TLSH digests and assign single-linkage cluster ids. +fn cluster_reads( + reads: Vec<(String, Result, String>)>, + threshold: i32, +) -> Vec { + // Phase 1 (parallel): hash + digest each input. Keep the TLSH object alongside its record + // for pairwise diffing; its concrete type is inferred (tlsh2 exposes no public alias). + let entries = reads + .par_iter() + .map(|(file, res)| match res { + Err(e) => ( + ClusterRecord { + file: file.clone(), + file_size: 0, + sha256: String::new(), + tlsh: None, + cluster_id: None, + warnings: vec![format!("read failed: {e}")], + }, + None, + ), + Ok(data) => { + let sha256 = sha256_hex(data); + let mut builder = TlshDefaultBuilder::new(); + builder.update(data); + match builder.build() { + Some(tlsh) => { + let bytes = tlsh.hash(); + let digest = String::from_utf8_lossy(&bytes) + .trim_end_matches('\0') + .to_string(); + ( + ClusterRecord { + file: file.clone(), + file_size: data.len() as u64, + sha256, + tlsh: Some(digest), + cluster_id: None, + warnings: Vec::new(), + }, + Some(tlsh), + ) + } + None => ( + ClusterRecord { + file: file.clone(), + file_size: data.len() as u64, + sha256, + tlsh: None, + cluster_id: None, + warnings: vec![TLSH_NULL_WARNING.to_string()], + }, + None, + ), + } + } + }) + .collect::>(); + + let (mut records, objs): (Vec, Vec<_>) = entries.into_iter().unzip(); + let n = records.len(); + + // Phase 2: discover edges (pairs within `threshold`) in parallel, then union sequentially. + // O(n^2) pairwise diffs — fine for the few-thousand-file corpora this targets; documented. + let objs_ref = &objs; + let edges: Vec<(usize, usize)> = (0..n) + .into_par_iter() + .flat_map_iter(move |i| { + let a = &objs_ref[i]; + ((i + 1)..n).filter_map(move |j| match (a, &objs_ref[j]) { + (Some(x), Some(y)) if x.diff(y, true) <= threshold => Some((i, j)), + _ => None, + }) + }) + .collect(); + + let mut uf = UnionFind::new(n); + for (i, j) in edges { + uf.union(i, j); + } + + // Phase 3: group indices that have a digest by their component root, keep components of + // size >= 2 as clusters, and number them deterministically (by each cluster's smallest + // file path) so output is independent of input order and thread scheduling. + let mut comps: HashMap> = HashMap::new(); + for (i, o) in objs.iter().enumerate() { + if o.is_some() { + let root = uf.find(i); + comps.entry(root).or_default().push(i); + } + } + let mut clusters: Vec> = comps.into_values().filter(|v| v.len() >= 2).collect(); + clusters.sort_by(|a, b| { + let ka = a.iter().map(|&i| &records[i].file).min(); + let kb = b.iter().map(|&i| &records[i].file).min(); + ka.cmp(&kb) + }); + for (idx, members) in clusters.iter().enumerate() { + let id = (idx as u32) + 1; + for &i in members { + records[i].cluster_id = Some(id); + } + } + + records +} + +fn sha256_hex(data: &[u8]) -> String { + let mut h = Sha256::new(); + h.update(data); + let digest = h.finalize(); + let mut out = String::with_capacity(64); + for b in digest { + out.push_str(&format!("{b:02x}")); + } + out +} + +/// Minimal union-find (disjoint set) with path compression and union by rank. +struct UnionFind { + parent: Vec, + rank: Vec, +} + +impl UnionFind { + fn new(n: usize) -> Self { + Self { + parent: (0..n).collect(), + rank: vec![0; n], + } + } + + fn find(&mut self, x: usize) -> usize { + let mut root = x; + while self.parent[root] != root { + root = self.parent[root]; + } + // Path compression. + let mut cur = x; + while self.parent[cur] != cur { + let next = self.parent[cur]; + self.parent[cur] = root; + cur = next; + } + root + } + + fn union(&mut self, a: usize, b: usize) { + let (ra, rb) = (self.find(a), self.find(b)); + if ra == rb { + return; + } + match self.rank[ra].cmp(&self.rank[rb]) { + std::cmp::Ordering::Less => self.parent[ra] = rb, + std::cmp::Ordering::Greater => self.parent[rb] = ra, + std::cmp::Ordering::Equal => { + self.parent[rb] = ra; + self.rank[ra] += 1; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Deterministic pseudo-random bytes: varied enough for TLSH to produce a digest. + fn varied(len: usize, seed: u64) -> Vec { + let mut x = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1); + (0..len) + .map(|_| { + x = x + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + (x >> 33) as u8 + }) + .collect() + } + + fn ok(name: &str, data: Vec) -> (String, Result, String>) { + (name.to_string(), Ok(data)) + } + + fn cluster_of(recs: &[ClusterRecord], name: &str) -> Option { + recs.iter().find(|r| r.file == name).unwrap().cluster_id + } + + #[test] + fn digest_is_deterministic_and_carries_t1_prefix() { + let data = varied(2048, 1); + let a = cluster_reads(vec![ok("a", data.clone())], 70); + let b = cluster_reads(vec![ok("a", data)], 70); + let digest = a[0].tlsh.as_ref().expect("expected a TLSH digest"); + assert_eq!( + a[0].tlsh, b[0].tlsh, + "same bytes must produce the same digest" + ); + assert!( + digest.starts_with("T1"), + "digest should carry the canonical T1 version prefix: {digest}" + ); + assert!(digest.len() >= 70, "unexpectedly short digest: {digest}"); + } + + #[test] + fn identical_files_share_a_cluster() { + let data = varied(4096, 2); + let recs = cluster_reads(vec![ok("a", data.clone()), ok("b", data)], 70); + assert!(recs[0].cluster_id.is_some()); + assert_eq!(recs[0].cluster_id, recs[1].cluster_id); + } + + #[test] + fn near_duplicate_shares_cluster_with_original() { + let a = varied(8192, 3); + let mut a2 = a.clone(); + // Perturb a small fraction of bytes: still a near-duplicate to TLSH. + for k in (0..a2.len()).step_by(997) { + a2[k] ^= 0xFF; + } + let recs = cluster_reads(vec![ok("a", a), ok("a2", a2)], 70); + assert!( + recs[0].cluster_id.is_some(), + "near-duplicates should cluster" + ); + assert_eq!(recs[0].cluster_id, recs[1].cluster_id); + } + + #[test] + fn cluster_ids_are_independent_of_input_order() { + let a = varied(4096, 5); + let b = a.clone(); // identical to a -> must always co-cluster + let c = varied(4096, 6); + let forward = cluster_reads( + vec![ok("a", a.clone()), ok("b", b.clone()), ok("c", c.clone())], + 70, + ); + let reversed = cluster_reads(vec![ok("c", c), ok("b", b), ok("a", a)], 70); + // a and b co-cluster regardless of order... + assert_eq!(cluster_of(&forward, "a"), cluster_of(&forward, "b")); + assert_eq!(cluster_of(&reversed, "a"), cluster_of(&reversed, "b")); + // ...and the deterministic (min-path) numbering gives them the same id both ways. + assert_eq!(cluster_of(&forward, "a"), cluster_of(&reversed, "a")); + } + + #[test] + fn tiny_input_yields_no_tlsh_with_warning() { + let recs = cluster_reads(vec![ok("t", vec![1, 2, 3, 4])], 70); + assert!(recs[0].tlsh.is_none()); + assert!(recs[0].cluster_id.is_none()); + assert!(!recs[0].warnings.is_empty()); + } + + #[test] + fn read_error_is_reported_and_not_clustered() { + let recs = cluster_reads(vec![("x".to_string(), Err("boom".to_string()))], 70); + assert!(recs[0].tlsh.is_none()); + assert!(recs[0].cluster_id.is_none()); + assert_eq!(recs[0].warnings.len(), 1); + assert!(recs[0].warnings[0].contains("boom")); + } + + #[test] + fn record_serializes_with_offsetinspect_style_field_names() { + let rec = ClusterRecord { + file: "f".into(), + file_size: 1, + sha256: "h".into(), + tlsh: Some("T1XX".into()), + cluster_id: Some(1), + warnings: vec![], + }; + let json = serde_json::to_string(&rec).unwrap(); + for key in [ + "\"File\":", + "\"FileSize\":", + "\"SHA256\":", + "\"Tlsh\":", + "\"ClusterId\":", + "\"Warnings\":", + ] { + assert!(json.contains(key), "missing {key} in {json}"); + } + // The wrong PascalCase casing for SHA256 must not appear. + assert!(!json.contains("\"Sha256\":")); + } +} diff --git a/src/main.rs b/src/main.rs index f629e8b..126e902 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +mod cluster; mod entropy; mod ioc; mod ordinals; @@ -69,6 +70,17 @@ enum Commands { #[arg(long)] recurse: bool, }, + /// TLSH similarity hashing + single-linkage clustering across a corpus. Groups files + /// that are variants of one another (repacks, families, near-duplicates). + Cluster { + /// File or directory (use --recurse for directories) or glob pattern. + path: String, + #[arg(long)] + recurse: bool, + /// Max TLSH distance for two files to share a cluster (lower = stricter; 0 = identical). + #[arg(long, default_value_t = 70)] + threshold: i32, + }, /// Match files against YARA rules; one record per matched string with its offset. /// Requires a build with `--features yara-scan`. Yara { @@ -198,6 +210,23 @@ fn main() { }); } } + Commands::Cluster { + path, + recurse, + threshold, + } => { + let files = expand_paths(&path, recurse); + // Whole-corpus operation: every file must be hashed before cluster ids can be + // assigned, so results are computed together rather than streamed per file. + let records = cluster::run(files, threshold); + if ndjson { + for record in &records { + emit_ndjson_line(record); + } + } else { + print_json(&records); + } + } Commands::Yara { path, rules,