diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..a658bd7 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# Sponsor button for this repository (GitHub Sponsors). +# https://docs.github.com/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/displaying-a-sponsor-button-in-your-repository +github: warpedatom diff --git a/.gitignore b/.gitignore index 1f8bf98..7335499 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,4 @@ .DS_Store Thumbs.db -# Keep Cargo.lock — this crate builds a binary, so the lock is committed. +# Keep Cargo.lock - this crate builds a binary, so the lock is committed. diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e6ce20..d225af0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,25 +3,39 @@ All notable changes to OffsetScan are documented in this file. The project follows semantic versioning. -## [Unreleased] +## [0.4.0] - 2026-07-27 -### Planned +### Added -- `offsetscan rich` — PE Rich-header parsing and rich hash (build-toolchain fingerprint for - clustering / attribution), the fast-follow to 0.3.0's TLSH similarity clustering. +- **`offsetscan rich` - PE Rich-header parsing and a build-toolchain fingerprint hash.** Parses + the undocumented Rich header the Microsoft linker writes between the DOS stub and the PE + header, reporting its `@comp.id` records (`ProductId`/`BuildId`/`Count`) and a `RichHash`. + `RichHash` is the MD5 of the decrypted Rich array (the `DanS` marker, its padding, and every + comp.id/count record - pefile's `RICH_HEADER.clear_data`), deliberately excluding the per-file + XOR key and the `Rich`/checksum trailer, so it is invariant across binaries built with the same + toolchain - a clustering / attribution primitive that complements imphash and TLSH. + - **Verified byte-identical to pefile:** `md5(pe.RICH_HEADER.clear_data)` matched OffsetScan's + `RichHash` on 118/118 System32 DLLs carrying a Rich header - 0 mismatches, 0 presence + disagreements. + - One record per file - `File`/`FileSize`/`HasRichHeader`/`RichHash`/`XorKey`/`EntryCount`/ + `Entries`/`Warnings`. A missing Rich header (non-MSVC or stripped binaries) is reported + cleanly with `HasRichHeader: false`, not as an error; a coincidental `Rich` marker with no + valid `DanS` behind it is skipped in favour of the genuine header. + - Per-file and streaming (`--ndjson`); no new dependencies (reuses `md-5`). OffsetScan-only - + no OffsetInspect equivalent, so it sits outside the JSON parity contract. ## [0.3.0] - 2026-07-27 ### Added -- **`offsetscan cluster` — TLSH similarity hashing and corpus clustering.** Computes a +- **`offsetscan cluster` - TLSH similarity hashing and corpus clustering.** Computes a canonical TLSH (`T1...`) locality-sensitive digest per file and groups files whose pairwise TLSH distance is within `--threshold` (default 70; lower = stricter, 0 = identical) into - single-linkage clusters. Emits one record per file — `File`/`FileSize`/`SHA256`/`Tlsh`/ - `ClusterId`/`Warnings` — where `ClusterId` is shared across near-duplicates and null for + single-linkage clusters. Emits one record per file - `File`/`FileSize`/`SHA256`/`Tlsh`/ + `ClusterId`/`Warnings` - where `ClusterId` is shared across near-duplicates and null for singletons. This is the first OffsetScan capability with no OffsetInspect equivalent: it is inherently corpus-scale, answering "which of these thousands of samples are variants of one - another?" (repacks, malware families, near-duplicates) — variants that a different SHA-256 + another?" (repacks, malware families, near-duplicates) - variants that a different SHA-256 hides but TLSH still groups. Verified on real PEs: a modified copy (different SHA-256) still clusters with its original, while an unrelated system DLL does not. - Files too small/uniform for TLSH, or unreadable, are reported with a warning and left @@ -30,8 +44,8 @@ The project follows semantic versioning. by each cluster's smallest member path), independent of input order or thread scheduling. - Digests carry the `T1` version prefix, so they are directly comparable with other canonical TLSH tooling. -- Depends on the pure-Rust `tlsh2` crate (`diff` feature) — no C toolchain or `libclang` - required, unlike the optional `yara-scan` feature — so `cargo install offsetscan` keeps +- Depends on the pure-Rust `tlsh2` crate (`diff` feature) - no C toolchain or `libclang` + required, unlike the optional `yara-scan` feature - so `cargo install offsetscan` keeps working with no system prerequisites. ### Notes @@ -46,7 +60,7 @@ The project follows semantic versioning. ### Documentation - Correct the crate description's schema reference. It previously claimed compatibility with - `OffsetInspect.Result`/`ThreatScanResult` — objects produced by `Invoke-OffsetInspect` and + `OffsetInspect.Result`/`ThreatScanResult` - objects produced by `Invoke-OffsetInspect` and `Invoke-OffsetThreatScan`, which OffsetScan does not emit. It is schema-compatible with the static-triage output (`Get-OffsetPEInfo`/`Get-OffsetEntropy`/`Get-OffsetString`/`Get-OffsetIOC`). No code changes. @@ -77,8 +91,8 @@ The project follows semantic versioning. ### Added - **`offsetscan yara` subcommand** (feature-gated behind `yara-scan`, off by default). - Matches files against YARA rule files and emits one record per matched string — - `File`/`Rule`/`StringId`/`Offset`/`OffsetHex`/`Data` — **verified byte-identical to + Matches files against YARA rule files and emits one record per matched string - + `File`/`Rule`/`StringId`/`Offset`/`OffsetHex`/`Data` - **verified byte-identical to OffsetInspect's `Invoke-OffsetYaraScan`** on the same rules and sample. Supports multiple `--rules` files, `--recurse`, a `--timeout`, and `--ndjson`. A default build (no feature) exposes the subcommand but returns a clear rebuild instruction. Schema field names are @@ -98,7 +112,7 @@ The project follows semantic versioning. ### Fixed - The strict PE path computed the overlay boundary **without** filtering out zero-raw-size - sections, unlike OffsetInspect's `Get-OIPEOverlayRange` and the lenient path — a latent + sections, unlike OffsetInspect's `Get-OIPEOverlayRange` and the lenient path - a latent inconsistency that could misplace the overlay for PEs with a zero-raw-size section (e.g. some packers' virtual sections). Both paths now use the filtered computation. @@ -107,12 +121,12 @@ The project follows semantic versioning. ### Fixed - **Structurally-damaged PEs are no longer rejected outright.** goblin does strict - full-structure validation, so a truncated or carved sample — headers intact but section - data cut off, common with partial downloads and file-carving — was reported as + full-structure validation, so a truncated or carved sample - headers intact but section + data cut off, common with partial downloads and file-carving - was reported as `IsPE: false` with every PE field null, losing all triage value. `parse_pe` now falls back to a lenient header-salvage parse (mirroring OffsetInspect's `ConvertTo-OIPEImage`) when goblin rejects a file, recovering machine, sections, entry point, overlay, and any - reachable imports/imphash. Valid PEs are unaffected — they always take the goblin path, + reachable imports/imphash. Valid PEs are unaffected - they always take the goblin path, verified unchanged across a 150-file corpus vs pefile. Across a 34-file malformed corpus (truncations, bit-flips, corrupted directories, bogus section counts, pure garbage), `IsPE` now agrees with OffsetInspect on all 34 (was 22/34), and neither engine crashes nor @@ -130,8 +144,8 @@ The project follows semantic versioning. - **imphash now resolves ordinal imports from `ws2_32`/`wsock32`/`oleaut32` to their real function names, matching pefile and VirusTotal.** Previously these rendered as `ord`, - so any binary importing those libraries by ordinal — a common malware networking - pattern — produced an imphash that did not match the value on VirusTotal, defeating the + so any binary importing those libraries by ordinal - a common malware networking + pattern - produced an imphash that did not match the value on VirusTotal, defeating the imphash's purpose of correlating a sample against threat intel. Verified against pefile across a 150-file corpus (44 exercising the special-ordinal path): 0 mismatches, and 15 binaries that previously diverged now agree. Other ordinal imports still render `ord`, @@ -170,9 +184,9 @@ The project follows semantic versioning. units and silently skipped any UTF-16LE string beginning at an odd offset. It now resynchronizes one byte at a time after a non-match, mirroring `Get-OIByteString`. This under-reported `PrintableStringCount` in the `ioc` panel and omitted hits from - `strings` — on `user32.dll`, 36 of 366 UTF-16LE strings were missing. Verified after + `strings` - on `user32.dll`, 36 of 366 UTF-16LE strings were missing. Verified after the fix: `offsetscan strings` and `Get-OffsetString` return set-identical results - (offset, encoding, and value) for `ntdll.dll` — 32,506 strings, zero differences on + (offset, encoding, and value) for `ntdll.dll` - 32,506 strings, zero differences on either side. - The UTF-16LE printability test now explicitly requires a zero high byte rather than range-checking the assembled `u16`, matching the reference implementation. diff --git a/Cargo.lock b/Cargo.lock index 569d468..1055b6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -488,7 +488,7 @@ dependencies = [ [[package]] name = "offsetscan" -version = "0.3.0" +version = "0.4.0" dependencies = [ "chrono", "clap", diff --git a/Cargo.toml b/Cargo.toml index 1c9b11a..eadb1ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "offsetscan" -version = "0.3.0" +version = "0.4.0" edition = "2021" authors = ["DreadHost Research"] description = "Standalone native corpus-scale engine for PE parsing, entropy, string extraction, imphash, and IOC panels (JSON schema-compatible with OffsetInspect's Get-OffsetPEInfo/Get-OffsetEntropy/Get-OffsetString/Get-OffsetIOC), plus TLSH similarity clustering for grouping related samples across a corpus." @@ -26,7 +26,7 @@ serde_json = "1" md-5 = "0.10" sha1 = "0.10" sha2 = "0.10" -# Parallel corpus scanning (thousands of files) — the actual perf payoff over PowerShell +# Parallel corpus scanning (thousands of files) - the actual perf payoff over PowerShell rayon = "1" # Directory/wildcard expansion, matching Invoke-OffsetThreatScanBatch's file-discovery semantics walkdir = "2" diff --git a/README.md b/README.md index 225c3e9..acd629f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # OffsetScan

- OffsetScan — native corpus-scale static-triage engine + OffsetScan - native corpus-scale static-triage engine

@@ -23,7 +23,7 @@ great for single-file, interactive analysis. OffsetScan exists for the other end of the workload: **thousands of files**, where PowerShell's per-file overhead adds up and a parallel, no-GC native core pays for itself. -OffsetScan does not touch AMSI or Microsoft Defender — that stays exactly +OffsetScan does not touch AMSI or Microsoft Defender - that stays exactly where it belongs, as Windows-only functionality in OffsetInspect. OffsetScan only does the read-only, cross-platform static-analysis layer: PE parsing, entropy, string extraction, hashing, and (optionally) YARA matching. @@ -45,17 +45,18 @@ interchangeable at the JSON layer: The struct definitions have been verified field-for-field against the authoritative `docs/OUTPUT-SCHEMA.md` and the real OffsetInspect 3.x objects, and cross-checked at runtime: `offsetscan ioc` and `Get-OffsetIOC` produce -identical panels for the same file — **including the imphash**. The +identical panels for the same file - **including the imphash**. The serialized field names (`MD5`/`SHA1`/`SHA256`/`IsPE`/`IsPE32Plus`) are locked by unit tests so the interchange contract can't silently drift. -One subcommand deliberately sits **outside** this contract: `offsetscan cluster` -(TLSH similarity clustering) has no OffsetInspect equivalent — it is inherently -corpus-scale, so it lives only here. See [Similarity clustering](#similarity-clustering). +Two subcommands deliberately sit **outside** this contract - neither has an OffsetInspect +equivalent, so they live only here: `offsetscan cluster` (TLSH similarity clustering, inherently +corpus-scale) and `offsetscan rich` (PE Rich-header build-toolchain fingerprint). See +[Similarity clustering](#similarity-clustering) and [Rich header](#rich-header). ### Verified parity -Both engines produce the same IOC panel for a file — captured at runtime against the same +Both engines produce the same IOC panel for a file - captured at runtime against the same Windows system DLL. `offsetscan ioc C:\Windows\System32\kernel32.dll`: @@ -109,13 +110,13 @@ while `ConvertTo-Json` emits a bare object with Windows `\` paths. String extraction is compared as a set, not just a count. For `ntdll.dll` (2,517,928 bytes), `offsetscan strings` and `Get-OffsetString` both return -**32,506** hits that agree exactly on offset, encoding, and value — zero entries +**32,506** hits that agree exactly on offset, encoding, and value - zero entries unique to either engine. One caveat when reproducing this: `Get-OffsetString` reads in 1 MiB windows by default and a string straddling a window seam is split in two, which inflates PowerShell's count by one per affected seam on files larger than the window. -That is an artifact of the windowed read, not a disagreement about content — +That is an artifact of the windowed read, not a disagreement about content - pass a `-WindowSize` large enough to hold the file to compare like for like. OffsetScan reads the whole file and is unaffected. @@ -139,24 +140,28 @@ offsetscan ioc ./sample.exe # Corpus mode (any subcommand): offsetscan ioc ./samples --recurse -# Streaming output for large corpora — one compact JSON object per line, +# Streaming output for large corpora - one compact JSON object per line, # emitted as each file finishes so peak memory stays flat: offsetscan ioc ./samples --recurse --ndjson # Flat CSV for spreadsheets/SIEM (ioc only; columns match the JSON field names): offsetscan ioc ./samples --recurse --csv > iocs.csv -# Similarity clustering — group variants/repacks/near-duplicates across a corpus: +# Similarity clustering - group variants/repacks/near-duplicates across a corpus: offsetscan cluster ./samples --recurse offsetscan cluster ./samples --recurse --threshold 40 # stricter (fewer, tighter clusters) + +# Rich-header build-toolchain fingerprint (RichHash + comp.id records): +offsetscan rich ./sample.exe +offsetscan rich ./samples --recurse --ndjson ``` By default all commands emit a pretty-printed JSON array to stdout, matching OffsetInspect's JSON-mode convention (always an array, even for one result). -Add `--ndjson` for newline-delimited JSON — pipe-friendly and constant-memory +Add `--ndjson` for newline-delimited JSON - pipe-friendly and constant-memory over hundreds of thousands of files. Add `--csv` (the `ioc` subcommand only, as the others produce nested data) for a flat header + one-row-per-file table whose -columns match the JSON field names — ready for Excel or a SIEM import. +columns match the JSON field names - ready for Excel or a SIEM import. ## Consuming from PowerShell @@ -177,7 +182,7 @@ that shape accepts OffsetScan's output as a drop-in, faster-at-scale alternative locality-sensitive digest for each file and groups files whose pairwise TLSH distance is within `--threshold` (default 70; lower = stricter, `0` = identical) into single-linkage clusters. It answers the question single-file inspection can't: *which of these thousands of -samples are variants of one another?* — repacks, malware families, near-duplicates. A modified +samples are variants of one another?* - repacks, malware families, near-duplicates. A modified copy has a different SHA-256, so exact-hash dedup misses it; TLSH still places it beside its original. @@ -197,22 +202,49 @@ One record per file: `ClusterId` is shared across near-duplicates and `null` for a singleton. Files too small or too uniform for TLSH (and unreadable files) come back with `Tlsh: null` and a `Warnings` entry rather than failing the run. Digests carry the `T1` version prefix, so they compare directly -with other canonical TLSH tooling. Clustering is deterministic — cluster numbering is ordered by +with other canonical TLSH tooling. Clustering is deterministic - cluster numbering is ordered by each cluster's smallest member path, so output doesn't depend on input order or thread timing. Notes: `cluster` is a whole-corpus operation (every file is hashed before ids are assigned), so unlike the other subcommands it isn't streamed as files finish; `--ndjson` still emits one record -per line, and `--csv` isn't supported for it. Pairwise comparison is `O(n²)` — sized for the +per line, and `--csv` isn't supported for it. Pairwise comparison is `O(n²)` - sized for the few-thousand-file corpora this targets. It depends only on the pure-Rust `tlsh2` crate, so `cargo install offsetscan` needs no system prerequisites (unlike the `yara-scan` feature). +## Rich header + +`offsetscan rich` parses the undocumented **Rich header** the Microsoft linker writes between +the DOS stub and the PE header, and reports a build-toolchain fingerprint. The header lists +`@comp.id` records - which build tool (`ProductId`), which build (`BuildId`), and how many +objects it contributed (`Count`) - so two samples compiled in the same environment share it even +when little else lines up. Useful for clustering and attribution alongside imphash and TLSH. + +```json +[ + { "File": "sample.exe", "FileSize": 836232, "HasRichHeader": true, + "RichHash": "1f67254359760775d87ef504d2f80fb1", "XorKey": "0x54B34DDD", + "EntryCount": 10, + "Entries": [ { "ProductId": 257, "BuildId": 33145, "Count": 4 }, … ], + "Warnings": [] } +] +``` + +`RichHash` is the MD5 of the **decrypted** Rich array (the `DanS` marker, its padding, and every +comp.id/count record - pefile's `RICH_HEADER.clear_data`). It deliberately excludes the per-file +XOR key and the `Rich`/checksum trailer, so it is **invariant across binaries built with the same +toolchain**, which is what makes it a clustering key. It is **verified byte-identical to pefile**: +`md5(pe.RICH_HEADER.clear_data)` matched OffsetScan's `RichHash` on 118/118 System32 DLLs that +carry a Rich header. A binary without one (non-MSVC or stripped) comes back with +`HasRichHeader: false` and no error; a stray `Rich` byte sequence with no valid `DanS` behind it +is skipped in favour of the genuine header. + ## What's intentionally NOT here -- AMSI / Microsoft Defender scanning — stays in OffsetInspect (Windows-only, +- AMSI / Microsoft Defender scanning - stays in OffsetInspect (Windows-only, needs the actual providers). -- Detection-boundary bisection search — that's a stateful, provider-driven +- Detection-boundary bisection search - that's a stateful, provider-driven workflow (`Invoke-OffsetThreatScan`), not a stateless corpus pass. -- ClamAV integration — `clamscan` process-spawning has no real parallel-corpus +- ClamAV integration - `clamscan` process-spawning has no real parallel-corpus benefit from a Rust rewrite; left in PowerShell. ## Status @@ -225,7 +257,7 @@ from the PE resource data directory. The `yara-scan` feature adds an `offsetscan yara` subcommand whose records (`File`/`Rule`/`StringId`/`Offset`/`OffsetHex`/`Data`) are verified identical to `Invoke-OffsetYaraScan`'s on the same rules and sample. It is **feature-gated and -off by default** — building it needs a C toolchain and `libclang` (the `yara` crate +off by default** - building it needs a C toolchain and `libclang` (the `yara` crate compiles a vendored `libyara` via bindgen), so the default binary (and everything on crates.io / the release page) does not include it. A CI job builds and tests the feature on Linux (installing `libclang` for bindgen). Non-printable match bytes are diff --git a/src/cluster.rs b/src/cluster.rs index c65bd1c..7df8f55 100644 --- a/src/cluster.rs +++ b/src/cluster.rs @@ -1,10 +1,10 @@ -//! `cluster` — TLSH similarity hashing and single-linkage clustering for corpus-scale triage. +//! `cluster` - TLSH similarity hashing and single-linkage clustering for corpus-scale triage. //! //! Computes a canonical TLSH (`T1...`) locality-sensitive digest per file and groups files //! whose pairwise TLSH distance is within a threshold into single-linkage clusters. This is //! the first OffsetScan capability with no OffsetInspect equivalent: it is inherently //! corpus-scale, answering "which of these thousands of samples are variants of one another?" -//! (repacks, malware families, near-duplicates) — a question a single-file inspector cannot. +//! (repacks, malware families, near-duplicates) - a question a single-file inspector cannot. //! //! It is OffsetScan-only output and does NOT participate in the OffsetInspect JSON parity //! contract (OffsetInspect emits no TLSH object), so it lives outside `schema.rs`. @@ -37,7 +37,7 @@ pub struct ClusterRecord { } /// Read files in parallel, then cluster. Read errors become a per-file warning rather than -/// aborting the run — a corpus scan should survive one unreadable sample. +/// aborting the run - a corpus scan should survive one unreadable sample. pub fn run(files: Vec, threshold: i32) -> Vec { let reads: Vec<(String, Result, String>)> = files .par_iter() @@ -115,7 +115,7 @@ fn cluster_reads( let n = records.len(); // Phase 2: discover edges (pairs within `threshold`) in parallel, then union sequentially. - // O(n^2) pairwise diffs — fine for the few-thousand-file corpora this targets; documented. + // O(n^2) pairwise diffs - fine for the few-thousand-file corpora this targets; documented. let objs_ref = &objs; let edges: Vec<(usize, usize)> = (0..n) .into_par_iter() diff --git a/src/entropy.rs b/src/entropy.rs index 6b17986..c2ecc71 100644 --- a/src/entropy.rs +++ b/src/entropy.rs @@ -1,4 +1,4 @@ -//! Per-window Shannon entropy — mirrors `Get-OffsetEntropy` (OffsetInspect.EntropyResult). +//! Per-window Shannon entropy - mirrors `Get-OffsetEntropy` (OffsetInspect.EntropyResult). //! //! Parity notes vs OffsetInspect: //! - Windows are CONTIGUOUS / non-overlapping (start += length), not a sliding step. @@ -12,7 +12,7 @@ fn round6(x: f64) -> f64 { let r = (x * 1_000_000.0).round() / 1_000_000.0; // Normalize IEEE negative zero to positive zero. Shannon entropy of a single-value // buffer computes -1 * log2(1) = -0.0, which serializes to JSON as "-0.0" and breaks - // exact parity with Get-OffsetEntropy (which emits 0) on all-identical-byte files — + // exact parity with Get-OffsetEntropy (which emits 0) on all-identical-byte files - // all-zero padding, sparse regions. `-0.0 == 0.0` is true, so this catches it. if r == 0.0 { 0.0 @@ -21,7 +21,7 @@ fn round6(x: f64) -> f64 { } } -/// Shannon entropy (bits/byte) of a byte slice — raw, unrounded. +/// Shannon entropy (bits/byte) of a byte slice - raw, unrounded. pub fn shannon_entropy(data: &[u8]) -> f64 { if data.is_empty() { return 0.0; diff --git a/src/ioc.rs b/src/ioc.rs index e78dc2d..5c47322 100644 --- a/src/ioc.rs +++ b/src/ioc.rs @@ -1,4 +1,4 @@ -//! Consolidated indicator panel — mirrors `Get-OffsetIOC` (OffsetInspect.IOC). +//! Consolidated indicator panel - mirrors `Get-OffsetIOC` (OffsetInspect.IOC). //! //! Parity notes vs OffsetInspect: //! - FLAT shape: PE fields (Machine/ImpHash/ImportedDllCount/HasOverlay/OverlaySize) diff --git a/src/main.rs b/src/main.rs index 126e902..725e3fb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ mod entropy; mod ioc; mod ordinals; mod pe; +mod rich; mod schema; mod strings; mod yara_scan; @@ -16,7 +17,7 @@ use std::{fs, io}; #[derive(Parser)] #[command( name = "offsetscan", - about = "Standalone native corpus-scale static-triage engine — schema-compatible with OffsetInspect.", + about = "Standalone native corpus-scale static-triage engine - schema-compatible with OffsetInspect.", version )] struct Cli { @@ -70,6 +71,13 @@ enum Commands { #[arg(long)] recurse: bool, }, + /// Parse the PE Rich header and compute a build-toolchain fingerprint hash (RichHash). + Rich { + /// File or directory (use --recurse for directories) or glob pattern. + path: String, + #[arg(long)] + recurse: bool, + }, /// TLSH similarity hashing + single-linkage clustering across a corpus. Groups files /// that are variants of one another (repacks, families, near-duplicates). Cluster { @@ -210,6 +218,13 @@ fn main() { }); } } + Commands::Rich { path, recurse } => { + let files = expand_paths(&path, recurse); + run(files, ndjson, move |f| { + let data = fs::read(f).ok()?; + Some(rich::parse_rich(&data, &f.to_string_lossy())) + }); + } Commands::Cluster { path, recurse, @@ -268,7 +283,7 @@ fn main() { /// Write IOC panels as CSV (header + one row per file) to stdout. The `csv` crate handles /// quoting/escaping (e.g. file paths containing commas), and serde carries the OffsetInspect -/// field names — including the `MD5`/`SHA1`/`SHA256`/`IsPE` renames — into the CSV header. +/// field names - including the `MD5`/`SHA1`/`SHA256`/`IsPE` renames - into the CSV header. fn write_csv(records: &[schema::Ioc]) { let mut wtr = csv::Writer::from_writer(io::stdout()); for record in records { diff --git a/src/pe.rs b/src/pe.rs index 07cced7..228f772 100644 --- a/src/pe.rs +++ b/src/pe.rs @@ -1,4 +1,4 @@ -//! PE header/section/import parsing — mirrors `Get-OffsetPEInfo` (OffsetInspect.PEInfo). +//! PE header/section/import parsing - mirrors `Get-OffsetPEInfo` (OffsetInspect.PEInfo). //! //! Parity notes vs OffsetInspect: //! - `Machine` is a DISPLAY string ("x64 (AMD64)", ...), not the raw hex id. @@ -7,7 +7,7 @@ //! - Imphash = MD5 of comma-joined lowercased `libbase.func` (libbase = dll name with //! .dll/.ocx/.sys stripped), per imported function in directory order; ordinal-only //! imports render as `ord`. Null (None) when there are no imports. -//! - `ResourceSize` is the PE data-directory #2 (Resource) SIZE field — a single u32, +//! - `ResourceSize` is the PE data-directory #2 (Resource) SIZE field - a single u32, //! NOT a resource-tree walk. //! - A PE section object carries NO entropy field. @@ -136,7 +136,7 @@ fn parse_pe_strict(data: &[u8], file_path: &str) -> Result { /// Assemble the final `PeInfo`, computing the overlay identically for both parse paths. /// The overlay boundary uses only sections with non-zero raw data, matching OffsetInspect's -/// `Get-OIPEOverlayRange` — a zero-raw-size section (e.g. a packer's virtual section) never +/// `Get-OIPEOverlayRange` - a zero-raw-size section (e.g. a packer's virtual section) never /// defines it. (The strict path previously omitted this filter, a latent inconsistency.) #[allow(clippy::too_many_arguments)] fn assemble_pe_info( diff --git a/src/rich.rs b/src/rich.rs new file mode 100644 index 0000000..d542b0b --- /dev/null +++ b/src/rich.rs @@ -0,0 +1,338 @@ +//! `rich` - PE "Rich" header parsing and a build-toolchain fingerprint hash. +//! +//! The Rich header is an undocumented block the Microsoft linker writes between the DOS stub +//! and the PE header. It records, as `@comp.id` records, which build tools (and how many +//! objects from each) produced the binary - a compiler/linker fingerprint. Malware analysts +//! use it for clustering and attribution: two samples from the same build environment share a +//! Rich header even when nothing else lines up. +//! +//! `RichHash` here is the MD5 of the **decrypted** Rich array (the `DanS` marker, its padding, +//! and every comp.id/count record - i.e. pefile's `RICH_HEADER.clear_data`). It deliberately +//! excludes the per-file XOR key and the `Rich`/checksum trailer, so it is invariant across +//! binaries built with the same toolchain sequence, which is what makes it useful for grouping. +//! +//! OffsetScan-only output: OffsetInspect has no Rich-header object, so this sits outside the +//! JSON parity contract and lives here rather than in `schema.rs`. + +use md5::Md5; +use serde::{Deserialize, Serialize}; +use sha2::Digest; + +/// "DanS" - the marker that starts the (decrypted) Rich array. +const DANS_MARKER: u32 = 0x536E_6144; +/// "Rich" - the plaintext marker that ends the Rich header, immediately followed by the XOR key. +const RICH_MARKER: &[u8; 4] = b"Rich"; +/// Offset of the DOS header's `e_lfanew` field (pointer to the PE header). +const E_LFANEW_OFFSET: usize = 0x3C; +/// The Rich header can only live after the DOS header (0x40) and before `e_lfanew`. +const DOS_HEADER_END: usize = 0x40; + +/// One `@comp.id` record: which build tool, which build, and how many objects it contributed. +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct RichEntry { + /// Upper 16 bits of the comp.id: the product/tool id. + pub product_id: u16, + /// Lower 16 bits of the comp.id: the tool build number. + pub build_id: u16, + /// How many objects this tool contributed. + pub count: u32, +} + +/// Rich-header result for one file. +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "PascalCase")] +pub struct RichInfo { + pub file: String, + pub file_size: u64, + pub has_rich_header: bool, + /// MD5 of the decrypted Rich array (toolchain-invariant), or null when there is no header. + #[serde(rename = "RichHash")] + pub rich_hash: Option, + /// The per-file XOR key (checksum) as `0x`-prefixed hex, or null when there is no header. + pub xor_key: Option, + pub entry_count: u32, + pub entries: Vec, + pub warnings: Vec, +} + +impl RichInfo { + fn empty(file: &str, file_size: u64, warnings: Vec) -> Self { + RichInfo { + file: file.to_string(), + file_size, + has_rich_header: false, + rich_hash: None, + xor_key: None, + entry_count: 0, + entries: Vec::new(), + warnings, + } + } +} + +/// Parse the Rich header of `data`. Never fails: a non-PE, a PE without a Rich header, or a +/// malformed header all return a `RichInfo` with `has_rich_header = false` (and a warning where +/// something was actually wrong, as opposed to simply absent - a missing Rich header is normal). +pub fn parse_rich(data: &[u8], file: &str) -> RichInfo { + let file_size = data.len() as u64; + + if data.len() < DOS_HEADER_END || &data[0..2] != b"MZ" { + return RichInfo::empty(file, file_size, vec!["not a PE (no MZ header)".to_string()]); + } + + let e_lfanew = u32::from_le_bytes([ + data[E_LFANEW_OFFSET], + data[E_LFANEW_OFFSET + 1], + data[E_LFANEW_OFFSET + 2], + data[E_LFANEW_OFFSET + 3], + ]) as usize; + + // The Rich header lives strictly between the DOS header and the PE header. + let search_end = e_lfanew.min(data.len()); + if search_end <= DOS_HEADER_END { + return RichInfo::empty(file, file_size, Vec::new()); + } + let region = &data[..search_end]; + + // A "Rich" marker can (rarely) appear coincidentally, so try each occurrence and keep the + // first whose XOR key decrypts a preceding dword back to "DanS" - the real header. + let mut start = DOS_HEADER_END; + while let Some(rel) = find_subslice(®ion[start..], RICH_MARKER) { + let rich_pos = start + rel; + start = rich_pos + 1; // advance so a failed match doesn't loop forever + if rich_pos + 8 > region.len() { + continue; // no room for the trailing key + } + let key = u32::from_le_bytes([ + region[rich_pos + 4], + region[rich_pos + 5], + region[rich_pos + 6], + region[rich_pos + 7], + ]); + + let Some(dans_pos) = find_dans_backwards(region, rich_pos, key) else { + continue; + }; + + return build_info(file, file_size, ®ion[dans_pos..rich_pos], key); + } + + // No Rich header - the common case for non-MSVC or stripped binaries. Not an error. + RichInfo::empty(file, file_size, Vec::new()) +} + +/// Walk backwards from just before `rich_pos` in dword steps, looking for a dword that decrypts +/// (XOR `key`) to the "DanS" marker. Returns the byte offset of that dword. +fn find_dans_backwards(region: &[u8], rich_pos: usize, key: u32) -> Option { + let mut i = rich_pos; + while i >= DOS_HEADER_END + 4 { + i -= 4; + let dw = u32::from_le_bytes([region[i], region[i + 1], region[i + 2], region[i + 3]]) ^ key; + if dw == DANS_MARKER { + return Some(i); + } + } + None +} + +/// Decrypt the Rich array `enc` (from the DanS dword up to, but not including, "Rich"), parse +/// its comp.id records, and hash the cleartext. +fn build_info(file: &str, file_size: u64, enc: &[u8], key: u32) -> RichInfo { + let mut warnings = Vec::new(); + if !enc.len().is_multiple_of(4) { + warnings.push("Rich data length is not dword-aligned; parsed best-effort".to_string()); + } + + // Decrypt to cleartext dwords. + let dwords: Vec = enc + .chunks_exact(4) + .map(|c| u32::from_le_bytes([c[0], c[1], c[2], c[3]]) ^ key) + .collect(); + + // Layout: [DanS][pad][pad][pad][comp0][count0][comp1][count1]... + let mut entries = Vec::new(); + if dwords.len() > 4 { + let mut idx = 4; + while idx + 1 < dwords.len() { + let comp = dwords[idx]; + let count = dwords[idx + 1]; + entries.push(RichEntry { + product_id: (comp >> 16) as u16, + build_id: (comp & 0xFFFF) as u16, + count, + }); + idx += 2; + } + } + + // RichHash = MD5 of the decrypted array (toolchain-invariant; excludes the XOR key/trailer). + let clear: Vec = dwords.iter().flat_map(|dw| dw.to_le_bytes()).collect(); + let mut hasher = Md5::default(); + Digest::update(&mut hasher, &clear); + let rich_hash = format!("{:x}", Digest::finalize(hasher)); + + RichInfo { + file: file.to_string(), + file_size, + has_rich_header: true, + rich_hash: Some(rich_hash), + xor_key: Some(format!("0x{key:08X}")), + entry_count: entries.len() as u32, + entries, + warnings, + } +} + +/// First index of `needle` within `haystack`, or None. +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + if needle.is_empty() || haystack.len() < needle.len() { + return None; + } + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a minimal buffer with a valid Rich header for the given comp.id records and key. + fn build_pe_with_rich(entries: &[(u16, u16, u32)], key: u32) -> Vec { + // Cleartext dwords: DanS, three pads, then (comp, count) pairs. + let mut clear = vec![DANS_MARKER, 0, 0, 0]; + for &(product_id, build_id, count) in entries { + clear.push(((product_id as u32) << 16) | (build_id as u32)); + clear.push(count); + } + // Encrypt with the key. + let enc: Vec = clear + .iter() + .flat_map(|dw| (dw ^ key).to_le_bytes()) + .collect(); + + let mut buf = vec![0u8; DOS_HEADER_END]; + buf[0] = b'M'; + buf[1] = b'Z'; + let mut body = Vec::new(); + body.extend_from_slice(&enc); + body.extend_from_slice(RICH_MARKER); + body.extend_from_slice(&key.to_le_bytes()); + // e_lfanew points just past the Rich header (bounds the search region). + let e_lfanew = (DOS_HEADER_END + body.len()) as u32; + buf[E_LFANEW_OFFSET..E_LFANEW_OFFSET + 4].copy_from_slice(&e_lfanew.to_le_bytes()); + buf.extend_from_slice(&body); + buf + } + + #[test] + fn parses_header_entries_and_key() { + let entries = [(0x00FF, 0x1234, 5), (0x00AA, 0x5678, 2)]; + let info = parse_rich(&build_pe_with_rich(&entries, 0xDEAD_BEEF), "t"); + assert!(info.has_rich_header); + assert_eq!(info.entry_count, 2); + assert_eq!(info.entries[0].product_id, 0x00FF); + assert_eq!(info.entries[0].build_id, 0x1234); + assert_eq!(info.entries[0].count, 5); + assert_eq!(info.entries[1].product_id, 0x00AA); + assert_eq!(info.xor_key.as_deref(), Some("0xDEADBEEF")); + assert!(info.rich_hash.is_some()); + assert!(info.warnings.is_empty()); + } + + #[test] + fn rich_hash_is_invariant_across_the_per_file_xor_key() { + // Same toolchain (identical comp.id records), different per-file keys -> same RichHash. + let entries = [(0x0102, 0x2000, 3), (0x0103, 0x2001, 1)]; + let a = parse_rich(&build_pe_with_rich(&entries, 0x1111_2222), "a"); + let b = parse_rich(&build_pe_with_rich(&entries, 0x9999_8888), "b"); + assert_ne!(a.xor_key, b.xor_key, "keys should differ"); + assert_eq!( + a.rich_hash, b.rich_hash, + "same toolchain must yield the same RichHash" + ); + } + + #[test] + fn different_toolchains_hash_differently() { + let a = parse_rich(&build_pe_with_rich(&[(1, 2, 3)], 0x1), "a"); + let b = parse_rich(&build_pe_with_rich(&[(4, 5, 6)], 0x1), "b"); + assert_ne!(a.rich_hash, b.rich_hash); + } + + #[test] + fn rich_hash_is_deterministic() { + let data = build_pe_with_rich(&[(7, 8, 9), (10, 11, 12)], 0xABCD_1234); + let a = parse_rich(&data, "t"); + let b = parse_rich(&data, "t"); + assert_eq!(a.rich_hash, b.rich_hash); + } + + #[test] + fn pe_without_rich_header_is_clean_not_an_error() { + let mut data = vec![0u8; 0x80]; + data[0] = b'M'; + data[1] = b'Z'; + data[E_LFANEW_OFFSET..E_LFANEW_OFFSET + 4].copy_from_slice(&0x80u32.to_le_bytes()); + let info = parse_rich(&data, "t"); + assert!(!info.has_rich_header); + assert!(info.rich_hash.is_none()); + assert_eq!(info.entry_count, 0); + assert!(info.warnings.is_empty(), "absent header is not a warning"); + } + + #[test] + fn non_pe_is_reported_with_a_warning() { + let info = parse_rich(b"this is not a PE file at all, no MZ here", "t"); + assert!(!info.has_rich_header); + assert!(!info.warnings.is_empty()); + } + + #[test] + fn a_false_rich_marker_is_skipped_for_the_real_one() { + // Prepend a decoy "Rich" (no valid DanS behind it) before a genuine header. + let real = build_pe_with_rich(&[(0x0100, 0x1000, 4)], 0x2222_3333); + let mut data = real.clone(); + // Splice a bogus "Rich" + key into the DOS stub area (bytes 0x08..0x10), which has no + // DanS behind it; parsing must fall through to the genuine header. + data[0x08..0x0C].copy_from_slice(RICH_MARKER); + data[0x0C..0x10].copy_from_slice(&0xFFFF_FFFFu32.to_le_bytes()); + let info = parse_rich(&data, "t"); + assert!(info.has_rich_header); + assert_eq!(info.entries[0].product_id, 0x0100); + assert_eq!(info.xor_key.as_deref(), Some("0x22223333")); + } + + #[test] + fn serializes_with_expected_field_names() { + let info = RichInfo { + file: "f".into(), + file_size: 1, + has_rich_header: true, + rich_hash: Some("abcd".into()), + xor_key: Some("0x1".into()), + entry_count: 1, + entries: vec![RichEntry { + product_id: 1, + build_id: 2, + count: 3, + }], + warnings: vec![], + }; + let json = serde_json::to_string(&info).unwrap(); + for key in [ + "\"File\":", + "\"HasRichHeader\":", + "\"RichHash\":", + "\"XorKey\":", + "\"EntryCount\":", + "\"Entries\":", + "\"ProductId\":", + "\"BuildId\":", + "\"Count\":", + ] { + assert!(json.contains(key), "missing {key} in {json}"); + } + } +} diff --git a/src/schema.rs b/src/schema.rs index 13af16b..6931e46 100644 --- a/src/schema.rs +++ b/src/schema.rs @@ -1,12 +1,12 @@ -//! Schema — the JSON contract with OffsetInspect (PowerShell). +//! Schema - the JSON contract with OffsetInspect (PowerShell). //! //! Field names/casing verified FIELD-FOR-FIELD against the actual OffsetInspect //! 3.0.0 result objects (Get-OffsetPEInfo, Get-OffsetEntropy, Get-OffsetString, -//! Get-OffsetIOC) and docs/OUTPUT-SCHEMA.md — not the README. +//! Get-OffsetIOC) and docs/OUTPUT-SCHEMA.md - not the README. //! //! PowerShell PSCustomObject property names are case-preserving, and consumers //! match on exact casing. `rename_all = "PascalCase"` handles most fields, but it -//! turns `md5`/`sha1`/`sha256`/`is_pe`/`is_pe32_plus` into `Md5`/`Sha1`/`IsPe`/... — +//! turns `md5`/`sha1`/`sha256`/`is_pe`/`is_pe32_plus` into `Md5`/`Sha1`/`IsPe`/... - //! WRONG. Those carry an explicit `#[serde(rename = ...)]` to match `MD5`/`SHA1`/ //! `SHA256`/`IsPE`/`IsPE32Plus`. //! @@ -16,7 +16,7 @@ //! model them separately against docs/OUTPUT-SCHEMA.md. // The ingestion structs below (OffsetResult / ThreatScanResult / BoundaryValidation / -// ProbeLogEntry) are deserialize-only — nothing in the binary constructs them yet. +// ProbeLogEntry) are deserialize-only - nothing in the binary constructs them yet. #![allow(dead_code)] use serde::{Deserialize, Serialize}; @@ -39,7 +39,7 @@ pub struct PeInfo { pub imported_dll_count: u32, pub imports: Vec, pub imp_hash: Option, - pub resource_size: u32, // PE data-directory #2 (Resource) size field — NOT a tree walk + pub resource_size: u32, // PE data-directory #2 (Resource) size field - NOT a tree walk pub has_overlay: bool, pub overlay_offset: Option, pub overlay_size: u64, // 0 when no overlay (not null) @@ -103,7 +103,7 @@ pub struct StringHit { pub value: String, } -/// `Get-OffsetIOC` -> OffsetInspect.IOC (FLAT — PE fields are inlined, not a nested object) +/// `Get-OffsetIOC` -> OffsetInspect.IOC (FLAT - PE fields are inlined, not a nested object) #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "PascalCase")] pub struct Ioc { @@ -132,13 +132,13 @@ pub struct Ioc { // INGESTION-ONLY. OffsetScan does NOT produce these. They are the shapes emitted // by Invoke-OffsetInspect (`OffsetInspect.Result`) and Invoke-OffsetThreatScan // (`OffsetInspect.ThreatScanResult`, Windows-only AMSI/Defender). Modeled here so -// a consumer/merge layer can deserialize OffsetInspect JSON — e.g. for feeding +// a consumer/merge layer can deserialize OffsetInspect JSON - e.g. for feeding // Export-OffsetThreatReport or Compare-OffsetThreatResult. Verified against // docs/OUTPUT-SCHEMA.md AND the actual pscustomobject construction (3.0.0). // Deeply-nested/variable arrays (ContextLines, HexDump, ProviderMetadata) are kept // as serde_json::Value because their per-row shape is not part of the stable contract. // All names are plain PascalCase words (FileSha256, not FileSHA256), so rename_all -// covers them — no explicit renames needed here. +// covers them - no explicit renames needed here. // --------------------------------------------------------------------------- /// `OffsetInspect.Result` (Invoke-OffsetInspect; also nested as ThreatScanResult.Inspection) @@ -177,7 +177,7 @@ pub struct OffsetResult { pub error: Option, } -/// `OffsetInspect.ThreatScanResult` (Invoke-OffsetThreatScan — Windows-only) +/// `OffsetInspect.ThreatScanResult` (Invoke-OffsetThreatScan - Windows-only) #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "PascalCase")] pub struct ThreatScanResult { diff --git a/src/strings.rs b/src/strings.rs index 7e06891..aa1d3ca 100644 --- a/src/strings.rs +++ b/src/strings.rs @@ -1,4 +1,4 @@ -//! Printable string extraction — mirrors `Get-OffsetString`. +//! Printable string extraction - mirrors `Get-OffsetString`. use crate::schema::StringHit; @@ -136,7 +136,7 @@ mod tests { fn extracts_utf16le_runs_starting_at_an_odd_offset() { // A single pad byte shifts "Data" (44 00 61 00 74 00 61 00) to offset 1. // Stepping two bytes from zero would inspect only even offsets and miss it - // entirely — the real-world divergence from Get-OffsetString that this guards. + // entirely - the real-world divergence from Get-OffsetString that this guards. let mut data = vec![0xFF]; data.extend_from_slice(&[0x44, 0x00, 0x61, 0x00, 0x74, 0x00, 0x61, 0x00]); let hits = extract_utf16le_strings(&data, 4); diff --git a/src/yara_scan.rs b/src/yara_scan.rs index 97c3ca2..3f0c4ad 100644 --- a/src/yara_scan.rs +++ b/src/yara_scan.rs @@ -1,4 +1,4 @@ -//! Optional YARA rule matching — mirrors `Invoke-OffsetYaraScan` +//! Optional YARA rule matching - mirrors `Invoke-OffsetYaraScan` //! (OffsetInspect.YaraMatch). Compiled in only with the `yara-scan` feature, //! which vendors libyara; the default build omits it entirely. //!