From ae5e208aa743ed67e98d8cdea5d62bf7725645b7 Mon Sep 17 00:00:00 2001 From: Akeem Jenkins Date: Sat, 25 Jul 2026 23:44:31 -0600 Subject: [PATCH] feat: implement OKF v0.2 spec (provenance, trust, lifecycle, attested computations) Co-Authored-By: Claude Fable 5 --- README.md | 42 ++- cmd/okf/main.go | 219 ++++++++++------ internal/bundle/bundle.go | 6 +- internal/bundle/bundle_test.go | 4 +- internal/concept/concept.go | 91 +++++-- internal/concept/concept_test.go | 2 +- internal/concept/v02.go | 193 ++++++++++++++ internal/concept/v02_test.go | 344 +++++++++++++++++++++++++ internal/graph/graph.go | 53 +++- internal/graph/v02_test.go | 92 +++++++ internal/index/index.go | 26 +- internal/index/index_test.go | 79 ++++++ internal/initbundle/initbundle.go | 5 +- internal/initbundle/initbundle_test.go | 18 ++ internal/search/search_test.go | 16 +- internal/validate/v02.go | 299 +++++++++++++++++++++ internal/validate/v02_test.go | 340 ++++++++++++++++++++++++ internal/validate/validate.go | 8 +- internal/validate/validate_test.go | 6 +- testdata/valid/datasets/ga4.md | 2 +- 20 files changed, 1707 insertions(+), 138 deletions(-) create mode 100644 internal/concept/v02.go create mode 100644 internal/concept/v02_test.go create mode 100644 internal/graph/v02_test.go create mode 100644 internal/index/index_test.go create mode 100644 internal/validate/v02.go create mode 100644 internal/validate/v02_test.go diff --git a/README.md b/README.md index 1288959..a4249fc 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,20 @@ # okf -A Go CLI toolkit for the [Open Knowledge Format (OKF)](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) — a vendor-neutral format for representing data catalog knowledge as plain markdown files with YAML frontmatter. +A Go CLI toolkit for the [Open Knowledge Format (OKF)](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md) - a vendor-neutral format for representing data catalog knowledge as plain markdown files with YAML frontmatter. `okf` creates, validates, lints, indexes, searches, and inspects OKF knowledge bundles. One static binary, no runtime dependencies, fast enough to validate millions of concepts. ## Why okf? -Google's reference OKF implementation is Python + Gemini + BigQuery — vendor-locked to Google's cloud. `okf` is the vendor-neutral alternative: a single Go binary that works anywhere, speaks JSON natively, and is designed to be driven by any AI agent on any provider. +Google's reference OKF implementation is Python + Gemini + BigQuery - vendor-locked to Google's cloud. `okf` is the vendor-neutral alternative: a single Go binary that works anywhere, speaks JSON natively, and is designed to be driven by any AI agent on any provider. **Agentic-first** means an AI agent can discover, understand, and drive the entire CLI without reading documentation or scraping text output. Three mechanisms make this work: -1. **`okf schema`** — emits a complete machine-readable description of every command: name, description, flags, arguments, output format, exit codes. One call and the agent knows the full CLI surface. +1. **`okf schema`** - emits a complete machine-readable description of every command: name, description, flags, arguments, output format, exit codes. One call and the agent knows the full CLI surface. -2. **JSON by default** — every command outputs structured JSON on stdout. No `--json` flag, no screen-scraping. Diagnostics go to stderr. +2. **JSON by default** - every command outputs structured JSON on stdout. No `--json` flag, no screen-scraping. Diagnostics go to stderr. -3. **Structured error envelopes** — all errors emit `{"error": {"kind":..., "code":..., "reason":..., "message":...}}` on stdout with a stable exit code. An agent can branch on the `kind` field to decide what to do next. +3. **Structured error envelopes** - all errors emit `{"error": {"kind":..., "code":..., "reason":..., "message":...}}` on stdout with a stable exit code. An agent can branch on the `kind` field to decide what to do next. ## Quick start @@ -37,7 +37,7 @@ okf graph ./my-bundle ### 1. AI-driven documentation pipeline -An AI agent creates a bundle, writes concept documents from a database schema or API spec, validates them, and generates navigation — all autonomously. +An AI agent creates a bundle, writes concept documents from a database schema or API spec, validates them, and generates navigation - all autonomously. ```bash okf init ./bundles/mydb # start from scratch @@ -112,7 +112,7 @@ Progressive disclosure (index.md) lets the agent navigate level by level instead | `okf validate ` | Validate a bundle against the OKF spec (exit 1 on errors) | | `okf lint ` | Check recommended fields and style (warnings only) | | `okf index ` | Generate index.md files (progressive disclosure) | -| `okf list ` | List all concepts with ID, type, title | +| `okf list ` | List all concepts with ID, type, title, status, trust tier | | `okf show ` | Show a single concept's full content as JSON | | `okf search [--tag] [--type] [--text]` | Search concepts by tag, type, or text | | `okf backlinks ` | List concepts that link to a given concept | @@ -131,7 +131,7 @@ Progressive disclosure (index.md) lets the agent navigate level by level instead ## What is OKF? -OKF is an open format from Google for representing knowledge — the metadata, context, and curated insight that surrounds data and systems. A bundle is a directory of markdown files with YAML frontmatter: +OKF is an open format from Google for representing knowledge - the metadata, context, and curated insight that surrounds data and systems. A bundle is a directory of markdown files with YAML frontmatter: ``` my-bundle/ @@ -170,18 +170,34 @@ Joined with [users](/tables/users.md) on user_id. The format is intentionally minimal: no schema registry, no central authority, no required tooling. If you can `cat` a file, you can read OKF; if you can `git clone` a repo, you can ship it. +## OKF v0.2 support + +`okf` implements [OKF v0.2](https://github.com/GoogleCloudPlatform/knowledge-catalog/blob/main/okf/SPEC.md), which makes provenance, trust, lifecycle, and attestation first-class: + +- **Provenance (§5.1)** - the `sources` family with per-source credibility signals (`author`, `usage_count`, `last_modified`) and `usage_window`; footnote labels are validated as join keys into `sources[].id` for per-claim attribution. +- **Trust (§5.2–5.3)** - `generated` and `verified` are parsed (a bare `verified` mapping is treated as a one-element list, as the spec requires) and every concept gets a derived trust tier: `unverified`, `machine-confirmed`, or `human-reviewed`. +- **Lifecycle (§5.4–5.5)** - `status` (draft/stable/deprecated, default stable) and `stale_after`; `validate` flags stale concepts and invalid values, and `show`/`list` expose the effective state. +- **Attested Computations (§10)** - the full contract (`runtime`, `parameters`, `computation`, `executor`, `attester`) is parsed and validated: `runtime` is required, the computation must exist inline or as a file, and contract paths that point at missing files are flagged. +- **Actor convention (§7)** - `generated.by` and `verified[].by` are checked against `human:`, `process:`, and `/` forms. +- **Versioning (§12)** - `okf init` declares `okf_version: "0.2"` in the root `index.md`; `okf index` preserves an existing declaration when regenerating. +- **v0.1 compatibility (§13)** - legacy `timestamp` and `# Citations` are still consumed (with migration warnings), and `generated.at` falls back to `timestamp`. + +Graph and backlinks also follow §6.2 path-valued references: `sources[].resource`, `computation`, `executor.resource`, and `attester.resource` that resolve to concepts become derivation edges. + +All four reference bundles in the upstream repository (acme_retail, crypto_bitcoin, ga4, stackoverflow) load and validate with zero errors. + ## Project status -Early development. The CLI surface is functional with 35 tests: +Early development. The CLI surface is functional: - `schema`, `init`, `validate`, `lint`, `index`, `list`, `show`, `search`, `backlinks`, `graph`, `version` Planned: -- `okf serve` — local HTTP server to browse a bundle interactively -- `okf render` — export a bundle as a self-contained HTML file -- `okf-go` — Go library package for embedding in applications +- `okf serve` - local HTTP server to browse a bundle interactively +- `okf render` - export a bundle as a self-contained HTML file +- `okf-go` - Go library package for embedding in applications ## License -Apache 2.0 — matching the upstream [Google knowledge-catalog](https://github.com/GoogleCloudPlatform/knowledge-catalog) repository. \ No newline at end of file +Apache 2.0 - matching the upstream [Google knowledge-catalog](https://github.com/GoogleCloudPlatform/knowledge-catalog) repository. \ No newline at end of file diff --git a/cmd/okf/main.go b/cmd/okf/main.go index dd199ca..b930727 100644 --- a/cmd/okf/main.go +++ b/cmd/okf/main.go @@ -13,6 +13,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/okfcli/okf/internal/backlinks" "github.com/okfcli/okf/internal/bundle" @@ -71,7 +72,7 @@ func main() { } } -const usage = `okf — Open Knowledge Format toolkit (v%s) +const usage = `okf - Open Knowledge Format toolkit (v%s) Usage: okf @@ -257,9 +258,11 @@ func runList(args []string) { concepts := make([]map[string]string, 0, len(b.Concepts)) for _, c := range b.Concepts { concepts = append(concepts, map[string]string{ - "id": c.ID, - "type": c.Frontmatter.Type, - "title": c.Frontmatter.Title, + "id": c.ID, + "type": c.Frontmatter.Type, + "title": c.Frontmatter.Title, + "status": c.Frontmatter.EffectiveStatus(), + "trust_tier": c.Frontmatter.TrustTier(), }) } @@ -288,19 +291,89 @@ func runShow(args []string) { tags = c.Frontmatter.Tags } + fm := c.Frontmatter + out := map[string]any{ + "id": c.ID, + "path": c.Path, + "type": fm.Type, + "title": fm.Title, + "description": fm.Description, + "resource": fm.Resource, + "tags": tags, + "body": c.Body, + // Trust and lifecycle (OKF v0.2 §5): always present so agents can + // branch on them without probing for key existence. + "status": fm.EffectiveStatus(), + "trust_tier": fm.TrustTier(), + "stale": fm.IsStale(time.Now()), + } + if fm.Generated != nil { + out["generated"] = map[string]any{"by": fm.Generated.By, "at": fm.Generated.At} + } + if len(fm.Verified) > 0 { + verified := make([]map[string]any, 0, len(fm.Verified)) + for _, v := range fm.Verified { + verified = append(verified, map[string]any{"by": v.By, "at": v.At}) + } + out["verified"] = verified + } + if fm.StaleAfter != "" { + out["stale_after"] = fm.StaleAfter + } + if len(fm.Sources) > 0 { + sources := make([]map[string]any, 0, len(fm.Sources)) + for _, s := range fm.Sources { + src := map[string]any{"resource": s.Resource} + if s.ID != "" { + src["id"] = s.ID + } + if s.Title != "" { + src["title"] = s.Title + } + if s.Author != "" { + src["author"] = s.Author + } + if s.UsageCount != nil { + src["usage_count"] = *s.UsageCount + } + if s.LastModified != "" { + src["last_modified"] = s.LastModified + } + if s.UsageWindow != nil { + src["usage_window"] = map[string]string{"from": s.UsageWindow.From, "to": s.UsageWindow.To} + } + sources = append(sources, src) + } + out["sources"] = sources + } + if fm.UsageWindow != nil { + out["usage_window"] = map[string]string{"from": fm.UsageWindow.From, "to": fm.UsageWindow.To} + } + // Attested Computation contract (OKF v0.2 §10). + if fm.Runtime != "" { + out["runtime"] = fm.Runtime + } + if len(fm.Parameters) > 0 { + params := make([]map[string]any, 0, len(fm.Parameters)) + for _, p := range fm.Parameters { + params = append(params, map[string]any{"name": p.Name, "type": p.Type, "required": p.Required}) + } + out["parameters"] = params + } + if fm.Computation != "" { + out["computation"] = fm.Computation + } + if fm.Executor != nil { + out["executor"] = map[string]any{"resource": fm.Executor.Resource, "receipt": fm.Executor.Receipt} + } + if fm.Attester != nil { + out["attester"] = map[string]any{"resource": fm.Attester.Resource} + } + outputJSON(map[string]any{ "command": "show", "bundle": b.Root, - "concept": map[string]any{ - "id": c.ID, - "path": c.Path, - "type": c.Frontmatter.Type, - "title": c.Frontmatter.Title, - "description": c.Frontmatter.Description, - "resource": c.Frontmatter.Resource, - "tags": tags, - "body": c.Body, - }, + "concept": out, }) } @@ -355,11 +428,11 @@ func runSearch(args []string) { } outputJSON(map[string]any{ - "command": "search", - "bundle": b.Root, - "filters": f, - "results": concepts, - "count": len(results), + "command": "search", + "bundle": b.Root, + "filters": f, + "results": concepts, + "count": len(results), }) } @@ -441,10 +514,10 @@ type schemaCommand struct { // schemaRoot is the top-level schema output. type schemaRoot struct { - Name string `json:"name"` - Version string `json:"version"` - Description string `json:"description"` - Commands []schemaCommand `json:"commands"` + Name string `json:"name"` + Version string `json:"version"` + Description string `json:"description"` + Commands []schemaCommand `json:"commands"` ExitCodes []cerr.ExitCodeDoc `json:"exit_codes"` } @@ -474,68 +547,68 @@ func buildSchemaRoot() schemaRoot { func allSchemaCommands() []schemaCommand { return []schemaCommand{ { - Name: "schema", - Short: "Print machine-readable CLI metadata as JSON", - Long: "Outputs a JSON document describing every command, its flags, arguments, output format, and exit codes. Pass a command name to describe just that command.", - Args: []schemaArg{{Name: "command", Required: false}}, - Stdout: "json", + Name: "schema", + Short: "Print machine-readable CLI metadata as JSON", + Long: "Outputs a JSON document describing every command, its flags, arguments, output format, and exit codes. Pass a command name to describe just that command.", + Args: []schemaArg{{Name: "command", Required: false}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeUsage}, }, { - Name: "init", - Short: "Create a new empty OKF bundle", - Long: "Creates a bundle directory with standard subdirectories (tables, datasets, playbooks), a root index.md, and a .gitignore. Fails if the directory already exists.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "init", + Short: "Create a new empty OKF bundle", + Long: "Creates a bundle directory with standard subdirectories (tables, datasets, playbooks), a root index.md, and a .gitignore. Fails if the directory already exists.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "validate", - Short: "Validate a bundle against the OKF spec", - Long: "Checks every concept for required frontmatter (type), recommended fields (title, description, tags), non-empty body, and valid cross-links. Exits 1 if any errors are found.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "validate", + Short: "Validate a bundle against the OKF spec", + Long: "Checks every concept against OKF v0.2: required frontmatter (type), recommended fields (title, description, tags), non-empty body, valid cross-links, the provenance/trust/lifecycle families (sources, generated, verified, status, stale_after), the Attested Computation contract (runtime, parameters, computation, executor, attester), reserved-file structure (index.md, log.md), and legacy v0.1 constructs (timestamp, # Citations). Exits 1 if any errors are found.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeValidation, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "lint", - Short: "Check recommended fields and style (warnings only)", - Long: "Same checks as validate but only emits warnings — errors are suppressed. Exits 0 even with warnings.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "lint", + Short: "Check recommended fields and style (warnings only)", + Long: "Same checks as validate but only emits warnings - errors are suppressed. Exits 0 even with warnings.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "index", - Short: "Generate index.md files (progressive disclosure)", - Long: "Writes index.md into every directory containing concept documents, providing progressive disclosure per OKF spec §6.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "index", + Short: "Generate index.md files (progressive disclosure)", + Long: "Writes index.md into every directory containing concept documents, providing progressive disclosure per OKF spec §6.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "list", - Short: "List all concepts in the bundle", - Long: "Lists every concept document with its ID, type, and title.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "list", + Short: "List all concepts in the bundle", + Long: "Lists every concept document with its ID, type, title, lifecycle status, and trust tier (unverified, machine-confirmed, or human-reviewed).", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "show", - Short: "Show a single concept's full content", - Long: "Returns the concept's ID, file path, frontmatter (type, title, description, resource, tags), and markdown body as JSON.", + Name: "show", + Short: "Show a single concept's full content", + Long: "Returns the concept's ID, file path, frontmatter (type, title, description, resource, tags), trust and lifecycle state (status, trust_tier, stale, generated, verified, stale_after), provenance (sources, usage_window), the Attested Computation contract when present (runtime, parameters, computation, executor, attester), and markdown body as JSON.", Args: []schemaArg{ {Name: "bundle", Required: true}, {Name: "concept-id", Required: true}, }, - Stdout: "json", + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeValidation, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "search", - Short: "Search concepts by tag, type, or text", - Long: "Filters concepts in a bundle by tag (--tag), frontmatter type (--type), or full-text search in title, description, and body (--text). Multiple filters are AND-combined. With no filters, returns all concepts.", + Name: "search", + Short: "Search concepts by tag, type, or text", + Long: "Filters concepts in a bundle by tag (--tag), frontmatter type (--type), or full-text search in title, description, and body (--text). Multiple filters are AND-combined. With no filters, returns all concepts.", Flags: []schemaFlag{ {Name: "tag", Type: "string", Default: "", Description: "filter by tag (case-insensitive)"}, {Name: "type", Type: "string", Default: "", Description: "filter by frontmatter type (case-insensitive)"}, @@ -544,32 +617,32 @@ func allSchemaCommands() []schemaCommand { Args: []schemaArg{ {Name: "bundle", Required: true}, }, - Stdout: "json", + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "backlinks", - Short: "List concepts that link to a given concept", - Long: "Returns the IDs of all concepts in the bundle that contain a markdown link to the specified concept. Deduplicates multiple links from the same source.", + Name: "backlinks", + Short: "List concepts that link to a given concept", + Long: "Returns the IDs of all concepts in the bundle that contain a markdown link to the specified concept. Deduplicates multiple links from the same source.", Args: []schemaArg{ {Name: "bundle", Required: true}, {Name: "concept-id", Required: true}, }, - Stdout: "json", + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "graph", - Short: "Print cross-link graph statistics", - Long: "Builds the directed cross-link graph from concept markdown links and prints nodes, edges, and summary statistics.", - Args: []schemaArg{{Name: "bundle", Required: true}}, - Stdout: "json", + Name: "graph", + Short: "Print cross-link graph statistics", + Long: "Builds the directed cross-link graph from concept markdown links and prints nodes, edges, and summary statistics.", + Args: []schemaArg{{Name: "bundle", Required: true}}, + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK, cerr.ExitCodeIO, cerr.ExitCodeUsage}, }, { - Name: "version", - Short: "Print version", - Stdout: "json", + Name: "version", + Short: "Print version", + Stdout: "json", ExitCodes: []int{cerr.ExitCodeOK}, }, } diff --git a/internal/bundle/bundle.go b/internal/bundle/bundle.go index 52ac3dc..5c03288 100644 --- a/internal/bundle/bundle.go +++ b/internal/bundle/bundle.go @@ -14,10 +14,10 @@ import ( // Bundle is a loaded OKF knowledge bundle. type Bundle struct { - Root string // absolute path to the bundle root - Concepts []*concept.Concept // all concept documents, sorted by ID + Root string // absolute path to the bundle root + Concepts []*concept.Concept // all concept documents, sorted by ID conceptByID map[string]*concept.Concept - Reserved []*concept.Concept // index.md / log.md files (parsed if present) + Reserved []*concept.Concept // index.md / log.md files (parsed if present) } // Load walks a bundle directory and parses every .md file. diff --git a/internal/bundle/bundle_test.go b/internal/bundle/bundle_test.go index 20382e3..9663cbc 100644 --- a/internal/bundle/bundle_test.go +++ b/internal/bundle/bundle_test.go @@ -21,8 +21,8 @@ func TestLoad_ValidBundle(t *testing.T) { } // Check concept IDs are derived correctly. wantIDs := map[string]bool{ - "datasets/ga4": false, - "tables/events_": false, + "datasets/ga4": false, + "tables/events_": false, "playbooks/freshness": false, } for _, c := range b.Concepts { diff --git a/internal/concept/concept.go b/internal/concept/concept.go index f35910b..ba80b0f 100644 --- a/internal/concept/concept.go +++ b/internal/concept/concept.go @@ -16,20 +16,36 @@ import ( // Frontmatter holds the recognized OKF frontmatter keys. Producers may include // arbitrary extra keys; they are preserved in Extensions. type Frontmatter struct { - Type string `yaml:"type"` - Title string `yaml:"title"` - Description string `yaml:"description"` - Resource string `yaml:"resource"` - Tags []string `yaml:"tags"` - Timestamp time.Time `yaml:"timestamp"` - Links StringList `yaml:"links"` - Extensions map[string]any `yaml:",inline"` + Type string `yaml:"type"` + Title string `yaml:"title"` + Description string `yaml:"description"` + Resource string `yaml:"resource"` + Tags TagList `yaml:"tags"` + Timestamp time.Time `yaml:"timestamp"` // legacy v0.1; superseded by generated.at (OKF §13.1) + Links StringList `yaml:"links"` + + // Provenance, trust, and lifecycle families (OKF v0.2 §5). + Sources []Source `yaml:"sources"` + UsageWindow *UsageWindow `yaml:"usage_window"` + Generated *Generated `yaml:"generated"` + Verified VerifiedList `yaml:"verified"` + Status string `yaml:"status"` + StaleAfter string `yaml:"stale_after"` + + // Attested Computation contract (OKF v0.2 §10). + Runtime string `yaml:"runtime"` + Parameters []Parameter `yaml:"parameters"` + Computation string `yaml:"computation"` + Executor *Executor `yaml:"executor"` + Attester *Attester `yaml:"attester"` + + Extensions map[string]any `yaml:",inline"` } // StringList is a []string that unmarshals from YAML flexibly: it accepts both // a single scalar string (links: /b) and a sequence of strings (links: [/b, /c]). // A missing/empty value stays a nil/empty slice, and a malformed shape (e.g. a -// mapping) is tolerated as an empty slice rather than failing the whole parse — +// mapping) is tolerated as an empty slice rather than failing the whole parse - // so one badly-shaped concept cannot take down an entire bundle load. type StringList []string @@ -44,7 +60,7 @@ func (s *StringList) UnmarshalYAML(value *yaml.Node) error { } var single string if err := value.Decode(&single); err != nil { - // Non-string scalar (e.g. a number) — tolerate as empty. + // Non-string scalar (e.g. a number) - tolerate as empty. *s = nil return nil } @@ -53,7 +69,7 @@ func (s *StringList) UnmarshalYAML(value *yaml.Node) error { case yaml.SequenceNode: var list []string if err := value.Decode(&list); err != nil { - // A sequence of non-strings — tolerate as empty rather than + // A sequence of non-strings - tolerate as empty rather than // failing the entire bundle load. *s = nil return nil @@ -61,23 +77,64 @@ func (s *StringList) UnmarshalYAML(value *yaml.Node) error { *s = StringList(list) return nil default: - // Mapping or any other unexpected shape — tolerate as empty. + // Mapping or any other unexpected shape - tolerate as empty. *s = nil return nil } } +// TagList is a []string that unmarshals from YAML flexibly. The spec form is +// a sequence (§4.1), but reference bundles in the wild emit a comma-separated +// scalar ("a, b, c"); consumers MUST NOT reject such documents (§11), so the +// scalar form is split on commas. Malformed shapes are tolerated as empty. +type TagList []string + +// UnmarshalYAML implements yaml.Unmarshaler, accepting sequence-or-scalar. +func (t *TagList) UnmarshalYAML(value *yaml.Node) error { + switch value.Kind { + case yaml.SequenceNode: + var list []string + if err := value.Decode(&list); err != nil { + *t = nil + return nil + } + *t = TagList(list) + return nil + case yaml.ScalarNode: + if value.Tag == "!!null" { + *t = nil + return nil + } + var single string + if err := value.Decode(&single); err != nil { + *t = nil + return nil + } + var tags []string + for _, part := range strings.Split(single, ",") { + if p := strings.TrimSpace(part); p != "" { + tags = append(tags, p) + } + } + *t = TagList(tags) + return nil + default: + *t = nil + return nil + } +} + // Concept is a parsed concept document. type Concept struct { // ID is the file path within the bundle with .md removed, e.g. "tables/users". - ID string - Path string // absolute path to the .md file on disk + ID string + Path string // absolute path to the .md file on disk Frontmatter Frontmatter - Body string - RawFront string // raw YAML block for round-tripping + Body string + RawFront string // raw YAML block for round-tripping } -// Reserved filenames per OKF spec §3.1 — they are NOT concepts. +// Reserved filenames per OKF spec §3.1 - they are NOT concepts. var ReservedNames = map[string]bool{ "index.md": true, "log.md": true, diff --git a/internal/concept/concept_test.go b/internal/concept/concept_test.go index ccecd30..a400830 100644 --- a/internal/concept/concept_test.go +++ b/internal/concept/concept_test.go @@ -41,7 +41,7 @@ func TestParse_LinksMissing(t *testing.T) { } func TestParse_LinksMalformedShapeTolerated(t *testing.T) { - // A mapping (or otherwise unexpected) shape must not fail the parse — it is + // A mapping (or otherwise unexpected) shape must not fail the parse - it is // tolerated as an empty list so one bad concept cannot kill a bundle load. raw := []byte("---\ntype: T\ntitle: A\nlinks:\n target: /b\n---\n\nbody") c, err := ParseBytes(raw, "a.md", "/tmp/a.md") diff --git a/internal/concept/v02.go b/internal/concept/v02.go new file mode 100644 index 0000000..1c17ea7 --- /dev/null +++ b/internal/concept/v02.go @@ -0,0 +1,193 @@ +// OKF v0.2 frontmatter families: provenance (§5.1), trust (§5.2–5.3), +// lifecycle (§5.4–5.5), the actor convention (§7), and the Attested +// Computation contract (§10). + +package concept + +import ( + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +// TypeAttestedComputation is the concept type carrying a sanctioned +// computation (OKF §10). +const TypeAttestedComputation = "Attested Computation" + +// Trust tiers derived from `verified` (OKF §5.3), lowest to highest. +const ( + TierUnverified = "unverified" + TierMachineConfirmed = "machine-confirmed" + TierHumanReviewed = "human-reviewed" +) + +// Actor kinds under the actor convention (OKF §7). +const ( + ActorHuman = "human" + ActorProcess = "process" + ActorAgent = "agent" + ActorUnknown = "unknown" +) + +// Source is one entry of the `sources` provenance family (OKF §5.1). +// Resource is REQUIRED within an entry; it names either a followable artifact +// (URL or path) or a population/scope descriptor. The credibility signals +// (author, usage_count, last_modified) are optional and objective; OKF records +// signals, not a score. +type Source struct { + ID string `yaml:"id"` + Resource string `yaml:"resource"` + Title string `yaml:"title"` + Author string `yaml:"author"` + UsageCount *int64 `yaml:"usage_count"` + LastModified string `yaml:"last_modified"` + UsageWindow *UsageWindow `yaml:"usage_window"` // per-entry override of the shared window +} + +// UsageWindow frames usage_count with a { from, to } date range (OKF §5.1). +type UsageWindow struct { + From string `yaml:"from"` + To string `yaml:"to"` +} + +// Generated records how the current content was produced (OKF §5.2). +type Generated struct { + By string `yaml:"by"` // REQUIRED within generated; an actor (§7) + At string `yaml:"at"` // ISO 8601 datetime of the last meaningful change +} + +// Verification is one verification event (OKF §5.2). +type Verification struct { + By string `yaml:"by"` + At string `yaml:"at"` +} + +// VerifiedList unmarshals the `verified` field. Consumers MUST treat a bare +// { by, at } mapping as a one-element list (OKF §5.2, §11), so both shapes are +// accepted. Malformed shapes are tolerated as empty rather than failing the +// whole bundle load, matching StringList's posture. +type VerifiedList []Verification + +// UnmarshalYAML implements yaml.Unmarshaler, accepting mapping-or-sequence. +func (v *VerifiedList) UnmarshalYAML(value *yaml.Node) error { + switch value.Kind { + case yaml.MappingNode: + var one Verification + if err := value.Decode(&one); err != nil { + *v = nil + return nil + } + *v = VerifiedList{one} + return nil + case yaml.SequenceNode: + var list []Verification + if err := value.Decode(&list); err != nil { + *v = nil + return nil + } + *v = VerifiedList(list) + return nil + default: + *v = nil + return nil + } +} + +// Parameter is one typed, named hole of an Attested Computation (OKF §10.2). +type Parameter struct { + Name string `yaml:"name"` + Type string `yaml:"type"` + Required bool `yaml:"required"` +} + +// Executor says how a computation is run and what evidence a run must return +// (OKF §10.2). +type Executor struct { + Resource string `yaml:"resource"` + Receipt []string `yaml:"receipt"` +} + +// Attester names deterministic (no-LLM) code that inspects a receipt and +// returns a verdict (OKF §10.2). +type Attester struct { + Resource string `yaml:"resource"` +} + +// ValidStatuses are the lifecycle states of §5.4. +var ValidStatuses = map[string]bool{"draft": true, "stable": true, "deprecated": true} + +// EffectiveStatus returns the concept's lifecycle status, defaulting to +// "stable" when absent (OKF §5.4). An invalid value is returned as written; +// validation flags it separately. +func (f *Frontmatter) EffectiveStatus() string { + if strings.TrimSpace(f.Status) == "" { + return "stable" + } + return f.Status +} + +// TrustTier derives the trust tier from `verified` (OKF §5.3): no verified key +// means unverified; only non-human actors means machine-confirmed; any +// human: actor means human-reviewed. +func (f *Frontmatter) TrustTier() string { + if len(f.Verified) == 0 { + return TierUnverified + } + for _, v := range f.Verified { + if ActorKind(v.By) == ActorHuman { + return TierHumanReviewed + } + } + return TierMachineConfirmed +} + +// GeneratedAt returns when the content last meaningfully changed: generated.at +// when present, falling back to the legacy v0.1 `timestamp` (OKF §13.1). +// ok is false when neither is present or parseable. +func (f *Frontmatter) GeneratedAt() (at time.Time, ok bool) { + if f.Generated != nil && f.Generated.At != "" { + if t, err := ParseDatetime(f.Generated.At); err == nil { + return t, true + } + } + if !f.Timestamp.IsZero() { + return f.Timestamp, true + } + return time.Time{}, false +} + +// IsStale reports whether the concept is stale as of today: stale when +// today >= stale_after (OKF §5.5). Absent or unparseable stale_after is never +// stale; validation flags the unparseable case separately. +func (f *Frontmatter) IsStale(today time.Time) bool { + if strings.TrimSpace(f.StaleAfter) == "" { + return false + } + d, err := time.Parse("2006-01-02", f.StaleAfter) + if err != nil { + return false + } + return !today.Before(d) +} + +// ActorKind classifies an actor string per the convention of §7: +// human:, process:, or / for agents and tools. +func ActorKind(actor string) string { + switch { + case strings.HasPrefix(actor, "human:") && len(actor) > len("human:"): + return ActorHuman + case strings.HasPrefix(actor, "process:") && len(actor) > len("process:"): + return ActorProcess + case strings.Contains(actor, "/") && !strings.ContainsAny(actor, " \t"): + return ActorAgent + default: + return ActorUnknown + } +} + +// ParseDatetime parses an ISO 8601 datetime as used by generated.at and +// verified[].at. +func ParseDatetime(s string) (time.Time, error) { + return time.Parse(time.RFC3339, s) +} diff --git a/internal/concept/v02_test.go b/internal/concept/v02_test.go new file mode 100644 index 0000000..e19ced6 --- /dev/null +++ b/internal/concept/v02_test.go @@ -0,0 +1,344 @@ +package concept + +import ( + "reflect" + "testing" + "time" +) + +// --- §5.2 generated --- + +func TestParse_Generated(t *testing.T) { + raw := []byte("---\ntype: T\ngenerated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-20T22:53:05Z }\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + g := c.Frontmatter.Generated + if g == nil { + t.Fatal("Generated = nil, want parsed mapping") + } + if g.By != "reference_agent/gemini-2.5-pro" { + t.Errorf("Generated.By = %q", g.By) + } + if g.At != "2026-06-20T22:53:05Z" { + t.Errorf("Generated.At = %q", g.At) + } +} + +// --- §5.2 verified: bare mapping MUST be treated as a one-element list --- + +func TestParse_VerifiedBareMapping(t *testing.T) { + raw := []byte("---\ntype: T\nverified: { by: human:ahormati, at: 2026-06-25T09:00:00Z }\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + v := c.Frontmatter.Verified + if len(v) != 1 { + t.Fatalf("Verified len = %d, want 1 (bare mapping is a one-element list, OKF §5.2)", len(v)) + } + if v[0].By != "human:ahormati" || v[0].At != "2026-06-25T09:00:00Z" { + t.Errorf("Verified[0] = %+v", v[0]) + } +} + +func TestParse_VerifiedList(t *testing.T) { + raw := []byte("---\ntype: T\nverified:\n - { by: human:ahormati, at: 2026-06-25T09:00:00Z }\n - { by: process:finance-nightly, at: 2026-06-26T02:00:00Z }\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + v := c.Frontmatter.Verified + if len(v) != 2 { + t.Fatalf("Verified len = %d, want 2", len(v)) + } + if v[1].By != "process:finance-nightly" { + t.Errorf("Verified[1].By = %q", v[1].By) + } +} + +func TestParse_VerifiedMalformedTolerated(t *testing.T) { + // A scalar (or otherwise unexpected shape) must not fail the whole parse. + raw := []byte("---\ntype: T\nverified: yes\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes should tolerate malformed verified, got: %v", err) + } + if len(c.Frontmatter.Verified) != 0 { + t.Errorf("Verified = %+v, want empty", c.Frontmatter.Verified) + } +} + +// --- §5.3 trust tiers --- + +func TestTrustTier(t *testing.T) { + cases := []struct { + name string + yaml string + want string + }{ + {"no verified key", "---\ntype: T\n---\n", TierUnverified}, + {"machine only", "---\ntype: T\nverified: { by: process:nightly, at: 2026-01-01T00:00:00Z }\n---\n", TierMachineConfirmed}, + {"agent only", "---\ntype: T\nverified: { by: agent/v1, at: 2026-01-01T00:00:00Z }\n---\n", TierMachineConfirmed}, + {"human present", "---\ntype: T\nverified:\n - { by: process:nightly, at: 2026-01-01T00:00:00Z }\n - { by: human:ahormati, at: 2026-01-02T00:00:00Z }\n---\n", TierHumanReviewed}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, err := ParseBytes([]byte(tc.yaml+"body"), "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + if got := c.Frontmatter.TrustTier(); got != tc.want { + t.Errorf("TrustTier() = %q, want %q", got, tc.want) + } + }) + } +} + +// --- §7 actor convention --- + +func TestActorKind(t *testing.T) { + cases := []struct{ actor, want string }{ + {"human:ahormati", ActorHuman}, + {"process:finance-nightly", ActorProcess}, + {"reference_agent/gemini-2.5-pro", ActorAgent}, + {"", ActorUnknown}, + {"just a name", ActorUnknown}, + } + for _, tc := range cases { + if got := ActorKind(tc.actor); got != tc.want { + t.Errorf("ActorKind(%q) = %q, want %q", tc.actor, got, tc.want) + } + } +} + +// --- §5.4 status --- + +func TestEffectiveStatus(t *testing.T) { + cases := []struct{ yaml, want string }{ + {"---\ntype: T\n---\n", "stable"}, // absent => stable + {"---\ntype: T\nstatus: draft\n---\n", "draft"}, + {"---\ntype: T\nstatus: deprecated\n---\n", "deprecated"}, + } + for _, tc := range cases { + c, err := ParseBytes([]byte(tc.yaml+"body"), "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + if got := c.Frontmatter.EffectiveStatus(); got != tc.want { + t.Errorf("EffectiveStatus() = %q, want %q", got, tc.want) + } + } +} + +// --- §5.5 stale_after --- + +func TestIsStale(t *testing.T) { + raw := []byte("---\ntype: T\nstale_after: 2026-09-23\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + day := func(s string) time.Time { + d, _ := time.Parse("2006-01-02", s) + return d + } + if c.Frontmatter.IsStale(day("2026-09-22")) { + t.Error("IsStale(day before) = true, want false") + } + if !c.Frontmatter.IsStale(day("2026-09-23")) { + t.Error("IsStale(same day) = false, want true (stale when today >= stale_after)") + } + if !c.Frontmatter.IsStale(day("2026-10-01")) { + t.Error("IsStale(after) = false, want true") + } +} + +func TestIsStale_AbsentNeverStale(t *testing.T) { + c, err := ParseBytes([]byte("---\ntype: T\n---\nbody"), "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + if c.Frontmatter.IsStale(time.Now()) { + t.Error("IsStale with no stale_after = true, want false") + } +} + +// --- §13.1 generated.at with legacy timestamp fallback --- + +func TestGeneratedAt_FallbackToLegacyTimestamp(t *testing.T) { + // v0.1 concept: only timestamp. + c, err := ParseBytes([]byte("---\ntype: T\ntimestamp: 2026-05-28T22:53:05Z\n---\nbody"), "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + at, ok := c.Frontmatter.GeneratedAt() + if !ok { + t.Fatal("GeneratedAt ok = false, want fallback to legacy timestamp (OKF §13.1)") + } + if at.Year() != 2026 || at.Month() != 5 { + t.Errorf("GeneratedAt = %v", at) + } + + // v0.2 concept: generated.at wins over timestamp. + c2, err := ParseBytes([]byte("---\ntype: T\ntimestamp: 2020-01-01T00:00:00Z\ngenerated: { by: a/1, at: 2026-06-20T22:53:05Z }\n---\nbody"), "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + at2, ok := c2.Frontmatter.GeneratedAt() + if !ok || at2.Year() != 2026 { + t.Errorf("GeneratedAt = %v ok=%v, want generated.at to win", at2, ok) + } +} + +// --- §5.1 sources and usage_window --- + +func TestParse_SourcesWithCredibilitySignals(t *testing.T) { + raw := []byte(`--- +type: T +sources: + - id: ga4-schema + resource: https://developers.google.com/analytics/bigquery/export-schema + title: GA4 BigQuery Export schema + author: team:ga4-docs + usage_count: 5000 + last_modified: 2026-05-30 + - resource: all queries in BigQuery project X +usage_window: { from: 2026-06-01, to: 2026-06-30 } +--- +body`) + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + s := c.Frontmatter.Sources + if len(s) != 2 { + t.Fatalf("Sources len = %d, want 2", len(s)) + } + first := s[0] + if first.ID != "ga4-schema" || first.Author != "team:ga4-docs" || first.Title != "GA4 BigQuery Export schema" { + t.Errorf("Sources[0] = %+v", first) + } + if first.UsageCount == nil || *first.UsageCount != 5000 { + t.Errorf("Sources[0].UsageCount = %v, want 5000", first.UsageCount) + } + if first.LastModified != "2026-05-30" { + t.Errorf("Sources[0].LastModified = %q", first.LastModified) + } + if s[1].Resource != "all queries in BigQuery project X" { + t.Errorf("Sources[1].Resource = %q (scope descriptors are valid resources)", s[1].Resource) + } + w := c.Frontmatter.UsageWindow + if w == nil || w.From != "2026-06-01" || w.To != "2026-06-30" { + t.Errorf("UsageWindow = %+v", w) + } +} + +func TestParse_SourceEntryUsageWindowOverride(t *testing.T) { + raw := []byte(`--- +type: T +sources: + - id: s1 + resource: dashboards/exec + usage_count: 10 + usage_window: { from: 2026-01-01, to: 2026-01-31 } +--- +body`) + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + w := c.Frontmatter.Sources[0].UsageWindow + if w == nil || w.From != "2026-01-01" { + t.Errorf("per-entry UsageWindow = %+v, want override present", w) + } +} + +// --- §10 attested computation contract --- + +func TestParse_AttestedComputationContract(t *testing.T) { + raw := []byte(`--- +type: Attested Computation +title: Revenue for fiscal year +runtime: bigquery +parameters: + - { name: year, type: integer, required: true } +computation: references/computations/lib/revenue.sql +executor: + resource: references/skills/run-on-bq.md + receipt: [job_id, executed_sql, result] +attester: + resource: references/attesters/revenue.py +--- +body`) + c, err := ParseBytes(raw, "computations/revenue.md", "/tmp/revenue.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + fm := c.Frontmatter + if fm.Type != TypeAttestedComputation { + t.Errorf("Type = %q, want %q", fm.Type, TypeAttestedComputation) + } + if fm.Runtime != "bigquery" { + t.Errorf("Runtime = %q", fm.Runtime) + } + wantParams := []Parameter{{Name: "year", Type: "integer", Required: true}} + if !reflect.DeepEqual(fm.Parameters, wantParams) { + t.Errorf("Parameters = %+v, want %+v", fm.Parameters, wantParams) + } + if fm.Computation != "references/computations/lib/revenue.sql" { + t.Errorf("Computation = %q", fm.Computation) + } + if fm.Executor == nil || fm.Executor.Resource != "references/skills/run-on-bq.md" { + t.Fatalf("Executor = %+v", fm.Executor) + } + if !reflect.DeepEqual(fm.Executor.Receipt, []string{"job_id", "executed_sql", "result"}) { + t.Errorf("Executor.Receipt = %v", fm.Executor.Receipt) + } + if fm.Attester == nil || fm.Attester.Resource != "references/attesters/revenue.py" { + t.Errorf("Attester = %+v", fm.Attester) + } +} + +// --- extensions still preserved alongside new fields --- + +func TestParse_ExtensionsPreservedWithV02Fields(t *testing.T) { + raw := []byte("---\ntype: T\nstatus: stable\ncustom_key: hello\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + if c.Frontmatter.Extensions["custom_key"] != "hello" { + t.Errorf("Extensions = %v, want custom_key preserved", c.Frontmatter.Extensions) + } + if _, leaked := c.Frontmatter.Extensions["status"]; leaked { + t.Error("status leaked into Extensions; should be a recognized field") + } +} + +// Upstream reference bundles emit tags as a comma-separated scalar rather +// than a YAML list. Consumers MUST NOT reject such documents (§11), so the +// scalar form is tolerated and split on commas. +func TestParse_TagsScalarTolerated(t *testing.T) { + raw := []byte("---\ntype: T\ntags: Stack Overflow, Q&A, developer\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes should tolerate scalar tags, got: %v", err) + } + want := []string{"Stack Overflow", "Q&A", "developer"} + if !reflect.DeepEqual([]string(c.Frontmatter.Tags), want) { + t.Errorf("Tags = %v, want %v", c.Frontmatter.Tags, want) + } +} + +func TestParse_TagsListStillWorks(t *testing.T) { + raw := []byte("---\ntype: T\ntags: [a, b]\n---\nbody") + c, err := ParseBytes(raw, "a.md", "/tmp/a.md") + if err != nil { + t.Fatalf("ParseBytes: %v", err) + } + if !reflect.DeepEqual([]string(c.Frontmatter.Tags), []string{"a", "b"}) { + t.Errorf("Tags = %v", c.Frontmatter.Tags) + } +} diff --git a/internal/graph/graph.go b/internal/graph/graph.go index 7689dc8..0593f2c 100644 --- a/internal/graph/graph.go +++ b/internal/graph/graph.go @@ -15,8 +15,8 @@ import ( // Graph is the directed cross-link graph of a bundle. type Graph struct { - Nodes []Node - Edges []Edge + Nodes []Node + Edges []Edge // Backlinks maps concept ID -> list of concept IDs that link to it. Backlinks map[string][]string } @@ -49,11 +49,21 @@ func Build(b *bundle.Bundle) *Graph { // uniformly. The edgeSet dedup handles overlaps. bodyLinks := validate.ExtractLinks(c.Body) fmLinks := validate.ExtractFrontmatterLinks(c.Frontmatter.Links) - links := make([]validate.Link, 0, len(bodyLinks)+len(fmLinks)) + pathRefs := pathValuedRefs(c) + links := make([]validate.Link, 0, len(bodyLinks)+len(fmLinks)+len(pathRefs)) links = append(links, bodyLinks...) links = append(links, fmLinks...) + links = append(links, pathRefs...) for _, link := range links { target := resolveLink(c.ID, link) + if target != "" && !b.HasConcept(target) { + // Bare root-relative fallback: contract paths in the wild (and + // in the spec's §10.2 example) omit the leading slash. The + // bundle-root interpretation only differs for relative links. + if rootTarget := resolveLink("", link); rootTarget != "" && b.HasConcept(rootTarget) { + target = rootTarget + } + } if target == "" || target == c.ID || !b.HasConcept(target) { continue // skip external, unresolved, self, and dangling links } @@ -78,10 +88,10 @@ func Build(b *bundle.Bundle) *Graph { // Stats returns summary statistics about the graph. type Stats struct { - NodeCount int - EdgeCount int - IsolatedNodes int // nodes with no inbound or outbound edges - MaxBacklinks int // highest number of backlinks on any single concept + NodeCount int + EdgeCount int + IsolatedNodes int // nodes with no inbound or outbound edges + MaxBacklinks int // highest number of backlinks on any single concept } // Stats computes summary statistics. @@ -119,6 +129,32 @@ func resolveLink(fromConceptID string, link validate.Link) string { return validate.ResolveLink(fromConceptID, link) } +// pathValuedRefs collects the path-valued frontmatter fields of OKF §6.2 that +// can reference other concepts: sources[].resource (a derivation edge per +// §5.1), computation, executor.resource, and attester.resource. Values that +// are not concept paths (external URLs, scope descriptors, non-.md files) +// simply fail to resolve downstream and produce no edge. +func pathValuedRefs(c *concept.Concept) []validate.Link { + fm := c.Frontmatter + var refs []validate.Link + add := func(target string) { + if strings.TrimSpace(target) != "" { + refs = append(refs, validate.Link{Target: target}) + } + } + for _, s := range fm.Sources { + add(s.Resource) + } + add(fm.Computation) + if fm.Executor != nil { + add(fm.Executor.Resource) + } + if fm.Attester != nil { + add(fm.Attester.Resource) + } + return refs +} + // Summary returns a human-readable summary string. func (g *Graph) Summary() string { s := g.Stats() @@ -131,6 +167,3 @@ func (g *Graph) Summary() string { } return sb.String() } - -// silence unused import warning for concept (used transitively via bundle). -var _ = concept.ConceptID diff --git a/internal/graph/v02_test.go b/internal/graph/v02_test.go new file mode 100644 index 0000000..3e8891e --- /dev/null +++ b/internal/graph/v02_test.go @@ -0,0 +1,92 @@ +package graph + +import ( + "os" + "path/filepath" + "testing" + + "github.com/okfcli/okf/internal/bundle" +) + +func loadBundle(t *testing.T, files map[string]string) *bundle.Bundle { + t.Helper() + dir := t.TempDir() + for path, content := range files { + full := filepath.Join(dir, path) + if err := os.MkdirAll(filepath.Dir(full), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(full, []byte(content), 0644); err != nil { + t.Fatalf("write: %v", err) + } + } + b, err := bundle.Load(dir) + if err != nil { + t.Fatalf("Load: %v", err) + } + return b +} + +func hasEdge(g *Graph, from, to string) bool { + for _, e := range g.Edges { + if e.From == from && e.To == to { + return true + } + } + return false +} + +// §5.1: when a sources[].resource points at another OKF concept, the +// derivation edge exists in the bundle graph. +func TestBuild_SourceResourceDerivationEdge(t *testing.T) { + b := loadBundle(t, map[string]string{ + "metrics/revenue.md": "---\ntype: Metric\ntitle: R\nsources:\n - id: dash\n resource: /dashboards/exec.md\n---\n\nbody", + "dashboards/exec.md": "---\ntype: Dashboard\ntitle: D\n---\n\nbody", + }) + g := Build(b) + if !hasEdge(g, "metrics/revenue", "dashboards/exec") { + t.Fatalf("expected derivation edge metrics/revenue -> dashboards/exec, edges = %+v", g.Edges) + } +} + +// §6.2: computation and executor.resource are path-valued and produce edges +// when they resolve to concepts; external URLs and non-concept files do not. +func TestBuild_ComputationContractEdges(t *testing.T) { + b := loadBundle(t, map[string]string{ + "computations/revenue.md": "---\ntype: Attested Computation\ntitle: R\nruntime: bigquery\nexecutor:\n resource: /references/skills/run-on-bq.md\nattester:\n resource: /references/attesters/rev.py\nsources:\n - id: pol\n resource: https://wiki.example/policy\n---\n\n# Computation\n\n SELECT 1\n", + "references/skills/run-on-bq.md": "---\ntype: Skill\ntitle: S\n---\n\nbody", + }) + g := Build(b) + if !hasEdge(g, "computations/revenue", "references/skills/run-on-bq") { + t.Fatalf("expected edge to executor skill concept, edges = %+v", g.Edges) + } + // The .py attester and the external URL are not concepts: no edges, no panic. + if len(g.Edges) != 1 { + t.Fatalf("expected exactly 1 edge, got %+v", g.Edges) + } +} + +// A scope-descriptor resource ("all queries in project X") must not produce +// an edge or a bogus node. +func TestBuild_ScopeDescriptorResourceIgnored(t *testing.T) { + b := loadBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\nsources:\n - resource: all queries in BigQuery project X\n---\n\nbody", + }) + g := Build(b) + if len(g.Edges) != 0 { + t.Fatalf("expected no edges, got %+v", g.Edges) + } +} + +// Bare root-relative contract paths (as used by upstream bundles and the +// spec's §10.2 example) still produce derivation edges via root fallback. +func TestBuild_RootRelativeContractPathEdge(t *testing.T) { + b := loadBundle(t, map[string]string{ + "computations/c.md": "---\ntype: Attested Computation\ntitle: C\nruntime: bigquery\nexecutor:\n resource: skills/run.md\n---\n\n# Computation\n\n SELECT 1\n", + "skills/run.md": "---\ntype: Skill\ntitle: S\n---\n\nbody", + }) + g := Build(b) + if !hasEdge(g, "computations/c", "skills/run") { + t.Fatalf("expected root-relative fallback edge, edges = %+v", g.Edges) + } +} diff --git a/internal/index/index.go b/internal/index/index.go index 8c406db..b278ae4 100644 --- a/internal/index/index.go +++ b/internal/index/index.go @@ -1,6 +1,6 @@ // Package index generates index.md files for OKF bundle directories. // Per OKF spec §6, index.md provides a directory listing for progressive -// disclosure — agents and humans navigate one level at a time. +// disclosure - agents and humans navigate one level at a time. package index import ( @@ -77,9 +77,18 @@ func generateForDir(root, dir string) error { sort.Strings(subdirs) var sb strings.Builder - sb.WriteString("# Index\n\n") relDir, _ := filepath.Rel(root, dir) relDir = filepath.ToSlash(relDir) + // A bundle-root index.md may declare okf_version in a frontmatter block - + // the only frontmatter an index file is permitted to carry (OKF §8, §12). + // Regeneration preserves an existing declaration; it never adds one, since + // the bundle may deliberately target an older revision. + if relDir == "." { + if ver := rootOKFVersion(filepath.Join(dir, "index.md")); ver != "" { + fmt.Fprintf(&sb, "---\nokf_version: %q\n---\n\n", ver) + } + } + sb.WriteString("# Index\n\n") if relDir == "." { sb.WriteString("Bundle root.\n\n") } else { @@ -112,6 +121,19 @@ func generateForDir(root, dir string) error { return os.WriteFile(indexPath, []byte(sb.String()), 0644) } +// rootOKFVersion reads an existing root index.md and returns its declared +// okf_version, or "" when the file or declaration is absent. +func rootOKFVersion(path string) string { + c, err := concept.ParseReserved(path, "index.md") + if err != nil { + return "" + } + if v, ok := c.Frontmatter.Extensions["okf_version"]; ok { + return fmt.Sprintf("%v", v) + } + return "" +} + type conceptInfo struct { filename string title string diff --git a/internal/index/index_test.go b/internal/index/index_test.go new file mode 100644 index 0000000..0dcfd4f --- /dev/null +++ b/internal/index/index_test.go @@ -0,0 +1,79 @@ +package index + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func writeFiles(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for path, content := range files { + full := filepath.Join(dir, path) + if err := os.MkdirAll(filepath.Dir(full), 0755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(full, []byte(content), 0644); err != nil { + t.Fatalf("write: %v", err) + } + } + return dir +} + +func TestGenerate_WritesIndexes(t *testing.T) { + dir := writeFiles(t, map[string]string{ + "tables/users.md": "---\ntype: Table\ntitle: Users\ndescription: d\n---\n\nbody", + }) + if err := Generate(dir); err != nil { + t.Fatalf("Generate: %v", err) + } + for _, p := range []string{"index.md", "tables/index.md"} { + if _, err := os.Stat(filepath.Join(dir, p)); err != nil { + t.Errorf("missing %s: %v", p, err) + } + } +} + +// §12: a bundle-root index.md may declare okf_version; regeneration must +// preserve an existing declaration rather than dropping it. +func TestGenerate_PreservesRootOkfVersion(t *testing.T) { + dir := writeFiles(t, map[string]string{ + "index.md": "---\nokf_version: \"0.2\"\n---\n# Index\n\nstale content\n", + "tables/users.md": "---\ntype: Table\ntitle: Users\ndescription: d\n---\n\nbody", + }) + if err := Generate(dir); err != nil { + t.Fatalf("Generate: %v", err) + } + got, err := os.ReadFile(filepath.Join(dir, "index.md")) + if err != nil { + t.Fatalf("read root index: %v", err) + } + s := string(got) + if !strings.HasPrefix(s, "---\n") || !strings.Contains(s, `okf_version: "0.2"`) { + t.Errorf("root index.md lost its okf_version declaration:\n%s", s) + } + if strings.Contains(s, "stale content") { + t.Errorf("root index.md body was not regenerated:\n%s", s) + } +} + +// A root index with no prior declaration must not gain one implicitly (the +// bundle may target v0.1), and non-root indexes never carry frontmatter (§8). +func TestGenerate_NoImplicitOkfVersion(t *testing.T) { + dir := writeFiles(t, map[string]string{ + "tables/users.md": "---\ntype: Table\ntitle: Users\ndescription: d\n---\n\nbody", + }) + if err := Generate(dir); err != nil { + t.Fatalf("Generate: %v", err) + } + root, _ := os.ReadFile(filepath.Join(dir, "index.md")) + if strings.Contains(string(root), "okf_version") { + t.Errorf("root index.md gained an implicit okf_version:\n%s", root) + } + sub, _ := os.ReadFile(filepath.Join(dir, "tables", "index.md")) + if strings.HasPrefix(string(sub), "---\n") { + t.Errorf("non-root index.md must not carry frontmatter:\n%s", sub) + } +} diff --git a/internal/initbundle/initbundle.go b/internal/initbundle/initbundle.go index 66e3f60..fe55667 100644 --- a/internal/initbundle/initbundle.go +++ b/internal/initbundle/initbundle.go @@ -31,8 +31,9 @@ func Create(dir string) error { } } - // Root index.md - indexContent := "# Index\n\nBundle root.\n\n## Subdirectories\n\n- [tables/](tables/index.md)\n- [datasets/](datasets/index.md)\n- [playbooks/](playbooks/index.md)\n" + // Root index.md. The frontmatter block declares the OKF spec revision the + // bundle targets - the only frontmatter an index.md may carry (OKF §8, §12). + indexContent := "---\nokf_version: \"0.2\"\n---\n\n# Index\n\nBundle root.\n\n## Subdirectories\n\n- [tables/](tables/index.md)\n- [datasets/](datasets/index.md)\n- [playbooks/](playbooks/index.md)\n" if err := os.WriteFile(filepath.Join(dir, "index.md"), []byte(indexContent), 0644); err != nil { return fmt.Errorf("write index.md: %w", err) } diff --git a/internal/initbundle/initbundle_test.go b/internal/initbundle/initbundle_test.go index a7bcb00..03ad86c 100644 --- a/internal/initbundle/initbundle_test.go +++ b/internal/initbundle/initbundle_test.go @@ -3,6 +3,7 @@ package initbundle import ( "os" "path/filepath" + "strings" "testing" ) @@ -70,3 +71,20 @@ func TestCreate_CreatesGitignore(t *testing.T) { t.Fatalf("missing .gitignore: %v", err) } } + +// §12: a freshly created bundle targets OKF v0.2 via a root index.md +// frontmatter declaration. +func TestCreate_RootIndexDeclaresOKFVersion(t *testing.T) { + dir := filepath.Join(t.TempDir(), "mybundle") + if err := Create(dir); err != nil { + t.Fatalf("Create: %v", err) + } + got, err := os.ReadFile(filepath.Join(dir, "index.md")) + if err != nil { + t.Fatalf("read index.md: %v", err) + } + s := string(got) + if !strings.HasPrefix(s, "---\n") || !strings.Contains(s, `okf_version: "0.2"`) { + t.Errorf("root index.md should declare okf_version \"0.2\" (OKF §12):\n%s", s) + } +} diff --git a/internal/search/search_test.go b/internal/search/search_test.go index ff8345e..26719a3 100644 --- a/internal/search/search_test.go +++ b/internal/search/search_test.go @@ -28,9 +28,9 @@ func TestSearch_ByTag(t *testing.T) { func TestSearch_ByType(t *testing.T) { b := testBundle(t, map[string]string{ - "tables/users.md": "---\ntype: Table\ntitle: Users\n---\n\nbody", - "datasets/ga4.md": "---\ntype: Dataset\ntitle: GA4\n---\n\nbody", - "tables/orders.md": "---\ntype: Table\ntitle: Orders\n---\n\nbody", + "tables/users.md": "---\ntype: Table\ntitle: Users\n---\n\nbody", + "datasets/ga4.md": "---\ntype: Dataset\ntitle: GA4\n---\n\nbody", + "tables/orders.md": "---\ntype: Table\ntitle: Orders\n---\n\nbody", }) results := Search(b, Filters{Type: "Table"}) @@ -41,8 +41,8 @@ func TestSearch_ByType(t *testing.T) { func TestSearch_ByText(t *testing.T) { b := testBundle(t, map[string]string{ - "tables/users.md": "---\ntype: Table\ntitle: Users\n---\n\n# Users\n\nOne row per authenticated user account.", - "tables/orders.md": "---\ntype: Table\ntitle: Orders\n---\n\n# Orders\n\nOne row per customer order.", + "tables/users.md": "---\ntype: Table\ntitle: Users\n---\n\n# Users\n\nOne row per authenticated user account.", + "tables/orders.md": "---\ntype: Table\ntitle: Orders\n---\n\n# Orders\n\nOne row per customer order.", "tables/products.md": "---\ntype: Table\ntitle: Products\n---\n\n# Products\n\nProduct catalog.", }) @@ -69,9 +69,9 @@ func TestSearch_TextMatchesTitleAndDescription(t *testing.T) { func TestSearch_CombinedFilters(t *testing.T) { b := testBundle(t, map[string]string{ - "tables/users.md": "---\ntype: Table\ntitle: Users\ntags: [auth]\n---\n\nuser accounts", - "tables/orders.md": "---\ntype: Table\ntitle: Orders\ntags: [auth]\n---\n\norder data", - "datasets/users.md": "---\ntype: Dataset\ntitle: Users\ntags: [auth]\n---\n\nuser dataset", + "tables/users.md": "---\ntype: Table\ntitle: Users\ntags: [auth]\n---\n\nuser accounts", + "tables/orders.md": "---\ntype: Table\ntitle: Orders\ntags: [auth]\n---\n\norder data", + "datasets/users.md": "---\ntype: Dataset\ntitle: Users\ntags: [auth]\n---\n\nuser dataset", }) results := Search(b, Filters{Type: "Table", Tag: "auth"}) diff --git a/internal/validate/v02.go b/internal/validate/v02.go new file mode 100644 index 0000000..4ed856f --- /dev/null +++ b/internal/validate/v02.go @@ -0,0 +1,299 @@ +// OKF v0.2 checks: provenance, trust, and lifecycle families (§5), the +// Attested Computation contract (§10), reserved-file structure (§8, §9), and +// the okf_version declaration (§12). +// +// Severity follows the spec's own language: violations of a MUST or a +// REQUIRED-within-family rule are errors; departures from SHOULD guidance and +// legacy v0.1 constructs (§13.1) are warnings. + +package validate + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/okfcli/okf/internal/bundle" + "github.com/okfcli/okf/internal/concept" +) + +// now is injectable for deterministic staleness tests. +var now = time.Now + +// knownOKFVersions are the spec revisions this tool understands (§12). +var knownOKFVersions = map[string]bool{"0.1": true, "0.2": true} + +// validateV02 runs the v0.2 family checks for one concept. +func validateV02(r *Report, b *bundle.Bundle, c *concept.Concept) { + fm := c.Frontmatter + + // §5.4 status is a fixed enum when present. + if s := strings.TrimSpace(fm.Status); s != "" && !concept.ValidStatuses[s] { + r.add(c.ID, SeverityError, fmt.Sprintf( + "frontmatter: 'status' must be draft, stable, or deprecated, got %q (OKF §5.4)", fm.Status)) + } + + // §5.5 stale_after is an absolute YYYY-MM-DD date. + if s := strings.TrimSpace(fm.StaleAfter); s != "" { + if _, err := time.Parse("2006-01-02", s); err != nil { + r.add(c.ID, SeverityError, fmt.Sprintf( + "frontmatter: 'stale_after' must be an absolute YYYY-MM-DD date, got %q (OKF §5.5)", fm.StaleAfter)) + } else if fm.IsStale(now()) { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "concept is stale: today >= stale_after (%s) (OKF §5.5)", fm.StaleAfter)) + } + } + + // §5.2 generated.by is REQUIRED within generated. + if fm.Generated != nil { + if strings.TrimSpace(fm.Generated.By) == "" { + r.add(c.ID, SeverityError, "frontmatter: 'generated' requires 'by' (OKF §5.2)") + } else { + checkActor(r, c.ID, "generated.by", fm.Generated.By) + } + if at := strings.TrimSpace(fm.Generated.At); at != "" { + if _, err := concept.ParseDatetime(at); err != nil { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "frontmatter: 'generated.at' is not an ISO 8601 datetime: %q (OKF §5.2)", fm.Generated.At)) + } + } + } + + // §5.2 verified events carry by and at. + for i, v := range fm.Verified { + if strings.TrimSpace(v.By) == "" || strings.TrimSpace(v.At) == "" { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "frontmatter: 'verified[%d]' should carry both 'by' and 'at' (OKF §5.2)", i)) + continue + } + checkActor(r, c.ID, fmt.Sprintf("verified[%d].by", i), v.By) + if _, err := concept.ParseDatetime(v.At); err != nil { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "frontmatter: 'verified[%d].at' is not an ISO 8601 datetime: %q (OKF §5.2)", i, v.At)) + } + } + + // §13.1 legacy v0.1 constructs. + if !fm.Timestamp.IsZero() { + r.add(c.ID, SeverityWarning, + "frontmatter: legacy 'timestamp' is superseded by 'generated.at' in OKF v0.2 (§13.1)") + } + if citationsHeading.MatchString(c.Body) { + r.add(c.ID, SeverityWarning, + "body: legacy '# Citations' list is superseded by the 'sources' frontmatter family in OKF v0.2 (§13.1)") + } + + validateSources(r, c) + if fm.Type == concept.TypeAttestedComputation { + validateComputationContract(r, b, c) + } +} + +// checkActor warns when an identity field does not follow the actor +// convention of §7 (human:, process:, /). +func checkActor(r *Report, id, field, actor string) { + if concept.ActorKind(actor) == concept.ActorUnknown { + r.add(id, SeverityWarning, fmt.Sprintf( + "frontmatter: '%s' (%q) does not follow the actor convention: human:, process:, or / (OKF §7)", + field, actor)) + } +} + +var ( + citationsHeading = regexp.MustCompile(`(?mi)^#{1,6}\s+Citations\s*$`) + // footnoteRef matches inline footnote references [^label] (not definitions). + footnoteRef = regexp.MustCompile(`\[\^([^\]\s]+)\](?::)?`) + isoDate = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) +) + +// validateSources checks the §5.1 provenance family: resource is REQUIRED +// within an entry, usage_count needs a framing usage_window, and body +// footnote labels must join into sources[].id. +func validateSources(r *Report, c *concept.Concept) { + fm := c.Frontmatter + ids := make(map[string]bool) + for i, s := range fm.Sources { + if strings.TrimSpace(s.Resource) == "" { + r.add(c.ID, SeverityError, fmt.Sprintf( + "frontmatter: 'sources[%d]' requires 'resource' (OKF §5.1)", i)) + } + if s.ID != "" { + ids[s.ID] = true + } + if s.UsageCount != nil && s.UsageWindow == nil && fm.UsageWindow == nil { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "frontmatter: 'sources[%d].usage_count' has no framing 'usage_window' (entry or shared) (OKF §5.1)", i)) + } + } + + // Per-claim attribution: footnote labels are join keys into sources[].id. + // Only meaningful when the concept declares source ids at all. + if len(ids) == 0 { + return + } + seen := make(map[string]bool) + for _, m := range footnoteRef.FindAllStringSubmatch(c.Body, -1) { + label := m[1] + if seen[label] || ids[label] { + seen[label] = true + continue + } + seen[label] = true + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "body: footnote label [^%s] has no matching 'sources[].id' - labels are the join key for per-claim attribution (OKF §5.1)", label)) + } +} + +// hasComputationSection reports whether the body carries a `# Computation` +// section with content (an inline fence or indented code, §10.3). +func hasComputationSection(body string) bool { + return regexp.MustCompile(`(?mi)^#{1,6}\s+Computation\s*$`).MatchString(body) +} + +// validateComputationContract checks the §10.2 contract of an Attested +// Computation concept. +func validateComputationContract(r *Report, b *bundle.Bundle, c *concept.Concept) { + fm := c.Frontmatter + + // runtime is REQUIRED for this type. + if strings.TrimSpace(fm.Runtime) == "" { + r.add(c.ID, SeverityError, + "frontmatter: 'runtime' is required for type Attested Computation (OKF §10.2)") + } + + hasInline := hasComputationSection(c.Body) + hasFile := strings.TrimSpace(fm.Computation) != "" + switch { + case hasInline && hasFile: + r.add(c.ID, SeverityWarning, + "computation is provided both inline (body Computation section) and via the 'computation' path; provide one (OKF §10.3)") + case !hasInline && !hasFile: + r.add(c.ID, SeverityWarning, + "no computation found: provide a body Computation section or a 'computation' path (OKF §10.3)") + } + + for i, p := range fm.Parameters { + if strings.TrimSpace(p.Name) == "" || strings.TrimSpace(p.Type) == "" { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "frontmatter: parameter %d should carry 'name' and 'type' (OKF §10.2)", i)) + } + } + + // Path-valued contract fields should point at material that exists (§6.2). + checkLocalPath(r, b, c, "computation", fm.Computation) + if fm.Executor != nil { + checkLocalPath(r, b, c, "executor.resource", fm.Executor.Resource) + } + if fm.Attester != nil { + checkLocalPath(r, b, c, "attester.resource", fm.Attester.Resource) + } +} + +// checkLocalPath warns when a path-valued field (§6.2) names a local file +// that does not exist in the bundle. URLs are not checked. Bundle-relative +// paths (leading /) resolve from the bundle root; bare relative paths are +// tried from the concept's own directory and then from the bundle root - the +// spec's §10.2 example and upstream bundles write contract paths root-relative +// without a leading slash. +func checkLocalPath(r *Report, b *bundle.Bundle, c *concept.Concept, field, path string) { + p := strings.TrimSpace(path) + if p == "" || strings.Contains(p, "://") || strings.HasPrefix(p, "mailto:") { + return + } + var candidates []string + if strings.HasPrefix(p, "/") { + candidates = []string{strings.TrimPrefix(p, "/")} + } else { + candidates = []string{normalizePath(dirOf(c.ID) + "/" + p), normalizePath(p)} + } + for _, rel := range candidates { + if _, err := os.Stat(filepath.Join(b.Root, filepath.FromSlash(rel))); err == nil { + return + } + } + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "'%s' points at %s, which does not exist in the bundle (OKF §6.2)", field, path)) +} + +// validateReserved checks index.md and log.md structure (§8, §9, §12). +func validateReserved(r *Report, b *bundle.Bundle) { + for _, c := range b.Reserved { + isIndex := c.ID == "index" || strings.HasSuffix(c.ID, "/index") + isLog := c.ID == "log" || strings.HasSuffix(c.ID, "/log") + switch { + case isIndex: + validateIndexFile(r, c) + case isLog: + validateLogFile(r, c) + } + } +} + +// validateIndexFile enforces §8: index files carry no frontmatter, except a +// bundle-root index.md, which may declare okf_version (§12) and nothing else. +func validateIndexFile(r *Report, c *concept.Concept) { + if c.RawFront == "" { + return + } + if c.ID != "index" { + r.add(c.ID, SeverityError, + "index.md files must not contain frontmatter; only the bundle-root index.md may carry 'okf_version' (OKF §8)") + return + } + + keys := frontmatterKeys(c) + for _, k := range keys { + if k != "okf_version" { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "root index.md frontmatter carries %q; only 'okf_version' is permitted (OKF §8, §12)", k)) + } + } + if v, ok := c.Frontmatter.Extensions["okf_version"]; ok { + ver := fmt.Sprintf("%v", v) + if !knownOKFVersions[ver] { + r.add(c.ID, SeverityWarning, fmt.Sprintf( + "root index.md declares okf_version %q, which this tool does not recognize (known: 0.1, 0.2) (OKF §12)", ver)) + } + } +} + +// frontmatterKeys lists the keys present in a reserved file's frontmatter. +// Reserved files decode all keys into Extensions except the typed fields; +// typed fields are reported when non-zero. +func frontmatterKeys(c *concept.Concept) []string { + var keys []string + for k := range c.Frontmatter.Extensions { + keys = append(keys, k) + } + fm := c.Frontmatter + for k, present := range map[string]bool{ + "type": fm.Type != "", + "title": fm.Title != "", + "description": fm.Description != "", + "resource": fm.Resource != "", + "tags": len(fm.Tags) > 0, + "status": fm.Status != "", + } { + if present { + keys = append(keys, k) + } + } + return keys +} + +// validateLogFile enforces §9: date headings MUST use ISO 8601 YYYY-MM-DD. +func validateLogFile(r *Report, c *concept.Concept) { + for _, line := range strings.Split(c.Body, "\n") { + if !strings.HasPrefix(line, "## ") { + continue + } + heading := strings.TrimSpace(strings.TrimPrefix(line, "## ")) + if !isoDate.MatchString(heading) { + r.add(c.ID, SeverityError, fmt.Sprintf( + "log.md date heading %q must use ISO 8601 YYYY-MM-DD form (OKF §9)", heading)) + } + } +} diff --git a/internal/validate/v02_test.go b/internal/validate/v02_test.go new file mode 100644 index 0000000..db36fa6 --- /dev/null +++ b/internal/validate/v02_test.go @@ -0,0 +1,340 @@ +package validate + +import ( + "strings" + "testing" + "time" +) + +// ok is a minimal fully-clean concept body (no warnings). +const okConcept = "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\n---\n\nbody" + +// findingWith reports whether the report contains a finding of the given +// severity whose message contains substr. +func findingWith(t *testing.T, r *Report, sev Severity, substr string) bool { + t.Helper() + for _, f := range r.Findings { + if f.Severity == sev && strings.Contains(f.Message, substr) { + return true + } + } + return false +} + +func mustFinding(t *testing.T, r *Report, sev Severity, substr string) { + t.Helper() + if !findingWith(t, r, sev, substr) { + t.Fatalf("expected %s finding containing %q, findings = %+v", sev, substr, r.Findings) + } +} + +func mustNotFinding(t *testing.T, r *Report, substr string) { + t.Helper() + for _, f := range r.Findings { + if strings.Contains(f.Message, substr) { + t.Fatalf("unexpected finding containing %q: %+v", substr, f) + } + } +} + +// --- §5.4 status --- + +func TestValidate_InvalidStatus(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nstatus: retired\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityError, "'status'") +} + +func TestValidate_ValidStatusesAccepted(t *testing.T) { + for _, s := range []string{"draft", "stable", "deprecated"} { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nstatus: " + s + "\n---\n\nbody", + }) + mustNotFinding(t, Validate(b), "'status'") + } +} + +// --- §5.5 stale_after --- + +func TestValidate_MalformedStaleAfter(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nstale_after: soon\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityError, "stale_after") +} + +func TestValidate_StaleConceptWarns(t *testing.T) { + defer func(orig func() time.Time) { now = orig }(now) + now = func() time.Time { return time.Date(2027, 1, 1, 0, 0, 0, 0, time.UTC) } + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nstale_after: 2026-09-23\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityWarning, "stale") +} + +func TestValidate_FreshConceptNoStaleWarning(t *testing.T) { + defer func(orig func() time.Time) { now = orig }(now) + now = func() time.Time { return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) } + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nstale_after: 2026-09-23\n---\n\nbody", + }) + mustNotFinding(t, Validate(b), "stale") +} + +// --- §5.2 generated / verified --- + +func TestValidate_GeneratedWithoutBy(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\ngenerated: { at: 2026-06-20T22:53:05Z }\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityError, "generated") +} + +func TestValidate_VerifiedEntryMissingFields(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nverified: { by: human:x }\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityWarning, "verified") +} + +func TestValidate_ActorConventionWarning(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\ngenerated: { by: just a name, at: 2026-06-20T22:53:05Z }\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityWarning, "actor") +} + +// --- §13.1 legacy fields --- + +func TestValidate_LegacyTimestampWarns(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\ntimestamp: 2026-05-28T22:53:05Z\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityWarning, "generated.at") +} + +func TestValidate_LegacyCitationsWarns(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\n---\n\n# Citations\n- https://example.com\n", + }) + mustFinding(t, Validate(b), SeverityWarning, "sources") +} + +// --- §5.1 sources --- + +func TestValidate_SourceEntryWithoutResource(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nsources:\n - id: s1\n title: no resource here\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityError, "resource") +} + +func TestValidate_UsageCountWithoutWindow(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nsources:\n - id: s1\n resource: https://example.com\n usage_count: 10\n---\n\nbody", + }) + mustFinding(t, Validate(b), SeverityWarning, "usage_window") +} + +func TestValidate_UsageCountWithSharedWindowOK(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nsources:\n - id: s1\n resource: https://example.com\n usage_count: 10\nusage_window: { from: 2026-06-01, to: 2026-06-30 }\n---\n\nbody", + }) + mustNotFinding(t, Validate(b), "usage_window") +} + +// Footnote labels join into sources[].id (§5.1): a cited label with no +// matching source id is a warning. +func TestValidate_FootnoteLabelWithoutMatchingSourceID(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nsources:\n - id: rev-policy\n resource: https://example.com\n---\n\nA claim.[^wrong-label]\n\n[^wrong-label]: something\n", + }) + mustFinding(t, Validate(b), SeverityWarning, "wrong-label") +} + +func TestValidate_FootnoteLabelMatchingSourceIDOK(t *testing.T) { + b := testBundle(t, map[string]string{ + "a.md": "---\ntype: T\ntitle: A\ndescription: d\ntags: [x]\nsources:\n - id: rev-policy\n resource: https://example.com\n---\n\nA claim.[^rev-policy]\n\n[^rev-policy]: Revenue recognition policy\n", + }) + mustNotFinding(t, Validate(b), "rev-policy") +} + +// --- §10 attested computations --- + +func TestValidate_AttestedComputationRequiresRuntime(t *testing.T) { + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\n---\n\n# Computation\n\n SELECT 1\n", + }) + mustFinding(t, Validate(b), SeverityError, "runtime") +} + +func TestValidate_AttestedComputationNeedsComputation(t *testing.T) { + // Neither a `computation` path nor a body `# Computation` section. + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\n---\n\nprose only\n", + }) + mustFinding(t, Validate(b), SeverityWarning, "Computation") +} + +func TestValidate_AttestedComputationInlineFenceOK(t *testing.T) { + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\n---\n\n# Computation\n\n SELECT 1\n", + }) + r := Validate(b) + mustNotFinding(t, r, "# Computation") + if r.HasErrors() { + t.Fatalf("unexpected errors: %+v", r.Findings) + } +} + +func TestValidate_AttestedComputationBothInlineAndFileWarns(t *testing.T) { + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\ncomputation: /lib/revenue.sql\n---\n\n# Computation\n\n SELECT 1\n", + "lib/revenue.sql": "SELECT 1", + }) + mustFinding(t, Validate(b), SeverityWarning, "both") +} + +func TestValidate_ParameterMissingNameOrType(t *testing.T) { + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\nparameters:\n - { name: year }\n---\n\n# Computation\n\n SELECT 1\n", + }) + mustFinding(t, Validate(b), SeverityWarning, "parameter") +} + +// Path-valued contract fields (§6.2) that point at missing local files warn. +func TestValidate_MissingAttesterFileWarns(t *testing.T) { + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\nattester:\n resource: attesters/revenue.py\n---\n\n# Computation\n\n SELECT 1\n", + }) + mustFinding(t, Validate(b), SeverityWarning, "attesters/revenue.py") +} + +func TestValidate_PresentAttesterFileOK(t *testing.T) { + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\nattester:\n resource: attesters/revenue.py\n---\n\n# Computation\n\n SELECT 1\n", + "attesters/revenue.py": "print('hi')", + }) + mustNotFinding(t, Validate(b), "attesters/revenue.py") +} + +func TestValidate_ExternalExecutorURLNotChecked(t *testing.T) { + b := testBundle(t, map[string]string{ + "c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\nexecutor:\n resource: https://example.com/runner\n---\n\n# Computation\n\n SELECT 1\n", + }) + mustNotFinding(t, Validate(b), "example.com") +} + +// --- §8 / §12 index.md frontmatter and okf_version --- + +func TestValidate_RootIndexOkfVersionKnown(t *testing.T) { + b := testBundle(t, map[string]string{ + "index.md": "---\nokf_version: \"0.2\"\n---\n# Index\n\n* [A](a.md) - a concept\n", + "a.md": okConcept, + }) + mustNotFinding(t, Validate(b), "okf_version") +} + +func TestValidate_RootIndexOkfVersionUnknownWarns(t *testing.T) { + b := testBundle(t, map[string]string{ + "index.md": "---\nokf_version: \"9.9\"\n---\n# Index\n\n* [A](a.md) - a concept\n", + "a.md": okConcept, + }) + mustFinding(t, Validate(b), SeverityWarning, "okf_version") +} + +func TestValidate_NonRootIndexWithFrontmatterErrors(t *testing.T) { + b := testBundle(t, map[string]string{ + "sub/index.md": "---\nokf_version: \"0.2\"\n---\n# Index\n", + "sub/a.md": okConcept, + }) + mustFinding(t, Validate(b), SeverityError, "frontmatter") +} + +func TestValidate_RootIndexExtraFrontmatterKeysWarn(t *testing.T) { + b := testBundle(t, map[string]string{ + "index.md": "---\nokf_version: \"0.2\"\ncustom: x\n---\n# Index\n", + "a.md": okConcept, + }) + mustFinding(t, Validate(b), SeverityWarning, "okf_version") +} + +// --- §9 log.md date headings --- + +func TestValidate_LogDateHeadingsISO(t *testing.T) { + b := testBundle(t, map[string]string{ + "log.md": "# Update Log\n\n## 2026-05-22\n* **Update**: something.\n\n## May 15th 2026\n* bad heading.\n", + "a.md": okConcept, + }) + mustFinding(t, Validate(b), SeverityError, "YYYY-MM-DD") +} + +func TestValidate_LogValidHeadingsOK(t *testing.T) { + b := testBundle(t, map[string]string{ + "log.md": "# Update Log\n\n## 2026-05-22\n* **Update**: something.\n", + "a.md": okConcept, + }) + mustNotFinding(t, Validate(b), "YYYY-MM-DD") +} + +// --- clean v0.2 concept has no v0.2 findings --- + +func TestValidate_FullV02ConceptClean(t *testing.T) { + defer func(orig func() time.Time) { now = orig }(now) + now = func() time.Time { return time.Date(2026, 7, 1, 0, 0, 0, 0, time.UTC) } + b := testBundle(t, map[string]string{ + "computations/revenue.md": `--- +type: Attested Computation +title: Revenue for fiscal year +description: Recognized revenue for a fiscal year. +tags: [finance] +status: stable +runtime: bigquery +parameters: + - { name: year, type: integer, required: true } +executor: + resource: /references/skills/run-on-bq.md + receipt: [job_id, executed_sql, result] +attester: + resource: /references/attesters/revenue.py +generated: { by: reference_agent/gemini-2.5-pro, at: 2026-06-20T22:53:05Z } +verified: { by: human:ahormati, at: 2026-06-25T09:00:00Z } +stale_after: 2026-09-23 +sources: + - id: rev-policy + resource: https://wiki.acme/finance/revenue-recognition + title: Revenue recognition policy +--- + +# Computation + + SELECT SUM(amount) AS revenue + +Per the recognition policy.[^rev-policy] + +[^rev-policy]: Revenue recognition policy +`, + "references/skills/run-on-bq.md": "---\ntype: Skill\ntitle: Run on BQ\ndescription: d\ntags: [x]\n---\n\nbody", + "references/attesters/revenue.py": "print('check')", + }) + r := Validate(b) + if len(r.Findings) != 0 { + t.Fatalf("expected clean report, findings = %+v", r.Findings) + } +} + +// Upstream bundles and the spec's own §10.2 example write contract paths +// relative to the bundle root without a leading slash (e.g. +// `references/skills/run-on-bq.md` from a concept in computations/). A path +// that resolves from either the concept's directory or the bundle root is +// not warned about. +func TestValidate_ContractPathRootRelativeFallback(t *testing.T) { + b := testBundle(t, map[string]string{ + "computations/c.md": "---\ntype: Attested Computation\ntitle: A\ndescription: d\ntags: [x]\nruntime: bigquery\nexecutor:\n resource: skills/run.md\nattester:\n resource: attesters/eq.py\n---\n\n# Computation\n\n SELECT 1\n", + "skills/run.md": "---\ntype: Skill\ntitle: S\ndescription: d\ntags: [x]\n---\n\nbody", + "attesters/eq.py": "print('check')", + }) + r := Validate(b) + mustNotFinding(t, r, "does not exist") +} diff --git a/internal/validate/validate.go b/internal/validate/validate.go index 27b5d67..bd825e3 100644 --- a/internal/validate/validate.go +++ b/internal/validate/validate.go @@ -52,8 +52,10 @@ func Validate(b *bundle.Bundle) *Report { for _, c := range b.Concepts { validateFrontmatter(r, c) validateBody(r, c) + validateV02(r, b, c) } validateLinks(r, b) + validateReserved(r, b) return r } @@ -80,7 +82,7 @@ func validateFrontmatter(r *Report, c *concept.Concept) { // timestamp, if present, should be a valid ISO 8601 datetime. // yaml.v3 already parses it into time.Time; a zero value with a non-empty // raw would indicate a parse issue, but we accept zero as "not set". - if !fm.Timestamp.IsZero() && fm.Timestamp.After(time.Now().Add(24 * 365 * time.Hour)) { + if !fm.Timestamp.IsZero() && fm.Timestamp.After(time.Now().Add(24*365*time.Hour)) { r.add(c.ID, SeverityWarning, "frontmatter: 'timestamp' is more than a year in the future") } } @@ -88,7 +90,7 @@ func validateFrontmatter(r *Report, c *concept.Concept) { // validateBody checks the markdown body for structural issues. func validateBody(r *Report, c *concept.Concept) { if strings.TrimSpace(c.Body) == "" { - r.add(c.ID, SeverityWarning, "body is empty — structural markdown is recommended (OKF §4.2)") + r.add(c.ID, SeverityWarning, "body is empty - structural markdown is recommended (OKF §4.2)") } } @@ -176,7 +178,7 @@ func ExtractLinks(body string) []Link { // ExtractFrontmatterLinks converts a frontmatter "links:" list (concept IDs or // absolute /paths) into Link structs. Empty strings are skipped. The Text is // empty (frontmatter links have no link text); the Target is preserved as-is, -// including any leading "/" — ResolveLink handles the stripping. +// including any leading "/" - ResolveLink handles the stripping. func ExtractFrontmatterLinks(links []string) []Link { out := make([]Link, 0, len(links)) for _, l := range links { diff --git a/internal/validate/validate_test.go b/internal/validate/validate_test.go index 4373f8d..272888b 100644 --- a/internal/validate/validate_test.go +++ b/internal/validate/validate_test.go @@ -154,8 +154,8 @@ func TestValidateLinks_RelativeLinkSuggestsAbsolutePath(t *testing.T) { // the absolute target organizations/cloaked exists. The error should // suggest the absolute path. b := testBundle(t, map[string]string{ - "pages/about.md": "---\ntype: Page\ntitle: About\n---\n\nSee [Cloaked](organizations/cloaked).", - "organizations/cloaked.md": "---\ntype: Org\ntitle: Cloaked\n---\n\nbody", + "pages/about.md": "---\ntype: Page\ntitle: About\n---\n\nSee [Cloaked](organizations/cloaked).", + "organizations/cloaked.md": "---\ntype: Org\ntitle: Cloaked\n---\n\nbody", }) r := &Report{} @@ -206,7 +206,7 @@ func TestValidateLinks_NonexistentConceptNoAbsolutePathSuggestion(t *testing.T) func TestValidateLinks_AlreadyAbsoluteBrokenNoSuggestion(t *testing.T) { // A link that is already written as an absolute path (/...) but does not - // resolve must NOT produce an "absolute path" suggestion — there is + // resolve must NOT produce an "absolute path" suggestion - there is // nothing to suggest, so it falls through to the plain broken-link message. b := testBundle(t, map[string]string{ "pages/about.md": "---\ntype: Page\ntitle: About\n---\n\nSee [Cloaked](/organizations/cloaked).", diff --git a/testdata/valid/datasets/ga4.md b/testdata/valid/datasets/ga4.md index ab07659..355abcf 100644 --- a/testdata/valid/datasets/ga4.md +++ b/testdata/valid/datasets/ga4.md @@ -16,4 +16,4 @@ See [events_20210101](/tables/events_.md) for the core events table. # Key Tables -- [events_](/tables/events_.md) — the core events table, one row per event. +- [events_](/tables/events_.md) - the core events table, one row per event.