From 78d380bce71414f8bb899d9eb0da91b06654bb4b Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Sat, 12 Sep 2026 15:48:39 -0700 Subject: [PATCH 1/8] merkle: tlv and tree structure --- std/merklelog/store.go | 150 +++ std/merklelog/store_badger.go | 312 +++++ std/merklelog/store_badger_test.go | 42 + std/merklelog/store_memory.go | 64 + std/merklelog/store_test.go | 143 +++ std/merklelog/tree.go | 494 ++++++++ std/merklelog/tree_test.go | 360 ++++++ std/ndn/merklelog/definitions.go | 96 ++ std/ndn/merklelog/zz_generated.go | 1887 ++++++++++++++++++++++++++++ 9 files changed, 3548 insertions(+) create mode 100644 std/merklelog/store.go create mode 100644 std/merklelog/store_badger.go create mode 100644 std/merklelog/store_badger_test.go create mode 100644 std/merklelog/store_memory.go create mode 100644 std/merklelog/store_test.go create mode 100644 std/merklelog/tree.go create mode 100644 std/merklelog/tree_test.go create mode 100644 std/ndn/merklelog/definitions.go create mode 100644 std/ndn/merklelog/zz_generated.go diff --git a/std/merklelog/store.go b/std/merklelog/store.go new file mode 100644 index 00000000..46ae775e --- /dev/null +++ b/std/merklelog/store.go @@ -0,0 +1,150 @@ +package merklelog + +import ( + "bytes" + "fmt" + "math/bits" + "time" +) + +// StoreState is the durable Merkle state after a complete append. +type StoreState struct { + TreeSize uint64 + RootHash []byte + Frontier [][]byte + LastIngestTime time.Duration +} + +// StoreSnapshot is one consistent view of the complete persisted log. +type StoreSnapshot struct { + State StoreState + Entries [][]byte + DataIndex map[[HashSize]byte]uint64 +} + +// StoreAppend describes one atomic transition from ExpectedSize to State. +type StoreAppend struct { + ExpectedSize uint64 + EntryWire []byte + State StoreState +} + +// Store persists raw log entries, their Data hash index, and derived tree state +// in one atomic append operation. +type Store interface { + Load() (*StoreSnapshot, error) + Append(update StoreAppend) error +} + +func emptyStoreState() StoreState { + return StoreState{RootHash: rootFromFrontier(nil)} +} + +func validateStoreState(state StoreState) error { + if len(state.RootHash) != HashSize { + return fmt.Errorf("root hash length is %d, want %d", len(state.RootHash), HashSize) + } + if state.LastIngestTime < 0 || state.LastIngestTime%time.Millisecond != 0 { + return fmt.Errorf("last ingestion time is invalid") + } + expectedLevels := bits.Len64(state.TreeSize) + if len(state.Frontier) != expectedLevels { + return fmt.Errorf( + "frontier has %d levels, want %d for tree size %d", + len(state.Frontier), + expectedLevels, + state.TreeSize, + ) + } + for level, subtreeHash := range state.Frontier { + occupied := state.TreeSize&(uint64(1)< 0 && entry.IngestTime <= current.LastIngestTime { + return nil, fmt.Errorf("ingestion time does not increase") + } + + nextFrontier := appendFrontier(current.Frontier, hashLeaf(update.EntryWire)) + expected := StoreState{ + TreeSize: current.TreeSize + 1, + RootHash: rootFromFrontier(nextFrontier), + Frontier: nextFrontier, + LastIngestTime: entry.IngestTime, + } + if !equalStoreState(expected, update.State) { + return nil, fmt.Errorf("append state does not match entry and current frontier") + } + return &expected, nil +} + +func cloneStoreState(state StoreState) StoreState { + return StoreState{ + TreeSize: state.TreeSize, + RootHash: bytes.Clone(state.RootHash), + Frontier: cloneHashes(state.Frontier), + LastIngestTime: state.LastIngestTime, + } +} + +func equalStoreState(left StoreState, right StoreState) bool { + if left.TreeSize != right.TreeSize || + left.LastIngestTime != right.LastIngestTime || + !bytes.Equal(left.RootHash, right.RootHash) || + len(left.Frontier) != len(right.Frontier) { + return false + } + for level := range left.Frontier { + if !bytes.Equal(left.Frontier[level], right.Frontier[level]) { + return false + } + } + return true +} + +func cloneStoreSnapshot(snapshot *StoreSnapshot) *StoreSnapshot { + ret := &StoreSnapshot{ + State: cloneStoreState(snapshot.State), + Entries: make([][]byte, len(snapshot.Entries)), + DataIndex: make(map[[HashSize]byte]uint64, len(snapshot.DataIndex)), + } + for i, entryWire := range snapshot.Entries { + ret.Entries[i] = bytes.Clone(entryWire) + } + for dataHash, leafIndex := range snapshot.DataIndex { + ret.DataIndex[dataHash] = leafIndex + } + return ret +} diff --git a/std/merklelog/store_badger.go b/std/merklelog/store_badger.go new file mode 100644 index 00000000..b2c735dd --- /dev/null +++ b/std/merklelog/store_badger.go @@ -0,0 +1,312 @@ +//go:build !js + +package merklelog + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "time" + + "github.com/dgraph-io/badger/v4" +) + +var ( + storeStateKey = []byte{0x00} + storeEntryKeyPfx = []byte{0x01} + storeHashKeyPfx = []byte{0x02} +) + +const ( + storeStateMagic = "MLOG" + storeStateVersion = byte(1) + storeStateHeader = 4 + 1 + 8 + 8 + HashSize + 1 +) + +// BadgerStore persists the Merkle log in a dedicated Badger database. +type BadgerStore struct { + db *badger.DB +} + +// NewBadgerStore opens or creates a persistent Merkle log store at path. +func NewBadgerStore(path string) (*BadgerStore, error) { + options := badger.DefaultOptions(path).WithSyncWrites(true) + db, err := badger.Open(options) + if err != nil { + return nil, err + } + return &BadgerStore{db: db}, nil +} + +// Close closes the underlying Badger database. +func (s *BadgerStore) Close() error { + return s.db.Close() +} + +// Load returns one transactionally consistent snapshot of the persisted log. +func (s *BadgerStore) Load() (snapshot *StoreSnapshot, err error) { + snapshot = &StoreSnapshot{ + State: emptyStoreState(), + DataIndex: make(map[[HashSize]byte]uint64), + } + err = s.db.View(func(txn *badger.Txn) error { + state, found, err := loadBadgerState(txn) + if err != nil { + return err + } + if !found { + hasEntries := badgerHasPrefix(txn, storeEntryKeyPfx) + hasIndex := badgerHasPrefix(txn, storeHashKeyPfx) + if hasEntries || hasIndex { + return fmt.Errorf("%w: records exist without tree state", ErrStoreCorrupt) + } + return nil + } + snapshot.State = state + snapshot.Entries = nil + expectedIndex := uint64(0) + opts := badger.DefaultIteratorOptions + it := txn.NewIterator(opts) + defer it.Close() + for it.Seek(storeEntryKeyPfx); it.ValidForPrefix(storeEntryKeyPfx); it.Next() { + key := it.Item().Key() + if len(key) != 1+8 { + return fmt.Errorf("%w: malformed entry key", ErrStoreCorrupt) + } + leafIndex := binary.BigEndian.Uint64(key[1:]) + if leafIndex != expectedIndex { + return fmt.Errorf("%w: found entry %d, expected %d", ErrStoreCorrupt, leafIndex, expectedIndex) + } + entryWire, err := it.Item().ValueCopy(nil) + if err != nil { + return err + } + snapshot.Entries = append(snapshot.Entries, entryWire) + expectedIndex++ + } + if expectedIndex != state.TreeSize { + return fmt.Errorf("%w: found %d entries for tree size %d", ErrStoreCorrupt, expectedIndex, state.TreeSize) + } + + for it.Seek(storeHashKeyPfx); it.ValidForPrefix(storeHashKeyPfx); it.Next() { + key := it.Item().Key() + if len(key) != 1+HashSize { + return fmt.Errorf("%w: malformed Data hash index key", ErrStoreCorrupt) + } + value, err := it.Item().ValueCopy(nil) + if err != nil { + return err + } + if len(value) != 8 { + return fmt.Errorf("%w: malformed Data hash index value", ErrStoreCorrupt) + } + leafIndex := binary.BigEndian.Uint64(value) + if leafIndex >= state.TreeSize { + return fmt.Errorf("%w: indexed leaf %d is outside tree", ErrStoreCorrupt, leafIndex) + } + var dataHash [HashSize]byte + copy(dataHash[:], key[1:]) + snapshot.DataIndex[dataHash] = leafIndex + } + return nil + }) + return snapshot, err +} + +// Append writes one complete tree transition in a Badger transaction. +func (s *BadgerStore) Append(update StoreAppend) error { + return s.db.Update(func(txn *badger.Txn) error { + current, found, err := loadBadgerState(txn) + if err != nil { + return err + } + if !found { + current = emptyStoreState() + if badgerHasPrefix(txn, storeEntryKeyPfx) || badgerHasPrefix(txn, storeHashKeyPfx) { + return fmt.Errorf("%w: records exist without tree state", ErrStoreCorrupt) + } + } + nextState, err := validateStoreAppend(current, update) + if err != nil { + return err + } + entry, _, err := decodeLogEntry(update.EntryWire) + if err != nil { + return err + } + + entryKey := badgerEntryKey(update.ExpectedSize) + if _, err := txn.Get(entryKey); err == nil { + return fmt.Errorf("%w: entry %d already exists", ErrStoreCorrupt, update.ExpectedSize) + } else if !errors.Is(err, badger.ErrKeyNotFound) { + return err + } + seen := make(map[[HashSize]byte]struct{}, len(entry.DataHashes)) + for _, dataHash := range entry.DataHashes { + key := dataHashKey(dataHash) + if _, ok := seen[key]; ok { + return fmt.Errorf("%w: duplicate within entry", ErrDuplicateDataHash) + } + seen[key] = struct{}{} + hashKey := badgerHashKey(key) + if item, err := txn.Get(hashKey); err == nil { + value, copyErr := item.ValueCopy(nil) + if copyErr != nil { + return copyErr + } + if len(value) != 8 { + return fmt.Errorf("%w: malformed Data hash index value", ErrStoreCorrupt) + } + return fmt.Errorf( + "%w at leaf %d", + ErrDuplicateDataHash, + binary.BigEndian.Uint64(value), + ) + } else if !errors.Is(err, badger.ErrKeyNotFound) { + return err + } + } + + if err := txn.Set(entryKey, update.EntryWire); err != nil { + return err + } + indexValue := make([]byte, 8) + binary.BigEndian.PutUint64(indexValue, update.ExpectedSize) + for key := range seen { + if err := txn.Set(badgerHashKey(key), indexValue); err != nil { + return err + } + } + stateValue, err := encodeStoreState(*nextState) + if err != nil { + return err + } + return txn.Set(storeStateKey, stateValue) + }) +} + +func loadBadgerState(txn *badger.Txn) (StoreState, bool, error) { + item, err := txn.Get(storeStateKey) + if errors.Is(err, badger.ErrKeyNotFound) { + return StoreState{}, false, nil + } + if err != nil { + return StoreState{}, false, err + } + value, err := item.ValueCopy(nil) + if err != nil { + return StoreState{}, false, err + } + state, err := decodeStoreState(value) + if err != nil { + return StoreState{}, false, fmt.Errorf("%w: %v", ErrStoreCorrupt, err) + } + return state, true, nil +} + +func encodeStoreState(state StoreState) ([]byte, error) { + if err := validateStoreState(state); err != nil { + return nil, err + } + if len(state.Frontier) > 64 { + return nil, fmt.Errorf("frontier has too many levels") + } + size := storeStateHeader + for _, subtreeHash := range state.Frontier { + size++ + if subtreeHash != nil { + size += HashSize + } + } + ret := make([]byte, size) + copy(ret, storeStateMagic) + ret[4] = storeStateVersion + binary.BigEndian.PutUint64(ret[5:13], state.TreeSize) + binary.BigEndian.PutUint64(ret[13:21], uint64(state.LastIngestTime/time.Millisecond)) + copy(ret[21:21+HashSize], state.RootHash) + ret[21+HashSize] = byte(len(state.Frontier)) + pos := storeStateHeader + for _, subtreeHash := range state.Frontier { + if subtreeHash == nil { + ret[pos] = 0 + pos++ + continue + } + ret[pos] = 1 + pos++ + copy(ret[pos:], subtreeHash) + pos += HashSize + } + return ret, nil +} + +func decodeStoreState(value []byte) (StoreState, error) { + if len(value) < storeStateHeader { + return StoreState{}, fmt.Errorf("state is too short") + } + if !bytes.Equal(value[:4], []byte(storeStateMagic)) || value[4] != storeStateVersion { + return StoreState{}, fmt.Errorf("unsupported state format") + } + lastMillis := binary.BigEndian.Uint64(value[13:21]) + if lastMillis > uint64(math.MaxInt64/int64(time.Millisecond)) { + return StoreState{}, fmt.Errorf("last ingestion time overflows time.Duration") + } + state := StoreState{ + TreeSize: binary.BigEndian.Uint64(value[5:13]), + LastIngestTime: time.Duration(lastMillis) * time.Millisecond, + RootHash: bytes.Clone(value[21 : 21+HashSize]), + } + frontierCount := int(value[21+HashSize]) + state.Frontier = make([][]byte, frontierCount) + pos := storeStateHeader + for level := range state.Frontier { + if pos >= len(value) { + return StoreState{}, fmt.Errorf("frontier level %d is missing", level) + } + present := value[pos] + pos++ + switch present { + case 0: + case 1: + if pos+HashSize > len(value) { + return StoreState{}, fmt.Errorf("frontier level %d hash is truncated", level) + } + state.Frontier[level] = bytes.Clone(value[pos : pos+HashSize]) + pos += HashSize + default: + return StoreState{}, fmt.Errorf("frontier level %d has invalid presence marker", level) + } + } + if pos != len(value) { + return StoreState{}, fmt.Errorf("state has %d trailing bytes", len(value)-pos) + } + if err := validateStoreState(state); err != nil { + return StoreState{}, err + } + return state, nil +} + +func badgerEntryKey(leafIndex uint64) []byte { + key := make([]byte, 1+8) + key[0] = storeEntryKeyPfx[0] + binary.BigEndian.PutUint64(key[1:], leafIndex) + return key +} + +func badgerHashKey(dataHash [HashSize]byte) []byte { + key := make([]byte, 1+HashSize) + key[0] = storeHashKeyPfx[0] + copy(key[1:], dataHash[:]) + return key +} + +func badgerHasPrefix(txn *badger.Txn, prefix []byte) bool { + opts := badger.DefaultIteratorOptions + opts.PrefetchValues = false + it := txn.NewIterator(opts) + defer it.Close() + it.Seek(prefix) + return it.ValidForPrefix(prefix) +} diff --git a/std/merklelog/store_badger_test.go b/std/merklelog/store_badger_test.go new file mode 100644 index 00000000..02c691dd --- /dev/null +++ b/std/merklelog/store_badger_test.go @@ -0,0 +1,42 @@ +//go:build !js + +package merklelog + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBadgerStoreRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "badger") + store, err := NewBadgerStore(path) + require.NoError(t, err) + + tree, entries, expectedRoot := populateStore(t, store) + require.Equal(t, uint64(len(entries)), tree.Size()) + require.NoError(t, store.Close()) + + store, err = NewBadgerStore(path) + require.NoError(t, err) + defer store.Close() + assertRestoredTree(t, store, entries, expectedRoot) +} + +func TestStoreStateEncoding(t *testing.T) { + tree := NewTree() + for size := 0; size <= 16; size++ { + state := tree.storeState() + encoded, err := encodeStoreState(state) + require.NoError(t, err) + decoded, err := decodeStoreState(encoded) + require.NoError(t, err) + require.True(t, equalStoreState(state, decoded)) + + if size < 16 { + _, err = tree.Append(testEntryWire(size)) + require.NoError(t, err) + } + } +} diff --git a/std/merklelog/store_memory.go b/std/merklelog/store_memory.go new file mode 100644 index 00000000..677cbba0 --- /dev/null +++ b/std/merklelog/store_memory.go @@ -0,0 +1,64 @@ +package merklelog + +import ( + "bytes" + "fmt" + "sync" +) + +// MemoryStore is an in-memory Store implementation for tests and ephemeral logs. +type MemoryStore struct { + mutex sync.Mutex + snapshot *StoreSnapshot +} + +// NewMemoryStore creates an empty in-memory log store. +func NewMemoryStore() *MemoryStore { + return &MemoryStore{ + snapshot: &StoreSnapshot{ + State: emptyStoreState(), + DataIndex: make(map[[HashSize]byte]uint64), + }, + } +} + +// Load returns a consistent copy of all stored state. +func (s *MemoryStore) Load() (*StoreSnapshot, error) { + s.mutex.Lock() + defer s.mutex.Unlock() + return cloneStoreSnapshot(s.snapshot), nil +} + +// Append atomically stores one entry, its hash index, and the resulting state. +func (s *MemoryStore) Append(update StoreAppend) error { + s.mutex.Lock() + defer s.mutex.Unlock() + + nextState, err := validateStoreAppend(s.snapshot.State, update) + if err != nil { + return err + } + entry, _, err := decodeLogEntry(update.EntryWire) + if err != nil { + return err + } + seen := make(map[[HashSize]byte]struct{}, len(entry.DataHashes)) + for _, dataHash := range entry.DataHashes { + key := dataHashKey(dataHash) + if _, ok := seen[key]; ok { + return fmt.Errorf("%w: duplicate within entry", ErrDuplicateDataHash) + } + if leafIndex, ok := s.snapshot.DataIndex[key]; ok { + return fmt.Errorf("%w at leaf %d", ErrDuplicateDataHash, leafIndex) + } + seen[key] = struct{}{} + } + + leafIndex := update.ExpectedSize + s.snapshot.Entries = append(s.snapshot.Entries, bytes.Clone(update.EntryWire)) + for key := range seen { + s.snapshot.DataIndex[key] = leafIndex + } + s.snapshot.State = cloneStoreState(*nextState) + return nil +} diff --git a/std/merklelog/store_test.go b/std/merklelog/store_test.go new file mode 100644 index 00000000..cfb4531e --- /dev/null +++ b/std/merklelog/store_test.go @@ -0,0 +1,143 @@ +package merklelog + +import ( + "bytes" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMemoryStoreRestart(t *testing.T) { + store := NewMemoryStore() + testStoreRestart(t, store, func() (Store, error) { + return store, nil + }) +} + +func TestStoreRejectsStaleWriterWithoutMutatingTree(t *testing.T) { + store := NewMemoryStore() + first, err := OpenTree(store) + require.NoError(t, err) + stale, err := OpenTree(store) + require.NoError(t, err) + + _, err = first.Append(testEntryWire(0)) + require.NoError(t, err) + staleRoot := stale.Root() + _, err = stale.Append(testEntryWire(1)) + require.ErrorIs(t, err, ErrStoreStateChanged) + require.Equal(t, uint64(0), stale.Size()) + require.Equal(t, staleRoot, stale.Root()) + _, found := stale.Lookup(testDataHash(1)) + require.False(t, found) +} + +func TestStoreFailureDoesNotMutateTree(t *testing.T) { + expectedErr := errors.New("write failed") + store := &failingStore{ + Store: NewMemoryStore(), + err: expectedErr, + } + tree, err := OpenTree(store) + require.NoError(t, err) + expectedRoot := tree.Root() + + _, err = tree.Append(testEntryWire(0)) + require.ErrorIs(t, err, expectedErr) + require.Equal(t, uint64(0), tree.Size()) + require.Equal(t, expectedRoot, tree.Root()) + _, found := tree.Lookup(testDataHash(0)) + require.False(t, found) +} + +func TestOpenTreeRejectsCorruptMemoryStore(t *testing.T) { + tests := map[string]func(*MemoryStore){ + "root": func(store *MemoryStore) { + store.snapshot.State.RootHash[0] ^= 0xff + }, + "missing entry": func(store *MemoryStore) { + store.snapshot.Entries = store.snapshot.Entries[:len(store.snapshot.Entries)-1] + }, + "hash index": func(store *MemoryStore) { + delete(store.snapshot.DataIndex, dataHashKey(testDataHash(0))) + }, + "frontier": func(store *MemoryStore) { + store.snapshot.State.Frontier[0][0] ^= 0xff + }, + } + + for name, corrupt := range tests { + t.Run(name, func(t *testing.T) { + store := NewMemoryStore() + tree, err := OpenTree(store) + require.NoError(t, err) + _, err = tree.Append(testEntryWire(0)) + require.NoError(t, err) + corrupt(store) + + _, err = OpenTree(store) + require.ErrorIs(t, err, ErrStoreCorrupt) + }) + } +} + +func testStoreRestart( + t *testing.T, + store Store, + reopen func() (Store, error), +) { + _, entries, expectedRoot := populateStore(t, store) + reopened, err := reopen() + require.NoError(t, err) + assertRestoredTree(t, reopened, entries, expectedRoot) +} + +func populateStore(t *testing.T, store Store) (*Tree, [][]byte, []byte) { + t.Helper() + tree, err := OpenTree(store) + require.NoError(t, err) + entries := make([][]byte, 9) + for i := range entries { + entry := testEntry(i) + entry.DataHashes = append(entry.DataHashes, testDataHash(i+100)) + entryWire := mustEncodeLogEntry(entry) + entries[i] = bytes.Clone(entryWire) + leafIndex, err := tree.Append(entryWire) + require.NoError(t, err) + require.Equal(t, uint64(i), leafIndex) + entryWire[len(entryWire)-1] ^= 0xff + } + expectedRoot := bytes.Clone(tree.Root().RootHash) + return tree, entries, expectedRoot +} + +func assertRestoredTree(t *testing.T, store Store, entries [][]byte, expectedRoot []byte) { + t.Helper() + tree, err := OpenTree(store) + require.NoError(t, err) + require.Equal(t, uint64(len(entries)), tree.Size()) + require.Equal(t, expectedRoot, tree.Root().RootHash) + lastIngestTime, ok := tree.LastIngestTime() + require.True(t, ok) + require.Equal(t, testEntry(len(entries)-1).IngestTime, lastIngestTime) + for i := range entries { + for _, dataHash := range [][]byte{testDataHash(i), testDataHash(i + 100)} { + leafIndex, found := tree.Lookup(dataHash) + require.True(t, found) + require.Equal(t, uint64(i), leafIndex) + proof, err := tree.InclusionProof(leafIndex) + require.NoError(t, err) + require.NoError(t, VerifyInclusion(dataHash, proof, tree.Root())) + } + } +} + +type failingStore struct { + Store + err error +} + +func (s *failingStore) Append(StoreAppend) error { + return s.err +} diff --git a/std/merklelog/tree.go b/std/merklelog/tree.go new file mode 100644 index 00000000..08df1261 --- /dev/null +++ b/std/merklelog/tree.go @@ -0,0 +1,494 @@ +// Package merklelog implements the cryptographic core of the Merkle history log. +package merklelog + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "math/bits" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + defn "github.com/named-data/ndnd/std/ndn/merklelog" +) + +// HashSize is the size of every leaf, node, and Data hash in bytes. +const HashSize = sha256.Size + +var ( + // ErrDuplicateDataHash indicates that a Data hash already belongs to a leaf. + ErrDuplicateDataHash = errors.New("data hash is already logged") + // ErrStoreCorrupt indicates that persisted state is internally inconsistent. + ErrStoreCorrupt = errors.New("merkle log store is corrupt") + // ErrStoreStateChanged indicates that the persisted tree changed unexpectedly. + ErrStoreStateChanged = errors.New("merkle log store state changed") +) + +// Tree is an append-only Merkle tree. It keeps proof material in memory and may +// optionally persist each append through a Store. Tree is not safe for +// concurrent use; the log service is responsible for serializing appends. +type Tree struct { + store Store + entries [][]byte + leafHashes [][]byte + dataIndex map[[HashSize]byte]uint64 + frontier [][]byte + lastIngestTime time.Duration +} + +// NewTree creates an empty in-memory Merkle tree. +func NewTree() *Tree { + return &Tree{dataIndex: make(map[[HashSize]byte]uint64)} +} + +// OpenTree restores a Merkle tree from store and verifies all persisted state. +func OpenTree(store Store) (*Tree, error) { + if store == nil { + return nil, fmt.Errorf("merkle log store is nil") + } + snapshot, err := store.Load() + if err != nil { + return nil, err + } + if snapshot == nil { + return nil, fmt.Errorf("%w: store returned a nil snapshot", ErrStoreCorrupt) + } + if uint64(len(snapshot.Entries)) != snapshot.State.TreeSize { + return nil, fmt.Errorf( + "%w: state has %d leaves but store returned %d entries", + ErrStoreCorrupt, + snapshot.State.TreeSize, + len(snapshot.Entries), + ) + } + + tree := NewTree() + for i, entryWire := range snapshot.Entries { + if _, err := tree.Append(entryWire); err != nil { + return nil, fmt.Errorf("%w: entry %d: %v", ErrStoreCorrupt, i, err) + } + } + if !equalStoreState(tree.storeState(), snapshot.State) { + return nil, fmt.Errorf("%w: persisted tree state does not match entries", ErrStoreCorrupt) + } + if !equalDataIndex(tree.dataIndex, snapshot.DataIndex) { + return nil, fmt.Errorf("%w: persisted Data hash index does not match entries", ErrStoreCorrupt) + } + + tree.store = store + return tree, nil +} + +// Size returns the number of entries in the tree. +func (t *Tree) Size() uint64 { + return uint64(len(t.entries)) +} + +// LastIngestTime returns the latest log-assigned ingestion time. The boolean is +// false when the tree is empty. +func (t *Tree) LastIngestTime() (time.Duration, bool) { + return t.lastIngestTime, t.Size() > 0 +} + +// Lookup returns the leaf containing dataHash. +func (t *Tree) Lookup(dataHash []byte) (uint64, bool) { + if len(dataHash) != HashSize { + return 0, false + } + leafIndex, ok := t.dataIndex[dataHashKey(dataHash)] + return leafIndex, ok +} + +// Append adds one raw LogEntry TLV and returns its zero-based leaf index. +func (t *Tree) Append(entryWire []byte) (uint64, error) { + entry, _, err := decodeLogEntry(entryWire) + if err != nil { + return 0, err + } + if t.Size() > 0 && entry.IngestTime <= t.lastIngestTime { + return 0, fmt.Errorf( + "ingestion time %s is not later than %s", + entry.IngestTime, + t.lastIngestTime, + ) + } + + seen := make(map[[HashSize]byte]struct{}, len(entry.DataHashes)) + for _, dataHash := range entry.DataHashes { + key := dataHashKey(dataHash) + if _, ok := seen[key]; ok { + return 0, fmt.Errorf("%w: duplicate within entry", ErrDuplicateDataHash) + } + if leafIndex, ok := t.dataIndex[key]; ok { + return 0, fmt.Errorf("%w at leaf %d", ErrDuplicateDataHash, leafIndex) + } + seen[key] = struct{}{} + } + + ownedWire := bytes.Clone(entryWire) + leafHash := hashLeaf(ownedWire) + nextFrontier := appendFrontier(t.frontier, leafHash) + leafIndex := t.Size() + nextState := StoreState{ + TreeSize: leafIndex + 1, + RootHash: rootFromFrontier(nextFrontier), + Frontier: nextFrontier, + LastIngestTime: entry.IngestTime, + } + if t.store != nil { + err = t.store.Append(StoreAppend{ + ExpectedSize: leafIndex, + EntryWire: ownedWire, + State: nextState, + }) + if err != nil { + return 0, err + } + } + + t.entries = append(t.entries, ownedWire) + t.leafHashes = append(t.leafHashes, leafHash) + for key := range seen { + t.dataIndex[key] = leafIndex + } + t.frontier = nextFrontier + t.lastIngestTime = entry.IngestTime + return leafIndex, nil +} + +// Root returns the current tree size and root hash. The empty-tree root is +// SHA-256 of the empty byte string. +func (t *Tree) Root() *defn.TreeRoot { + return &defn.TreeRoot{ + TreeSize: t.Size(), + RootHash: rootFromFrontier(t.frontier), + } +} + +// InclusionProof returns the raw entry and bottom-up sibling path for leafIndex. +func (t *Tree) InclusionProof(leafIndex uint64) (*defn.InclusionProof, error) { + if leafIndex >= t.Size() { + return nil, fmt.Errorf("leaf index %d is outside tree of size %d", leafIndex, t.Size()) + } + _, entryValue, err := decodeLogEntry(t.entries[leafIndex]) + if err != nil { + return nil, fmt.Errorf("%w: entry %d: %v", ErrStoreCorrupt, leafIndex, err) + } + + return &defn.InclusionProof{ + Entry: enc.Wire{entryValue}, + LeafIndex: leafIndex, + SiblingHashes: inclusionPath(t.leafHashes, leafIndex), + }, nil +} + +// EncodeLogEntry returns the complete canonical LogEntry TLV. +func EncodeLogEntry(entry *defn.LogEntry) ([]byte, error) { + if err := validateEntry(entry); err != nil { + return nil, err + } + return wrapLogEntryValue(entry.Bytes()), nil +} + +// ParseLogEntry parses and validates one complete canonical LogEntry TLV. +func ParseLogEntry(entryWire []byte) (*defn.LogEntry, error) { + entry, _, err := decodeLogEntry(entryWire) + return entry, err +} + +// ParseProofEntry parses and validates the raw LogEntry carried by proof. +func ParseProofEntry(proof *defn.InclusionProof) (*defn.LogEntry, error) { + if proof == nil { + return nil, fmt.Errorf("inclusion proof is nil") + } + entry, _, err := decodeLogEntry(wrapLogEntryValue(proof.Entry.Join())) + return entry, err +} + +// LeafHash computes SHA-256(0x00 || rawLogEntryTLV). +func LeafHash(entryWire []byte) ([]byte, error) { + if _, _, err := decodeLogEntry(entryWire); err != nil { + return nil, err + } + return hashLeaf(entryWire), nil +} + +// VerifyInclusion verifies that dataHash occurs in the raw proof entry and that +// the entry's inclusion path reconstructs root. +func VerifyInclusion(dataHash []byte, proof *defn.InclusionProof, root *defn.TreeRoot) error { + if len(dataHash) != HashSize { + return fmt.Errorf("data hash length is %d, want %d", len(dataHash), HashSize) + } + if proof == nil { + return fmt.Errorf("inclusion proof is nil") + } + if root == nil { + return fmt.Errorf("tree root is nil") + } + if root.TreeSize == 0 { + return fmt.Errorf("an empty tree cannot contain an inclusion proof") + } + if len(root.RootHash) != HashSize { + return fmt.Errorf("root hash length is %d, want %d", len(root.RootHash), HashSize) + } + if proof.LeafIndex >= root.TreeSize { + return fmt.Errorf("leaf index %d is outside tree of size %d", proof.LeafIndex, root.TreeSize) + } + + entryWire := wrapLogEntryValue(proof.Entry.Join()) + entry, _, err := decodeLogEntry(entryWire) + if err != nil { + return fmt.Errorf("invalid proof entry: %w", err) + } + found := false + for _, entryHash := range entry.DataHashes { + if bytes.Equal(dataHash, entryHash) { + found = true + break + } + } + if !found { + return fmt.Errorf("data hash is not present in proof entry") + } + for i, siblingHash := range proof.SiblingHashes { + if len(siblingHash) != HashSize { + return fmt.Errorf("sibling hash %d length is %d, want %d", i, len(siblingHash), HashSize) + } + } + + proofRoot, err := rootFromInclusionPath( + hashLeaf(entryWire), + proof.LeafIndex, + root.TreeSize, + proof.SiblingHashes, + ) + if err != nil { + return err + } + if !bytes.Equal(proofRoot, root.RootHash) { + return fmt.Errorf("inclusion proof does not match tree root") + } + return nil +} + +func decodeLogEntry(entryWire []byte) (*defn.LogEntry, []byte, error) { + reader := enc.NewBufferView(entryWire) + typ, err := reader.ReadTLNum() + if err != nil { + return nil, nil, fmt.Errorf("failed to read LogEntry type: %w", err) + } + if typ != defn.TypeLogEntry { + return nil, nil, fmt.Errorf("LogEntry type is %d, want %d", typ, defn.TypeLogEntry) + } + length, err := reader.ReadTLNum() + if err != nil { + return nil, nil, fmt.Errorf("failed to read LogEntry length: %w", err) + } + if length != enc.TLNum(reader.Length()-reader.Pos()) { + return nil, nil, fmt.Errorf( + "LogEntry length is %d, but %d bytes remain", + length, + reader.Length()-reader.Pos(), + ) + } + entryValue, err := reader.ReadBuf(int(length)) + if err != nil { + return nil, nil, fmt.Errorf("failed to read LogEntry value: %w", err) + } + entry, err := defn.ParseLogEntry(enc.NewBufferView(entryValue), false) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse LogEntry: %w", err) + } + if err := validateEntry(entry); err != nil { + return nil, nil, err + } + if !bytes.Equal(entryWire, wrapLogEntryValue(entry.Bytes())) { + return nil, nil, fmt.Errorf("LogEntry TLV is not canonical") + } + return entry, bytes.Clone(entryValue), nil +} + +func validateEntry(entry *defn.LogEntry) error { + if entry == nil { + return fmt.Errorf("LogEntry is nil") + } + if entry.IngestTime < 0 { + return fmt.Errorf("ingestion time cannot be negative") + } + if entry.IngestTime%time.Millisecond != 0 { + return fmt.Errorf("ingestion time must have millisecond precision") + } + if len(entry.DataHashes) == 0 { + return fmt.Errorf("LogEntry has no Data hashes") + } + for i, dataHash := range entry.DataHashes { + if len(dataHash) != HashSize { + return fmt.Errorf("Data hash %d length is %d, want %d", i, len(dataHash), HashSize) + } + } + return nil +} + +func wrapLogEntryValue(entryValue []byte) []byte { + typLen := defn.TypeLogEntry.EncodingLength() + length := enc.TLNum(len(entryValue)) + ret := make([]byte, typLen+length.EncodingLength()+len(entryValue)) + pos := defn.TypeLogEntry.EncodeInto(ret) + pos += length.EncodeInto(ret[pos:]) + copy(ret[pos:], entryValue) + return ret +} + +func hashLeaf(entryWire []byte) []byte { + h := sha256.New() + h.Write([]byte{0x00}) + h.Write(entryWire) + return h.Sum(nil) +} + +func treeHash(leafHashes [][]byte) []byte { + switch len(leafHashes) { + case 0: + hash := sha256.Sum256(nil) + return hash[:] + case 1: + return bytes.Clone(leafHashes[0]) + default: + split := int(largestPowerOfTwoLessThan(uint64(len(leafHashes)))) + return nodeHash(treeHash(leafHashes[:split]), treeHash(leafHashes[split:])) + } +} + +func inclusionPath(leafHashes [][]byte, leafIndex uint64) [][]byte { + if len(leafHashes) == 1 { + return nil + } + + split := largestPowerOfTwoLessThan(uint64(len(leafHashes))) + if leafIndex < split { + path := inclusionPath(leafHashes[:split], leafIndex) + return append(path, treeHash(leafHashes[split:])) + } + + path := inclusionPath(leafHashes[split:], leafIndex-split) + return append(path, treeHash(leafHashes[:split])) +} + +func rootFromInclusionPath( + leafHash []byte, + leafIndex uint64, + treeSize uint64, + siblingHashes [][]byte, +) ([]byte, error) { + if treeSize == 1 { + if len(siblingHashes) != 0 { + return nil, fmt.Errorf("inclusion proof has %d unused sibling hashes", len(siblingHashes)) + } + return bytes.Clone(leafHash), nil + } + if len(siblingHashes) == 0 { + return nil, fmt.Errorf("inclusion proof is missing a sibling hash") + } + + split := largestPowerOfTwoLessThan(treeSize) + siblingHash := siblingHashes[len(siblingHashes)-1] + remaining := siblingHashes[:len(siblingHashes)-1] + if leafIndex < split { + leftHash, err := rootFromInclusionPath(leafHash, leafIndex, split, remaining) + if err != nil { + return nil, err + } + return nodeHash(leftHash, siblingHash), nil + } + + rightHash, err := rootFromInclusionPath(leafHash, leafIndex-split, treeSize-split, remaining) + if err != nil { + return nil, err + } + return nodeHash(siblingHash, rightHash), nil +} + +func appendFrontier(frontier [][]byte, leafHash []byte) [][]byte { + next := cloneHashes(frontier) + carry := bytes.Clone(leafHash) + for level := 0; ; level++ { + if level == len(next) { + next = append(next, carry) + return next + } + if next[level] == nil { + next[level] = carry + return next + } + carry = nodeHash(next[level], carry) + next[level] = nil + } +} + +func rootFromFrontier(frontier [][]byte) []byte { + var root []byte + for _, subtreeHash := range frontier { + if subtreeHash == nil { + continue + } + if root == nil { + root = bytes.Clone(subtreeHash) + } else { + root = nodeHash(subtreeHash, root) + } + } + if root == nil { + hash := sha256.Sum256(nil) + return hash[:] + } + return root +} + +func largestPowerOfTwoLessThan(value uint64) uint64 { + return uint64(1) << (bits.Len64(value-1) - 1) +} + +func nodeHash(leftHash []byte, rightHash []byte) []byte { + h := sha256.New() + h.Write([]byte{0x01}) + h.Write(leftHash) + h.Write(rightHash) + return h.Sum(nil) +} + +func dataHashKey(dataHash []byte) [HashSize]byte { + var key [HashSize]byte + copy(key[:], dataHash) + return key +} + +func cloneHashes(hashes [][]byte) [][]byte { + ret := make([][]byte, len(hashes)) + for i, hash := range hashes { + ret[i] = bytes.Clone(hash) + } + return ret +} + +func (t *Tree) storeState() StoreState { + return StoreState{ + TreeSize: t.Size(), + RootHash: rootFromFrontier(t.frontier), + Frontier: cloneHashes(t.frontier), + LastIngestTime: t.lastIngestTime, + } +} + +func equalDataIndex(left map[[HashSize]byte]uint64, right map[[HashSize]byte]uint64) bool { + if len(left) != len(right) { + return false + } + for dataHash, leafIndex := range left { + rightIndex, ok := right[dataHash] + if !ok || rightIndex != leafIndex { + return false + } + } + return true +} diff --git a/std/merklelog/tree_test.go b/std/merklelog/tree_test.go new file mode 100644 index 00000000..4a4f8812 --- /dev/null +++ b/std/merklelog/tree_test.go @@ -0,0 +1,360 @@ +package merklelog + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "strconv" + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + defn "github.com/named-data/ndnd/std/ndn/merklelog" + "github.com/stretchr/testify/require" +) + +func TestTreeRootVectors(t *testing.T) { + expectedRoots := []string{ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "3e49ed62039635b1f465bb6feb941b748a037f7461d1e29728db61011e51aebf", + "e9405c348553e9feb5a6234d6e91598090b7b16c5979110f2290f1d010a8114f", + "0ebbdd04404e790bcd16c3ac9265bdd68ab9aa577c67590b25c0c6b2f545acf0", + "fbb2aa61e73070546d195d57e57ed1cf3a9c1073e86afd129f41367a06dc0ff1", + "38ad2533c3b04b36c0f9c776011a2aea486d75edb710c8981f0a17f168a5262a", + "faec623ba14e1fe04b2c556578ab74756573c5db0824b6a05b67ad2532da50e3", + "1d80070a35eaa80d58c83c4f3b5a2c367c61f2233cafa40791bd8d856d9ccc0c", + "c80e2f1be2197852e733b9b0a6eb6d475e7d66871c95f4d67c20437ca04753b7", + } + + tree := NewTree() + for size, expectedRoot := range expectedRoots { + root := tree.Root() + require.Equal(t, uint64(size), root.TreeSize) + require.Equal(t, expectedRoot, hex.EncodeToString(root.RootHash)) + + if size < len(expectedRoots)-1 { + leafIndex, err := tree.Append(testEntryWire(size)) + require.NoError(t, err) + require.Equal(t, uint64(size), leafIndex) + } + } +} + +func TestLogEntryAndLeafHashVectors(t *testing.T) { + entryWire := testEntryWire(0) + require.Equal(t, + "fd1e122afd1e160203e8fd1e0020e501858e369df59267c5d1e0d0591806880984af98708c9406efca8055647634", + hex.EncodeToString(entryWire), + ) + + entry, err := ParseLogEntry(entryWire) + require.NoError(t, err) + require.Equal(t, testEntry(0), entry) + + leafHash, err := LeafHash(entryWire) + require.NoError(t, err) + require.Equal(t, + "3e49ed62039635b1f465bb6feb941b748a037f7461d1e29728db61011e51aebf", + hex.EncodeToString(leafHash), + ) +} + +func TestInclusionProofs(t *testing.T) { + for size := 1; size <= 16; size++ { + t.Run(strconv.Itoa(size), func(t *testing.T) { + tree := NewTree() + entries := make([]*defn.LogEntry, size) + for i := range entries { + entries[i] = testEntry(i) + _, err := tree.Append(mustEncodeLogEntry(entries[i])) + require.NoError(t, err) + } + + root := tree.Root() + for leafIndex, entry := range entries { + proof, err := tree.InclusionProof(uint64(leafIndex)) + require.NoError(t, err) + require.NoError(t, VerifyInclusion(entry.DataHashes[0], proof, root)) + parsedEntry, err := ParseProofEntry(proof) + require.NoError(t, err) + require.Equal(t, entry, parsedEntry) + } + }) + } +} + +func TestInclusionProofEncodingPreservesEntryWire(t *testing.T) { + tree := NewTree() + entryWire := testEntryWire(0) + _, err := tree.Append(entryWire) + require.NoError(t, err) + proof, err := tree.InclusionProof(0) + require.NoError(t, err) + + parsed, err := defn.ParseInclusionProof(enc.NewWireView(proof.Encode()), false) + require.NoError(t, err) + require.Equal(t, entryWire, wrapLogEntryValue(parsed.Entry.Join())) + require.NoError(t, VerifyInclusion(testDataHash(0), parsed, tree.Root())) +} + +func TestVerifyInclusionRejectsMutations(t *testing.T) { + tree := NewTree() + entries := make([]*defn.LogEntry, 5) + for i := range entries { + entries[i] = testEntry(i) + entries[i].DataHashes = append(entries[i].DataHashes, testDataHash(i+100)) + _, err := tree.Append(mustEncodeLogEntry(entries[i])) + require.NoError(t, err) + } + + dataHash := entries[4].DataHashes[0] + proof, err := tree.InclusionProof(4) + require.NoError(t, err) + root := tree.Root() + require.NoError(t, VerifyInclusion(dataHash, proof, root)) + + t.Run("requested hash", func(t *testing.T) { + mutated := bytes.Clone(dataHash) + mutated[0] ^= 0xff + require.Error(t, VerifyInclusion(mutated, proof, root)) + }) + + t.Run("entry hash", func(t *testing.T) { + mutated := cloneProof(proof) + entry := mustParseProofEntry(mutated) + entry.DataHashes[1][0] ^= 0xff + setProofEntry(mutated, entry) + require.Error(t, VerifyInclusion(dataHash, mutated, root)) + }) + + t.Run("ingestion time", func(t *testing.T) { + mutated := cloneProof(proof) + entry := mustParseProofEntry(mutated) + entry.IngestTime += time.Millisecond + setProofEntry(mutated, entry) + require.Error(t, VerifyInclusion(dataHash, mutated, root)) + }) + + t.Run("sibling hash", func(t *testing.T) { + mutated := cloneProof(proof) + mutated.SiblingHashes[0][0] ^= 0xff + require.Error(t, VerifyInclusion(dataHash, mutated, root)) + }) + + t.Run("leaf index", func(t *testing.T) { + mutated := cloneProof(proof) + mutated.LeafIndex-- + require.Error(t, VerifyInclusion(dataHash, mutated, root)) + }) + + t.Run("tree size", func(t *testing.T) { + mutated := cloneRoot(root) + mutated.TreeSize++ + require.Error(t, VerifyInclusion(dataHash, proof, mutated)) + }) + + t.Run("root hash", func(t *testing.T) { + mutated := cloneRoot(root) + mutated.RootHash[0] ^= 0xff + require.Error(t, VerifyInclusion(dataHash, proof, mutated)) + }) + + t.Run("missing sibling", func(t *testing.T) { + mutated := cloneProof(proof) + mutated.SiblingHashes = mutated.SiblingHashes[1:] + require.Error(t, VerifyInclusion(dataHash, mutated, root)) + }) + + t.Run("extra sibling", func(t *testing.T) { + mutated := cloneProof(proof) + mutated.SiblingHashes = append(mutated.SiblingHashes, make([]byte, HashSize)) + require.Error(t, VerifyInclusion(dataHash, mutated, root)) + }) +} + +func TestTreeCopiesCallerData(t *testing.T) { + tree := NewTree() + entryWire := testEntryWire(0) + originalWire := bytes.Clone(entryWire) + _, err := tree.Append(entryWire) + require.NoError(t, err) + + expectedRoot := tree.Root() + entryWire[len(entryWire)-1] ^= 0xff + require.Equal(t, expectedRoot, tree.Root()) + + proof, err := tree.InclusionProof(0) + require.NoError(t, err) + proof.Entry[0][0] ^= 0xff + + freshProof, err := tree.InclusionProof(0) + require.NoError(t, err) + require.Equal(t, originalWire, wrapLogEntryValue(freshProof.Entry.Join())) + require.NoError(t, VerifyInclusion(testDataHash(0), freshProof, tree.Root())) + + root := tree.Root() + root.RootHash[0] ^= 0xff + require.Equal(t, expectedRoot, tree.Root()) +} + +func TestTreeRejectsInvalidInputs(t *testing.T) { + invalidEntries := map[string]*defn.LogEntry{ + "nil": nil, + "negative time": {IngestTime: -time.Millisecond, DataHashes: [][]byte{testDataHash(0)}}, + "sub-millisecond time": {IngestTime: time.Nanosecond, DataHashes: [][]byte{testDataHash(0)}}, + "no hashes": {IngestTime: time.Second}, + "short hash": {IngestTime: time.Second, DataHashes: [][]byte{{0x01}}}, + } + for name, entry := range invalidEntries { + t.Run(name, func(t *testing.T) { + _, err := EncodeLogEntry(entry) + require.Error(t, err) + }) + } + + validWire := testEntryWire(0) + invalidWires := map[string][]byte{ + "nil": nil, + "wrong type": append([]byte{0x01}, validWire[3:]...), + "wrong length": append(bytes.Clone(validWire[:3]), append([]byte{0x01}, validWire[4:]...)...), + "truncated": validWire[:len(validWire)-1], + "non-canonical": append([]byte{0xfd, 0x1e, 0x12, 0xfd, 0x00, validWire[3]}, validWire[4:]...), + } + for name, entryWire := range invalidWires { + t.Run(name, func(t *testing.T) { + tree := NewTree() + _, err := tree.Append(entryWire) + require.Error(t, err) + require.Zero(t, tree.Size()) + }) + } + + tree := NewTree() + _, err := tree.Append(validWire) + require.NoError(t, err) + _, err = tree.InclusionProof(1) + require.Error(t, err) +} + +func TestTreeRejectsDuplicateHashesAndNonIncreasingTime(t *testing.T) { + tree := NewTree() + _, err := tree.Append(testEntryWire(0)) + require.NoError(t, err) + + duplicate := testEntry(1) + duplicate.DataHashes[0] = testDataHash(0) + _, err = tree.Append(mustEncodeLogEntry(duplicate)) + require.ErrorIs(t, err, ErrDuplicateDataHash) + + withinEntry := testEntry(1) + withinEntry.DataHashes = append(withinEntry.DataHashes, withinEntry.DataHashes[0]) + _, err = tree.Append(mustEncodeLogEntry(withinEntry)) + require.ErrorIs(t, err, ErrDuplicateDataHash) + + nonIncreasing := testEntry(1) + nonIncreasing.IngestTime = testEntry(0).IngestTime + _, err = tree.Append(mustEncodeLogEntry(nonIncreasing)) + require.Error(t, err) + require.Equal(t, uint64(1), tree.Size()) +} + +func TestVerifyInclusionRejectsMalformedProofs(t *testing.T) { + tree := NewTree() + entry := testEntry(0) + _, err := tree.Append(mustEncodeLogEntry(entry)) + require.NoError(t, err) + proof, err := tree.InclusionProof(0) + require.NoError(t, err) + root := tree.Root() + + tests := map[string]func() ([]byte, *defn.InclusionProof, *defn.TreeRoot){ + "short requested hash": func() ([]byte, *defn.InclusionProof, *defn.TreeRoot) { + return []byte{0x01}, proof, root + }, + "nil proof": func() ([]byte, *defn.InclusionProof, *defn.TreeRoot) { + return entry.DataHashes[0], nil, root + }, + "nil root": func() ([]byte, *defn.InclusionProof, *defn.TreeRoot) { + return entry.DataHashes[0], proof, nil + }, + "empty tree": func() ([]byte, *defn.InclusionProof, *defn.TreeRoot) { + return entry.DataHashes[0], proof, NewTree().Root() + }, + "short root hash": func() ([]byte, *defn.InclusionProof, *defn.TreeRoot) { + return entry.DataHashes[0], proof, &defn.TreeRoot{TreeSize: 1, RootHash: []byte{0x01}} + }, + "nil entry": func() ([]byte, *defn.InclusionProof, *defn.TreeRoot) { + mutated := cloneProof(proof) + mutated.Entry = nil + return entry.DataHashes[0], mutated, root + }, + "short sibling hash": func() ([]byte, *defn.InclusionProof, *defn.TreeRoot) { + mutated := cloneProof(proof) + mutated.SiblingHashes = [][]byte{{0x01}} + return entry.DataHashes[0], mutated, root + }, + } + + for name, makeInput := range tests { + t.Run(name, func(t *testing.T) { + dataHash, proof, root := makeInput() + require.Error(t, VerifyInclusion(dataHash, proof, root)) + }) + } +} + +func testEntry(index int) *defn.LogEntry { + return &defn.LogEntry{ + IngestTime: time.Duration(1000+index) * time.Millisecond, + DataHashes: [][]byte{testDataHash(index)}, + } +} + +func testEntryWire(index int) []byte { + return mustEncodeLogEntry(testEntry(index)) +} + +func testDataHash(index int) []byte { + hash := sha256.Sum256([]byte("data-" + strconv.Itoa(index))) + return hash[:] +} + +func mustEncodeLogEntry(entry *defn.LogEntry) []byte { + wire, err := EncodeLogEntry(entry) + if err != nil { + panic(err) + } + return wire +} + +func mustParseProofEntry(proof *defn.InclusionProof) *defn.LogEntry { + entry, err := ParseProofEntry(proof) + if err != nil { + panic(err) + } + return entry +} + +func setProofEntry(proof *defn.InclusionProof, entry *defn.LogEntry) { + entryWire := mustEncodeLogEntry(entry) + _, entryValue, err := decodeLogEntry(entryWire) + if err != nil { + panic(err) + } + proof.Entry = enc.Wire{entryValue} +} + +func cloneProof(proof *defn.InclusionProof) *defn.InclusionProof { + return &defn.InclusionProof{ + Entry: enc.Wire{bytes.Clone(proof.Entry.Join())}, + LeafIndex: proof.LeafIndex, + SiblingHashes: cloneHashes(proof.SiblingHashes), + } +} + +func cloneRoot(root *defn.TreeRoot) *defn.TreeRoot { + return &defn.TreeRoot{ + TreeSize: root.TreeSize, + RootHash: bytes.Clone(root.RootHash), + } +} diff --git a/std/ndn/merklelog/definitions.go b/std/ndn/merklelog/definitions.go new file mode 100644 index 00000000..6942b3e5 --- /dev/null +++ b/std/ndn/merklelog/definitions.go @@ -0,0 +1,96 @@ +//go:generate gondn_tlv_gen +package merklelog + +import ( + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/types/optional" +) + +// TypeLogEntry is the outer TLV type included in every Merkle leaf hash. +const TypeLogEntry enc.TLNum = 0x1E12 + +const ( + // AppendStatusOK indicates that the hash was added to a new log entry. + AppendStatusOK uint64 = iota + // AppendStatusDuplicate indicates that the hash already has a log entry. + AppendStatusDuplicate + // AppendStatusFailed indicates that the hash could not be logged. + AppendStatusFailed +) + +const ( + // CheckStatusIncluded indicates that the response contains an inclusion proof. + CheckStatusIncluded uint64 = iota + // CheckStatusNotFound indicates that the hash is absent from the returned tree. + CheckStatusNotFound +) + +type AppendRequest struct { + //+field:sequence:[]byte:binary:[]byte + DataHashes [][]byte `tlv:"0x1E00"` +} + +type AppendResponse struct { + //+field:sequence:*AppendResult:struct:AppendResult + Results []*AppendResult `tlv:"0x1E02"` +} + +type AppendResult struct { + //+field:binary + DataHash []byte `tlv:"0x1E00"` + //+field:natural + Status uint64 `tlv:"0x1E04"` + //+field:natural:optional + LeafIndex optional.Optional[uint64] `tlv:"0x1E06"` +} + +type CheckRequest struct { + //+field:sequence:[]byte:binary:[]byte + DataHashes [][]byte `tlv:"0x1E00"` +} + +type CheckResponse struct { + //+field:struct:TreeRoot + Root *TreeRoot `tlv:"0x1E08"` + //+field:sequence:*CheckResult:struct:CheckResult + Results []*CheckResult `tlv:"0x1E0A"` +} + +type CheckResult struct { + //+field:binary + DataHash []byte `tlv:"0x1E00"` + //+field:natural + Status uint64 `tlv:"0x1E04"` + //+field:struct:InclusionProof + Proof *InclusionProof `tlv:"0x1E0C"` +} + +type TreeRoot struct { + //+field:natural + TreeSize uint64 `tlv:"0x1E0E"` + //+field:binary + RootHash []byte `tlv:"0x1E10"` +} + +type InclusionProof struct { + // Entry contains the unmodified TLV-VALUE of the proven LogEntry. Keeping + // this as wire allows the verifier to reconstruct and hash the exact entry + // encoding instead of re-encoding parsed fields. + //+field:wire + Entry enc.Wire `tlv:"0x1E12"` + //+field:natural + LeafIndex uint64 `tlv:"0x1E06"` + //+field:sequence:[]byte:binary:[]byte + SiblingHashes [][]byte `tlv:"0x1E14"` +} + +type LogEntry struct { + // IngestTime is the log-assigned Unix timestamp, represented at millisecond + // precision as a duration since the Unix epoch. + //+field:time + IngestTime time.Duration `tlv:"0x1E16"` + //+field:sequence:[]byte:binary:[]byte + DataHashes [][]byte `tlv:"0x1E00"` +} diff --git a/std/ndn/merklelog/zz_generated.go b/std/ndn/merklelog/zz_generated.go new file mode 100644 index 00000000..64e4485e --- /dev/null +++ b/std/ndn/merklelog/zz_generated.go @@ -0,0 +1,1887 @@ +// Code generated by ndn tlv codegen DO NOT EDIT. +package merklelog + +import ( + "encoding/binary" + "io" + + "time" + + enc "github.com/named-data/ndnd/std/encoding" +) + +type AppendRequestEncoder struct { + Length uint + + DataHashes_subencoder []struct { + } +} + +type AppendRequestParsingContext struct { +} + +func (encoder *AppendRequestEncoder) Init(value *AppendRequest) { + { + DataHashes_l := len(value.DataHashes) + encoder.DataHashes_subencoder = make([]struct { + }, DataHashes_l) + for i := 0; i < DataHashes_l; i++ { + pseudoEncoder := &encoder.DataHashes_subencoder[i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: value.DataHashes[i], + } + { + encoder := pseudoEncoder + value := &pseudoValue + + _ = encoder + _ = value + } + } + } + + l := uint(0) + if value.DataHashes != nil { + for seq_i, seq_v := range value.DataHashes { + pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.DataHashes != nil { + l += 3 + l += uint(enc.TLNum(len(value.DataHashes)).EncodingLength()) + l += uint(len(value.DataHashes)) + } + _ = encoder + _ = value + } + } + } + encoder.Length = l + +} + +func (context *AppendRequestParsingContext) Init() { + +} + +func (encoder *AppendRequestEncoder) EncodeInto(value *AppendRequest, buf []byte) { + + pos := uint(0) + + if value.DataHashes != nil { + for seq_i, seq_v := range value.DataHashes { + pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.DataHashes != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7680)) + pos += 3 + pos += uint(enc.TLNum(len(value.DataHashes)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.DataHashes) + pos += uint(len(value.DataHashes)) + } + _ = encoder + _ = value + } + } + } +} + +func (encoder *AppendRequestEncoder) Encode(value *AppendRequest) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *AppendRequestParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*AppendRequest, error) { + + var handled_DataHashes bool = false + + progress := -1 + _ = progress + + value := &AppendRequest{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7680: + if true { + handled = true + handled_DataHashes = true + if value.DataHashes == nil { + value.DataHashes = make([][]byte, 0) + } + { + pseudoValue := struct { + DataHashes []byte + }{} + { + value := &pseudoValue + value.DataHashes = make([]byte, l) + _, err = reader.ReadFull(value.DataHashes) + _ = value + } + value.DataHashes = append(value.DataHashes, pseudoValue.DataHashes) + } + progress-- + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_DataHashes && err == nil { + // sequence - skip + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *AppendRequest) Encode() enc.Wire { + encoder := AppendRequestEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *AppendRequest) Bytes() []byte { + return value.Encode().Join() +} + +func ParseAppendRequest(reader enc.WireView, ignoreCritical bool) (*AppendRequest, error) { + context := AppendRequestParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type AppendResponseEncoder struct { + Length uint + + Results_subencoder []struct { + Results_encoder AppendResultEncoder + } +} + +type AppendResponseParsingContext struct { + Results_context AppendResultParsingContext +} + +func (encoder *AppendResponseEncoder) Init(value *AppendResponse) { + { + Results_l := len(value.Results) + encoder.Results_subencoder = make([]struct { + Results_encoder AppendResultEncoder + }, Results_l) + for i := 0; i < Results_l; i++ { + pseudoEncoder := &encoder.Results_subencoder[i] + pseudoValue := struct { + Results *AppendResult + }{ + Results: value.Results[i], + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.Results != nil { + encoder.Results_encoder.Init(value.Results) + } + _ = encoder + _ = value + } + } + } + + l := uint(0) + if value.Results != nil { + for seq_i, seq_v := range value.Results { + pseudoEncoder := &encoder.Results_subencoder[seq_i] + pseudoValue := struct { + Results *AppendResult + }{ + Results: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.Results != nil { + l += 3 + l += uint(enc.TLNum(encoder.Results_encoder.Length).EncodingLength()) + l += encoder.Results_encoder.Length + } + _ = encoder + _ = value + } + } + } + encoder.Length = l + +} + +func (context *AppendResponseParsingContext) Init() { + context.Results_context.Init() +} + +func (encoder *AppendResponseEncoder) EncodeInto(value *AppendResponse, buf []byte) { + + pos := uint(0) + + if value.Results != nil { + for seq_i, seq_v := range value.Results { + pseudoEncoder := &encoder.Results_subencoder[seq_i] + pseudoValue := struct { + Results *AppendResult + }{ + Results: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.Results != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7682)) + pos += 3 + pos += uint(enc.TLNum(encoder.Results_encoder.Length).EncodeInto(buf[pos:])) + if encoder.Results_encoder.Length > 0 { + encoder.Results_encoder.EncodeInto(value.Results, buf[pos:]) + pos += encoder.Results_encoder.Length + } + } + _ = encoder + _ = value + } + } + } +} + +func (encoder *AppendResponseEncoder) Encode(value *AppendResponse) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *AppendResponseParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*AppendResponse, error) { + + var handled_Results bool = false + + progress := -1 + _ = progress + + value := &AppendResponse{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7682: + if true { + handled = true + handled_Results = true + if value.Results == nil { + value.Results = make([]*AppendResult, 0) + } + { + pseudoValue := struct { + Results *AppendResult + }{} + { + value := &pseudoValue + value.Results, err = context.Results_context.Parse(reader.Delegate(int(l)), ignoreCritical) + _ = value + } + value.Results = append(value.Results, pseudoValue.Results) + } + progress-- + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_Results && err == nil { + // sequence - skip + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *AppendResponse) Encode() enc.Wire { + encoder := AppendResponseEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *AppendResponse) Bytes() []byte { + return value.Encode().Join() +} + +func ParseAppendResponse(reader enc.WireView, ignoreCritical bool) (*AppendResponse, error) { + context := AppendResponseParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type AppendResultEncoder struct { + Length uint +} + +type AppendResultParsingContext struct { +} + +func (encoder *AppendResultEncoder) Init(value *AppendResult) { + + l := uint(0) + if value.DataHash != nil { + l += 3 + l += uint(enc.TLNum(len(value.DataHash)).EncodingLength()) + l += uint(len(value.DataHash)) + } + l += 3 + l += uint(1 + enc.Nat(value.Status).EncodingLength()) + if optval, ok := value.LeafIndex.Get(); ok { + l += 3 + l += uint(1 + enc.Nat(optval).EncodingLength()) + } + encoder.Length = l + +} + +func (context *AppendResultParsingContext) Init() { + +} + +func (encoder *AppendResultEncoder) EncodeInto(value *AppendResult, buf []byte) { + + pos := uint(0) + + if value.DataHash != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7680)) + pos += 3 + pos += uint(enc.TLNum(len(value.DataHash)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.DataHash) + pos += uint(len(value.DataHash)) + } + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7684)) + pos += 3 + + buf[pos] = byte(enc.Nat(value.Status).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + if optval, ok := value.LeafIndex.Get(); ok { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7686)) + pos += 3 + + buf[pos] = byte(enc.Nat(optval).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + + } +} + +func (encoder *AppendResultEncoder) Encode(value *AppendResult) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *AppendResultParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*AppendResult, error) { + + var handled_DataHash bool = false + var handled_Status bool = false + var handled_LeafIndex bool = false + + progress := -1 + _ = progress + + value := &AppendResult{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7680: + if true { + handled = true + handled_DataHash = true + value.DataHash = make([]byte, l) + _, err = reader.ReadFull(value.DataHash) + } + case 7684: + if true { + handled = true + handled_Status = true + value.Status = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.Status = uint64(value.Status<<8) | uint64(x) + } + } + } + case 7686: + if true { + handled = true + handled_LeafIndex = true + { + optval := uint64(0) + optval = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + optval = uint64(optval<<8) | uint64(x) + } + } + value.LeafIndex.Set(optval) + } + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_DataHash && err == nil { + value.DataHash = nil + } + if !handled_Status && err == nil { + err = enc.ErrSkipRequired{Name: "Status", TypeNum: 7684} + } + if !handled_LeafIndex && err == nil { + value.LeafIndex.Unset() + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *AppendResult) Encode() enc.Wire { + encoder := AppendResultEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *AppendResult) Bytes() []byte { + return value.Encode().Join() +} + +func ParseAppendResult(reader enc.WireView, ignoreCritical bool) (*AppendResult, error) { + context := AppendResultParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type CheckRequestEncoder struct { + Length uint + + DataHashes_subencoder []struct { + } +} + +type CheckRequestParsingContext struct { +} + +func (encoder *CheckRequestEncoder) Init(value *CheckRequest) { + { + DataHashes_l := len(value.DataHashes) + encoder.DataHashes_subencoder = make([]struct { + }, DataHashes_l) + for i := 0; i < DataHashes_l; i++ { + pseudoEncoder := &encoder.DataHashes_subencoder[i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: value.DataHashes[i], + } + { + encoder := pseudoEncoder + value := &pseudoValue + + _ = encoder + _ = value + } + } + } + + l := uint(0) + if value.DataHashes != nil { + for seq_i, seq_v := range value.DataHashes { + pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.DataHashes != nil { + l += 3 + l += uint(enc.TLNum(len(value.DataHashes)).EncodingLength()) + l += uint(len(value.DataHashes)) + } + _ = encoder + _ = value + } + } + } + encoder.Length = l + +} + +func (context *CheckRequestParsingContext) Init() { + +} + +func (encoder *CheckRequestEncoder) EncodeInto(value *CheckRequest, buf []byte) { + + pos := uint(0) + + if value.DataHashes != nil { + for seq_i, seq_v := range value.DataHashes { + pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.DataHashes != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7680)) + pos += 3 + pos += uint(enc.TLNum(len(value.DataHashes)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.DataHashes) + pos += uint(len(value.DataHashes)) + } + _ = encoder + _ = value + } + } + } +} + +func (encoder *CheckRequestEncoder) Encode(value *CheckRequest) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *CheckRequestParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*CheckRequest, error) { + + var handled_DataHashes bool = false + + progress := -1 + _ = progress + + value := &CheckRequest{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7680: + if true { + handled = true + handled_DataHashes = true + if value.DataHashes == nil { + value.DataHashes = make([][]byte, 0) + } + { + pseudoValue := struct { + DataHashes []byte + }{} + { + value := &pseudoValue + value.DataHashes = make([]byte, l) + _, err = reader.ReadFull(value.DataHashes) + _ = value + } + value.DataHashes = append(value.DataHashes, pseudoValue.DataHashes) + } + progress-- + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_DataHashes && err == nil { + // sequence - skip + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *CheckRequest) Encode() enc.Wire { + encoder := CheckRequestEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *CheckRequest) Bytes() []byte { + return value.Encode().Join() +} + +func ParseCheckRequest(reader enc.WireView, ignoreCritical bool) (*CheckRequest, error) { + context := CheckRequestParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type CheckResponseEncoder struct { + Length uint + + Root_encoder TreeRootEncoder + Results_subencoder []struct { + Results_encoder CheckResultEncoder + } +} + +type CheckResponseParsingContext struct { + Root_context TreeRootParsingContext + Results_context CheckResultParsingContext +} + +func (encoder *CheckResponseEncoder) Init(value *CheckResponse) { + if value.Root != nil { + encoder.Root_encoder.Init(value.Root) + } + { + Results_l := len(value.Results) + encoder.Results_subencoder = make([]struct { + Results_encoder CheckResultEncoder + }, Results_l) + for i := 0; i < Results_l; i++ { + pseudoEncoder := &encoder.Results_subencoder[i] + pseudoValue := struct { + Results *CheckResult + }{ + Results: value.Results[i], + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.Results != nil { + encoder.Results_encoder.Init(value.Results) + } + _ = encoder + _ = value + } + } + } + + l := uint(0) + if value.Root != nil { + l += 3 + l += uint(enc.TLNum(encoder.Root_encoder.Length).EncodingLength()) + l += encoder.Root_encoder.Length + } + if value.Results != nil { + for seq_i, seq_v := range value.Results { + pseudoEncoder := &encoder.Results_subencoder[seq_i] + pseudoValue := struct { + Results *CheckResult + }{ + Results: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.Results != nil { + l += 3 + l += uint(enc.TLNum(encoder.Results_encoder.Length).EncodingLength()) + l += encoder.Results_encoder.Length + } + _ = encoder + _ = value + } + } + } + encoder.Length = l + +} + +func (context *CheckResponseParsingContext) Init() { + context.Root_context.Init() + context.Results_context.Init() +} + +func (encoder *CheckResponseEncoder) EncodeInto(value *CheckResponse, buf []byte) { + + pos := uint(0) + + if value.Root != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7688)) + pos += 3 + pos += uint(enc.TLNum(encoder.Root_encoder.Length).EncodeInto(buf[pos:])) + if encoder.Root_encoder.Length > 0 { + encoder.Root_encoder.EncodeInto(value.Root, buf[pos:]) + pos += encoder.Root_encoder.Length + } + } + if value.Results != nil { + for seq_i, seq_v := range value.Results { + pseudoEncoder := &encoder.Results_subencoder[seq_i] + pseudoValue := struct { + Results *CheckResult + }{ + Results: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.Results != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7690)) + pos += 3 + pos += uint(enc.TLNum(encoder.Results_encoder.Length).EncodeInto(buf[pos:])) + if encoder.Results_encoder.Length > 0 { + encoder.Results_encoder.EncodeInto(value.Results, buf[pos:]) + pos += encoder.Results_encoder.Length + } + } + _ = encoder + _ = value + } + } + } +} + +func (encoder *CheckResponseEncoder) Encode(value *CheckResponse) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *CheckResponseParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*CheckResponse, error) { + + var handled_Root bool = false + var handled_Results bool = false + + progress := -1 + _ = progress + + value := &CheckResponse{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7688: + if true { + handled = true + handled_Root = true + value.Root, err = context.Root_context.Parse(reader.Delegate(int(l)), ignoreCritical) + } + case 7690: + if true { + handled = true + handled_Results = true + if value.Results == nil { + value.Results = make([]*CheckResult, 0) + } + { + pseudoValue := struct { + Results *CheckResult + }{} + { + value := &pseudoValue + value.Results, err = context.Results_context.Parse(reader.Delegate(int(l)), ignoreCritical) + _ = value + } + value.Results = append(value.Results, pseudoValue.Results) + } + progress-- + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_Root && err == nil { + value.Root = nil + } + if !handled_Results && err == nil { + // sequence - skip + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *CheckResponse) Encode() enc.Wire { + encoder := CheckResponseEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *CheckResponse) Bytes() []byte { + return value.Encode().Join() +} + +func ParseCheckResponse(reader enc.WireView, ignoreCritical bool) (*CheckResponse, error) { + context := CheckResponseParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type CheckResultEncoder struct { + Length uint + + Proof_encoder InclusionProofEncoder +} + +type CheckResultParsingContext struct { + Proof_context InclusionProofParsingContext +} + +func (encoder *CheckResultEncoder) Init(value *CheckResult) { + + if value.Proof != nil { + encoder.Proof_encoder.Init(value.Proof) + } + + l := uint(0) + if value.DataHash != nil { + l += 3 + l += uint(enc.TLNum(len(value.DataHash)).EncodingLength()) + l += uint(len(value.DataHash)) + } + l += 3 + l += uint(1 + enc.Nat(value.Status).EncodingLength()) + if value.Proof != nil { + l += 3 + l += uint(enc.TLNum(encoder.Proof_encoder.Length).EncodingLength()) + l += encoder.Proof_encoder.Length + } + encoder.Length = l + +} + +func (context *CheckResultParsingContext) Init() { + + context.Proof_context.Init() +} + +func (encoder *CheckResultEncoder) EncodeInto(value *CheckResult, buf []byte) { + + pos := uint(0) + + if value.DataHash != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7680)) + pos += 3 + pos += uint(enc.TLNum(len(value.DataHash)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.DataHash) + pos += uint(len(value.DataHash)) + } + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7684)) + pos += 3 + + buf[pos] = byte(enc.Nat(value.Status).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + if value.Proof != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7692)) + pos += 3 + pos += uint(enc.TLNum(encoder.Proof_encoder.Length).EncodeInto(buf[pos:])) + if encoder.Proof_encoder.Length > 0 { + encoder.Proof_encoder.EncodeInto(value.Proof, buf[pos:]) + pos += encoder.Proof_encoder.Length + } + } +} + +func (encoder *CheckResultEncoder) Encode(value *CheckResult) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *CheckResultParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*CheckResult, error) { + + var handled_DataHash bool = false + var handled_Status bool = false + var handled_Proof bool = false + + progress := -1 + _ = progress + + value := &CheckResult{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7680: + if true { + handled = true + handled_DataHash = true + value.DataHash = make([]byte, l) + _, err = reader.ReadFull(value.DataHash) + } + case 7684: + if true { + handled = true + handled_Status = true + value.Status = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.Status = uint64(value.Status<<8) | uint64(x) + } + } + } + case 7692: + if true { + handled = true + handled_Proof = true + value.Proof, err = context.Proof_context.Parse(reader.Delegate(int(l)), ignoreCritical) + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_DataHash && err == nil { + value.DataHash = nil + } + if !handled_Status && err == nil { + err = enc.ErrSkipRequired{Name: "Status", TypeNum: 7684} + } + if !handled_Proof && err == nil { + value.Proof = nil + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *CheckResult) Encode() enc.Wire { + encoder := CheckResultEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *CheckResult) Bytes() []byte { + return value.Encode().Join() +} + +func ParseCheckResult(reader enc.WireView, ignoreCritical bool) (*CheckResult, error) { + context := CheckResultParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type TreeRootEncoder struct { + Length uint +} + +type TreeRootParsingContext struct { +} + +func (encoder *TreeRootEncoder) Init(value *TreeRoot) { + + l := uint(0) + l += 3 + l += uint(1 + enc.Nat(value.TreeSize).EncodingLength()) + if value.RootHash != nil { + l += 3 + l += uint(enc.TLNum(len(value.RootHash)).EncodingLength()) + l += uint(len(value.RootHash)) + } + encoder.Length = l + +} + +func (context *TreeRootParsingContext) Init() { + +} + +func (encoder *TreeRootEncoder) EncodeInto(value *TreeRoot, buf []byte) { + + pos := uint(0) + + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7694)) + pos += 3 + + buf[pos] = byte(enc.Nat(value.TreeSize).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + if value.RootHash != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7696)) + pos += 3 + pos += uint(enc.TLNum(len(value.RootHash)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.RootHash) + pos += uint(len(value.RootHash)) + } +} + +func (encoder *TreeRootEncoder) Encode(value *TreeRoot) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *TreeRootParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*TreeRoot, error) { + + var handled_TreeSize bool = false + var handled_RootHash bool = false + + progress := -1 + _ = progress + + value := &TreeRoot{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7694: + if true { + handled = true + handled_TreeSize = true + value.TreeSize = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.TreeSize = uint64(value.TreeSize<<8) | uint64(x) + } + } + } + case 7696: + if true { + handled = true + handled_RootHash = true + value.RootHash = make([]byte, l) + _, err = reader.ReadFull(value.RootHash) + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_TreeSize && err == nil { + err = enc.ErrSkipRequired{Name: "TreeSize", TypeNum: 7694} + } + if !handled_RootHash && err == nil { + value.RootHash = nil + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *TreeRoot) Encode() enc.Wire { + encoder := TreeRootEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *TreeRoot) Bytes() []byte { + return value.Encode().Join() +} + +func ParseTreeRoot(reader enc.WireView, ignoreCritical bool) (*TreeRoot, error) { + context := TreeRootParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type InclusionProofEncoder struct { + Length uint + + Entry_length uint + + SiblingHashes_subencoder []struct { + } +} + +type InclusionProofParsingContext struct { +} + +func (encoder *InclusionProofEncoder) Init(value *InclusionProof) { + if value.Entry != nil { + encoder.Entry_length = 0 + for _, c := range value.Entry { + encoder.Entry_length += uint(len(c)) + } + } + + { + SiblingHashes_l := len(value.SiblingHashes) + encoder.SiblingHashes_subencoder = make([]struct { + }, SiblingHashes_l) + for i := 0; i < SiblingHashes_l; i++ { + pseudoEncoder := &encoder.SiblingHashes_subencoder[i] + pseudoValue := struct { + SiblingHashes []byte + }{ + SiblingHashes: value.SiblingHashes[i], + } + { + encoder := pseudoEncoder + value := &pseudoValue + + _ = encoder + _ = value + } + } + } + + l := uint(0) + if value.Entry != nil { + l += 3 + l += uint(enc.TLNum(encoder.Entry_length).EncodingLength()) + l += encoder.Entry_length + } + l += 3 + l += uint(1 + enc.Nat(value.LeafIndex).EncodingLength()) + if value.SiblingHashes != nil { + for seq_i, seq_v := range value.SiblingHashes { + pseudoEncoder := &encoder.SiblingHashes_subencoder[seq_i] + pseudoValue := struct { + SiblingHashes []byte + }{ + SiblingHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.SiblingHashes != nil { + l += 3 + l += uint(enc.TLNum(len(value.SiblingHashes)).EncodingLength()) + l += uint(len(value.SiblingHashes)) + } + _ = encoder + _ = value + } + } + } + encoder.Length = l + +} + +func (context *InclusionProofParsingContext) Init() { + +} + +func (encoder *InclusionProofEncoder) EncodeInto(value *InclusionProof, buf []byte) { + + pos := uint(0) + + if value.Entry != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7698)) + pos += 3 + pos += uint(enc.TLNum(encoder.Entry_length).EncodeInto(buf[pos:])) + for _, w := range value.Entry { + copy(buf[pos:], w) + pos += uint(len(w)) + } + } + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7686)) + pos += 3 + + buf[pos] = byte(enc.Nat(value.LeafIndex).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + if value.SiblingHashes != nil { + for seq_i, seq_v := range value.SiblingHashes { + pseudoEncoder := &encoder.SiblingHashes_subencoder[seq_i] + pseudoValue := struct { + SiblingHashes []byte + }{ + SiblingHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.SiblingHashes != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7700)) + pos += 3 + pos += uint(enc.TLNum(len(value.SiblingHashes)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.SiblingHashes) + pos += uint(len(value.SiblingHashes)) + } + _ = encoder + _ = value + } + } + } +} + +func (encoder *InclusionProofEncoder) Encode(value *InclusionProof) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *InclusionProofParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*InclusionProof, error) { + + var handled_Entry bool = false + var handled_LeafIndex bool = false + var handled_SiblingHashes bool = false + + progress := -1 + _ = progress + + value := &InclusionProof{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7698: + if true { + handled = true + handled_Entry = true + value.Entry, err = reader.ReadWire(int(l)) + } + case 7686: + if true { + handled = true + handled_LeafIndex = true + value.LeafIndex = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + value.LeafIndex = uint64(value.LeafIndex<<8) | uint64(x) + } + } + } + case 7700: + if true { + handled = true + handled_SiblingHashes = true + if value.SiblingHashes == nil { + value.SiblingHashes = make([][]byte, 0) + } + { + pseudoValue := struct { + SiblingHashes []byte + }{} + { + value := &pseudoValue + value.SiblingHashes = make([]byte, l) + _, err = reader.ReadFull(value.SiblingHashes) + _ = value + } + value.SiblingHashes = append(value.SiblingHashes, pseudoValue.SiblingHashes) + } + progress-- + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_Entry && err == nil { + value.Entry = nil + } + if !handled_LeafIndex && err == nil { + err = enc.ErrSkipRequired{Name: "LeafIndex", TypeNum: 7686} + } + if !handled_SiblingHashes && err == nil { + // sequence - skip + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *InclusionProof) Encode() enc.Wire { + encoder := InclusionProofEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *InclusionProof) Bytes() []byte { + return value.Encode().Join() +} + +func ParseInclusionProof(reader enc.WireView, ignoreCritical bool) (*InclusionProof, error) { + context := InclusionProofParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} + +type LogEntryEncoder struct { + Length uint + + DataHashes_subencoder []struct { + } +} + +type LogEntryParsingContext struct { +} + +func (encoder *LogEntryEncoder) Init(value *LogEntry) { + + { + DataHashes_l := len(value.DataHashes) + encoder.DataHashes_subencoder = make([]struct { + }, DataHashes_l) + for i := 0; i < DataHashes_l; i++ { + pseudoEncoder := &encoder.DataHashes_subencoder[i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: value.DataHashes[i], + } + { + encoder := pseudoEncoder + value := &pseudoValue + + _ = encoder + _ = value + } + } + } + + l := uint(0) + l += 3 + l += uint(1 + enc.Nat(uint64(value.IngestTime/time.Millisecond)).EncodingLength()) + if value.DataHashes != nil { + for seq_i, seq_v := range value.DataHashes { + pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.DataHashes != nil { + l += 3 + l += uint(enc.TLNum(len(value.DataHashes)).EncodingLength()) + l += uint(len(value.DataHashes)) + } + _ = encoder + _ = value + } + } + } + encoder.Length = l + +} + +func (context *LogEntryParsingContext) Init() { + +} + +func (encoder *LogEntryEncoder) EncodeInto(value *LogEntry, buf []byte) { + + pos := uint(0) + + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7702)) + pos += 3 + + buf[pos] = byte(enc.Nat(uint64(value.IngestTime / time.Millisecond)).EncodeInto(buf[pos+1:])) + pos += uint(1 + buf[pos]) + if value.DataHashes != nil { + for seq_i, seq_v := range value.DataHashes { + pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] + pseudoValue := struct { + DataHashes []byte + }{ + DataHashes: seq_v, + } + { + encoder := pseudoEncoder + value := &pseudoValue + if value.DataHashes != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7680)) + pos += 3 + pos += uint(enc.TLNum(len(value.DataHashes)).EncodeInto(buf[pos:])) + copy(buf[pos:], value.DataHashes) + pos += uint(len(value.DataHashes)) + } + _ = encoder + _ = value + } + } + } +} + +func (encoder *LogEntryEncoder) Encode(value *LogEntry) enc.Wire { + + wire := make(enc.Wire, 1) + wire[0] = make([]byte, encoder.Length) + buf := wire[0] + encoder.EncodeInto(value, buf) + + return wire +} + +func (context *LogEntryParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*LogEntry, error) { + + var handled_IngestTime bool = false + var handled_DataHashes bool = false + + progress := -1 + _ = progress + + value := &LogEntry{} + var err error + var startPos int + for { + startPos = reader.Pos() + if startPos >= reader.Length() { + break + } + typ := enc.TLNum(0) + l := enc.TLNum(0) + typ, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + l, err = reader.ReadTLNum() + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} + } + + err = nil + if handled := false; true { + switch typ { + case 7702: + if true { + handled = true + handled_IngestTime = true + { + timeInt := uint64(0) + timeInt = uint64(0) + { + for i := 0; i < int(l); i++ { + x := byte(0) + x, err = reader.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + break + } + timeInt = uint64(timeInt<<8) | uint64(x) + } + } + value.IngestTime = time.Duration(timeInt) * time.Millisecond + } + } + case 7680: + if true { + handled = true + handled_DataHashes = true + if value.DataHashes == nil { + value.DataHashes = make([][]byte, 0) + } + { + pseudoValue := struct { + DataHashes []byte + }{} + { + value := &pseudoValue + value.DataHashes = make([]byte, l) + _, err = reader.ReadFull(value.DataHashes) + _ = value + } + value.DataHashes = append(value.DataHashes, pseudoValue.DataHashes) + } + progress-- + } + default: + if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { + return nil, enc.ErrUnrecognizedField{TypeNum: typ} + } + handled = true + err = reader.Skip(int(l)) + } + if err == nil && !handled { + } + if err != nil { + return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} + } + } + } + + startPos = reader.Pos() + err = nil + + if !handled_IngestTime && err == nil { + err = enc.ErrSkipRequired{Name: "IngestTime", TypeNum: 7702} + } + if !handled_DataHashes && err == nil { + // sequence - skip + } + + if err != nil { + return nil, err + } + + return value, nil +} + +func (value *LogEntry) Encode() enc.Wire { + encoder := LogEntryEncoder{} + encoder.Init(value) + return encoder.Encode(value) +} + +func (value *LogEntry) Bytes() []byte { + return value.Encode().Join() +} + +func ParseLogEntry(reader enc.WireView, ignoreCritical bool) (*LogEntry, error) { + context := LogEntryParsingContext{} + context.Init() + return context.Parse(reader, ignoreCritical) +} From e360730d796825fe41aa91f92785d2260c076543 Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Sat, 12 Sep 2026 16:07:16 -0700 Subject: [PATCH 2/8] merkle: add daemon --- cmd/cmd.go | 18 +++++ merkle/cmd.go | 46 +++++++++++ merkle/cmd/main.go | 7 ++ merkle/config.go | 95 +++++++++++++++++++++++ merkle/merkle.go | 164 +++++++++++++++++++++++++++++++++++++++ merkle/merkle.sample.yml | 13 ++++ 6 files changed, 343 insertions(+) create mode 100644 merkle/cmd.go create mode 100644 merkle/cmd/main.go create mode 100644 merkle/config.go create mode 100644 merkle/merkle.go create mode 100644 merkle/merkle.sample.yml diff --git a/cmd/cmd.go b/cmd/cmd.go index 58ff0328..a747a595 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -3,6 +3,7 @@ package cmd import ( dv "github.com/named-data/ndnd/dv/cmd" fw "github.com/named-data/ndnd/fw/cmd" + "github.com/named-data/ndnd/merkle" "github.com/named-data/ndnd/repo" "github.com/named-data/ndnd/std/utils" "github.com/named-data/ndnd/tools" @@ -41,6 +42,7 @@ func init() { CmdNDNd.AddCommand(cmdDv()) CmdNDNd.AddCommand(cmdDaemon) CmdNDNd.AddCommand(cmdRepo()) + CmdNDNd.AddCommand(cmdMerkle()) CmdNDNd.AddGroup(&cobra.Group{ID: "sec", Title: "Security Tools"}) CmdNDNd.AddCommand(sec.CmdSec()) @@ -121,3 +123,19 @@ func cmdRepo() *cobra.Command { return cmdRepo } + +// cmdMerkle creates the top-level command for the Merkle history log daemon. +func cmdMerkle() *cobra.Command { + cmdMerkle := &cobra.Command{ + Use: "merkle", + Short: "Merkle History Log", + GroupID: "daemons", + } + + cmdMerkle.AddGroup(&cobra.Group{ID: "run", Title: "Merkle History Log Daemon"}) + merkle.CmdMerkle.Use = "run CONFIG-FILE" + merkle.CmdMerkle.Short = "Start the Merkle History Log Daemon" + cmdMerkle.AddCommand(merkle.CmdMerkle) + + return cmdMerkle +} diff --git a/merkle/cmd.go b/merkle/cmd.go new file mode 100644 index 00000000..d2fd1c1f --- /dev/null +++ b/merkle/cmd.go @@ -0,0 +1,46 @@ +package merkle + +import ( + "errors" + "os" + "os/signal" + "syscall" + + "github.com/named-data/ndnd/std/log" + "github.com/named-data/ndnd/std/utils" + "github.com/named-data/ndnd/std/utils/toolutils" + "github.com/spf13/cobra" +) + +// CmdMerkle starts the Merkle history log daemon. +var CmdMerkle = &cobra.Command{ + Use: "merkle CONFIG-FILE", + Short: "Merkle history log", + GroupID: "run", + Version: utils.NDNdVersion, + Args: cobra.ExactArgs(1), + Run: run, +} + +func run(cmd *cobra.Command, args []string) { + config := struct { + Merkle *Config `json:"merkle"` + }{ + Merkle: DefaultConfig(), + } + toolutils.ReadYaml(&config, args[0]) + if config.Merkle == nil { + log.Fatal(nil, "Configuration error", "err", errors.New("merkle configuration is missing")) + } + + service := NewLog(config.Merkle) + if err := service.Start(); err != nil { + log.Fatal(nil, "Failed to start Merkle history log", "err", err) + } + defer service.Stop() + + sigChannel := make(chan os.Signal, 1) + signal.Notify(sigChannel, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sigChannel) + <-sigChannel +} diff --git a/merkle/cmd/main.go b/merkle/cmd/main.go new file mode 100644 index 00000000..f47d9de0 --- /dev/null +++ b/merkle/cmd/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/named-data/ndnd/merkle" + +func main() { + merkle.CmdMerkle.Execute() +} diff --git a/merkle/config.go b/merkle/config.go new file mode 100644 index 00000000..95387c6c --- /dev/null +++ b/merkle/config.go @@ -0,0 +1,95 @@ +package merkle + +import ( + "fmt" + "os" + "path/filepath" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/security/trust_schema" +) + +// Config contains the configuration for a Merkle history log service. +type Config struct { + // Name is the service prefix. + Name string `json:"name"` + // StorageDir contains the packet cache and persistent Merkle log. + StorageDir string `json:"storage_dir"` + // KeyChainUri specifies the keychain location. + KeyChainUri string `json:"keychain"` + // TrustSchema is the path to a compiled LVS trust schema. + TrustSchema string `json:"trust_schema"` + // TrustAnchors lists the full names of validation trust anchors. + TrustAnchors []string `json:"trust_anchors"` + + nameN enc.Name + trustSchema *trust_schema.LvsSchema + trustAnchors []enc.Name +} + +// Parse validates the configuration and prepares its storage directory. +func (c *Config) Parse() error { + name, err := enc.NameFromStr(c.Name) + if err != nil { + return fmt.Errorf("failed to parse Merkle log name (%s): %w", c.Name, err) + } + if len(name) == 0 { + return fmt.Errorf("merkle log name must not be empty") + } + if c.StorageDir == "" { + return fmt.Errorf("storage-dir must be set") + } + if c.KeyChainUri == "" { + return fmt.Errorf("keychain must be set") + } + if c.TrustSchema == "" { + return fmt.Errorf("trust-schema must be set") + } + if len(c.TrustAnchors) == 0 { + return fmt.Errorf("no trust anchors provided") + } + + anchors := make([]enc.Name, len(c.TrustAnchors)) + for i, anchor := range c.TrustAnchors { + anchors[i], err = enc.NameFromStr(anchor) + if err != nil { + return fmt.Errorf("failed to parse trust anchor name (%s): %w", anchor, err) + } + if len(anchors[i]) == 0 { + return fmt.Errorf("trust anchor name must not be empty") + } + } + + trustSchemaPath, err := filepath.Abs(c.TrustSchema) + if err != nil { + return fmt.Errorf("failed to get absolute trust schema path: %w", err) + } + trustSchemaWire, err := os.ReadFile(trustSchemaPath) + if err != nil { + return fmt.Errorf("failed to read trust schema: %w", err) + } + schema, err := trust_schema.NewLvsSchema(trustSchemaWire) + if err != nil { + return fmt.Errorf("failed to parse trust schema: %w", err) + } + + storageDir, err := filepath.Abs(c.StorageDir) + if err != nil { + return fmt.Errorf("failed to get absolute storage path: %w", err) + } + if err := os.MkdirAll(storageDir, 0755); err != nil { + return fmt.Errorf("failed to create storage directory: %w", err) + } + + c.nameN = name + c.StorageDir = storageDir + c.TrustSchema = trustSchemaPath + c.trustSchema = schema + c.trustAnchors = anchors + return nil +} + +// DefaultConfig returns a configuration whose required fields are unset. +func DefaultConfig() *Config { + return &Config{} +} diff --git a/merkle/merkle.go b/merkle/merkle.go new file mode 100644 index 00000000..56917a61 --- /dev/null +++ b/merkle/merkle.go @@ -0,0 +1,164 @@ +// Package merkle implements the Merkle history log daemon. +package merkle + +import ( + "errors" + "fmt" + "path/filepath" + + "github.com/named-data/ndnd/std/engine" + "github.com/named-data/ndnd/std/log" + "github.com/named-data/ndnd/std/merklelog" + "github.com/named-data/ndnd/std/ndn" + "github.com/named-data/ndnd/std/object" + "github.com/named-data/ndnd/std/object/storage" + sec "github.com/named-data/ndnd/std/security" + "github.com/named-data/ndnd/std/security/keychain" +) + +const ( + packetStoreDir = "packets" + logStoreDir = "log" +) + +// Log is a Merkle history log service. +type Log struct { + config *Config + + engine ndn.Engine + client ndn.Client + + packetStore *storage.BadgerStore + logStore *merklelog.BadgerStore + tree *merklelog.Tree + + keychain ndn.KeyChain + trust *sec.TrustConfig + + clientStarted bool + started bool +} + +// NewLog creates a stopped Merkle history log service. +func NewLog(config *Config) *Log { + return &Log{config: config} +} + +// String returns the service's log identifier. +func (*Log) String() string { + return "merkle-log" +} + +// Start opens persistent state and starts the network-facing service. +func (m *Log) Start() (err error) { + if m.started || m.packetStore != nil || m.logStore != nil || m.engine != nil || m.client != nil { + return fmt.Errorf("merkle log is already started") + } + if m.config == nil { + return fmt.Errorf("merkle log configuration is nil") + } + if err := m.config.Parse(); err != nil { + return err + } + + log.Info(m, "Starting Merkle history log", "dir", m.config.StorageDir) + defer func() { + if err != nil { + err = errors.Join(err, m.stop()) + } + }() + + m.packetStore, err = storage.NewBadgerStore(filepath.Join(m.config.StorageDir, packetStoreDir)) + if err != nil { + return fmt.Errorf("open packet store: %w", err) + } + m.logStore, err = merklelog.NewBadgerStore(filepath.Join(m.config.StorageDir, logStoreDir)) + if err != nil { + return fmt.Errorf("open Merkle log store: %w", err) + } + m.tree, err = merklelog.OpenTree(m.logStore) + if err != nil { + return fmt.Errorf("restore Merkle tree: %w", err) + } + + m.keychain, err = keychain.NewKeyChain(m.config.KeyChainUri, m.packetStore) + if err != nil { + return fmt.Errorf("open keychain: %w", err) + } + m.trust, err = sec.NewTrustConfig( + m.keychain, + m.config.trustSchema, + m.config.trustAnchors, + ) + if err != nil { + return fmt.Errorf("create trust configuration: %w", err) + } + m.trust.UseDataNameFwHint = true + + m.engine = engine.NewBasicEngine(engine.NewDefaultFace()) + if err = m.engine.Start(); err != nil { + return fmt.Errorf("start NDN engine: %w", err) + } + + m.client = object.NewClient(m.engine, m.packetStore, m.trust) + if err = m.client.Start(); err != nil { + return fmt.Errorf("start Object client: %w", err) + } + m.clientStarted = true + m.client.AnnouncePrefix(ndn.Announcement{ + Name: m.config.nameN, + Expose: true, + }) + m.started = true + return nil +} + +// Stop stops the network service and closes its persistent stores. +func (m *Log) Stop() error { + if m.started { + log.Info(m, "Stopping Merkle history log") + } + return m.stop() +} + +func (m *Log) stop() error { + var errs []error + if m.client != nil { + if m.started { + m.client.WithdrawPrefix(m.config.nameN, nil) + } + if m.clientStarted { + if err := m.client.Stop(); err != nil { + errs = append(errs, fmt.Errorf("stop Object client: %w", err)) + } + } + m.client = nil + m.clientStarted = false + } + if m.engine != nil { + if m.engine.IsRunning() { + if err := m.engine.Stop(); err != nil { + errs = append(errs, fmt.Errorf("stop NDN engine: %w", err)) + } + } + m.engine = nil + } + if m.logStore != nil { + if err := m.logStore.Close(); err != nil { + errs = append(errs, fmt.Errorf("close Merkle log store: %w", err)) + } + m.logStore = nil + } + if m.packetStore != nil { + if err := m.packetStore.Close(); err != nil { + errs = append(errs, fmt.Errorf("close packet store: %w", err)) + } + m.packetStore = nil + } + + m.tree = nil + m.keychain = nil + m.trust = nil + m.started = false + return errors.Join(errs...) +} diff --git a/merkle/merkle.sample.yml b/merkle/merkle.sample.yml new file mode 100644 index 00000000..5e0b4513 --- /dev/null +++ b/merkle/merkle.sample.yml @@ -0,0 +1,13 @@ +merkle: + # [required] Name of the Merkle history log service + name: /ndnd/merkle + # [required] Directory for the packet cache and persistent log + storage_dir: /etc/ndn/merkle/storage + # [required] Keychain URI for security + # - Example: dir:///absolute/path/to/keychain + keychain: "dir:///etc/ndn/merkle/keys" + # [required] Compiled LVS trust schema for log requests and responses + trust_schema: /etc/ndn/merkle/schema.tlv + # [required] Full names of validation trust anchors + trust_anchors: + - "/ndn/KEY/%27%C4%B2%2A%9F%7B%81%27/ndn/v=1651246789556" From f0132ff92315b3d782e330efb896a6463befddbd Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Sat, 12 Sep 2026 17:39:50 -0700 Subject: [PATCH 3/8] merkle: basic append and check api --- merkle/api.go | 207 +++++++++++++++++++++++++++++++++++++ merkle/merkle.go | 40 +++++++- std/merklelog/client.go | 217 +++++++++++++++++++++++++++++++++++++++ std/object/client_cmd.go | 4 +- 4 files changed, 462 insertions(+), 6 deletions(-) create mode 100644 merkle/api.go create mode 100644 std/merklelog/client.go diff --git a/merkle/api.go b/merkle/api.go new file mode 100644 index 00000000..23d11916 --- /dev/null +++ b/merkle/api.go @@ -0,0 +1,207 @@ +package merkle + +import ( + "bytes" + "fmt" + "math" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/log" + "github.com/named-data/ndnd/std/merklelog" + defn "github.com/named-data/ndnd/std/ndn/merklelog" + "github.com/named-data/ndnd/std/types/optional" +) + +const requestFreshnessWindow = time.Minute + +func (m *Log) onAppend(name enc.Name, content enc.Wire, reply func(enc.Wire) error) { + if err := m.validateRequestName(name, merklelog.AppendPrefix(m.config.nameN)); err != nil { + log.Debug(m, "Rejected append request", "err", err) + return + } + request, err := defn.ParseAppendRequest(enc.NewWireView(content), false) + if err != nil { + log.Debug(m, "Failed to parse append request", "err", err) + return + } + go func() { + response, appendErr := m.append(request) + if appendErr != nil { + log.Error(m, "Failed to append Data hashes", "err", appendErr) + } + if response != nil { + if err := reply(response.Encode()); err != nil { + log.Warn(m, "Failed to reply to append request", "err", err) + } + } + }() +} + +func (m *Log) onCheck(name enc.Name, content enc.Wire, reply func(enc.Wire) error) { + if err := m.validateRequestName(name, merklelog.CheckPrefix(m.config.nameN)); err != nil { + log.Debug(m, "Rejected check request", "err", err) + return + } + request, err := defn.ParseCheckRequest(enc.NewWireView(content), false) + if err != nil { + log.Debug(m, "Failed to parse check request", "err", err) + return + } + go func() { + response, err := m.check(request) + if err != nil { + log.Debug(m, "Rejected check request", "err", err) + return + } + if err := reply(response.Encode()); err != nil { + log.Warn(m, "Failed to reply to check request", "err", err) + } + }() +} + +func (m *Log) append(request *defn.AppendRequest) (*defn.AppendResponse, error) { + if request == nil || len(request.DataHashes) == 0 { + return nil, fmt.Errorf("append request has no Data hashes") + } + + m.treeMutex.Lock() + defer m.treeMutex.Unlock() + if m.tree == nil { + return nil, fmt.Errorf("Merkle tree is not available") + } + + response := &defn.AppendResponse{Results: make([]*defn.AppendResult, len(request.DataHashes))} + pending := make(map[[merklelog.HashSize]byte][]int) + newHashes := make([][]byte, 0, len(request.DataHashes)) + for i, dataHash := range request.DataHashes { + result := &defn.AppendResult{ + DataHash: bytes.Clone(dataHash), + Status: defn.AppendStatusFailed, + } + response.Results[i] = result + if len(dataHash) != merklelog.HashSize { + continue + } + if leafIndex, ok := m.tree.Lookup(dataHash); ok { + result.Status = defn.AppendStatusDuplicate + result.LeafIndex = optional.Some(leafIndex) + continue + } + + var key [merklelog.HashSize]byte + copy(key[:], dataHash) + if indexes, ok := pending[key]; ok { + result.Status = defn.AppendStatusDuplicate + pending[key] = append(indexes, i) + continue + } + result.Status = defn.AppendStatusOK + pending[key] = []int{i} + newHashes = append(newHashes, bytes.Clone(dataHash)) + } + if len(newHashes) == 0 { + return response, nil + } + + entryWire, err := merklelog.EncodeLogEntry(&defn.LogEntry{ + IngestTime: m.nextIngestTime(), + DataHashes: newHashes, + }) + if err != nil { + markAppendFailed(response, pending) + return response, err + } + leafIndex, err := m.tree.Append(entryWire) + if err != nil { + markAppendFailed(response, pending) + return response, err + } + for _, indexes := range pending { + for _, i := range indexes { + response.Results[i].LeafIndex = optional.Some(leafIndex) + } + } + return response, nil +} + +func (m *Log) check(request *defn.CheckRequest) (*defn.CheckResponse, error) { + if request == nil || len(request.DataHashes) == 0 { + return nil, fmt.Errorf("check request has no Data hashes") + } + for i, dataHash := range request.DataHashes { + if len(dataHash) != merklelog.HashSize { + return nil, fmt.Errorf( + "Data hash %d length is %d, want %d", + i, + len(dataHash), + merklelog.HashSize, + ) + } + } + + m.treeMutex.Lock() + defer m.treeMutex.Unlock() + if m.tree == nil { + return nil, fmt.Errorf("Merkle tree is not available") + } + + response := &defn.CheckResponse{ + Root: m.tree.Root(), + Results: make([]*defn.CheckResult, len(request.DataHashes)), + } + for i, dataHash := range request.DataHashes { + result := &defn.CheckResult{ + DataHash: bytes.Clone(dataHash), + Status: defn.CheckStatusNotFound, + } + response.Results[i] = result + if leafIndex, ok := m.tree.Lookup(dataHash); ok { + proof, err := m.tree.InclusionProof(leafIndex) + if err != nil { + return nil, err + } + result.Status = defn.CheckStatusIncluded + result.Proof = proof + } + } + return response, nil +} + +func (m *Log) validateRequestName(name enc.Name, commandPrefix enc.Name) error { + if !commandPrefix.IsPrefix(name) || len(name) < len(commandPrefix)+2 { + return fmt.Errorf("request name does not match %s//t=", commandPrefix) + } + timestamp := name.At(-1) + if !timestamp.IsTimestamp() { + return fmt.Errorf("request name has no timestamp component") + } + timestampValue := timestamp.NumberVal() + if len(timestamp.Val) != enc.Nat(timestampValue).EncodingLength() || timestampValue > math.MaxInt64 { + return fmt.Errorf("request timestamp is not canonical") + } + requestTime := time.UnixMilli(int64(timestampValue)) + now := m.now() + if requestTime.Before(now.Add(-requestFreshnessWindow)) || + requestTime.After(now.Add(requestFreshnessWindow)) { + return fmt.Errorf("request timestamp is outside the permitted window") + } + return nil +} + +func (m *Log) nextIngestTime() time.Duration { + next := time.Duration(m.now().UnixMilli()) * time.Millisecond + if last, ok := m.tree.LastIngestTime(); ok && next <= last { + return last + time.Millisecond + } + return next +} + +func markAppendFailed(response *defn.AppendResponse, pending map[[merklelog.HashSize]byte][]int) { + for _, indexes := range pending { + for _, i := range indexes { + response.Results[i].Status = defn.AppendStatusFailed + response.Results[i].LeafIndex.Unset() + } + } +} diff --git a/merkle/merkle.go b/merkle/merkle.go index 56917a61..ab428a8d 100644 --- a/merkle/merkle.go +++ b/merkle/merkle.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "path/filepath" + "sync" + "time" "github.com/named-data/ndnd/std/engine" "github.com/named-data/ndnd/std/log" @@ -35,13 +37,21 @@ type Log struct { keychain ndn.KeyChain trust *sec.TrustConfig - clientStarted bool - started bool + treeMutex sync.Mutex + now func() time.Time + + clientStarted bool + appendHandlerAttached bool + checkHandlerAttached bool + started bool } // NewLog creates a stopped Merkle history log service. func NewLog(config *Config) *Log { - return &Log{config: config} + return &Log{ + config: config, + now: time.Now, + } } // String returns the service's log identifier. @@ -105,6 +115,14 @@ func (m *Log) Start() (err error) { return fmt.Errorf("start Object client: %w", err) } m.clientStarted = true + if err = m.client.AttachCommandHandler(merklelog.AppendPrefix(m.config.nameN), m.onAppend); err != nil { + return fmt.Errorf("attach append handler: %w", err) + } + m.appendHandlerAttached = true + if err = m.client.AttachCommandHandler(merklelog.CheckPrefix(m.config.nameN), m.onCheck); err != nil { + return fmt.Errorf("attach check handler: %w", err) + } + m.checkHandlerAttached = true m.client.AnnouncePrefix(ndn.Announcement{ Name: m.config.nameN, Expose: true, @@ -127,6 +145,18 @@ func (m *Log) stop() error { if m.started { m.client.WithdrawPrefix(m.config.nameN, nil) } + if m.checkHandlerAttached { + if err := m.client.DetachCommandHandler(merklelog.CheckPrefix(m.config.nameN)); err != nil { + errs = append(errs, fmt.Errorf("detach check handler: %w", err)) + } + m.checkHandlerAttached = false + } + if m.appendHandlerAttached { + if err := m.client.DetachCommandHandler(merklelog.AppendPrefix(m.config.nameN)); err != nil { + errs = append(errs, fmt.Errorf("detach append handler: %w", err)) + } + m.appendHandlerAttached = false + } if m.clientStarted { if err := m.client.Stop(); err != nil { errs = append(errs, fmt.Errorf("stop Object client: %w", err)) @@ -143,12 +173,15 @@ func (m *Log) stop() error { } m.engine = nil } + m.treeMutex.Lock() if m.logStore != nil { if err := m.logStore.Close(); err != nil { errs = append(errs, fmt.Errorf("close Merkle log store: %w", err)) } m.logStore = nil } + m.tree = nil + m.treeMutex.Unlock() if m.packetStore != nil { if err := m.packetStore.Close(); err != nil { errs = append(errs, fmt.Errorf("close packet store: %w", err)) @@ -156,7 +189,6 @@ func (m *Log) stop() error { m.packetStore = nil } - m.tree = nil m.keychain = nil m.trust = nil m.started = false diff --git a/std/merklelog/client.go b/std/merklelog/client.go new file mode 100644 index 00000000..802477be --- /dev/null +++ b/std/merklelog/client.go @@ -0,0 +1,217 @@ +package merklelog + +import ( + "bytes" + "crypto/sha256" + "fmt" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/ndn" + defn "github.com/named-data/ndnd/std/ndn/merklelog" +) + +var ( + appendKeyword = enc.NewKeywordComponent("append") + checkKeyword = enc.NewKeywordComponent("check") +) + +// AppendPrefix returns the append command prefix under logPrefix. +func AppendPrefix(logPrefix enc.Name) enc.Name { + return logPrefix.Clone().Append(appendKeyword) +} + +// CheckPrefix returns the check command prefix under logPrefix. +func CheckPrefix(logPrefix enc.Name) enc.Name { + return logPrefix.Clone().Append(checkKeyword) +} + +// Client sends signed requests to a Merkle history log. The supplied Object +// client must use a TrustConfig that signs requests and strictly validates log +// response signatures. +type Client struct { + client ndn.Client + appendName enc.Name + checkName enc.Name + requester enc.Name + now func() time.Time +} + +// NewClient creates a client for logPrefix. requester identifies the request +// signer in command Data names. +func NewClient(client ndn.Client, logPrefix enc.Name, requester enc.Name) (*Client, error) { + if client == nil { + return nil, fmt.Errorf("Object client is nil") + } + if len(logPrefix) == 0 { + return nil, fmt.Errorf("log prefix is empty") + } + if len(requester) == 0 { + return nil, fmt.Errorf("requester name is empty") + } + return &Client{ + client: client, + appendName: AppendPrefix(logPrefix), + checkName: CheckPrefix(logPrefix), + requester: requester.Clone(), + now: time.Now, + }, nil +} + +// Append submits Data packet hashes to the log. +func (c *Client) Append(dataHashes [][]byte, callback func(*defn.AppendResponse, error)) { + hashes, err := copyRequestHashes(dataHashes) + if err != nil { + callback(nil, err) + return + } + requestName, err := c.requestName(c.appendName) + if err != nil { + callback(nil, err) + return + } + c.client.ExpressCommand( + c.appendName, + requestName, + (&defn.AppendRequest{DataHashes: hashes}).Encode(), + func(wire enc.Wire, err error) { + if err != nil { + callback(nil, err) + return + } + response, err := defn.ParseAppendResponse(enc.NewWireView(wire), false) + if err != nil { + callback(nil, fmt.Errorf("parse append response: %w", err)) + return + } + if err := validateAppendResponse(hashes, response); err != nil { + callback(nil, err) + return + } + callback(response, nil) + }, + ) +} + +// Check retrieves and verifies inclusion proofs for Data packet hashes. +func (c *Client) Check(dataHashes [][]byte, callback func(*defn.CheckResponse, error)) { + hashes, err := copyRequestHashes(dataHashes) + if err != nil { + callback(nil, err) + return + } + requestName, err := c.requestName(c.checkName) + if err != nil { + callback(nil, err) + return + } + c.client.ExpressCommand( + c.checkName, + requestName, + (&defn.CheckRequest{DataHashes: hashes}).Encode(), + func(wire enc.Wire, err error) { + if err != nil { + callback(nil, err) + return + } + response, err := defn.ParseCheckResponse(enc.NewWireView(wire), false) + if err != nil { + callback(nil, fmt.Errorf("parse check response: %w", err)) + return + } + if err := validateCheckResponse(hashes, response); err != nil { + callback(nil, err) + return + } + callback(response, nil) + }, + ) +} + +func (c *Client) requestName(commandPrefix enc.Name) (enc.Name, error) { + timestamp := c.now().UnixMilli() + if timestamp < 0 { + return nil, fmt.Errorf("request time is before the Unix epoch") + } + return commandPrefix.Clone(). + Append(c.requester...). + Append(enc.NewTimestampComponent(uint64(timestamp))), nil +} + +func copyRequestHashes(dataHashes [][]byte) ([][]byte, error) { + if len(dataHashes) == 0 { + return nil, fmt.Errorf("request has no Data hashes") + } + ret := make([][]byte, len(dataHashes)) + for i, dataHash := range dataHashes { + if len(dataHash) != HashSize { + return nil, fmt.Errorf("Data hash %d length is %d, want %d", i, len(dataHash), HashSize) + } + ret[i] = bytes.Clone(dataHash) + } + return ret, nil +} + +func validateAppendResponse(dataHashes [][]byte, response *defn.AppendResponse) error { + if response == nil { + return fmt.Errorf("append response is nil") + } + if len(response.Results) != len(dataHashes) { + return fmt.Errorf("append response has %d results, want %d", len(response.Results), len(dataHashes)) + } + for i, result := range response.Results { + if result == nil || !bytes.Equal(result.DataHash, dataHashes[i]) { + return fmt.Errorf("append result %d does not match requested Data hash", i) + } + hasLeafIndex := result.LeafIndex.IsSet() + switch result.Status { + case defn.AppendStatusOK, defn.AppendStatusDuplicate: + if !hasLeafIndex { + return fmt.Errorf("append result %d has no leaf index", i) + } + case defn.AppendStatusFailed: + if hasLeafIndex { + return fmt.Errorf("failed append result %d has a leaf index", i) + } + default: + return fmt.Errorf("append result %d has unknown status %d", i, result.Status) + } + } + return nil +} + +func validateCheckResponse(dataHashes [][]byte, response *defn.CheckResponse) error { + if response == nil || response.Root == nil { + return fmt.Errorf("check response has no tree root") + } + if len(response.Root.RootHash) != HashSize { + return fmt.Errorf("root hash length is %d, want %d", len(response.Root.RootHash), HashSize) + } + if response.Root.TreeSize == 0 { + emptyRoot := sha256.Sum256(nil) + if !bytes.Equal(response.Root.RootHash, emptyRoot[:]) { + return fmt.Errorf("empty tree has an invalid root hash") + } + } + if len(response.Results) != len(dataHashes) { + return fmt.Errorf("check response has %d results, want %d", len(response.Results), len(dataHashes)) + } + for i, result := range response.Results { + if result == nil || !bytes.Equal(result.DataHash, dataHashes[i]) { + return fmt.Errorf("check result %d does not match requested Data hash", i) + } + switch result.Status { + case defn.CheckStatusIncluded: + if err := VerifyInclusion(dataHashes[i], result.Proof, response.Root); err != nil { + return fmt.Errorf("check result %d has invalid inclusion proof: %w", i, err) + } + case defn.CheckStatusNotFound: + if result.Proof != nil { + return fmt.Errorf("not-found check result %d has an inclusion proof", i) + } + default: + return fmt.Errorf("check result %d has unknown status %d", i, result.Status) + } + } + return nil +} diff --git a/std/object/client_cmd.go b/std/object/client_cmd.go index 5773fb88..68103365 100644 --- a/std/object/client_cmd.go +++ b/std/object/client_cmd.go @@ -93,9 +93,9 @@ func (c *Client) ExpressCommand(dest enc.Name, name enc.Name, cmd enc.Wire, call callback(nil, fmt.Errorf("command failed: %s", args.Result)) return } - c.Validate(args.Data, data.Wire, func(valid bool, err error) { + c.Validate(args.Data, args.SigCovered, func(valid bool, err error) { if !valid { - callback(nil, fmt.Errorf("command data validation failed: %w", err)) + callback(nil, fmt.Errorf("command response validation failed: %w", err)) return } callback(args.Data.Content(), nil) From 16e7b858b76f5a688ee08354251ba31c5d9ff088 Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Sun, 13 Sep 2026 01:02:24 -0700 Subject: [PATCH 4/8] sec: expose packet wire to expiry policies --- std/ndn/client.go | 3 + std/ndn/security.go | 3 + std/object/client_consume.go | 2 + std/object/client_consume_seg.go | 1 + std/object/client_trust.go | 1 + std/security/cert_cache.go | 41 ++++++--- std/security/trust_config.go | 139 ++++++++++++++++++------------- 7 files changed, 123 insertions(+), 67 deletions(-) diff --git a/std/ndn/client.go b/std/ndn/client.go index feb01242..c0a8b31c 100644 --- a/std/ndn/client.go +++ b/std/ndn/client.go @@ -158,6 +158,9 @@ type ExpressRArgs struct { type ValidateExtArgs struct { // Data packet to validate. Data Data + // RawData is the complete wire encoding of Data. It may be nil when the + // caller only has the parsed packet. + RawData enc.Wire // Signature covered wire. SigCovered enc.Wire // Callback for the result. diff --git a/std/ndn/security.go b/std/ndn/security.go index 7ce9ac8c..5128d1f7 100644 --- a/std/ndn/security.go +++ b/std/ndn/security.go @@ -55,6 +55,9 @@ type SigChecker func(name enc.Name, sigCovered enc.Wire, sig Signature) bool type CertExpiredCallbackArgs struct { // Data is the packet whose validation depends on Cert's validity. Data Data + // RawData is the complete wire encoding of Data. It may be nil when the + // validation caller only supplied the parsed packet. + RawData enc.Wire // Cert is the certificate or cross-schema packet authorizing Data. Cert Data } diff --git a/std/object/client_consume.go b/std/object/client_consume.go index 5d764ad4..acad3957 100644 --- a/std/object/client_consume.go +++ b/std/object/client_consume.go @@ -132,6 +132,7 @@ func (c *Client) fetchMetadata( } c.ValidateExt(ndn.ValidateExtArgs{ Data: args.Data, + RawData: args.RawData, SigCovered: args.SigCovered, OnCertExpired: onCertExpired, Callback: func(valid bool, err error) { @@ -187,6 +188,7 @@ func (c *Client) fetchDataByPrefix( } c.ValidateExt(ndn.ValidateExtArgs{ Data: args.Data, + RawData: args.RawData, SigCovered: args.SigCovered, OnCertExpired: onCertExpired, Callback: func(valid bool, err error) { diff --git a/std/object/client_consume_seg.go b/std/object/client_consume_seg.go index 9191aa95..463c0577 100644 --- a/std/object/client_consume_seg.go +++ b/std/object/client_consume_seg.go @@ -287,6 +287,7 @@ func (s *rrSegFetcher) handleResult(args ndn.ExpressCallbackArgs, state *Consume func (s *rrSegFetcher) handleData(args ndn.ExpressCallbackArgs, state *ConsumeState) { s.client.ValidateExt(ndn.ValidateExtArgs{ Data: args.Data, + RawData: args.RawData, SigCovered: args.SigCovered, OnCertExpired: state.args.OnCertExpired, Callback: func(valid bool, err error) { diff --git a/std/object/client_trust.go b/std/object/client_trust.go index 5f8a333e..c440c7d8 100644 --- a/std/object/client_trust.go +++ b/std/object/client_trust.go @@ -42,6 +42,7 @@ func (c *Client) ValidateExt(args ndn.ValidateExtArgs) { c.trust.Validate(sec.TrustConfigValidateArgs{ Data: args.Data, + RawData: args.RawData, DataSigCov: args.SigCovered, Callback: args.Callback, OverrideName: overrideName, diff --git a/std/security/cert_cache.go b/std/security/cert_cache.go index 8fb2e9c7..c64149ac 100644 --- a/std/security/cert_cache.go +++ b/std/security/cert_cache.go @@ -8,7 +8,7 @@ import ( "github.com/named-data/ndnd/std/ndn" ) -// CertCache is a memcache for certificates. +// CertCache is a memcache for certificates and their validation evidence. // It stores certificates by their name and key locator. // Only the most recent certificate is stored. // The cache is thread-safe. @@ -19,14 +19,21 @@ type CertCache struct { type certCacheEntry struct { data ndn.Data sigCovered enc.Wire + rawData enc.Wire expiry time.Time } -// CertListCache stores validated CertList Data packets keyed by prefix and full name. +// CertListCache stores validated CertList Data packets and their raw wire, +// keyed by prefix and full name. type CertListCache struct { cache sync.Map } +type certListCacheEntry struct { + data ndn.Data + rawData enc.Wire +} + // NewCertListCache creates a new CertListCache. func NewCertListCache() *CertListCache { return &CertListCache{} @@ -34,28 +41,39 @@ func NewCertListCache() *CertListCache { // Get returns a cached CertList for the given prefix or full name. func (clc *CertListCache) Get(prefix enc.Name) (ndn.Data, bool) { + entry, ok := clc.get(prefix) + return entry.data, ok +} + +func (clc *CertListCache) get(prefix enc.Name) (certListCacheEntry, bool) { if v, ok := clc.cache.Load(prefix.TlvStr()); ok { - if data, ok := v.(ndn.Data); ok { - return data, true + if entry, ok := v.(certListCacheEntry); ok { + return entry, true } } - return nil, false + return certListCacheEntry{}, false } // Put stores a CertList, preferring newer versions. func (clc *CertListCache) Put(anchorKeyName enc.Name, data ndn.Data) { + clc.put(anchorKeyName, data, nil) +} + +// put stores a CertList with its complete wire, when available. +func (clc *CertListCache) put(anchorKeyName enc.Name, data ndn.Data, rawData enc.Wire) { prefix, err := CertListPrefix(anchorKeyName) if err != nil { return } key := prefix.TlvStr() if v, ok := clc.cache.Load(key); ok { - if old, ok := v.(ndn.Data); ok && !isCertListNewer(old, data) { + if old, ok := v.(certListCacheEntry); ok && !isCertListNewer(old.data, data) { return } } - clc.cache.Store(key, data) - clc.cache.Store(data.Name().TlvStr(), data) + entry := certListCacheEntry{data: data, rawData: rawData} + clc.cache.Store(key, entry) + clc.cache.Store(data.Name().TlvStr(), entry) } func isCertListNewer(old, new ndn.Data) bool { @@ -91,11 +109,11 @@ func (cc *CertCache) get(name enc.Name) (certCacheEntry, bool) { // Put stores certificate data without signature verification evidence. func (cc *CertCache) Put(cert ndn.Data) { - cc.put(cert, nil) + cc.put(cert, nil, nil) } -// put stores a certificate with the wire covered by its signature. -func (cc *CertCache) put(cert ndn.Data, sigCovered enc.Wire) { +// put stores a certificate with the wire needed to identify and revalidate it. +func (cc *CertCache) put(cert ndn.Data, sigCovered enc.Wire, rawData enc.Wire) { _, expiry := cert.Signature().Validity() if !expiry.IsSet() { return // huh? @@ -104,6 +122,7 @@ func (cc *CertCache) put(cert ndn.Data, sigCovered enc.Wire) { entry := certCacheEntry{ data: cert, sigCovered: sigCovered, + rawData: rawData, expiry: expiry.Unwrap(), } diff --git a/std/security/trust_config.go b/std/security/trust_config.go index 7504b31b..ded0002a 100644 --- a/std/security/trust_config.go +++ b/std/security/trust_config.go @@ -12,7 +12,6 @@ import ( "github.com/named-data/ndnd/std/security/signer" "github.com/named-data/ndnd/std/security/trust_schema" "github.com/named-data/ndnd/std/types/optional" - "github.com/named-data/ndnd/std/utils" ) // TrustConfig is the configuration of the trust module. @@ -26,7 +25,8 @@ type TrustConfig struct { // roots are the full names of the trust anchors. roots []enc.Name - // certCache stores certificate data and its signature-covered wire. + // certCache stores certificate data and the wire needed to identify and + // revalidate it. // Cache hits are revalidated unless the certificate is a trust anchor. certCache *CertCache @@ -68,7 +68,7 @@ func NewTrustConfig(keyChain ndn.KeyChain, schema ndn.TrustSchema, roots []enc.N if err != nil { return nil, fmt.Errorf("failed to parse trust anchor %s: %w", root, err) } - certCache.put(certData, certSigCov) + certCache.put(certData, certSigCov, enc.Wire{certBytes}) } } @@ -109,6 +109,9 @@ func (tc *TrustConfig) SetSchema(schema ndn.TrustSchema) { type TrustConfigValidateArgs struct { // Data is the packet to validate. Data ndn.Data + // RawData is the complete wire encoding of Data. It may be nil when the + // caller only has the parsed packet. + RawData enc.Wire // DataSigCov is the signature covered data wire. DataSigCov enc.Wire @@ -136,8 +139,8 @@ type TrustConfigValidateArgs struct { certExpiryHandled bool // certSigCov is the signature covered certificate wire. certSigCov enc.Wire - // certRaw is the raw certificate bytes (if fetched). - certRaw enc.Wire + // certWire is the complete certificate wire used as packet identity. + certWire enc.Wire // certIsValid indicates if the certificate has been already validated. certIsValid bool @@ -233,8 +236,9 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { if !args.certExpiryHandled && (args.crossSchemaExpired || certDataExpired || CertIsExpired(args.cert)) { runCertExpiryPolicy(args.OnCertExpired, ndn.CertExpiredCallbackArgs{ - Data: args.Data, - Cert: args.cert, + Data: args.Data, + RawData: args.RawData, + Cert: args.cert, }, func(err error) { if err != nil { args.Callback(false, err) @@ -255,6 +259,7 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { } else if args.Data.CrossSchema() != nil { tc.validateCrossSchema(TrustConfigValidateArgs{ Data: args.Data, + RawData: args.RawData, DataSigCov: args.DataSigCov, Fetch: args.Fetch, @@ -302,23 +307,13 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { return } - // Monkey patch the callback to store the cert in - // keychain and cache if the validation passes. + // Monkey patch the callback to store the cert after validation passes. origCallback := args.Callback args.Callback = func(valid bool, err error) { if valid && err == nil { // Cache the certificate and the wire needed to revalidate its chain. - tc.certCache.put(args.cert, args.certSigCov) - - // Keychain is not thread safe for inserts - if len(args.certRaw) > 0 { - tc.mutex.Lock() - err := tc.keychain.InsertCert(args.certRaw.Join()) - tc.mutex.Unlock() - if err != nil { // broken keychain - log.Error(tc, "Failed to insert certificate to keychain", "name", args.cert.Name(), "err", err) - } - } + tc.certCache.put(args.cert, args.certSigCov, args.certWire) + tc.storeCertIfMissing(args.cert, args.certWire) } else { log.Warn(tc, "Received invalid certificate", "name", args.cert.Name(), "err", err) } @@ -329,6 +324,7 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { // Recursively validate the certificate tc.Validate(TrustConfigValidateArgs{ Data: args.cert, + RawData: args.certWire, DataSigCov: args.certSigCov, Fetch: args.Fetch, @@ -339,7 +335,7 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { cert: nil, certSigCov: nil, - certRaw: nil, + certWire: nil, certIsValid: false, crossSchemaIsValid: false, @@ -359,7 +355,7 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { // The code below might seem to have a lot of redundancy - this is intentional. args.cert = nil args.certSigCov = nil - args.certRaw = nil + args.certWire = nil args.certIsValid = false args.certExpiryHandled = false args.crossSchemaIsValid = false @@ -370,6 +366,7 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { (tc.isTrustAnchor(cached.data.Name()) || len(cached.sigCovered) > 0) { args.cert = cached.data args.certSigCov = cached.sigCovered + args.certWire = cached.rawData args.certIsValid = tc.isTrustAnchor(cached.data.Name()) // Continue validation with cached cert @@ -421,7 +418,7 @@ func (tc *TrustConfig) Validate(args TrustConfigValidateArgs) { // Call again with the fetched cert args.cert = res.Data args.certSigCov = res.SigCovered - args.certRaw = utils.If(!res.IsLocal, res.RawData, nil) // prevent double insert + args.certWire = res.RawData args.certIsValid = tc.isTrustAnchor(res.Data.Name()) // Continue validation with fetched cert @@ -447,23 +444,25 @@ func (tc *TrustConfig) validateCrossSchema(args TrustConfigValidateArgs) { // Check validity period of the cross schema. if CertIsExpired(crossData) { runCertExpiryPolicy(args.OnCertExpired, ndn.CertExpiredCallbackArgs{ - Data: args.Data, - Cert: crossData, + Data: args.Data, + RawData: args.RawData, + Cert: crossData, }, func(err error) { if err != nil { args.Callback(false, err) return } - tc.validateCrossSchemaData(args, crossData, crossDataSigCov, true) + tc.validateCrossSchemaData(args, crossData, crossWire, crossDataSigCov, true) }) return } - tc.validateCrossSchemaData(args, crossData, crossDataSigCov, false) + tc.validateCrossSchemaData(args, crossData, crossWire, crossDataSigCov, false) } func (tc *TrustConfig) validateCrossSchemaData( args TrustConfigValidateArgs, crossData ndn.Data, + crossDataRaw enc.Wire, crossDataSigCov enc.Wire, crossSchemaExpired bool, ) { @@ -488,6 +487,7 @@ func (tc *TrustConfig) validateCrossSchemaData( // Validate the cross schema signer to sign the original data tc.Validate(TrustConfigValidateArgs{ Data: crossData, + RawData: crossDataRaw, DataSigCov: crossDataSigCov, Fetch: args.Fetch, @@ -507,8 +507,9 @@ func (tc *TrustConfig) handleSelfSignedCert(args TrustConfigValidateArgs, keyLoc } if !args.certExpiryHandled && (args.crossSchemaExpired || certDataExpired) { runCertExpiryPolicy(args.OnCertExpired, ndn.CertExpiredCallbackArgs{ - Data: args.Data, - Cert: args.Data, + Data: args.Data, + RawData: args.RawData, + Cert: args.Data, }, func(err error) { if err != nil { args.Callback(false, err) @@ -552,7 +553,7 @@ func (tc *TrustConfig) handleSelfSignedCert(args TrustConfigValidateArgs, keyLoc tc.exploreCertList(certListArgs{ args: args, anchorCert: args.Data, - anchorRaw: args.certRaw, + anchorRaw: args.RawData, anchorKey: anchorKeyName, visitedLists: map[string]struct{}{}, visitedCerts: map[string]struct{}{}, @@ -564,15 +565,9 @@ func (tc *TrustConfig) PromoteAnchor(cert ndn.Data, raw enc.Wire) { if cert == nil { return } - tc.certCache.Put(cert) + tc.certCache.put(cert, nil, raw) name := cert.Name() - - // Persist the trust anchor if not already present and raw is available. - if len(raw) > 0 { - tc.mutex.Lock() - _ = tc.keychain.InsertCert(raw.Join()) - tc.mutex.Unlock() - } + tc.storeCertIfMissing(cert, raw) tc.mutex.Lock() defer tc.mutex.Unlock() @@ -584,6 +579,23 @@ func (tc *TrustConfig) PromoteAnchor(cert ndn.Data, raw enc.Wire) { tc.roots = append(tc.roots, name) } +// storeCertIfMissing persists a certificate without inserting it twice. +func (tc *TrustConfig) storeCertIfMissing(cert ndn.Data, wire enc.Wire) { + if cert == nil || len(wire) == 0 { + return + } + + tc.mutex.Lock() + stored, err := tc.keychain.Store().Get(cert.Name(), false) + if err == nil && len(stored) == 0 { + err = tc.keychain.InsertCert(wire.Join()) + } + tc.mutex.Unlock() + if err != nil { + log.Error(tc, "Failed to store certificate", "name", cert.Name(), "err", err) + } +} + func (tc *TrustConfig) isTrustedAnchorKey(keyLocator enc.Name) bool { tc.mutex.RLock() defer tc.mutex.RUnlock() @@ -616,6 +628,7 @@ type certListArgs struct { anchorRaw enc.Wire anchorKey enc.Name listData ndn.Data + listRaw enc.Wire visitedLists map[string]struct{} visitedCerts map[string]struct{} } @@ -628,8 +641,8 @@ func (tc *TrustConfig) exploreCertList(args certListArgs, prefix enc.Name) { } args.visitedLists[key] = struct{}{} - if cached, ok := tc.certListCache.Get(prefix); ok { - tc.processCertList(args, cached, nil, nil) + if cached, ok := tc.certListCache.get(prefix); ok { + tc.processCertList(args, cached.data, nil, cached.rawData) return } @@ -652,12 +665,21 @@ func (tc *TrustConfig) exploreCertList(args certListArgs, prefix enc.Name) { return } - raw := utils.If(!res.IsLocal, res.RawData, nil) - tc.processCertList(args, res.Data, res.SigCovered, raw) + tc.processCertList( + args, + res.Data, + res.SigCovered, + res.RawData, + ) }) } -func (tc *TrustConfig) processCertList(args certListArgs, listData ndn.Data, listSigCov enc.Wire, raw enc.Wire) { +func (tc *TrustConfig) processCertList( + args certListArgs, + listData ndn.Data, + listSigCov enc.Wire, + listRaw enc.Wire, +) { if listData == nil { args.args.Callback(false, fmt.Errorf("certlist missing")) return @@ -672,7 +694,7 @@ func (tc *TrustConfig) processCertList(args certListArgs, listData ndn.Data, lis args.args.Callback(false, fmt.Errorf("certlist invalid")) return } - tc.certListCache.Put(args.anchorKey, listData) + tc.certListCache.put(args.anchorKey, listData, listRaw) } names, err := DecodeCertList(listData.Content()) @@ -680,12 +702,19 @@ func (tc *TrustConfig) processCertList(args certListArgs, listData ndn.Data, lis args.args.Callback(false, fmt.Errorf("certlist invalid: %w", err)) return } - if len(raw) > 0 { - if err := tc.keychain.Store().Put(listData.Name(), raw.Join()); err != nil { + if len(listRaw) > 0 { + tc.mutex.Lock() + stored, err := tc.keychain.Store().Get(listData.Name(), false) + if err == nil && len(stored) == 0 { + err = tc.keychain.Store().Put(listData.Name(), listRaw.Join()) + } + tc.mutex.Unlock() + if err != nil { log.Warn(tc, "Failed to store CertList", "name", listData.Name(), "err", err) } } args.listData = listData + args.listRaw = listRaw tc.tryListedCerts(args, names, 0) } @@ -709,7 +738,7 @@ func (tc *TrustConfig) tryListedCerts(args certListArgs, names []enc.Name, idx i if cached, ok := tc.certCache.get(name); ok && (tc.isTrustAnchor(cached.data.Name()) || len(cached.sigCovered) > 0) { - tc.validateListedCert(args, names, idx, cached.data, cached.sigCovered, nil) + tc.validateListedCert(args, names, idx, cached.data, cached.sigCovered, cached.rawData) return } @@ -742,7 +771,7 @@ func (tc *TrustConfig) validateListedCert( idx int, cert ndn.Data, certSigCov enc.Wire, - certRaw enc.Wire, + certWire enc.Wire, ) { next := func() { tc.tryListedCerts(args, names, idx+1) @@ -766,18 +795,15 @@ func (tc *TrustConfig) validateListedCert( tc.validateCertListSigner(args, cert, func() { tc.Validate(TrustConfigValidateArgs{ Data: cert, + RawData: certWire, DataSigCov: certSigCov, Fetch: args.args.Fetch, UseDataNameFwHint: args.args.UseDataNameFwHint, Callback: func(valid bool, err error) { if valid && err == nil { - tc.certCache.put(cert, certSigCov) - if len(certRaw) > 0 { - tc.mutex.Lock() - _ = tc.keychain.InsertCert(certRaw.Join()) - tc.mutex.Unlock() - } + tc.certCache.put(cert, certSigCov, certWire) + tc.storeCertIfMissing(cert, certWire) tc.PromoteAnchor(args.anchorCert, args.anchorRaw) args.args.Callback(true, nil) return @@ -801,8 +827,9 @@ func (tc *TrustConfig) validateCertListSigner(args certListArgs, cert ndn.Data, } runCertExpiryPolicy(args.args.OnCertExpired, ndn.CertExpiredCallbackArgs{ - Data: args.listData, - Cert: cert, + Data: args.listData, + RawData: args.listRaw, + Cert: cert, }, func(err error) { if err != nil { onReject() From c48030df3787ac3a19db9b47bf7f8d42e1a2aa25 Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Sun, 13 Sep 2026 11:02:10 -0700 Subject: [PATCH 5/8] merkle: e2e logic flow --- merkle/api.go | 107 ++++++---- merkle/merkle.go | 4 +- std/merklelog/client.go | 81 ++++---- std/merklelog/policy.go | 72 +++++++ std/ndn/merklelog/definitions.go | 9 +- std/ndn/merklelog/zz_generated.go | 313 +++--------------------------- std/security/trust_config_test.go | 26 ++- 7 files changed, 234 insertions(+), 378 deletions(-) create mode 100644 std/merklelog/policy.go diff --git a/merkle/api.go b/merkle/api.go index 23d11916..fda82bf8 100644 --- a/merkle/api.go +++ b/merkle/api.go @@ -9,6 +9,7 @@ import ( enc "github.com/named-data/ndnd/std/encoding" "github.com/named-data/ndnd/std/log" "github.com/named-data/ndnd/std/merklelog" + "github.com/named-data/ndnd/std/ndn" defn "github.com/named-data/ndnd/std/ndn/merklelog" "github.com/named-data/ndnd/std/types/optional" ) @@ -38,24 +39,66 @@ func (m *Log) onAppend(name enc.Name, content enc.Wire, reply func(enc.Wire) err }() } -func (m *Log) onCheck(name enc.Name, content enc.Wire, reply func(enc.Wire) error) { - if err := m.validateRequestName(name, merklelog.CheckPrefix(m.config.nameN)); err != nil { - log.Debug(m, "Rejected check request", "err", err) +func (m *Log) onCheck(args ndn.InterestHandlerArgs) { + name := args.Interest.Name().Clone() + prefix := merklelog.CheckPrefix(m.config.nameN) + if !prefix.IsPrefix(name) { return } - request, err := defn.ParseCheckRequest(enc.NewWireView(content), false) - if err != nil { - log.Debug(m, "Failed to parse check request", "err", err) + hashIndex := len(prefix) + if len(name) <= hashIndex || + name[hashIndex].Typ != enc.TypeGenericNameComponent || + len(name[hashIndex].Val) != merklelog.HashSize { + log.Debug(m, "Rejected check Interest", "name", name) + return + } + + // Serve segments of proof objects that were produced by an earlier lookup. + if len(name) == len(prefix)+3 && name.At(-2).IsVersion() && name.At(-1).IsSegment() { + wire, err := m.packetStore.Get(name, false) + if err != nil { + log.Warn(m, "Failed to read check response", "name", name, "err", err) + } else if wire != nil { + _ = args.Reply(enc.Wire{wire}) + } + return + } + if len(name) != len(prefix)+1 || !args.Interest.CanBePrefix() { + log.Debug(m, "Rejected check Interest", "name", name) return } + dataHash := bytes.Clone(name[hashIndex].Val) + go func() { - response, err := m.check(request) + response, err := m.check(dataHash) + if err != nil { + log.Debug(m, "Rejected check Interest", "err", err) + return + } + + objectName := name.Append(enc.NewVersionComponent(response.Root.TreeSize)) + segmentName := objectName.Append(enc.NewSegmentComponent(0)) + wire, err := m.packetStore.Get(segmentName, false) + if err == nil && wire == nil { + _, err = m.client.Produce(ndn.ProduceArgs{ + Name: objectName, + Content: response.Encode(), + NoMetadata: true, + }) + if err == nil { + wire, err = m.packetStore.Get(segmentName, false) + } + } if err != nil { - log.Debug(m, "Rejected check request", "err", err) + log.Warn(m, "Failed to produce check response", "name", name, "err", err) return } - if err := reply(response.Encode()); err != nil { - log.Warn(m, "Failed to reply to check request", "err", err) + if wire == nil { + log.Warn(m, "Check response has no first segment", "name", name) + return + } + if err := args.Reply(enc.Wire{wire}); err != nil { + log.Warn(m, "Failed to reply to check Interest", "err", err) } }() } @@ -125,19 +168,13 @@ func (m *Log) append(request *defn.AppendRequest) (*defn.AppendResponse, error) return response, nil } -func (m *Log) check(request *defn.CheckRequest) (*defn.CheckResponse, error) { - if request == nil || len(request.DataHashes) == 0 { - return nil, fmt.Errorf("check request has no Data hashes") - } - for i, dataHash := range request.DataHashes { - if len(dataHash) != merklelog.HashSize { - return nil, fmt.Errorf( - "Data hash %d length is %d, want %d", - i, - len(dataHash), - merklelog.HashSize, - ) - } +func (m *Log) check(dataHash []byte) (*defn.CheckResponse, error) { + if len(dataHash) != merklelog.HashSize { + return nil, fmt.Errorf( + "Data hash length is %d, want %d", + len(dataHash), + merklelog.HashSize, + ) } m.treeMutex.Lock() @@ -147,23 +184,19 @@ func (m *Log) check(request *defn.CheckRequest) (*defn.CheckResponse, error) { } response := &defn.CheckResponse{ - Root: m.tree.Root(), - Results: make([]*defn.CheckResult, len(request.DataHashes)), - } - for i, dataHash := range request.DataHashes { - result := &defn.CheckResult{ + Root: m.tree.Root(), + Result: &defn.CheckResult{ DataHash: bytes.Clone(dataHash), Status: defn.CheckStatusNotFound, + }, + } + if leafIndex, ok := m.tree.Lookup(dataHash); ok { + proof, err := m.tree.InclusionProof(leafIndex) + if err != nil { + return nil, err } - response.Results[i] = result - if leafIndex, ok := m.tree.Lookup(dataHash); ok { - proof, err := m.tree.InclusionProof(leafIndex) - if err != nil { - return nil, err - } - result.Status = defn.CheckStatusIncluded - result.Proof = proof - } + response.Result.Status = defn.CheckStatusIncluded + response.Result.Proof = proof } return response, nil } diff --git a/merkle/merkle.go b/merkle/merkle.go index ab428a8d..1c7bc91e 100644 --- a/merkle/merkle.go +++ b/merkle/merkle.go @@ -119,7 +119,7 @@ func (m *Log) Start() (err error) { return fmt.Errorf("attach append handler: %w", err) } m.appendHandlerAttached = true - if err = m.client.AttachCommandHandler(merklelog.CheckPrefix(m.config.nameN), m.onCheck); err != nil { + if err = m.engine.AttachHandler(merklelog.CheckPrefix(m.config.nameN), m.onCheck); err != nil { return fmt.Errorf("attach check handler: %w", err) } m.checkHandlerAttached = true @@ -146,7 +146,7 @@ func (m *Log) stop() error { m.client.WithdrawPrefix(m.config.nameN, nil) } if m.checkHandlerAttached { - if err := m.client.DetachCommandHandler(merklelog.CheckPrefix(m.config.nameN)); err != nil { + if err := m.engine.DetachHandler(merklelog.CheckPrefix(m.config.nameN)); err != nil { errs = append(errs, fmt.Errorf("detach check handler: %w", err)) } m.checkHandlerAttached = false diff --git a/std/merklelog/client.go b/std/merklelog/client.go index 802477be..eaa75c51 100644 --- a/std/merklelog/client.go +++ b/std/merklelog/client.go @@ -21,14 +21,14 @@ func AppendPrefix(logPrefix enc.Name) enc.Name { return logPrefix.Clone().Append(appendKeyword) } -// CheckPrefix returns the check command prefix under logPrefix. +// CheckPrefix returns the check object prefix under logPrefix. func CheckPrefix(logPrefix enc.Name) enc.Name { return logPrefix.Clone().Append(checkKeyword) } -// Client sends signed requests to a Merkle history log. The supplied Object -// client must use a TrustConfig that signs requests and strictly validates log -// response signatures. +// Client appends to and queries a Merkle history log. The supplied Object +// client must use a TrustConfig that signs append requests and strictly +// validates log response signatures. type Client struct { client ndn.Client appendName enc.Name @@ -93,39 +93,42 @@ func (c *Client) Append(dataHashes [][]byte, callback func(*defn.AppendResponse, ) } -// Check retrieves and verifies inclusion proofs for Data packet hashes. -func (c *Client) Check(dataHashes [][]byte, callback func(*defn.CheckResponse, error)) { - hashes, err := copyRequestHashes(dataHashes) - if err != nil { - callback(nil, err) +// Check retrieves and verifies the inclusion proof for one Data packet hash. +func (c *Client) Check(dataHash []byte, callback func(*defn.CheckResponse, error)) { + if len(dataHash) != HashSize { + callback(nil, fmt.Errorf("Data hash length is %d, want %d", len(dataHash), HashSize)) return } - requestName, err := c.requestName(c.checkName) - if err != nil { - callback(nil, err) - return - } - c.client.ExpressCommand( - c.checkName, - requestName, - (&defn.CheckRequest{DataHashes: hashes}).Encode(), - func(wire enc.Wire, err error) { - if err != nil { - callback(nil, err) + hash := bytes.Clone(dataHash) + name := c.checkName.Clone().Append(enc.NewGenericBytesComponent(hash)) + c.client.ConsumeExt(ndn.ConsumeExtArgs{ + Name: name, + NoMetadata: true, + Callback: func(state ndn.ConsumeState) { + if err := state.Error(); err != nil { + callback(nil, fmt.Errorf("consume check response: %w", err)) return } - response, err := defn.ParseCheckResponse(enc.NewWireView(wire), false) + response, err := defn.ParseCheckResponse(enc.NewWireView(state.Content()), false) if err != nil { callback(nil, fmt.Errorf("parse check response: %w", err)) return } - if err := validateCheckResponse(hashes, response); err != nil { + if response.Root != nil && response.Root.TreeSize != state.Version() { + callback(nil, fmt.Errorf( + "check response tree size %d does not match object version %d", + response.Root.TreeSize, + state.Version(), + )) + return + } + if err := validateCheckResponse(hash, response); err != nil { callback(nil, err) return } callback(response, nil) }, - ) + }) } func (c *Client) requestName(commandPrefix enc.Name) (enc.Name, error) { @@ -180,7 +183,7 @@ func validateAppendResponse(dataHashes [][]byte, response *defn.AppendResponse) return nil } -func validateCheckResponse(dataHashes [][]byte, response *defn.CheckResponse) error { +func validateCheckResponse(dataHash []byte, response *defn.CheckResponse) error { if response == nil || response.Root == nil { return fmt.Errorf("check response has no tree root") } @@ -193,25 +196,21 @@ func validateCheckResponse(dataHashes [][]byte, response *defn.CheckResponse) er return fmt.Errorf("empty tree has an invalid root hash") } } - if len(response.Results) != len(dataHashes) { - return fmt.Errorf("check response has %d results, want %d", len(response.Results), len(dataHashes)) + result := response.Result + if result == nil || !bytes.Equal(result.DataHash, dataHash) { + return fmt.Errorf("check result does not match requested Data hash") } - for i, result := range response.Results { - if result == nil || !bytes.Equal(result.DataHash, dataHashes[i]) { - return fmt.Errorf("check result %d does not match requested Data hash", i) + switch result.Status { + case defn.CheckStatusIncluded: + if err := VerifyInclusion(dataHash, result.Proof, response.Root); err != nil { + return fmt.Errorf("check result has invalid inclusion proof: %w", err) } - switch result.Status { - case defn.CheckStatusIncluded: - if err := VerifyInclusion(dataHashes[i], result.Proof, response.Root); err != nil { - return fmt.Errorf("check result %d has invalid inclusion proof: %w", i, err) - } - case defn.CheckStatusNotFound: - if result.Proof != nil { - return fmt.Errorf("not-found check result %d has an inclusion proof", i) - } - default: - return fmt.Errorf("check result %d has unknown status %d", i, result.Status) + case defn.CheckStatusNotFound: + if result.Proof != nil { + return fmt.Errorf("not-found check result has an inclusion proof") } + default: + return fmt.Errorf("check result has unknown status %d", result.Status) } return nil } diff --git a/std/merklelog/policy.go b/std/merklelog/policy.go new file mode 100644 index 00000000..b5cee749 --- /dev/null +++ b/std/merklelog/policy.go @@ -0,0 +1,72 @@ +package merklelog + +import ( + "crypto/sha256" + "fmt" + "time" + + "github.com/named-data/ndnd/std/ndn" + defn "github.com/named-data/ndnd/std/ndn/merklelog" +) + +// NewCertExpiredPolicy creates an expiry policy backed by a Merkle history +// log. The log client's Object client must strictly validate log responses; +// log response validation must not use this policy recursively. +func NewCertExpiredPolicy(client *Client) ndn.CertExpiredCallback { + return func(args ndn.CertExpiredCallbackArgs, complete func(error)) { + if client == nil { + complete(fmt.Errorf("Merkle log client is nil")) + return + } + if args.Data == nil { + complete(fmt.Errorf("Data packet is nil")) + return + } + if args.RawData.Length() == 0 { + complete(fmt.Errorf("Data packet wire is unavailable: %s", args.Data.Name())) + return + } + if args.Cert == nil || args.Cert.Signature() == nil { + complete(fmt.Errorf("certificate is missing or unsigned")) + return + } + + notBefore, notAfter := args.Cert.Signature().Validity() + validFrom, hasValidFrom := notBefore.Get() + validUntil, hasValidUntil := notAfter.Get() + if !hasValidFrom || !hasValidUntil || validUntil.Before(validFrom) { + complete(fmt.Errorf("certificate has an invalid validity period: %s", args.Cert.Name())) + return + } + + dataHash := sha256.Sum256(args.RawData.Join()) + client.Check(dataHash[:], func(response *defn.CheckResponse, err error) { + if err != nil { + complete(fmt.Errorf("Merkle log check failed: %w", err)) + return + } + // Check has verified the response shape, requested hash, and proof. + result := response.Result + if result.Status != defn.CheckStatusIncluded { + complete(fmt.Errorf("Data packet is not present in the Merkle log: %s", args.Data.Name())) + return + } + + // Parse the verified entry to apply the policy to its ingestion time. + entry, err := ParseProofEntry(result.Proof) + if err != nil { + complete(fmt.Errorf("parse proven log entry: %w", err)) + return + } + ingestTime := time.Unix(0, int64(entry.IngestTime)) + if ingestTime.Before(validFrom) || ingestTime.After(validUntil) { + complete(fmt.Errorf( + "Data packet was logged outside certificate validity: %s", + args.Data.Name(), + )) + return + } + complete(nil) + }) + } +} diff --git a/std/ndn/merklelog/definitions.go b/std/ndn/merklelog/definitions.go index 6942b3e5..3b73fe9e 100644 --- a/std/ndn/merklelog/definitions.go +++ b/std/ndn/merklelog/definitions.go @@ -46,16 +46,11 @@ type AppendResult struct { LeafIndex optional.Optional[uint64] `tlv:"0x1E06"` } -type CheckRequest struct { - //+field:sequence:[]byte:binary:[]byte - DataHashes [][]byte `tlv:"0x1E00"` -} - type CheckResponse struct { //+field:struct:TreeRoot Root *TreeRoot `tlv:"0x1E08"` - //+field:sequence:*CheckResult:struct:CheckResult - Results []*CheckResult `tlv:"0x1E0A"` + //+field:struct:CheckResult + Result *CheckResult `tlv:"0x1E0A"` } type CheckResult struct { diff --git a/std/ndn/merklelog/zz_generated.go b/std/ndn/merklelog/zz_generated.go index 64e4485e..20754699 100644 --- a/std/ndn/merklelog/zz_generated.go +++ b/std/ndn/merklelog/zz_generated.go @@ -609,243 +609,24 @@ func ParseAppendResult(reader enc.WireView, ignoreCritical bool) (*AppendResult, return context.Parse(reader, ignoreCritical) } -type CheckRequestEncoder struct { - Length uint - - DataHashes_subencoder []struct { - } -} - -type CheckRequestParsingContext struct { -} - -func (encoder *CheckRequestEncoder) Init(value *CheckRequest) { - { - DataHashes_l := len(value.DataHashes) - encoder.DataHashes_subencoder = make([]struct { - }, DataHashes_l) - for i := 0; i < DataHashes_l; i++ { - pseudoEncoder := &encoder.DataHashes_subencoder[i] - pseudoValue := struct { - DataHashes []byte - }{ - DataHashes: value.DataHashes[i], - } - { - encoder := pseudoEncoder - value := &pseudoValue - - _ = encoder - _ = value - } - } - } - - l := uint(0) - if value.DataHashes != nil { - for seq_i, seq_v := range value.DataHashes { - pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] - pseudoValue := struct { - DataHashes []byte - }{ - DataHashes: seq_v, - } - { - encoder := pseudoEncoder - value := &pseudoValue - if value.DataHashes != nil { - l += 3 - l += uint(enc.TLNum(len(value.DataHashes)).EncodingLength()) - l += uint(len(value.DataHashes)) - } - _ = encoder - _ = value - } - } - } - encoder.Length = l - -} - -func (context *CheckRequestParsingContext) Init() { - -} - -func (encoder *CheckRequestEncoder) EncodeInto(value *CheckRequest, buf []byte) { - - pos := uint(0) - - if value.DataHashes != nil { - for seq_i, seq_v := range value.DataHashes { - pseudoEncoder := &encoder.DataHashes_subencoder[seq_i] - pseudoValue := struct { - DataHashes []byte - }{ - DataHashes: seq_v, - } - { - encoder := pseudoEncoder - value := &pseudoValue - if value.DataHashes != nil { - buf[pos] = 253 - binary.BigEndian.PutUint16(buf[pos+1:], uint16(7680)) - pos += 3 - pos += uint(enc.TLNum(len(value.DataHashes)).EncodeInto(buf[pos:])) - copy(buf[pos:], value.DataHashes) - pos += uint(len(value.DataHashes)) - } - _ = encoder - _ = value - } - } - } -} - -func (encoder *CheckRequestEncoder) Encode(value *CheckRequest) enc.Wire { - - wire := make(enc.Wire, 1) - wire[0] = make([]byte, encoder.Length) - buf := wire[0] - encoder.EncodeInto(value, buf) - - return wire -} - -func (context *CheckRequestParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*CheckRequest, error) { - - var handled_DataHashes bool = false - - progress := -1 - _ = progress - - value := &CheckRequest{} - var err error - var startPos int - for { - startPos = reader.Pos() - if startPos >= reader.Length() { - break - } - typ := enc.TLNum(0) - l := enc.TLNum(0) - typ, err = reader.ReadTLNum() - if err != nil { - return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} - } - l, err = reader.ReadTLNum() - if err != nil { - return nil, enc.ErrFailToParse{TypeNum: 0, Err: err} - } - - err = nil - if handled := false; true { - switch typ { - case 7680: - if true { - handled = true - handled_DataHashes = true - if value.DataHashes == nil { - value.DataHashes = make([][]byte, 0) - } - { - pseudoValue := struct { - DataHashes []byte - }{} - { - value := &pseudoValue - value.DataHashes = make([]byte, l) - _, err = reader.ReadFull(value.DataHashes) - _ = value - } - value.DataHashes = append(value.DataHashes, pseudoValue.DataHashes) - } - progress-- - } - default: - if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { - return nil, enc.ErrUnrecognizedField{TypeNum: typ} - } - handled = true - err = reader.Skip(int(l)) - } - if err == nil && !handled { - } - if err != nil { - return nil, enc.ErrFailToParse{TypeNum: typ, Err: err} - } - } - } - - startPos = reader.Pos() - err = nil - - if !handled_DataHashes && err == nil { - // sequence - skip - } - - if err != nil { - return nil, err - } - - return value, nil -} - -func (value *CheckRequest) Encode() enc.Wire { - encoder := CheckRequestEncoder{} - encoder.Init(value) - return encoder.Encode(value) -} - -func (value *CheckRequest) Bytes() []byte { - return value.Encode().Join() -} - -func ParseCheckRequest(reader enc.WireView, ignoreCritical bool) (*CheckRequest, error) { - context := CheckRequestParsingContext{} - context.Init() - return context.Parse(reader, ignoreCritical) -} - type CheckResponseEncoder struct { Length uint - Root_encoder TreeRootEncoder - Results_subencoder []struct { - Results_encoder CheckResultEncoder - } + Root_encoder TreeRootEncoder + Result_encoder CheckResultEncoder } type CheckResponseParsingContext struct { - Root_context TreeRootParsingContext - Results_context CheckResultParsingContext + Root_context TreeRootParsingContext + Result_context CheckResultParsingContext } func (encoder *CheckResponseEncoder) Init(value *CheckResponse) { if value.Root != nil { encoder.Root_encoder.Init(value.Root) } - { - Results_l := len(value.Results) - encoder.Results_subencoder = make([]struct { - Results_encoder CheckResultEncoder - }, Results_l) - for i := 0; i < Results_l; i++ { - pseudoEncoder := &encoder.Results_subencoder[i] - pseudoValue := struct { - Results *CheckResult - }{ - Results: value.Results[i], - } - { - encoder := pseudoEncoder - value := &pseudoValue - if value.Results != nil { - encoder.Results_encoder.Init(value.Results) - } - _ = encoder - _ = value - } - } + if value.Result != nil { + encoder.Result_encoder.Init(value.Result) } l := uint(0) @@ -854,26 +635,10 @@ func (encoder *CheckResponseEncoder) Init(value *CheckResponse) { l += uint(enc.TLNum(encoder.Root_encoder.Length).EncodingLength()) l += encoder.Root_encoder.Length } - if value.Results != nil { - for seq_i, seq_v := range value.Results { - pseudoEncoder := &encoder.Results_subencoder[seq_i] - pseudoValue := struct { - Results *CheckResult - }{ - Results: seq_v, - } - { - encoder := pseudoEncoder - value := &pseudoValue - if value.Results != nil { - l += 3 - l += uint(enc.TLNum(encoder.Results_encoder.Length).EncodingLength()) - l += encoder.Results_encoder.Length - } - _ = encoder - _ = value - } - } + if value.Result != nil { + l += 3 + l += uint(enc.TLNum(encoder.Result_encoder.Length).EncodingLength()) + l += encoder.Result_encoder.Length } encoder.Length = l @@ -881,7 +646,7 @@ func (encoder *CheckResponseEncoder) Init(value *CheckResponse) { func (context *CheckResponseParsingContext) Init() { context.Root_context.Init() - context.Results_context.Init() + context.Result_context.Init() } func (encoder *CheckResponseEncoder) EncodeInto(value *CheckResponse, buf []byte) { @@ -898,30 +663,14 @@ func (encoder *CheckResponseEncoder) EncodeInto(value *CheckResponse, buf []byte pos += encoder.Root_encoder.Length } } - if value.Results != nil { - for seq_i, seq_v := range value.Results { - pseudoEncoder := &encoder.Results_subencoder[seq_i] - pseudoValue := struct { - Results *CheckResult - }{ - Results: seq_v, - } - { - encoder := pseudoEncoder - value := &pseudoValue - if value.Results != nil { - buf[pos] = 253 - binary.BigEndian.PutUint16(buf[pos+1:], uint16(7690)) - pos += 3 - pos += uint(enc.TLNum(encoder.Results_encoder.Length).EncodeInto(buf[pos:])) - if encoder.Results_encoder.Length > 0 { - encoder.Results_encoder.EncodeInto(value.Results, buf[pos:]) - pos += encoder.Results_encoder.Length - } - } - _ = encoder - _ = value - } + if value.Result != nil { + buf[pos] = 253 + binary.BigEndian.PutUint16(buf[pos+1:], uint16(7690)) + pos += 3 + pos += uint(enc.TLNum(encoder.Result_encoder.Length).EncodeInto(buf[pos:])) + if encoder.Result_encoder.Length > 0 { + encoder.Result_encoder.EncodeInto(value.Result, buf[pos:]) + pos += encoder.Result_encoder.Length } } } @@ -939,7 +688,7 @@ func (encoder *CheckResponseEncoder) Encode(value *CheckResponse) enc.Wire { func (context *CheckResponseParsingContext) Parse(reader enc.WireView, ignoreCritical bool) (*CheckResponse, error) { var handled_Root bool = false - var handled_Results bool = false + var handled_Result bool = false progress := -1 _ = progress @@ -975,22 +724,8 @@ func (context *CheckResponseParsingContext) Parse(reader enc.WireView, ignoreCri case 7690: if true { handled = true - handled_Results = true - if value.Results == nil { - value.Results = make([]*CheckResult, 0) - } - { - pseudoValue := struct { - Results *CheckResult - }{} - { - value := &pseudoValue - value.Results, err = context.Results_context.Parse(reader.Delegate(int(l)), ignoreCritical) - _ = value - } - value.Results = append(value.Results, pseudoValue.Results) - } - progress-- + handled_Result = true + value.Result, err = context.Result_context.Parse(reader.Delegate(int(l)), ignoreCritical) } default: if !ignoreCritical && ((typ <= 31) || ((typ & 1) == 1)) { @@ -1013,8 +748,8 @@ func (context *CheckResponseParsingContext) Parse(reader enc.WireView, ignoreCri if !handled_Root && err == nil { value.Root = nil } - if !handled_Results && err == nil { - // sequence - skip + if !handled_Result && err == nil { + value.Result = nil } if err != nil { diff --git a/std/security/trust_config_test.go b/std/security/trust_config_test.go index 7519ef76..08e98976 100644 --- a/std/security/trust_config_test.go +++ b/std/security/trust_config_test.go @@ -1,6 +1,7 @@ package security_test import ( + "bytes" "crypto/elliptic" _ "embed" "fmt" @@ -104,9 +105,21 @@ func validateSync(opts ValidateSyncOptions) bool { require.NoError(tcTestT, err) data, sigCov, err := spec.Spec{}.ReadData(enc.NewWireView(dataW.Wire)) require.NoError(tcTestT, err) + onCertExpired := opts.onCertExpired + rawDataMismatch := false + if onCertExpired != nil { + onCertExpired = func(args ndn.CertExpiredCallbackArgs, complete func(error)) { + if args.Data.Name().Equal(data.Name()) && + !bytes.Equal(dataW.Wire.Join(), args.RawData.Join()) { + rawDataMismatch = true + } + opts.onCertExpired(args, complete) + } + } ch := make(chan bool) go tcTestTrustConfig.Validate(sec.TrustConfigValidateArgs{ Data: data, + RawData: dataW.Wire, DataSigCov: sigCov, Fetch: fetchFun, Callback: func(valid bool, err error) { @@ -114,9 +127,11 @@ func validateSync(opts ValidateSyncOptions) bool { ch <- valid close(ch) }, - OnCertExpired: opts.onCertExpired, + OnCertExpired: onCertExpired, }) - return <-ch + valid := <-ch + require.False(tcTestT, rawDataMismatch) + return valid } // Helper to validate certificates @@ -565,6 +580,7 @@ func testTrustConfigIntra(t *testing.T, schema ndn.TrustSchema) { require.Equal(t, "/test/alice/app/test/bob/expired-invite", crossSchemaExpiryArgs[0].Data.Name().String()) require.True(t, expiredInviteData.Name().Equal(crossSchemaExpiryArgs[0].Cert.Name())) require.True(t, expiredInviteData.Name().Equal(crossSchemaExpiryArgs[1].Data.Name())) + require.Equal(t, expiredInvite.Join(), crossSchemaExpiryArgs[1].RawData.Join()) require.True(t, aliceCertData.Name().Equal(crossSchemaExpiryArgs[1].Cert.Name())) require.False(t, validateSync(ValidateSyncOptions{ @@ -853,6 +869,7 @@ func testTrustConfigIntra(t *testing.T, schema ndn.TrustSchema) { require.Equal(t, "/test/eve/data3", callbackArgs[0].Data.Name().String()) require.True(t, eveCertData.Name().Equal(callbackArgs[0].Cert.Name())) require.True(t, eveCertData.Name().Equal(callbackArgs[1].Data.Name())) + require.Equal(t, eveCertWire.Join(), callbackArgs[1].RawData.Join()) require.True(t, rootCertData.Name().Equal(callbackArgs[1].Cert.Name())) } @@ -1132,6 +1149,7 @@ func testTrustConfigInter(t *testing.T, schema ndn.TrustSchema) { require.Error(t, err) require.Len(t, expiryArgs, 1) require.True(t, freshPreAnchorData.Name().Equal(expiryArgs[0].Data.Name())) + require.Equal(t, freshPreAnchorWire.Join(), expiryArgs[0].RawData.Join()) require.True(t, expiringOwnerData.Name().Equal(expiryArgs[0].Cert.Name())) require.Equal(t, 0, tcTestFetchCount) }) @@ -1286,8 +1304,10 @@ func testTrustConfigInter(t *testing.T, schema ndn.TrustSchema) { require.Error(t, err) require.Len(t, rejectedExpiryArgs, 2) require.True(t, expiredListData.Name().Equal(rejectedExpiryArgs[0].Data.Name())) + require.Equal(t, expiredListWire.Wire.Join(), rejectedExpiryArgs[0].RawData.Join()) require.True(t, expiredPreAnchorData.Name().Equal(rejectedExpiryArgs[0].Cert.Name())) require.True(t, expiredPreAnchorData.Name().Equal(rejectedExpiryArgs[1].Data.Name())) + require.Equal(t, expiredPreAnchorWire.Join(), rejectedExpiryArgs[1].RawData.Join()) require.True(t, ownerCertData.Name().Equal(rejectedExpiryArgs[1].Cert.Name())) require.Equal(t, 1, tcTestFetchCount) // CertList; target certificate is reloaded locally. @@ -1301,8 +1321,10 @@ func testTrustConfigInter(t *testing.T, schema ndn.TrustSchema) { require.Error(t, err) require.Len(t, expiryArgs, 2) require.True(t, expiredListData.Name().Equal(expiryArgs[0].Data.Name())) + require.Equal(t, expiredListWire.Wire.Join(), expiryArgs[0].RawData.Join()) require.True(t, expiredPreAnchorData.Name().Equal(expiryArgs[0].Cert.Name())) require.True(t, expiredPreAnchorData.Name().Equal(expiryArgs[1].Data.Name())) + require.Equal(t, expiredPreAnchorWire.Join(), expiryArgs[1].RawData.Join()) require.True(t, ownerCertData.Name().Equal(expiryArgs[1].Cert.Name())) require.Equal(t, 0, tcTestFetchCount) // CertList is cached; target is reloaded locally. From 30dcbaf3a1f3e81f80cca89a8a15f67ca7421059 Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Sun, 13 Sep 2026 12:09:01 -0700 Subject: [PATCH 6/8] merkle: add more tests --- merkle/api_test.go | 243 +++++++++++++++++ merkle/e2e_test.go | 498 +++++++++++++++++++++++++++++++++++ std/merklelog/client_test.go | 174 ++++++++++++ std/merklelog/policy_test.go | 308 ++++++++++++++++++++++ 4 files changed, 1223 insertions(+) create mode 100644 merkle/api_test.go create mode 100644 merkle/e2e_test.go create mode 100644 std/merklelog/client_test.go create mode 100644 std/merklelog/policy_test.go diff --git a/merkle/api_test.go b/merkle/api_test.go new file mode 100644 index 00000000..dcc4f279 --- /dev/null +++ b/merkle/api_test.go @@ -0,0 +1,243 @@ +package merkle + +import ( + "crypto/sha256" + "errors" + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/merklelog" + "github.com/named-data/ndnd/std/ndn" + defn "github.com/named-data/ndnd/std/ndn/merklelog" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + "github.com/named-data/ndnd/std/object" + "github.com/named-data/ndnd/std/object/storage" + "github.com/named-data/ndnd/std/security/signer" + "github.com/stretchr/testify/require" +) + +func TestAppendAndCheck(t *testing.T) { + m := newTestLog(t) + hash0 := apiTestHash(0) + hash1 := apiTestHash(1) + + response, err := m.append(&defn.AppendRequest{ + DataHashes: [][]byte{hash0, hash1, hash0, {0x01}}, + }) + require.NoError(t, err) + require.Len(t, response.Results, 4) + require.Equal(t, defn.AppendStatusOK, response.Results[0].Status) + require.Equal(t, defn.AppendStatusOK, response.Results[1].Status) + require.Equal(t, defn.AppendStatusDuplicate, response.Results[2].Status) + require.Equal(t, defn.AppendStatusFailed, response.Results[3].Status) + for _, i := range []int{0, 1, 2} { + require.Equal(t, uint64(0), response.Results[i].LeafIndex.Unwrap()) + } + require.False(t, response.Results[3].LeafIndex.IsSet()) + require.Equal(t, uint64(1), m.tree.Size()) + + check, err := m.check(hash0) + require.NoError(t, err) + require.Equal(t, uint64(1), check.Root.TreeSize) + require.Equal(t, defn.CheckStatusIncluded, check.Result.Status) + require.NoError(t, merklelog.VerifyInclusion(hash0, check.Result.Proof, check.Root)) + + missing, err := m.check(apiTestHash(99)) + require.NoError(t, err) + require.Equal(t, defn.CheckStatusNotFound, missing.Result.Status) + require.Nil(t, missing.Result.Proof) + + second, err := m.check(hash1) + require.NoError(t, err) + require.Equal(t, defn.CheckStatusIncluded, second.Result.Status) + require.NoError(t, merklelog.VerifyInclusion(hash1, second.Result.Proof, second.Root)) +} + +func TestAppendReplayAndMonotonicTime(t *testing.T) { + m := newTestLog(t) + firstNow := m.now() + + first, err := m.append(&defn.AppendRequest{DataHashes: [][]byte{apiTestHash(0)}}) + require.NoError(t, err) + require.Equal(t, defn.AppendStatusOK, first.Results[0].Status) + firstIngestTime, ok := m.tree.LastIngestTime() + require.True(t, ok) + require.Equal(t, time.Duration(firstNow.UnixMilli())*time.Millisecond, firstIngestTime) + + replay, err := m.append(&defn.AppendRequest{DataHashes: [][]byte{apiTestHash(0)}}) + require.NoError(t, err) + require.Equal(t, defn.AppendStatusDuplicate, replay.Results[0].Status) + require.Equal(t, uint64(1), m.tree.Size()) + + m.now = func() time.Time { return firstNow.Add(-time.Hour) } + second, err := m.append(&defn.AppendRequest{DataHashes: [][]byte{apiTestHash(1)}}) + require.NoError(t, err) + require.Equal(t, defn.AppendStatusOK, second.Results[0].Status) + secondIngestTime, ok := m.tree.LastIngestTime() + require.True(t, ok) + require.Equal(t, firstIngestTime+time.Millisecond, secondIngestTime) +} + +func TestCheckHandlerServesObject(t *testing.T) { + m := newTestLog(t) + packetStore, err := storage.NewBadgerStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, packetStore.Close()) }) + m.packetStore = packetStore + m.client = &checkProducer{store: packetStore} + + hash := apiTestHash(0) + _, err = m.append(&defn.AppendRequest{DataHashes: [][]byte{hash}}) + require.NoError(t, err) + checkName := merklelog.CheckPrefix(m.config.nameN). + Append(enc.NewGenericBytesComponent(hash)) + + firstReply := make(chan enc.Wire, 1) + m.onCheck(ndn.InterestHandlerArgs{ + Interest: makeCheckInterest(t, checkName, true), + Reply: func(wire enc.Wire) error { + firstReply <- wire + return nil + }, + }) + + var firstWire enc.Wire + select { + case firstWire = <-firstReply: + case <-time.After(time.Second): + t.Fatal("check handler did not reply") + } + data, _, err := spec.Spec{}.ReadData(enc.NewWireView(firstWire)) + require.NoError(t, err) + expectedName := checkName. + Append(enc.NewVersionComponent(m.tree.Size())). + Append(enc.NewSegmentComponent(0)) + require.True(t, expectedName.Equal(data.Name())) + response, err := defn.ParseCheckResponse(enc.NewWireView(data.Content()), false) + require.NoError(t, err) + require.NoError(t, merklelog.VerifyInclusion(hash, response.Result.Proof, response.Root)) + + segmentReply := make(chan enc.Wire, 1) + m.onCheck(ndn.InterestHandlerArgs{ + Interest: makeCheckInterest(t, expectedName, false), + Reply: func(wire enc.Wire) error { + segmentReply <- wire + return nil + }, + }) + select { + case segmentWire := <-segmentReply: + require.Equal(t, firstWire.Join(), segmentWire.Join()) + case <-time.After(time.Second): + t.Fatal("check handler did not serve the stored segment") + } +} + +func TestAppendStoreFailure(t *testing.T) { + storeErr := errors.New("store failure") + tree, err := merklelog.OpenTree(&failingAppendStore{ + Store: merklelog.NewMemoryStore(), + err: storeErr, + }) + require.NoError(t, err) + m := newTestLog(t) + m.tree = tree + + response, err := m.append(&defn.AppendRequest{DataHashes: [][]byte{apiTestHash(0)}}) + require.ErrorIs(t, err, storeErr) + require.Equal(t, defn.AppendStatusFailed, response.Results[0].Status) + require.False(t, response.Results[0].LeafIndex.IsSet()) + require.Zero(t, tree.Size()) +} + +func TestRequestNameValidation(t *testing.T) { + m := newTestLog(t) + prefix := merklelog.AppendPrefix(m.config.nameN) + nowMillis := uint64(m.now().UnixMilli()) + valid := prefix.Clone(). + Append(enc.NewGenericComponent("requester")). + Append(enc.NewTimestampComponent(nowMillis)) + require.NoError(t, m.validateRequestName(valid, prefix)) + + tests := map[string]enc.Name{ + "wrong command": merklelog.CheckPrefix(m.config.nameN).Clone(). + Append(enc.NewGenericComponent("requester")). + Append(enc.NewTimestampComponent(nowMillis)), + "no requester": prefix.Clone().Append(enc.NewTimestampComponent(nowMillis)), + "no timestamp": prefix.Clone(). + Append(enc.NewGenericComponent("requester"), enc.NewGenericComponent("time")), + "stale": prefix.Clone(). + Append(enc.NewGenericComponent("requester")). + Append(enc.NewTimestampComponent(uint64(m.now().Add(-time.Minute - time.Millisecond).UnixMilli()))), + "future": prefix.Clone(). + Append(enc.NewGenericComponent("requester")). + Append(enc.NewTimestampComponent(uint64(m.now().Add(time.Minute + time.Millisecond).UnixMilli()))), + "non-canonical timestamp": prefix.Clone(). + Append(enc.NewGenericComponent("requester")). + Append(enc.NewBytesComponent(enc.TypeTimestampNameComponent, []byte{0, 1})), + } + for name, requestName := range tests { + t.Run(name, func(t *testing.T) { + require.Error(t, m.validateRequestName(requestName, prefix)) + }) + } +} + +func TestRejectInvalidRequests(t *testing.T) { + m := newTestLog(t) + response, err := m.append(nil) + require.Error(t, err) + require.Nil(t, response) + + check, err := m.check([]byte{0x01}) + require.Error(t, err) + require.Nil(t, check) +} + +type failingAppendStore struct { + merklelog.Store + err error +} + +type checkProducer struct { + ndn.Client + store ndn.Store +} + +func (c *checkProducer) Produce(args ndn.ProduceArgs) (enc.Name, error) { + return object.Produce(args, c.store, signer.NewSha256Signer()) +} + +func makeCheckInterest(t *testing.T, name enc.Name, canBePrefix bool) ndn.Interest { + t.Helper() + encoded, err := spec.Spec{}.MakeInterest( + name, + &ndn.InterestConfig{CanBePrefix: canBePrefix}, + nil, + nil, + ) + require.NoError(t, err) + interest, _, err := spec.Spec{}.ReadInterest(enc.NewWireView(encoded.Wire)) + require.NoError(t, err) + return interest +} + +func (s *failingAppendStore) Append(merklelog.StoreAppend) error { + return s.err +} + +func newTestLog(t *testing.T) *Log { + t.Helper() + name, err := enc.NameFromStr("/operator/merkle") + require.NoError(t, err) + m := NewLog(&Config{nameN: name}) + m.tree = merklelog.NewTree() + m.now = func() time.Time { return time.UnixMilli(1700000000000) } + return m +} + +func apiTestHash(index byte) []byte { + hash := sha256.Sum256([]byte{index}) + return hash[:] +} diff --git a/merkle/e2e_test.go b/merkle/e2e_test.go new file mode 100644 index 00000000..245ca269 --- /dev/null +++ b/merkle/e2e_test.go @@ -0,0 +1,498 @@ +package merkle + +import ( + "bytes" + "crypto/sha256" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/engine/basic" + "github.com/named-data/ndnd/std/merklelog" + "github.com/named-data/ndnd/std/ndn" + defn "github.com/named-data/ndnd/std/ndn/merklelog" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + "github.com/named-data/ndnd/std/object" + "github.com/named-data/ndnd/std/object/storage" + sec "github.com/named-data/ndnd/std/security" + "github.com/named-data/ndnd/std/security/keychain" + "github.com/named-data/ndnd/std/security/signer" + "github.com/stretchr/testify/require" +) + +const e2eOperationTimeout = 5 * time.Second + +func TestExpiredCertificateMerkleLogEndToEnd(t *testing.T) { + fixture := newE2EFixture(t) + validFrom := time.Now().Add(-time.Hour).Truncate(time.Millisecond) + validUntil := time.Now().Add(3 * time.Second).Truncate(time.Millisecond) + + accepted := fixture.newEvidence(t, "/app/accepted", validFrom, validUntil) + missingData := fixture.newEvidence(t, "/app/missing-data", validFrom, validUntil) + missingCert := fixture.newEvidence(t, "/app/missing-cert", validFrom, validUntil) + outsideValidity := fixture.newEvidence( + t, + "/app/outside-validity", + time.Now().Add(-2*time.Hour), + time.Now().Add(-time.Hour), + ) + loggedHashes := [][]byte{ + accepted.dataHash, + accepted.certHash, + missingData.certHash, + missingCert.dataHash, + outsideValidity.dataHash, + outsideValidity.certHash, + } + + appendResponse, err := fixture.append(loggedHashes) + require.NoError(t, err) + require.Len(t, appendResponse.Results, len(loggedHashes)) + for i, result := range appendResponse.Results { + require.Equal(t, defn.AppendStatusOK, result.Status, "append result %d", i) + require.True(t, result.LeafIndex.IsSet(), "append result %d has no leaf index", i) + require.Equal(t, uint64(0), result.LeafIndex.Unwrap(), "append result %d", i) + } + require.Equal(t, uint64(1), fixture.service.tree.Size()) + + // Repeating an authenticated append must preserve the original evidence. + duplicateResponse, err := fixture.append([][]byte{accepted.dataHash, accepted.certHash}) + require.NoError(t, err) + for i, result := range duplicateResponse.Results { + require.Equal(t, defn.AppendStatusDuplicate, result.Status, "duplicate result %d", i) + require.Equal(t, uint64(0), result.LeafIndex.Unwrap(), "duplicate result %d", i) + } + require.Equal(t, uint64(1), fixture.service.tree.Size()) + + // Loading a second Tree from the durable store must reconstruct the same + // root and make every committed hash independently provable. + restored, err := merklelog.OpenTree(fixture.logStore) + require.NoError(t, err) + require.Equal(t, fixture.service.tree.Size(), restored.Size()) + require.True(t, bytes.Equal(fixture.service.tree.Root().RootHash, restored.Root().RootHash)) + for _, dataHash := range loggedHashes { + leafIndex, ok := restored.Lookup(dataHash) + require.True(t, ok, "restored tree is missing %x", dataHash) + proof, err := restored.InclusionProof(leafIndex) + require.NoError(t, err) + require.NoError(t, merklelog.VerifyInclusion(dataHash, proof, restored.Root())) + } + + ingestTime, ok := fixture.service.tree.LastIngestTime() + require.True(t, ok) + loggedAt := time.Unix(0, int64(ingestTime)) + require.False(t, loggedAt.Before(validFrom)) + require.False(t, loggedAt.After(validUntil)) + if wait := time.Until(validUntil) + time.Millisecond; wait > 0 { + time.Sleep(wait) + } + for _, evidence := range []*e2eEvidence{accepted, missingData, missingCert, outsideValidity} { + require.True(t, sec.CertIsExpired(evidence.cert)) + } + + t.Run("accepts two proven relations", func(t *testing.T) { + require.NoError(t, fixture.validate(accepted, merklelog.NewCertExpiredPolicy(fixture.logClient))) + + // Recursive validation must retrieve evidence for both relations: + // Data <- expired child certificate and child certificate <- root. + for _, dataHash := range [][]byte{accepted.dataHash, accepted.certHash} { + checkName := merklelog.CheckPrefix(fixture.logPrefix). + Append(enc.NewGenericBytesComponent(dataHash)) + wire, err := fixture.serverStore.Get(checkName, true) + require.NoError(t, err) + require.NotEmpty(t, wire, "validation did not retrieve proof for %x", dataHash) + } + }) + + t.Run("rejects expired chain without policy", func(t *testing.T) { + err := fixture.validate(accepted, nil) + require.ErrorContains(t, err, "certificate is expired") + }) + + t.Run("rejects unlogged data", func(t *testing.T) { + err := fixture.validate(missingData, merklelog.NewCertExpiredPolicy(fixture.logClient)) + require.ErrorContains(t, err, "not present in the Merkle log") + }) + + t.Run("rejects unlogged certificate", func(t *testing.T) { + err := fixture.validate(missingCert, merklelog.NewCertExpiredPolicy(fixture.logClient)) + require.ErrorContains(t, err, "not present in the Merkle log") + }) + + t.Run("rejects evidence logged outside validity", func(t *testing.T) { + err := fixture.validate(outsideValidity, merklelog.NewCertExpiredPolicy(fixture.logClient)) + require.ErrorContains(t, err, "logged outside certificate validity") + }) + + t.Run("rejects different raw packet wire", func(t *testing.T) { + altered := *accepted + altered.rawData = enc.Wire{bytes.Clone(accepted.rawData.Join())} + altered.rawData[0][len(altered.rawData[0])-1] ^= 0xff + err := fixture.validate(&altered, merklelog.NewCertExpiredPolicy(fixture.logClient)) + require.ErrorContains(t, err, "not present in the Merkle log") + }) + + t.Run("proof does not bypass signature validation", func(t *testing.T) { + altered := *accepted + altered.sigCovered = enc.Wire{[]byte("invalid signature input")} + err := fixture.validate(&altered, merklelog.NewCertExpiredPolicy(fixture.logClient)) + require.ErrorContains(t, err, "signature is invalid") + }) + + t.Run("check returns included and not-found results", func(t *testing.T) { + included, err := fixture.check(accepted.dataHash) + require.NoError(t, err) + require.Equal(t, defn.CheckStatusIncluded, included.Result.Status) + require.NoError(t, merklelog.VerifyInclusion( + accepted.dataHash, + included.Result.Proof, + included.Root, + )) + + missingHash := sha256.Sum256([]byte("not logged")) + notFound, err := fixture.check(missingHash[:]) + require.NoError(t, err) + require.Equal(t, defn.CheckStatusNotFound, notFound.Result.Status) + require.Nil(t, notFound.Result.Proof) + }) + + t.Run("rejects tampered check response", func(t *testing.T) { + dataHash := sha256.Sum256([]byte("tampered check response")) + response, err := fixture.check(dataHash[:]) + require.NoError(t, err) + require.Equal(t, defn.CheckStatusNotFound, response.Result.Status) + + segmentName := merklelog.CheckPrefix(fixture.logPrefix). + Append(enc.NewGenericBytesComponent(dataHash[:])). + Append(enc.NewVersionComponent(response.Root.TreeSize)). + Append(enc.NewSegmentComponent(0)) + wire, err := fixture.serverStore.Get(segmentName, false) + require.NoError(t, err) + require.NotEmpty(t, wire) + wire[len(wire)-1] ^= 0xff + require.NoError(t, fixture.serverStore.Put(segmentName, wire)) + + _, err = fixture.check(dataHash[:]) + require.Error(t, err) + }) +} + +type e2eFixture struct { + logPrefix enc.Name + service *Log + serverStore *storage.BadgerStore + logStore *merklelog.BadgerStore + consumerClient ndn.Client + consumerKeyChain ndn.KeyChain + logClient *merklelog.Client + rootSigner ndn.Signer +} + +type e2eEvidence struct { + data ndn.Data + rawData enc.Wire + sigCovered enc.Wire + cert ndn.Data + dataHash []byte + certHash []byte +} + +func newE2EFixture(t *testing.T) *e2eFixture { + t.Helper() + logPrefix := e2eName(t, "/operator/merkle") + serverFace, consumerFace := newE2EFacePair() + serverEngine := basic.NewEngine(serverFace, basic.NewTimer()) + consumerEngine := basic.NewEngine(consumerFace, basic.NewTimer()) + + serverStore, err := storage.NewBadgerStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, serverStore.Close()) }) + logStore, err := merklelog.NewBadgerStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, logStore.Close()) }) + tree, err := merklelog.OpenTree(logStore) + require.NoError(t, err) + + require.NoError(t, serverEngine.Start()) + t.Cleanup(func() { + if serverEngine.IsRunning() { + require.NoError(t, serverEngine.Stop()) + } + if consumerEngine.IsRunning() { + require.NoError(t, consumerEngine.Stop()) + } + require.Eventually(t, func() bool { + return !serverEngine.IsRunning() && !consumerEngine.IsRunning() + }, time.Second, time.Millisecond) + }) + require.NoError(t, consumerEngine.Start()) + + now := time.Now() + rootSigner, err := signer.KeygenEd25519(sec.MakeKeyName(e2eName(t, "/root"))) + require.NoError(t, err) + rootWire, rootCert := e2eSelfSign(t, rootSigner, now.Add(-time.Hour), now.Add(time.Hour)) + logSigner, err := signer.KeygenEd25519(sec.MakeKeyName(e2eName(t, "/operator/logger"))) + require.NoError(t, err) + logWire, logCert := e2eSelfSign(t, logSigner, now.Add(-time.Hour), now.Add(time.Hour)) + + serverKeyChain := keychain.NewKeyChainMem(serverStore) + require.NoError(t, serverKeyChain.InsertCert(rootWire.Join())) + require.NoError(t, serverKeyChain.InsertKey(logSigner)) + serverTrust, err := sec.NewTrustConfig( + serverKeyChain, + e2eTrustSchema{signer: logSigner}, + []enc.Name{rootCert.Name()}, + ) + require.NoError(t, err) + serverClient := object.NewClient(serverEngine, serverStore, serverTrust) + require.NoError(t, serverClient.Start()) + t.Cleanup(func() { require.NoError(t, serverClient.Stop()) }) + + consumerStore := storage.NewMemoryStore() + consumerKeyChain := keychain.NewKeyChainMem(consumerStore) + require.NoError(t, consumerKeyChain.InsertKey(rootSigner)) + require.NoError(t, consumerKeyChain.InsertCert(rootWire.Join())) + require.NoError(t, consumerKeyChain.InsertCert(logWire.Join())) + consumerTrust, err := sec.NewTrustConfig( + consumerKeyChain, + e2eTrustSchema{signer: rootSigner}, + []enc.Name{rootCert.Name(), logCert.Name()}, + ) + require.NoError(t, err) + consumerClient := object.NewClient(consumerEngine, consumerStore, consumerTrust) + require.NoError(t, consumerClient.Start()) + t.Cleanup(func() { require.NoError(t, consumerClient.Stop()) }) + + service := NewLog(&Config{nameN: logPrefix}) + service.engine = serverEngine + service.client = serverClient + service.packetStore = serverStore + service.logStore = logStore + service.tree = tree + require.NoError(t, serverClient.AttachCommandHandler(merklelog.AppendPrefix(logPrefix), service.onAppend)) + t.Cleanup(func() { + require.NoError(t, serverClient.DetachCommandHandler(merklelog.AppendPrefix(logPrefix))) + }) + require.NoError(t, serverEngine.AttachHandler(merklelog.CheckPrefix(logPrefix), service.onCheck)) + t.Cleanup(func() { + require.NoError(t, serverEngine.DetachHandler(merklelog.CheckPrefix(logPrefix))) + }) + + logClient, err := merklelog.NewClient(consumerClient, logPrefix, rootSigner.KeyName()) + require.NoError(t, err) + return &e2eFixture{ + logPrefix: logPrefix, + service: service, + serverStore: serverStore, + logStore: logStore, + consumerClient: consumerClient, + consumerKeyChain: consumerKeyChain, + logClient: logClient, + rootSigner: rootSigner, + } +} + +func (f *e2eFixture) newEvidence( + t *testing.T, + identity string, + validFrom time.Time, + validUntil time.Time, +) *e2eEvidence { + t.Helper() + childSigner, err := signer.KeygenEd25519(sec.MakeKeyName(e2eName(t, identity))) + require.NoError(t, err) + childKey, err := signer.MarshalSecretToData(childSigner) + require.NoError(t, err) + childWire, err := sec.SignCert(sec.SignCertArgs{ + Signer: f.rootSigner, + Data: childKey, + IssuerId: enc.NewGenericComponent("root"), + NotBefore: validFrom, + NotAfter: validUntil, + }) + require.NoError(t, err) + childCert, _, err := spec.Spec{}.ReadData(enc.NewWireView(childWire)) + require.NoError(t, err) + require.NoError(t, f.consumerKeyChain.InsertCert(childWire.Join())) + + dataWire, err := spec.Spec{}.MakeData( + e2eName(t, identity+"/data"), + &ndn.DataConfig{}, + enc.Wire{[]byte(identity)}, + signer.AsContextSigner(childSigner), + ) + require.NoError(t, err) + data, dataSigCovered, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire.Wire)) + require.NoError(t, err) + return &e2eEvidence{ + data: data, + rawData: dataWire.Wire, + sigCovered: dataSigCovered, + cert: childCert, + dataHash: e2eDataHash(dataWire.Wire), + certHash: e2eDataHash(childWire), + } +} + +func (f *e2eFixture) append(dataHashes [][]byte) (*defn.AppendResponse, error) { + type result struct { + response *defn.AppendResponse + err error + } + completed := make(chan result, 1) + f.logClient.Append(dataHashes, func(response *defn.AppendResponse, err error) { + completed <- result{response: response, err: err} + }) + select { + case ret := <-completed: + return ret.response, ret.err + case <-time.After(e2eOperationTimeout): + return nil, fmt.Errorf("append timed out") + } +} + +func (f *e2eFixture) check(dataHash []byte) (*defn.CheckResponse, error) { + type result struct { + response *defn.CheckResponse + err error + } + completed := make(chan result, 1) + f.logClient.Check(dataHash, func(response *defn.CheckResponse, err error) { + completed <- result{response: response, err: err} + }) + select { + case ret := <-completed: + return ret.response, ret.err + case <-time.After(e2eOperationTimeout): + return nil, fmt.Errorf("check timed out") + } +} + +func (f *e2eFixture) validate( + evidence *e2eEvidence, + policy ndn.CertExpiredCallback, +) error { + completed := make(chan error, 1) + f.consumerClient.ValidateExt(ndn.ValidateExtArgs{ + Data: evidence.data, + RawData: evidence.rawData, + SigCovered: evidence.sigCovered, + OnCertExpired: policy, + Callback: func(valid bool, err error) { + if !valid && err == nil { + err = fmt.Errorf("validation failed") + } + completed <- err + }, + }) + select { + case err := <-completed: + return err + case <-time.After(e2eOperationTimeout): + return fmt.Errorf("validation timed out") + } +} + +func e2eDataHash(wire enc.Wire) []byte { + hash := sha256.Sum256(wire.Join()) + return hash[:] +} + +func e2eSelfSign( + t *testing.T, + packetSigner ndn.Signer, + notBefore time.Time, + notAfter time.Time, +) (enc.Wire, ndn.Data) { + t.Helper() + wire, err := sec.SelfSign(sec.SignCertArgs{ + Signer: packetSigner, + NotBefore: notBefore, + NotAfter: notAfter, + }) + require.NoError(t, err) + cert, _, err := spec.Spec{}.ReadData(enc.NewWireView(wire)) + require.NoError(t, err) + return wire, cert +} + +func e2eName(t *testing.T, value string) enc.Name { + t.Helper() + name, err := enc.NameFromStr(value) + require.NoError(t, err) + return name +} + +type e2eTrustSchema struct { + signer ndn.Signer +} + +func (e2eTrustSchema) Check(enc.Name, enc.Name) bool { return true } + +func (s e2eTrustSchema) Suggest(enc.Name, ndn.KeyChain) ndn.Signer { + return s.signer +} + +type e2eFace struct { + running atomic.Bool + peer *e2eFace + mutex sync.RWMutex + onPacket func([]byte) +} + +func newE2EFacePair() (*e2eFace, *e2eFace) { + left := &e2eFace{} + right := &e2eFace{} + left.peer = right + right.peer = left + return left, right +} + +func (*e2eFace) String() string { return "e2e-face" } + +func (f *e2eFace) IsRunning() bool { return f.running.Load() } + +func (*e2eFace) IsLocal() bool { return true } + +func (f *e2eFace) OnPacket(onPacket func([]byte)) { + f.mutex.Lock() + f.onPacket = onPacket + f.mutex.Unlock() +} + +func (*e2eFace) OnError(func(error)) {} + +func (f *e2eFace) Open() error { + if !f.running.CompareAndSwap(false, true) { + return fmt.Errorf("face is already running") + } + return nil +} + +func (f *e2eFace) Close() error { + if !f.running.CompareAndSwap(true, false) { + return fmt.Errorf("face is not running") + } + return nil +} + +func (f *e2eFace) Send(wire enc.Wire) error { + if !f.running.Load() || !f.peer.running.Load() { + return fmt.Errorf("face is not running") + } + f.peer.mutex.RLock() + onPacket := f.peer.onPacket + f.peer.mutex.RUnlock() + if onPacket == nil { + return fmt.Errorf("peer has no packet handler") + } + onPacket(bytes.Clone(wire.Join())) + return nil +} + +func (*e2eFace) OnUp(func()) func() { return func() {} } + +func (*e2eFace) OnDown(func()) func() { return func() {} } diff --git a/std/merklelog/client_test.go b/std/merklelog/client_test.go new file mode 100644 index 00000000..b9726e6b --- /dev/null +++ b/std/merklelog/client_test.go @@ -0,0 +1,174 @@ +package merklelog + +import ( + "crypto/sha256" + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/ndn" + defn "github.com/named-data/ndnd/std/ndn/merklelog" + "github.com/named-data/ndnd/std/types/optional" + "github.com/stretchr/testify/require" +) + +func TestClientAppend(t *testing.T) { + hash := clientTestHash(0) + mock := &commandClientMock{ + response: (&defn.AppendResponse{Results: []*defn.AppendResult{{ + DataHash: hash, + Status: defn.AppendStatusOK, + LeafIndex: optional.Some(uint64(7)), + }}}).Encode(), + } + logPrefix := clientTestName(t, "/operator/merkle") + requester := clientTestName(t, "/requester") + client, err := NewClient(mock, logPrefix, requester) + require.NoError(t, err) + client.now = func() time.Time { return time.UnixMilli(1700000000000) } + + var response *defn.AppendResponse + client.Append([][]byte{hash}, func(ret *defn.AppendResponse, err error) { + require.NoError(t, err) + response = ret + }) + require.NotNil(t, response) + require.Equal(t, uint64(7), response.Results[0].LeafIndex.Unwrap()) + require.True(t, mock.dest.Equal(AppendPrefix(logPrefix))) + require.True(t, AppendPrefix(logPrefix).IsPrefix(mock.name)) + require.Equal(t, requester, mock.name[len(AppendPrefix(logPrefix)):len(mock.name)-1]) + require.True(t, mock.name.At(-1).IsTimestamp()) + require.Equal(t, uint64(1700000000000), mock.name.At(-1).NumberVal()) + + request, err := defn.ParseAppendRequest(enc.NewWireView(mock.command), false) + require.NoError(t, err) + require.Equal(t, [][]byte{hash}, request.DataHashes) +} + +func TestClientCheckVerifiesProof(t *testing.T) { + hash := clientTestHash(0) + tree := NewTree() + entryWire, err := EncodeLogEntry(&defn.LogEntry{ + IngestTime: time.Second, + DataHashes: [][]byte{hash}, + }) + require.NoError(t, err) + _, err = tree.Append(entryWire) + require.NoError(t, err) + proof, err := tree.InclusionProof(0) + require.NoError(t, err) + + mock := &commandClientMock{ + response: (&defn.CheckResponse{ + Root: tree.Root(), + Result: &defn.CheckResult{ + DataHash: hash, + Status: defn.CheckStatusIncluded, + Proof: proof, + }, + }).Encode(), + responseVersion: tree.Size(), + } + client, err := NewClient( + mock, + clientTestName(t, "/operator/merkle"), + clientTestName(t, "/requester"), + ) + require.NoError(t, err) + + var response *defn.CheckResponse + client.Check(hash, func(ret *defn.CheckResponse, err error) { + require.NoError(t, err) + response = ret + }) + require.NotNil(t, response) + require.NoError(t, VerifyInclusion(hash, response.Result.Proof, response.Root)) + checkName := CheckPrefix(clientTestName(t, "/operator/merkle")). + Append(enc.NewGenericBytesComponent(hash)) + require.True(t, checkName.Equal(mock.consumeName)) + require.Equal(t, enc.TypeGenericNameComponent, mock.consumeName.At(-1).Typ) +} + +func TestClientRejectsInvalidResponses(t *testing.T) { + hash := clientTestHash(0) + + require.Error(t, validateAppendResponse([][]byte{hash}, &defn.AppendResponse{ + Results: []*defn.AppendResult{{ + DataHash: hash, + Status: defn.AppendStatusOK, + }}, + })) + require.Error(t, validateCheckResponse(hash, &defn.CheckResponse{ + Root: &defn.TreeRoot{RootHash: make([]byte, HashSize)}, + Result: &defn.CheckResult{ + DataHash: hash, + Status: defn.CheckStatusIncluded, + }, + })) + require.Error(t, validateCheckResponse(hash, &defn.CheckResponse{ + Root: &defn.TreeRoot{RootHash: make([]byte, HashSize)}, + Result: &defn.CheckResult{ + DataHash: hash, + Status: defn.CheckStatusNotFound, + }, + })) +} + +type commandClientMock struct { + ndn.Client + dest enc.Name + name enc.Name + command enc.Wire + consumeName enc.Name + response enc.Wire + responseVersion uint64 + err error +} + +func (m *commandClientMock) ExpressCommand( + dest enc.Name, + name enc.Name, + command enc.Wire, + callback func(enc.Wire, error), +) { + m.dest = dest + m.name = name + m.command = command + callback(m.response, m.err) +} + +func (m *commandClientMock) ConsumeExt(args ndn.ConsumeExtArgs) { + m.consumeName = args.Name + args.Callback(&consumeStateMock{ + name: args.Name.Append(enc.NewVersionComponent(m.responseVersion)), + content: m.response, + err: m.err, + }) +} + +type consumeStateMock struct { + name enc.Name + content enc.Wire + err error +} + +func (s *consumeStateMock) Name() enc.Name { return s.name } +func (s *consumeStateMock) Version() uint64 { return s.name.At(-1).NumberVal() } +func (s *consumeStateMock) IsComplete() bool { return true } +func (s *consumeStateMock) Progress() int { return 1 } +func (s *consumeStateMock) ProgressMax() int { return 1 } +func (s *consumeStateMock) Error() error { return s.err } +func (s *consumeStateMock) Content() enc.Wire { return s.content } +func (s *consumeStateMock) Cancel() {} + +func clientTestName(t *testing.T, value string) enc.Name { + t.Helper() + name, err := enc.NameFromStr(value) + require.NoError(t, err) + return name +} + +func clientTestHash(index byte) []byte { + hash := sha256.Sum256([]byte{index}) + return hash[:] +} diff --git a/std/merklelog/policy_test.go b/std/merklelog/policy_test.go new file mode 100644 index 00000000..d412cf5a --- /dev/null +++ b/std/merklelog/policy_test.go @@ -0,0 +1,308 @@ +package merklelog + +import ( + "bytes" + "crypto/sha256" + "errors" + "fmt" + "testing" + "time" + + enc "github.com/named-data/ndnd/std/encoding" + "github.com/named-data/ndnd/std/ndn" + defn "github.com/named-data/ndnd/std/ndn/merklelog" + spec "github.com/named-data/ndnd/std/ndn/spec_2022" + "github.com/named-data/ndnd/std/object/storage" + sec "github.com/named-data/ndnd/std/security" + "github.com/named-data/ndnd/std/security/keychain" + "github.com/named-data/ndnd/std/security/signer" + "github.com/named-data/ndnd/std/types/optional" + "github.com/stretchr/testify/require" +) + +func TestCertExpiredPolicyAcceptsIngestTimeWithinValidity(t *testing.T) { + rawData := enc.Wire{[]byte("exact Data wire")} + validFrom := time.UnixMilli(1_700_000_000_000) + validUntil := validFrom.Add(time.Hour) + cert := policyTestCert(t, validFrom, validUntil) + + for _, ingestTime := range []time.Time{ + validFrom, + validFrom.Add(30 * time.Minute), + validUntil, + } { + t.Run(ingestTime.String(), func(t *testing.T) { + client := policyTestClient(t, rawData, ingestTime) + err := runPolicy(NewCertExpiredPolicy(client), ndn.CertExpiredCallbackArgs{ + Data: cert, + RawData: rawData, + Cert: cert, + }) + require.NoError(t, err) + }) + } +} + +func TestCertExpiredPolicyRejectsInvalidEvidence(t *testing.T) { + rawData := enc.Wire{[]byte("exact Data wire")} + validFrom := time.UnixMilli(1_700_000_000_000) + validUntil := validFrom.Add(time.Hour) + cert := policyTestCert(t, validFrom, validUntil) + args := ndn.CertExpiredCallbackArgs{Data: cert, RawData: rawData, Cert: cert} + + t.Run("outside validity", func(t *testing.T) { + client := policyTestClient(t, rawData, validUntil.Add(time.Millisecond)) + require.ErrorContains(t, runPolicy(NewCertExpiredPolicy(client), args), "outside certificate validity") + }) + + t.Run("not found", func(t *testing.T) { + dataHash := sha256.Sum256(rawData.Join()) + emptyTree := NewTree() + mock := &commandClientMock{response: (&defn.CheckResponse{ + Root: emptyTree.Root(), + Result: &defn.CheckResult{ + DataHash: dataHash[:], + Status: defn.CheckStatusNotFound, + }, + }).Encode()} + client, err := NewClient(mock, clientTestName(t, "/operator/merkle"), clientTestName(t, "/requester")) + require.NoError(t, err) + require.ErrorContains(t, runPolicy(NewCertExpiredPolicy(client), args), "not present") + }) + + t.Run("different wire", func(t *testing.T) { + client := policyTestClient(t, rawData, validFrom) + mismatchedArgs := args + mismatchedArgs.RawData = enc.Wire{[]byte("different Data wire")} + require.ErrorContains(t, runPolicy(NewCertExpiredPolicy(client), mismatchedArgs), "does not match requested Data hash") + }) + + t.Run("check error", func(t *testing.T) { + client, err := NewClient( + &commandClientMock{err: errors.New("timeout")}, + clientTestName(t, "/operator/merkle"), + clientTestName(t, "/requester"), + ) + require.NoError(t, err) + require.ErrorContains(t, runPolicy(NewCertExpiredPolicy(client), args), "timeout") + }) +} + +func TestCertExpiredPolicyRejectsMissingInputs(t *testing.T) { + validFrom := time.UnixMilli(1_700_000_000_000) + cert := policyTestCert(t, validFrom, validFrom.Add(time.Hour)) + client := policyTestClient(t, enc.Wire{[]byte("wire")}, validFrom) + policy := NewCertExpiredPolicy(client) + + require.ErrorContains(t, runPolicy(policy, ndn.CertExpiredCallbackArgs{}), "Data packet is nil") + require.ErrorContains(t, runPolicy(policy, ndn.CertExpiredCallbackArgs{ + Data: cert, + Cert: cert, + }), "wire is unavailable") + + noValidityWire, err := spec.Spec{}.MakeData( + clientTestName(t, "/certificate"), + &ndn.DataConfig{}, + nil, + signer.NewSha256Signer(), + ) + require.NoError(t, err) + noValidity, _, err := spec.Spec{}.ReadData(enc.NewWireView(noValidityWire.Wire)) + require.NoError(t, err) + require.ErrorContains(t, runPolicy(policy, ndn.CertExpiredCallbackArgs{ + Data: cert, + RawData: enc.Wire{[]byte("wire")}, + Cert: noValidity, + }), "invalid validity period") + + require.ErrorContains(t, runPolicy(NewCertExpiredPolicy(nil), ndn.CertExpiredCallbackArgs{}), "client is nil") +} + +func TestCertExpiredPolicyValidatesDataAndCertificateSeparately(t *testing.T) { + now := time.Now() + rootSigner, err := signer.KeygenEd25519(sec.MakeKeyName(clientTestName(t, "/root"))) + require.NoError(t, err) + rootWire, err := sec.SelfSign(sec.SignCertArgs{ + Signer: rootSigner, + NotBefore: now.Add(-3 * time.Hour), + NotAfter: now.Add(time.Hour), + }) + require.NoError(t, err) + rootCert, _, err := spec.Spec{}.ReadData(enc.NewWireView(rootWire)) + require.NoError(t, err) + + childSigner, err := signer.KeygenEd25519(sec.MakeKeyName(clientTestName(t, "/app"))) + require.NoError(t, err) + childKey, err := signer.MarshalSecretToData(childSigner) + require.NoError(t, err) + childWire, err := sec.SignCert(sec.SignCertArgs{ + Signer: rootSigner, + Data: childKey, + IssuerId: enc.NewGenericComponent("root"), + NotBefore: now.Add(-2 * time.Hour), + NotAfter: now.Add(-time.Hour), + }) + require.NoError(t, err) + childCert, childSigCovered, err := spec.Spec{}.ReadData(enc.NewWireView(childWire)) + require.NoError(t, err) + + dataWire, err := spec.Spec{}.MakeData( + clientTestName(t, "/app/data"), + &ndn.DataConfig{}, + enc.Wire{[]byte("payload")}, + signer.AsContextSigner(childSigner), + ) + require.NoError(t, err) + data, dataSigCovered, err := spec.Spec{}.ReadData(enc.NewWireView(dataWire.Wire)) + require.NoError(t, err) + + dataHash := sha256.Sum256(dataWire.Wire.Join()) + childHash := sha256.Sum256(childWire.Join()) + tree := NewTree() + entryWire, err := EncodeLogEntry(&defn.LogEntry{ + IngestTime: time.Duration(now.Add(-90*time.Minute).UnixMilli()) * time.Millisecond, + DataHashes: [][]byte{dataHash[:], childHash[:]}, + }) + require.NoError(t, err) + _, err = tree.Append(entryWire) + require.NoError(t, err) + + logTransport := &proofCommandClient{tree: tree} + logClient, err := NewClient( + logTransport, + clientTestName(t, "/operator/merkle"), + clientTestName(t, "/requester"), + ) + require.NoError(t, err) + + keychainStore := storage.NewMemoryStore() + keyChain := keychain.NewKeyChainMem(keychainStore) + require.NoError(t, keyChain.InsertCert(rootWire.Join())) + trust, err := sec.NewTrustConfig(keyChain, allowAllTrustSchema{}, []enc.Name{rootCert.Name()}) + require.NoError(t, err) + + validate := func(sigCovered enc.Wire) error { + result := make(chan error, 1) + trust.Validate(sec.TrustConfigValidateArgs{ + Data: data, + RawData: dataWire.Wire, + DataSigCov: sigCovered, + Fetch: func(name enc.Name, _ *ndn.InterestConfig, callback ndn.ExpressCallbackFunc) { + if name.IsPrefix(childCert.Name()) { + callback(ndn.ExpressCallbackArgs{ + Result: ndn.InterestResultData, + Data: childCert, + RawData: childWire, + SigCovered: childSigCovered, + }) + return + } + callback(ndn.ExpressCallbackArgs{Error: fmt.Errorf("certificate not found: %s", name)}) + }, + Callback: func(valid bool, err error) { + if !valid && err == nil { + err = errors.New("validation failed") + } + result <- err + }, + OnCertExpired: NewCertExpiredPolicy(logClient), + }) + return <-result + } + + require.ErrorContains(t, validate(enc.Wire{[]byte("invalid signature input")}), "signature is invalid") + require.Equal(t, [][]byte{dataHash[:]}, logTransport.queries) + require.NoError(t, validate(dataSigCovered)) + require.Equal(t, [][]byte{dataHash[:], dataHash[:], childHash[:]}, logTransport.queries) +} + +func policyTestClient(t *testing.T, rawData enc.Wire, ingestTime time.Time) *Client { + t.Helper() + dataHash := sha256.Sum256(rawData.Join()) + tree := NewTree() + entryWire, err := EncodeLogEntry(&defn.LogEntry{ + IngestTime: time.Duration(ingestTime.UnixMilli()) * time.Millisecond, + DataHashes: [][]byte{dataHash[:]}, + }) + require.NoError(t, err) + _, err = tree.Append(entryWire) + require.NoError(t, err) + proof, err := tree.InclusionProof(0) + require.NoError(t, err) + + mock := &commandClientMock{response: (&defn.CheckResponse{ + Root: tree.Root(), + Result: &defn.CheckResult{ + DataHash: dataHash[:], + Status: defn.CheckStatusIncluded, + Proof: proof, + }, + }).Encode(), responseVersion: tree.Size()} + client, err := NewClient(mock, clientTestName(t, "/operator/merkle"), clientTestName(t, "/requester")) + require.NoError(t, err) + return client +} + +func policyTestCert(t *testing.T, validFrom, validUntil time.Time) ndn.Data { + t.Helper() + wire, err := spec.Spec{}.MakeData( + clientTestName(t, "/certificate"), + &ndn.DataConfig{ + SigNotBefore: optional.Some(validFrom), + SigNotAfter: optional.Some(validUntil), + }, + nil, + signer.NewSha256Signer(), + ) + require.NoError(t, err) + cert, _, err := spec.Spec{}.ReadData(enc.NewWireView(wire.Wire)) + require.NoError(t, err) + return cert +} + +func runPolicy(policy ndn.CertExpiredCallback, args ndn.CertExpiredCallbackArgs) error { + result := make(chan error, 1) + policy(args, func(err error) { result <- err }) + select { + case err := <-result: + return err + case <-time.After(time.Second): + return errors.New("policy did not complete") + } +} + +type allowAllTrustSchema struct{} + +func (allowAllTrustSchema) Check(enc.Name, enc.Name) bool { return true } +func (allowAllTrustSchema) Suggest(enc.Name, ndn.KeyChain) ndn.Signer { return nil } + +type proofCommandClient struct { + ndn.Client + tree *Tree + queries [][]byte +} + +func (c *proofCommandClient) ConsumeExt(args ndn.ConsumeExtArgs) { + dataHash := args.Name.At(-1).Val + response := &defn.CheckResponse{ + Root: c.tree.Root(), + Result: &defn.CheckResult{ + DataHash: bytes.Clone(dataHash), + Status: defn.CheckStatusNotFound, + }, + } + c.queries = append(c.queries, bytes.Clone(dataHash)) + if leafIndex, ok := c.tree.Lookup(dataHash); ok { + response.Result.Status = defn.CheckStatusIncluded + proof, err := c.tree.InclusionProof(leafIndex) + if err != nil { + args.Callback(&consumeStateMock{err: err}) + return + } + response.Result.Proof = proof + } + args.Callback(&consumeStateMock{ + name: args.Name.Append(enc.NewVersionComponent(c.tree.Size())), + content: response.Encode(), + }) +} From e87c160aa6050585f327216d54cc98988947faeb Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Sun, 13 Sep 2026 16:15:38 -0700 Subject: [PATCH 7/8] merkle: design doc --- docs/merkle-log-design.md | 557 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 557 insertions(+) create mode 100644 docs/merkle-log-design.md diff --git a/docs/merkle-log-design.md b/docs/merkle-log-design.md new file mode 100644 index 00000000..4d15fe2a --- /dev/null +++ b/docs/merkle-log-design.md @@ -0,0 +1,557 @@ +# Merkle History Log Protocol and Expired-Certificate Policy + +Status: Experimental Design Specification + +Design revision: 1 (not encoded on the wire) + +## Abstract + +An expired certificate does not necessarily make every packet signed by its +key unusable. A consumer can validate historical Data if it has trustworthy +evidence that the exact packet existed while its signing certificate was +valid. + +The Merkle history log records SHA-256 hashes of complete Data packets as +encoded on the wire. An authorized requester explicitly submits hashes; the +log assigns an ingestion time and commits them to an append-only Merkle tree. +A consumer later retrieves a signed root and inclusion proof. An asynchronous +history policy accepts an expired validation relation only when the current +packet was logged within the authorizing certificate's validity period. +Normal trust-schema and signature validation still applies. + +This revision trusts one log operator. It does not provide consistency proofs, +witnesses, gossip, or protection against equivocation by that operator. + +## 1. Scope and system model + +This revision provides: + +- explicit authenticated submission of one or more packet hashes; +- one log-assigned timestamp and at most one leaf per append request; +- first-ingestion semantics for duplicate hashes; +- a durable Merkle tree and hash-to-leaf index; +- public single-hash checks with signed, possibly segmented responses; and +- an asynchronous expiry policy that works through certificate recursion. + +It does not discover packets automatically, retrieve packets represented by +submitted hashes, prove that the log clock is honest, or provide a +cryptographic non-inclusion proof. + +```text + explicit submission + +------------------+ signed command +----------------------+ + | Append requester | ------------------> | Merkle log | + | hashes exact wire| | | + +------------------+ public check | | + | Data consumer | ------------------> | | + | + trust policy | <------------------ | +----------------+ | + | + expiry policy | signed proof object| | Merkle tree | | + +------------------+ | +----------------+ | + +----------------------+ +``` + +The requester computes hashes from the exact packet wire. The log +authenticates requesters, assigns ingestion times, atomically extends the +tree, and signs responses. Consumers authenticate responses and verify proofs +under their configured trust policy. The application chooses the expiry +policy and any larger operation deadline. + +The log receives hashes rather than packets, so it cannot prove that the +corresponding bytes were observed. Authentication only binds a submission to +an authorized requester; the admission policy decides whose observation +claims to trust. + +## 2. Packet and tree model + +### 2.1. Exact packet identity + +For complete Data wire `W`: + +```text +DataHash = SHA256(W) +``` + +`W` includes the outer Data TLV, Name, MetaInfo, Content, SignatureInfo, and +SignatureValue exactly as encoded. Certificates are Data packets and use the +same calculation. This is the digest used by an NDN +ImplicitSha256DigestComponent, although this protocol transports it as a +generic name component and in `DataHash` TLVs. + +Hashing only Content, only signature-covered wire, or a re-encoded parsed Data +packet is invalid. Re-encoding MUST NOT be assumed to reproduce the received +bytes. The validation interface MUST preserve the exact received Data wire; +missing exact wire MUST cause the history policy to reject. + +### 2.2. Log entries and batching + +One accepted append request creates at most one leaf: + +```text +LogEntry = IngestTime 1*DataHash +``` + +Only the first occurrence of each new 32-byte hash is placed in the entry, in +request order. Existing hashes and later duplicates within the request MUST NOT +be inserted again. + +Consequently, multiple hashes may share one leaf, timestamp, and leaf index: + +```text +AppendRequest [H1, H2, H3] + | + v +Leaf 7 = LogEntry(Time, H1, H2, H3) +``` + +A check for any of those hashes returns the complete `LogEntry`, including the +other hashes. Batching therefore reduces leaf count but exposes which hashes +were submitted together. + +`IngestTime` is an integer count of whole milliseconds since the Unix epoch in +the inclusive range 0 through 9,223,372,036,854. Sub-millisecond values are +invalid. The on-the-wire encoding is the minimum-length unsigned integer that +represents the value. + +### 2.3. Hashing and tree shape + +```text +LeafHash(E) = SHA256(0x00 || CompleteLogEntryTLV(E)) +NodeHash(L, R) = SHA256(0x01 || L || R) +EmptyRoot = SHA256(empty byte string) +``` + +The domain bytes distinguish leaves from interior nodes. The complete leaf +input includes the `LogEntry` TLV with type `0x1E12`, its canonical length, +and its raw TLV-VALUE. A proof carries the unmodified TLV-VALUE; the verifier +MUST restore the fixed outer type and canonical length before hashing. + +For more than one leaf, the split point is the largest power of two strictly +smaller than the leaf count. For five leaves: + +```text + Root + / \ + N(0..3) L4 + / \ + N(0..1) N(2..3) + / \ / \ + L0 L1 L2 L3 +``` + +Leaves MUST be zero-indexed. Sibling hashes are ordered from the leaf upward. +The path for `L2` above is `[L3, N(0..1), L4]`. The verifier MUST derive +left/right placement from `LeafIndex` and `TreeSize`, and MUST reject missing +or extra siblings, invalid lengths, out-of-range indexes, and root mismatches. + +### 2.4. Time and complexity + +The log MUST assign each accepted leaf an `IngestTime` strictly greater than +every previously assigned `IngestTime`. The log MUST guarantee this property +across all observable appends, regardless of clock behaviour or transient +failures. It does not make the clock trustworthy. + +Incremental root calculation, proof size, and proof verification require +`O(log N)` hash work or space. + +## 3. TLV wire format + +### 3.1. Assigned types + +```text +Type name Hex Decimal +------------------------------------ +DataHash 0x1E00 7680 +AppendResult 0x1E02 7682 +Status 0x1E04 7684 +LeafIndex 0x1E06 7686 +TreeRoot 0x1E08 7688 +CheckResult 0x1E0A 7690 +InclusionProof 0x1E0C 7692 +TreeSize 0x1E0E 7694 +RootHash 0x1E10 7696 +LogEntry 0x1E12 7698 +SiblingHash 0x1E14 7700 +IngestTime 0x1E16 7702 +``` + +Append request, append response, and check response are content models without +additional outer wrapper TLVs. + +### 3.2. Grammar + +```text +AppendRequest = 1*DataHash + +AppendResponse = 1*AppendResult +AppendResult = DataHash Status [LeafIndex] + +CheckResponse = TreeRoot CheckResult +TreeRoot = TreeSize RootHash +CheckResult = DataHash Status [InclusionProof] +InclusionProof = LogEntry LeafIndex *SiblingHash + +LogEntry = IngestTime 1*DataHash +``` + +Every hash is 32 bytes. `TreeSize` and `LeafIndex` are unsigned natural +numbers. + +Append status values and invariants are: + +```text +Value Name LeafIndex Meaning +---------------------------------------------------------- +0 OK REQUIRED hash was placed in a new leaf +1 DUPLICATE REQUIRED hash already belongs to an existing leaf +2 FAILED MUST be absent hash was not appended +``` + +Check status values are: + +```text +Value Name InclusionProof +-------------------------------- +0 INCLUDED REQUIRED +1 NOT_FOUND MUST be absent +``` + +The response `DataHash` MUST equal the request. `NOT_FOUND` is a signed log +statement for that root, not a Merkle non-inclusion proof or a statement about +future roots. + +### 3.3. Protocol names + +Let `P` be the log prefix. `append` and `check` are keyword components of type +32. A queried hash is a generic component of type 8. + +```text +Append command prefix: + P / 32=append + +Embedded signed AppendRequest Data name: + P / 32=append / / t= + +Append Interest name: + P / 32=append / params-sha256= + +Check Interest name: + P / 32=check / 8=<32-byte-DataHash> + +Check object and segments: + P / 32=check / 8= / v= + P / 32=check / 8= / v= / seg= +``` + +The append Interest application parameters contain the complete signed request +Data. NDN encoding adds the ParametersSha256DigestComponent; the log MUST NOT +require a second request digest. Append response Data uses the complete +Interest name. A check object version MUST equal its `TreeSize`. + +## 4. Append protocol + +```text +Requester Merkle log + | | + | hash exact packet wire | + | build and sign AppendRequest Data | + |-- Interest + application parameters ----->| + | | authenticate request + | | validate name/time + | | classify hashes + | | persist one update + |<--------- signed AppendResponse -----------| + | validate signature and result fields | +``` + +The log authenticates the complete embedded request Data under its configured +trust policy. The request Data name MUST: + +1. begin with `P/32=append`; +2. contain at least one requester-name component; and +3. end in a canonical timestamp within one minute of log time. + +The timestamp is a freshness check, not replay prevention. The log MUST NOT +use it as `IngestTime`. + +An append command that fails authentication, has unparsable content, has a +malformed name, has a timestamp outside the freshness window, or contains zero +`DataHash` TLVs MUST receive no response. The log MUST NOT partially process +such a command. A parseable `DataHash` TLV whose value is not 32 bytes is an +invalid item rather than a malformed command and receives `FAILED` below. + +For each requested hash, the log reports: + +- `FAILED` for a non-32-byte value; +- `DUPLICATE` and the existing leaf for a hash already in the log; +- `OK` for the first occurrence of each previously-unseen hash; or +- `DUPLICATE` with the new leaf index for a later occurrence within the same + request of a hash that was `OK` earlier in the same request. + +If any new hashes remain after classification, they form one `LogEntry` and +one atomic tree update. Every `OK` result and every within-request +later-occurrence receives that new leaf index. If persistence fails, results +dependent on the new entry become `FAILED`; pre-existing duplicates remain +`DUPLICATE`. If nothing is new, no leaf or `IngestTime` is created. Results +remain in request order. + +The client validates the signed response, result count, corresponding hashes, +status values, and `LeafIndex` presence for `OK` and `DUPLICATE`. Append +responses contain no root or proof. + +Append requests and responses are not segmented. Their fully encoded Interest +and Data packets MUST NOT exceed the 8,800-byte maximum NDN packet size. The +command transport rejects an oversized packet; a requester MUST split a larger +submission into multiple commands, each becoming a separate batching and +timestamp boundary. Retrying a newly signed command is safe because hashes have +first-ingestion semantics. Check responses use segmentation as described below. + +## 5. Check protocol + +```text +Consumer/expiry policy Merkle log + | | + |-- Interest P/32=check/8= ---------->| + | CanBePrefix=true, MustBeFresh=true | + | | snapshot root + | | lookup hash -> leaf + | | build proof/status + |<-- ...//v=/seg=0 -------------| + |-- additional segment Interests ---------->| + |<-- signed segments ------------------------| + | validate, assemble, parse, verify | +``` + +The initial Interest contains exactly one generic 32-byte hash after +`P/32=check` and sets `CanBePrefix`. Exact names ending in version and segment +are used to retrieve already-produced segments. Other shapes receive no +response. + +The log evaluates each check against one atomic tree snapshot and resolves the +requested hash to its first leaf index. If present, it returns the complete +leaf `LogEntry`, its index, and the sibling path. Otherwise it returns +`NOT_FOUND` with the snapshot root. Multiple hashes from one append may map to +the same leaf. + +The response is a metadata-free, standard NDN segmented object. Every segment +MUST be signed and authenticated, and all segments MUST represent the same +tree snapshot. An unversioned query MUST use the latest tree size. + +For an empty tree (`TreeSize = 0`), every check returns `NOT_FOUND` with the +empty-tree root. A versioned segment Interest whose `TreeSize` is no longer +available, or whose segment number is past the last segment, MUST receive no +response. + +Before succeeding, the client MUST: + +1. validate Object Data signatures under the log trust configuration; +2. parse the response content and confirm a 32-byte root; +3. confirm the object version equals `TreeSize`; +4. confirm the returned hash equals the requested hash; +5. confirm a known status with correct proof presence; and +6. validate the inclusion proof for `INCLUDED`. + +For an empty tree, the root MUST equal `SHA256("")`. + +## 6. Persistence + +The log MUST durably preserve ordered entries, the hash-to-first-leaf mapping, +tree size, root, and last ingestion time. Appends are atomic: any external +observation refers to either the complete state before the append or the +complete state after the append, never an intermediate state. + +The log MUST ensure its current state is consistent with the ordered entries. +Inconsistencies — including non-contiguous leaves, duplicate first-ingestion +records, non-monotonic ingestion times, invalid indexes, or a mismatched tree +root — MUST cause the log to refuse any operation that depends on the +inconsistent state. + +## 7. Expired-certificate integration + +The asynchronous history policy requires the current Data, its exact received +wire, the authorizing certificate, and a complete ordered validity interval on +that certificate. + +```text +H = SHA256(DataWire) +R = Check(H) + +require R.Status == INCLUDED +require inclusion proof verifies H against R.Root +E = parse R.Proof.LogEntry +require Cert.NotBefore <= E.IngestTime <= Cert.NotAfter + +complete(success) +``` + +Both validity endpoints are inclusive. The policy does not use the signer's +asserted `SignatureTime`. The response and proof are verified before the policy +evaluates ingestion time. Missing wire or evidence, invalid validity, retrieval +failure, response-signature failure, proof failure, and out-of-range time all +reject the relation. + +The policy starts the check and completes asynchronously. The application MAY +bound the larger validation operation with a deadline. + +### 7.1. Recursive validation + +For Data `D` signed by child certificate `C1`, itself signed by `C2`: + +```text + relation 1 relation 2 + D <-------------- C1 <----------------- C2 + | | + | +-- Check(SHA256(wire(C1))) + +-- Check(SHA256(wire(D))) +``` + +If `C1` is expired, the callback MAY be invoked twice: + +1. `Data=D, Cert=C1`: `D` MUST have been logged during `C1` validity. +2. `Data=C1, Cert=C2`: the exact `C1` packet MUST have been logged during `C2` + validity. + +Every affected packet-to-authorizer relation requires separate evidence. The +same contract applies to expired cross-schema relations: the current Data wire +is hashed and the authorizer's validity is applied. + +After policy acceptance, the validator still checks the schema and packet +signature. A proof cannot authorize a forbidden signer or repair an invalid +signature. + +### 7.2. Wire preservation and completion + +Exact packet wire MUST remain associated with the packet through every step of +recursive validation and any intermediate storage. A non-anchor certificate +retained as evidence is not, by itself, a reusable validation decision; its +chain and expiry relation are revalidated each time the evidence is consumed. +An asynchronous policy MUST resolve each validation relation exactly once. + +### 7.3. Trust-anchor promotion + +Use of the history policy MUST NOT by itself prevent trust-anchor promotion. +After an authorized `CertList` (see [certlist.md](certlist.md)) supplies a +listed certificate for the same key and that certificate's complete validation +succeeds, including any required history checks, the self-signed anchor +candidate MAY be promoted. Promotion is a decision of the trust configuration's +anchor-admission policy; the history policy itself MUST NOT promote. Once +promoted, the certificate MUST be treated like another trust anchor under that +trust configuration. Ordinary non-anchor retention MUST NOT perform this +promotion. + +## 8. Trust and security considerations + +Configured trust policies authorize append command Data and response-signing +identities. Consumers MUST authenticate log responses independently. The +history policy MUST NOT be used recursively to justify an expired log-response +signer through the same log. Supplying the independent response trust policy is +the consumer's responsibility; an accept-all validator does not conform to +this design. + +The log operator is trusted to assign honest times and maintain one history. A +proof authenticates membership relative to a signed response root, but the +operator can still backdate entries, equivocate between clients, or lie in a +`NOT_FOUND` response. Independent checkpoints and auditors are needed to +address those threats. + +The policy supplements normal validation; it does not replace schema, +key-locator, certificate-chain, or signature checks. Its packet binding relies +on SHA-256 and on the caller supplying exact wire that actually corresponds to +the parsed Data and signature-covered wire. + +This revision defines no append rate limit, tree-pruning policy, or +check-response retention duration. Deployments require admission and resource +controls. + +Unauthenticated commands, unparsable command content, invalid command names, +empty append requests, and malformed check names MUST receive no response. +Parseable append requests containing a `DataHash` with an invalid length +receive a per-item `FAILED` result. Clients MUST reject malformed, mismatched, +unsigned, untrusted, or cryptographically invalid responses. There is no +machine-readable protocol error object in this revision. + +## 9. Auditor model (Discussion) + +Status: discussion only. This section is non-normative and outside the current +design. + +A stateless consumer cannot determine whether the log has rewritten history or +shown different roots to different clients. A possible future model assigns +that work to independent, stateful auditors: + +```text + +------------------+ signed append +------------------+ + | Append requester | -------------------> | Merkle log | + +------------------+ +------------------+ + | | entries, + | verify own inclusion | checkpoints + v v + +------------------+ approved root +------------------+ + | Data consumer | <------------------- | Auditor(s) | + | + expiry policy | | retained history | + +------------------+ +------------------+ + | + | gossip + v + +------------------+ + | Other auditors | + | or witnesses | + +------------------+ +``` + +The policies remain distinct: + +- Log admission policy: who may append and which key signs responses. +- Auditor policy: whether log transitions are append-only, timely, and + consistent with previously accepted state. +- Consumer history policy: whether approved evidence permits one expired + validation relation. + +A future signed checkpoint might contain: + +```text +LogCheckpoint { + LogName + TreeSize + RootHash + CheckpointTime + LastIngestTime +} +``` + +Given an authenticated entry stream, an auditor could verify the log signature, +stable roots at repeated sizes, a consistency proof from its prior root, +canonical new entries, unique hashes, strictly increasing ingestion times, and +configured clock-skew limits. It could then sign an approval for the +checkpoint. Conflicting signed states would be retained and reported rather +than silently replaced. + +A consumer could require an exact proof root approved by one auditor or a +`k-of-n` quorum, subject to maximum approval age and clock-skew policy: + +```text +exact packet hash + inclusion proof + approved root + valid ingestion time + + normal schema and signature validation +``` + +Append requesters can separately monitor inclusion of their own submissions, +but cannot detect a split view without gossip. + +This model would require signed checkpoint publication, consistency proofs, +entry retrieval by index or incremental feed, approval encoding, gossip, a +minimum-age requirement on checkpoint acceptance, and checks against a +specified tree size. This revision's `Check` operation always proves against +the latest root. An auditor retrieving entries would also see every hash in +each batched leaf. + +## 10. Open design issues + +Deferred work includes: + +1. a proactive append batch bound that accounts for response-signing overhead; +2. signed checkpoints and consistency proofs; +3. auditor or witness approvals and gossip; +4. cryptographic or independently verifiable non-inclusion; +5. requester retry semantics; +6. scalable proof generation; +7. check-response retention and garbage collection; and +8. explicit version negotiation and stable type allocation. From 0cf95769dfe4e670ad64ec9b209245f35f156538 Mon Sep 17 00:00:00 2001 From: Tianyuan Yu Date: Thu, 17 Sep 2026 02:01:09 +0000 Subject: [PATCH 8/8] merkle: doc update --- docs/merkle-log-design.md | 176 +++++++++++++++++++++++++++----------- 1 file changed, 128 insertions(+), 48 deletions(-) diff --git a/docs/merkle-log-design.md b/docs/merkle-log-design.md index 4d15fe2a..1110f65d 100644 --- a/docs/merkle-log-design.md +++ b/docs/merkle-log-design.md @@ -4,6 +4,12 @@ Status: Experimental Design Specification Design revision: 1 (not encoded on the wire) +This specification describes the current [log service](../merkle/api.go), +[Merkle core and client](../std/merklelog/), +[wire models](../std/ndn/merklelog/definitions.go), and +[trust validator](../std/security/trust_config.go). Implementation limits are +called out below; the auditor model remains future work. + ## Abstract An expired certificate does not necessarily make every packet signed by its @@ -109,8 +115,9 @@ were submitted together. `IngestTime` is an integer count of whole milliseconds since the Unix epoch in the inclusive range 0 through 9,223,372,036,854. Sub-millisecond values are -invalid. The on-the-wire encoding is the minimum-length unsigned integer that -represents the value. +invalid. The on-the-wire encoding is a big-endian NDN non-negative integer, +using the shortest permitted width (1, 2, 4, or 8 bytes) that represents the +value. ### 2.3. Hashing and tree shape @@ -125,6 +132,14 @@ input includes the `LogEntry` TLV with type `0x1E12`, its canonical length, and its raw TLV-VALUE. A proof carries the unmodified TLV-VALUE; the verifier MUST restore the fixed outer type and canonical length before hashing. +The tree accepts only complete entries matching their canonical encoding: one +`IngestTime` followed by the ordered `DataHash` TLVs, with canonical type, +length, and integer encodings. The proof verifier applies the same check after +restoring the outer wrapper, so the preserved TLV-VALUE must be canonical too. +Reordered fields, additional fields, repeated `IngestTime`, or non-canonical +inner encodings are rejected. Canonicality is checked by comparison with a +re-encoding; the hash input retains the received entry value. + For more than one leaf, the split point is the largest power of two strictly smaller than the leaf count. For five leaves: @@ -145,13 +160,20 @@ or extra siblings, invalid lengths, out-of-range indexes, and root mismatches. ### 2.4. Time and complexity -The log MUST assign each accepted leaf an `IngestTime` strictly greater than -every previously assigned `IngestTime`. The log MUST guarantee this property -across all observable appends, regardless of clock behaviour or transient -failures. It does not make the clock trustworthy. - -Incremental root calculation, proof size, and proof verification require -`O(log N)` hash work or space. +The log assigns each new leaf the current Unix time truncated to milliseconds, +or the last committed ingestion time plus one millisecond, whichever is later. +The tree MUST reject an entry whose time is not strictly later than its +predecessor's. The last committed time is restored on restart. An append that +fails before committing does not reserve a timestamp; a later attempt may +reuse that candidate time. Monotonicity therefore applies to committed leaves +and does not make the clock trustworthy or bound its drift from wall time. + +For `N` leaves and `B` hashes in the proven entry, incremental root calculation +uses `O(log N)` tree-hash work. The sibling path contains `O(log N)` hashes, +but the proof also carries the entire entry: total proof size and verification +work are `O(B + log N)`. The current implementation keeps entries, leaf hashes, +and the hash index in memory. It generates a proof by recomputing sibling +subtrees from leaf hashes, requiring `O(N)` hash work per proof. ## 3. TLV wire format @@ -193,8 +215,10 @@ InclusionProof = LogEntry LeafIndex *SiblingHash LogEntry = IngestTime 1*DataHash ``` -Every hash is 32 bytes. `TreeSize` and `LeafIndex` are unsigned natural -numbers. +Valid hashes are 32 bytes. An append result with status `FAILED` can echo a +request's invalid-length `DataHash` as described in Section 4. `TreeSize`, +`LeafIndex`, and `Status` are unsigned NDN non-negative integers; the encoder +uses the shortest permitted width of 1, 2, 4, or 8 bytes. Append status values and invariants are: @@ -202,7 +226,8 @@ Append status values and invariants are: Value Name LeafIndex Meaning ---------------------------------------------------------- 0 OK REQUIRED hash was placed in a new leaf -1 DUPLICATE REQUIRED hash already belongs to an existing leaf +1 DUPLICATE REQUIRED hash was already logged, including earlier + in this request 2 FAILED MUST be absent hash was not appended ``` @@ -295,15 +320,22 @@ dependent on the new entry become `FAILED`; pre-existing duplicates remain remain in request order. The client validates the signed response, result count, corresponding hashes, -status values, and `LeafIndex` presence for `OK` and `DUPLICATE`. Append -responses contain no root or proof. +status values, `LeafIndex` presence for `OK` and `DUPLICATE`, and its absence +for `FAILED`. The supplied client rejects empty submissions and invalid-length +hashes locally, before sending a command. Append responses contain no root or +proof. Append requests and responses are not segmented. Their fully encoded Interest and Data packets MUST NOT exceed the 8,800-byte maximum NDN packet size. The -command transport rejects an oversized packet; a requester MUST split a larger +current Merkle client and command handler do not preflight those sizes or split +batches automatically; oversized-packet handling depends on the transport. A +requester MUST allow for both request and response overhead and split a larger submission into multiple commands, each becoming a separate batching and -timestamp boundary. Retrying a newly signed command is safe because hashes have -first-ingestion semantics. Check responses use segmentation as described below. +timestamp boundary. The append commits before the response is encoded and +sent, so a response failure does not roll back the append. Retrying a newly +signed command is safe because hashes have first-ingestion semantics. The +append client performs no automatic retries. Check responses use segmentation +as described below. ## 5. Check protocol @@ -322,9 +354,10 @@ Consumer/expiry policy Merkle log ``` The initial Interest contains exactly one generic 32-byte hash after -`P/32=check` and sets `CanBePrefix`. Exact names ending in version and segment -are used to retrieve already-produced segments. Other shapes receive no -response. +`P/32=check` and sets `CanBePrefix` and `MustBeFresh`. The service requires +`CanBePrefix` for this name shape but does not enforce `MustBeFresh`. Exact +names ending in version and segment are used to retrieve already-produced +segments. Other shapes receive no response. The log evaluates each check against one atomic tree snapshot and resolves the requested hash to its first leaf index. If present, it returns the complete @@ -334,12 +367,22 @@ the same leaf. The response is a metadata-free, standard NDN segmented object. Every segment MUST be signed and authenticated, and all segments MUST represent the same -tree snapshot. An unversioned query MUST use the latest tree size. +tree snapshot. An unversioned query that reaches the service uses the tree +size at the time the service takes its snapshot. + +The service uses the Object producer's default four-second `FreshnessPeriod` +and reuses stored segments for an already-produced hash/version pair. A fresh +cached response may satisfy an unversioned Interest even after the tree has +grown. `MustBeFresh` does not guarantee the latest tree size or establish a +checkpoint age. The client verifies the returned snapshot but does not track +previously observed roots or enforce a minimum tree size. For an empty tree (`TreeSize = 0`), every check returns `NOT_FOUND` with the empty-tree root. A versioned segment Interest whose `TreeSize` is no longer available, or whose segment number is past the last segment, MUST receive no -response. +response. Versioned requests only look up cached segments; they do not generate +proofs for arbitrary historical tree sizes. Previously produced segments can +remain available after the tree grows or the service restarts. Before succeeding, the client MUST: @@ -354,16 +397,30 @@ For an empty tree, the root MUST equal `SHA256("")`. ## 6. Persistence -The log MUST durably preserve ordered entries, the hash-to-first-leaf mapping, -tree size, root, and last ingestion time. Appends are atomic: any external -observation refers to either the complete state before the append or the -complete state after the append, never an intermediate state. - -The log MUST ensure its current state is consistent with the ordered entries. -Inconsistencies — including non-contiguous leaves, duplicate first-ingestion -records, non-monotonic ingestion times, invalid indexes, or a mismatched tree -root — MUST cause the log to refuse any operation that depends on the -inconsistent state. +The service stores ordered entries, the hash-to-first-leaf mapping, tree size, +root, frontier, and last ingestion time in a dedicated Badger database with +synchronous writes. Each append updates these records in one transaction. +Only after the store reports success does the in-memory tree advance. The +service serializes appends and check snapshots with one mutex, so a snapshot +contains either the complete state before an append or the complete state +after it. + +On opening the tree, the implementation replays all entries and compares the +reconstructed state and hash index with the persisted records. Non-contiguous +leaves, duplicate hashes, non-increasing ingestion times, invalid indexes, or +inconsistent root, frontier, or last ingestion time cause startup to fail. + +Each store append checks the expected tree size, stored frontier/root state, +new entry, and affected index records. A rejected write leaves the in-memory +tree unchanged. This is not a continuous integrity audit: checks and duplicate +lookups use the in-memory tree without rereading the database. A store error +does not disable subsequent reads or automatically reload the tree. The +service assumes a single writer and an atomic store with reliable commit +reporting; recovery from an ambiguous commit outcome is not implemented. + +Signed check objects use a separate packet database. Their creation and +retention are independent of the log transaction; a committed leaf remains +committed even if producing or sending its response fails. ## 7. Expired-certificate integration @@ -394,6 +451,13 @@ bound the larger validation operation with a deadline. ### 7.1. Recursive validation +The validator invokes the policy when the authorizing certificate is outside +its validity period, when the current Data is itself a certificate outside its +validity period, or when an expired cross-schema packet is being recursively +validated. The implementation's expiry predicate also includes not-yet-valid +certificates and certificates with missing validity endpoints. The history +policy still requires the authorizer's complete, ordered validity interval. + For Data `D` signed by child certificate `C1`, itself signed by `C2`: ```text @@ -404,7 +468,8 @@ For Data `D` signed by child certificate `C1`, itself signed by `C2`: +-- Check(SHA256(wire(D))) ``` -If `C1` is expired, the callback MAY be invoked twice: +If `C1` is expired and is not already a trust anchor, successful recursive +validation requires two policy acceptances: 1. `Data=D, Cert=C1`: `D` MUST have been logged during `C1` validity. 2. `Data=C1, Cert=C2`: the exact `C1` packet MUST have been logged during `C2` @@ -425,18 +490,29 @@ recursive validation and any intermediate storage. A non-anchor certificate retained as evidence is not, by itself, a reusable validation decision; its chain and expiry relation are revalidated each time the evidence is consumed. An asynchronous policy MUST resolve each validation relation exactly once. +The validator resumes a relation only on the first completion; subsequent +completion calls are ignored. There is no built-in policy deadline. ### 7.3. Trust-anchor promotion -Use of the history policy MUST NOT by itself prevent trust-anchor promotion. -After an authorized `CertList` (see [certlist.md](certlist.md)) supplies a -listed certificate for the same key and that certificate's complete validation -succeeds, including any required history checks, the self-signed anchor -candidate MAY be promoted. Promotion is a decision of the trust configuration's -anchor-admission policy; the history policy itself MUST NOT promote. Once -promoted, the certificate MUST be treated like another trust anchor under that -trust configuration. Ordinary non-anchor retention MUST NOT perform this -promotion. +Use of the history policy does not prevent trust-anchor promotion. The current +trust validator promotes a self-signed anchor candidate after an authorized +`CertList` (see [certlist.md](certlist.md)) supplies a certificate with the same +key content and that certificate validates successfully, or is already a +trust anchor. The history policy itself does not perform the promotion. + +If the listed certificate is expired, the `CertList` packet must also pass the +policy with `Data=CertList, Cert=listed certificate`. Its exact wire must have +been logged during the listed certificate's validity. Recursive validation of +the listed certificate applies any additional history checks. An expired +self-signed candidate also requires evidence for its own wire against its own +validity period before the validator explores the `CertList`. + +Once promoted, the candidate is treated as a trust anchor in that trust +configuration. Later validations can terminate their chain at it without +revalidating the admitting `CertList` and listed certificate. Expiry checks for +the current packet-to-anchor relation still apply. Ordinary non-anchor +retention does not perform this promotion. ## 8. Trust and security considerations @@ -539,9 +615,11 @@ but cannot detect a split view without gossip. This model would require signed checkpoint publication, consistency proofs, entry retrieval by index or incremental feed, approval encoding, gossip, a minimum-age requirement on checkpoint acceptance, and checks against a -specified tree size. This revision's `Check` operation always proves against -the latest root. An auditor retrieving entries would also see every hash in -each batched leaf. +specified tree size. This revision's unversioned `Check` takes a current +snapshot when it reaches the service, but caches may return an earlier +snapshot. It does not support generating a proof at a caller-selected tree +size. An auditor retrieving entries would also see every hash in each batched +leaf. ## 10. Open design issues @@ -551,7 +629,9 @@ Deferred work includes: 2. signed checkpoints and consistency proofs; 3. auditor or witness approvals and gossip; 4. cryptographic or independently verifiable non-inclusion; -5. requester retry semantics; +5. configurable requester retries and recovery from ambiguous commit outcomes; 6. scalable proof generation; -7. check-response retention and garbage collection; and -8. explicit version negotiation and stable type allocation. +7. check-response retention and garbage collection; +8. explicit version negotiation and stable type allocation; +9. checkpoint freshness and minimum-tree-size requirements; and +10. continuous store-integrity auditing and service behaviour after store errors.