From 43b4d73dc35301417cf08acfd79dd768dba3f517 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 20:39:27 +0900 Subject: [PATCH 001/162] store: add migration version import export --- kv/leader_routed_store.go | 24 ++ kv/shard_store.go | 12 + store/lsm_migration.go | 349 ++++++++++++++++++++++++++++ store/lsm_store.go | 3 +- store/migration_versions.go | 379 +++++++++++++++++++++++++++++++ store/migration_versions_test.go | 217 ++++++++++++++++++ store/mvcc_store.go | 16 +- store/store.go | 60 +++++ 8 files changed, 1053 insertions(+), 7 deletions(-) create mode 100644 store/lsm_migration.go create mode 100644 store/migration_versions.go create mode 100644 store/migration_versions_test.go diff --git a/kv/leader_routed_store.go b/kv/leader_routed_store.go index d484431a1..3f6004242 100644 --- a/kv/leader_routed_store.go +++ b/kv/leader_routed_store.go @@ -503,6 +503,30 @@ func (s *LeaderRoutedStore) Compact(ctx context.Context, minTS uint64) error { return errors.WithStack(s.local.Compact(ctx, minTS)) } +func (s *LeaderRoutedStore) ExportVersions(ctx context.Context, opts store.ExportVersionsOptions) (store.ExportVersionsResult, error) { + if s == nil || s.local == nil { + return store.ExportVersionsResult{}, errors.WithStack(store.ErrNotSupported) + } + result, err := s.local.ExportVersions(ctx, opts) + return result, errors.WithStack(err) +} + +func (s *LeaderRoutedStore) ImportVersions(ctx context.Context, opts store.ImportVersionsOptions) (store.ImportVersionsResult, error) { + if s == nil || s.local == nil { + return store.ImportVersionsResult{}, errors.WithStack(store.ErrNotSupported) + } + result, err := s.local.ImportVersions(ctx, opts) + return result, errors.WithStack(err) +} + +func (s *LeaderRoutedStore) MigrationHLCFloor(ctx context.Context, jobID uint64) (uint64, error) { + if s == nil || s.local == nil { + return 0, errors.WithStack(store.ErrNotSupported) + } + floor, err := s.local.MigrationHLCFloor(ctx, jobID) + return floor, errors.WithStack(err) +} + func (s *LeaderRoutedStore) Snapshot() (store.Snapshot, error) { if s == nil || s.local == nil { return nil, errors.WithStack(store.ErrNotSupported) diff --git a/kv/shard_store.go b/kv/shard_store.go index bb0c15035..175c81224 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -1591,6 +1591,18 @@ func (s *ShardStore) Snapshot() (store.Snapshot, error) { return nil, store.ErrNotSupported } +func (s *ShardStore) ExportVersions(context.Context, store.ExportVersionsOptions) (store.ExportVersionsResult, error) { + return store.ExportVersionsResult{}, store.ErrNotSupported +} + +func (s *ShardStore) ImportVersions(context.Context, store.ImportVersionsOptions) (store.ImportVersionsResult, error) { + return store.ImportVersionsResult{}, store.ErrNotSupported +} + +func (s *ShardStore) MigrationHLCFloor(context.Context, uint64) (uint64, error) { + return 0, store.ErrNotSupported +} + func (s *ShardStore) Restore(_ io.Reader) error { return store.ErrNotSupported } diff --git a/store/lsm_migration.go b/store/lsm_migration.go new file mode 100644 index 000000000..88d56e9fe --- /dev/null +++ b/store/lsm_migration.go @@ -0,0 +1,349 @@ +package store + +import ( + "bytes" + "context" + "math" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +func (s *pebbleStore) ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { + pos, err := decodeExportCursor(opts.Cursor) + if err != nil { + return ExportVersionsResult{}, err + } + if opts.MaxVersions <= 0 { + return ExportVersionsResult{Done: true}, nil + } + + s.dbMu.RLock() + defer s.dbMu.RUnlock() + + iter, err := s.db.NewIter(pebbleExportIterOptions(opts)) + if err != nil { + return ExportVersionsResult{}, errors.WithStack(err) + } + defer iter.Close() + + seek, err := pebbleExportSeekKey(opts, pos) + if err != nil { + return ExportVersionsResult{}, err + } + result := newExportVersionsResult(opts.MaxVersions) + if err := s.runPebbleExportLoop(ctx, iter, seek, opts, pos, &result); err != nil { + if errors.Is(err, errExportChunkFull) { + return result, nil + } + if errors.Is(err, errExportReachedEnd) { + result.Done = true + result.NextCursor = nil + return result, nil + } + return result, err + } + if err := iter.Error(); err != nil { + return ExportVersionsResult{}, errors.WithStack(err) + } + result.Done = true + result.NextCursor = nil + return result, nil +} + +func (s *pebbleStore) runPebbleExportLoop( + ctx context.Context, + iter *pebble.Iterator, + seek []byte, + opts ExportVersionsOptions, + pos exportCursorPosition, + result *ExportVersionsResult, +) error { + for ok := iter.SeekGE(seek); ok; { + advance, done, err := s.exportPebbleIteratorPosition(ctx, iter, opts, pos, result) + if err != nil { + return err + } + if !done { + return errExportChunkFull + } + if advance { + ok = iter.Next() + continue + } + ok = iter.Valid() + } + return nil +} + +func pebbleExportIterOptions(opts ExportVersionsOptions) *pebble.IterOptions { + iterOpts := &pebble.IterOptions{ + LowerBound: encodeKey(opts.StartKey, math.MaxUint64), + } + if opts.EndKey != nil { + iterOpts.UpperBound = encodeKey(opts.EndKey, math.MaxUint64) + } + return iterOpts +} + +func pebbleExportSeekKey(opts ExportVersionsOptions, pos exportCursorPosition) ([]byte, error) { + if len(pos.key) == 0 { + return encodeKey(opts.StartKey, math.MaxUint64), nil + } + if opts.StartKey != nil && bytes.Compare(pos.key, opts.StartKey) < 0 { + return nil, errors.WithStack(ErrInvalidExportCursor) + } + if opts.EndKey != nil && bytes.Compare(pos.key, opts.EndKey) >= 0 { + return nil, errors.WithStack(ErrInvalidExportCursor) + } + return encodeKey(pos.key, pos.commitTS), nil +} + +func (s *pebbleStore) exportPebbleIteratorPosition( + ctx context.Context, + iter *pebble.Iterator, + opts ExportVersionsOptions, + pos exportCursorPosition, + result *ExportVersionsResult, +) (advance bool, done bool, err error) { + if err := ctx.Err(); err != nil { + return false, false, errors.WithStack(err) + } + rawKey := iter.Key() + if isPebbleMetaKey(rawKey) { + return true, true, nil + } + userKey, commitTS := decodeKeyView(rawKey) + if userKey == nil || pebbleExportCursorEqual(pos, userKey, commitTS) { + return true, true, nil + } + if opts.EndKey != nil && bytes.Compare(userKey, opts.EndKey) >= 0 { + return false, true, errExportReachedEnd + } + if commitTS <= opts.MinCommitTSExclusive { + _ = s.skipToNextUserKey(iter, userKey) + return false, true, nil + } + done, err = s.exportPebbleVersion(iter, opts, userKey, commitTS, result) + return true, done, err +} + +func pebbleExportCursorEqual(pos exportCursorPosition, userKey []byte, commitTS uint64) bool { + return len(pos.key) > 0 && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS +} + +func (s *pebbleStore) exportPebbleVersion( + iter *pebble.Iterator, + opts ExportVersionsOptions, + userKey []byte, + commitTS uint64, + result *ExportVersionsResult, +) (bool, error) { + tag := exportCursorTagScanned + rawValue := iter.Value() + result.ScannedBytes += versionExportSize(userKey, len(rawValue)) + if shouldExportPebbleVersion(opts, userKey, commitTS) { + version, err := s.decodeExportedPebbleVersion(iter, userKey, commitTS, opts.KeyFamily) + if err != nil { + return false, err + } + result.Versions = append(result.Versions, version) + result.AcceptedRows++ + tag = exportCursorTagEmitted + } + result.NextCursor = encodeExportCursor(userKey, commitTS, tag) + if finishExportIfLimited(opts, result) { + result.Done = false + return false, nil + } + return true, nil +} + +func shouldExportPebbleVersion(opts ExportVersionsOptions, userKey []byte, commitTS uint64) bool { + if opts.AcceptKey != nil && !opts.AcceptKey(userKey) { + return false + } + return opts.MaxCommitTSInclusive == 0 || commitTS <= opts.MaxCommitTSInclusive +} + +func (s *pebbleStore) decodeExportedPebbleVersion(iter *pebble.Iterator, userKey []byte, commitTS uint64, keyFamily uint32) (MVCCVersion, error) { + sv, err := decodeValue(iter.Value()) + if err != nil { + return MVCCVersion{}, errors.WithStack(err) + } + var value []byte + if sv.Tombstone { + value = nil + } else { + value, err = s.decryptForKey(iter.Key(), sv, sv.Value) + if err != nil { + return MVCCVersion{}, err + } + } + return MVCCVersion{ + Key: bytes.Clone(userKey), + CommitTS: commitTS, + Tombstone: sv.Tombstone, + Value: bytes.Clone(value), + KeyFamily: keyFamily, + ExpireAt: sv.ExpireAt, + }, nil +} + +func (s *pebbleStore) ImportVersions(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + + s.applyMu.Lock() + defer s.applyMu.Unlock() + + duplicate, ackedCursor, err := s.validatePebbleImportBatch(opts) + if err != nil { + return ImportVersionsResult{}, err + } + if duplicate { + return ImportVersionsResult{AckedCursor: ackedCursor, Duplicate: true}, nil + } + + batchMax := importBatchMaxTS(opts.Versions) + newLastTS, err := s.commitPebbleImportBatch(opts, batchMax) + if err != nil { + return ImportVersionsResult{}, errors.WithStack(err) + } + if batchMax > 0 { + s.lastCommitTS = newLastTS + } + s.log.InfoContext(ctx, "import_versions", + "job_id", opts.JobID, + "bracket_id", opts.BracketID, + "batch_seq", opts.BatchSeq, + "versions", len(opts.Versions), + "max_imported_ts", batchMax, + ) + return ImportVersionsResult{AckedCursor: bytes.Clone(opts.Cursor), MaxImportedTS: batchMax}, nil +} + +func (s *pebbleStore) validatePebbleImportBatch(opts ImportVersionsOptions) (bool, []byte, error) { + existing, hasExisting, err := s.readMigrationImportAck(opts.JobID, opts.BracketID) + if err != nil { + return false, nil, err + } + duplicate, err := validateNextImportBatch(existing, hasExisting, opts.BatchSeq) + if err != nil { + return false, nil, err + } + if duplicate { + return true, bytes.Clone(existing.cursor), nil + } + for _, version := range opts.Versions { + if err := validateImportVersion(version); err != nil { + return false, nil, err + } + } + return false, nil, nil +} + +func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchMax uint64) (uint64, error) { + batch := s.db.NewBatch() + defer batch.Close() + if err := s.applyImportVersionsBatch(batch, opts.Versions); err != nil { + return 0, err + } + if err := batch.Set(migrationAckKey(opts.JobID, opts.BracketID), encodeMigrationImportAck(migrationImportAck{ + batchSeq: opts.BatchSeq, + cursor: opts.Cursor, + }), nil); err != nil { + return 0, errors.WithStack(err) + } + unlock, newLastTS, err := s.stageMigrationClockMetadataIfNeeded(batch, opts.JobID, batchMax) + if err != nil { + return 0, err + } + defer unlock() + if err := batch.Commit(s.directApplyWriteOpts()); err != nil { + return 0, errors.WithStack(err) + } + return newLastTS, nil +} + +func (s *pebbleStore) stageMigrationClockMetadataIfNeeded(batch *pebble.Batch, jobID, batchMax uint64) (func(), uint64, error) { + if batchMax == 0 { + return func() {}, 0, nil + } + return s.stageMigrationClockMetadata(batch, jobID, batchMax) +} + +func (s *pebbleStore) applyImportVersionsBatch(batch *pebble.Batch, versions []MVCCVersion) error { + for _, version := range versions { + k := encodeKey(version.Key, version.CommitTS) + var encoded []byte + if version.Tombstone { + encoded = encodeValue(nil, true, 0, encStateCleartext) + } else { + body, encState, err := s.encryptForKey(k, version.Value, version.ExpireAt, true) + if err != nil { + return err + } + encoded = encodeValue(body, false, version.ExpireAt, encState) + } + if err := batch.Set(k, encoded, nil); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func (s *pebbleStore) stageMigrationClockMetadata(batch *pebble.Batch, jobID, batchMax uint64) (func(), uint64, error) { + s.mtx.Lock() + unlock := func() { s.mtx.Unlock() } + newLastTS := s.lastCommitTS + if batchMax > newLastTS { + newLastTS = batchMax + } + if err := setPebbleUint64InBatch(batch, metaLastCommitTSBytes, newLastTS); err != nil { + unlock() + return nil, 0, err + } + floor, err := s.readMigrationHLCFloorLocked(jobID) + if err != nil { + unlock() + return nil, 0, err + } + if batchMax > floor { + if err := setPebbleUint64InBatch(batch, migrationHLCFloorKey(jobID), batchMax); err != nil { + unlock() + return nil, 0, err + } + } + return unlock, newLastTS, nil +} + +func (s *pebbleStore) readMigrationImportAck(jobID, bracketID uint64) (migrationImportAck, bool, error) { + val, closer, err := s.db.Get(migrationAckKey(jobID, bracketID)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return migrationImportAck{}, false, nil + } + return migrationImportAck{}, false, errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + ack, ok := decodeMigrationImportAck(val) + if !ok { + return migrationImportAck{}, false, errors.New("corrupt migration import ack") + } + return ack, true, nil +} + +func (s *pebbleStore) readMigrationHLCFloorLocked(jobID uint64) (uint64, error) { + return readPebbleUint64(s.db, migrationHLCFloorKey(jobID)) +} + +func (s *pebbleStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64, error) { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + floor, err := s.readMigrationHLCFloorLocked(jobID) + if err != nil { + return 0, err + } + return floor, nil +} diff --git a/store/lsm_store.go b/store/lsm_store.go index d818b7bde..18913cddc 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -551,7 +551,8 @@ func isPebbleMetaKey(rawKey []byte) bool { return bytes.Equal(rawKey, metaLastCommitTSBytes) || bytes.Equal(rawKey, metaMinRetainedTSBytes) || bytes.Equal(rawKey, metaPendingMinRetainedTSBytes) || - bytes.Equal(rawKey, metaAppliedIndexBytes) + bytes.Equal(rawKey, metaAppliedIndexBytes) || + isMigrationMetadataKey(rawKey) } func (s *pebbleStore) findMaxCommitTS() (uint64, error) { diff --git a/store/migration_versions.go b/store/migration_versions.go new file mode 100644 index 000000000..45f221279 --- /dev/null +++ b/store/migration_versions.go @@ -0,0 +1,379 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + + "github.com/cockroachdb/errors" + "github.com/emirpasic/gods/maps/treemap" +) + +const ( + exportCursorTagEmitted byte = iota + exportCursorTagScanned + + migrationAckPrefix = "!migstage|ack|" + migrationHLCFloorPrefix = "!migstage|hlc_floor|" + migrationUint64Bytes = 8 + migrationAckKeyIDBytes = 2 * migrationUint64Bytes + exportVersionSizeOverhead = 24 +) + +type exportCursorPosition struct { + key []byte + commitTS uint64 + tag byte +} + +type migrationImportAck struct { + batchSeq uint64 + cursor []byte +} + +func encodeExportCursor(key []byte, commitTS uint64, tag byte) []byte { + var buf []byte + buf = binary.AppendUvarint(buf, lenAsUint64(len(key))) + buf = append(buf, key...) + buf = binary.AppendUvarint(buf, commitTS) + buf = append(buf, tag) + return buf +} + +func decodeExportCursor(cursor []byte) (exportCursorPosition, error) { + if len(cursor) == 0 { + return exportCursorPosition{}, nil + } + keyLen, n := binary.Uvarint(cursor) + if n <= 0 { + return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) + } + rest := cursor[n:] + if keyLen > lenAsUint64(len(rest)) { + return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) + } + key := bytes.Clone(rest[:keyLen]) + rest = rest[keyLen:] + commitTS, n := binary.Uvarint(rest) + if n <= 0 { + return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) + } + rest = rest[n:] + if len(rest) != 1 { + return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) + } + tag := rest[0] + if tag != exportCursorTagEmitted && tag != exportCursorTagScanned { + return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) + } + return exportCursorPosition{key: key, commitTS: commitTS, tag: tag}, nil +} + +func migrationAckKey(jobID, bracketID uint64) []byte { + key := make([]byte, len(migrationAckPrefix)+migrationAckKeyIDBytes) + copy(key, migrationAckPrefix) + binary.BigEndian.PutUint64(key[len(migrationAckPrefix):], jobID) + binary.BigEndian.PutUint64(key[len(migrationAckPrefix)+migrationUint64Bytes:], bracketID) + return key +} + +func migrationHLCFloorKey(jobID uint64) []byte { + key := make([]byte, len(migrationHLCFloorPrefix)+migrationUint64Bytes) + copy(key, migrationHLCFloorPrefix) + binary.BigEndian.PutUint64(key[len(migrationHLCFloorPrefix):], jobID) + return key +} + +func isMigrationMetadataKey(rawKey []byte) bool { + return (len(rawKey) == len(migrationAckPrefix)+migrationAckKeyIDBytes && bytes.HasPrefix(rawKey, []byte(migrationAckPrefix))) || + (len(rawKey) == len(migrationHLCFloorPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationHLCFloorPrefix))) +} + +func encodeMigrationImportAck(ack migrationImportAck) []byte { + buf := make([]byte, 0, migrationUint64Bytes+binary.MaxVarintLen64+len(ack.cursor)) + buf = binary.BigEndian.AppendUint64(buf, ack.batchSeq) + buf = binary.AppendUvarint(buf, lenAsUint64(len(ack.cursor))) + buf = append(buf, ack.cursor...) + return buf +} + +func decodeMigrationImportAck(data []byte) (migrationImportAck, bool) { + if len(data) < migrationUint64Bytes { + return migrationImportAck{}, false + } + ack := migrationImportAck{batchSeq: binary.BigEndian.Uint64(data[:migrationUint64Bytes])} + cursorLen, n := binary.Uvarint(data[migrationUint64Bytes:]) + if n <= 0 { + return migrationImportAck{}, false + } + rest := data[migrationUint64Bytes+n:] + if cursorLen != lenAsUint64(len(rest)) { + return migrationImportAck{}, false + } + ack.cursor = bytes.Clone(rest) + return ack, true +} + +func validateImportVersion(version MVCCVersion) error { + if version.CommitTS == 0 { + return errors.New("migration import version has zero commit_ts") + } + if version.Tombstone { + if version.ExpireAt != 0 { + return errors.New("migration import tombstone carries expire_at") + } + if len(version.Value) != 0 { + return errors.New("migration import tombstone carries value") + } + return nil + } + return validateValueSize(version.Value) +} + +func versionExportSize(key []byte, valueLen int) uint64 { + return lenAsUint64(len(key)) + lenAsUint64(valueLen) + exportVersionSizeOverhead +} + +func lenAsUint64(n int) uint64 { + if n <= 0 { + return 0 + } + return uint64(n) //nolint:gosec // slice lengths are non-negative and bounded by addressable memory. +} + +func importBatchMaxTS(versions []MVCCVersion) uint64 { + var maxTS uint64 + for _, version := range versions { + if version.CommitTS > maxTS { + maxTS = version.CommitTS + } + } + return maxTS +} + +func validateNextImportBatch(existing migrationImportAck, hasExisting bool, batchSeq uint64) (duplicate bool, err error) { + if hasExisting { + if batchSeq <= existing.batchSeq { + return true, nil + } + if batchSeq != existing.batchSeq+1 { + return false, errors.WithStack(ErrImportBatchGap) + } + return false, nil + } + if batchSeq != 1 { + return false, errors.WithStack(ErrImportBatchGap) + } + return false, nil +} + +func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { + pos, err := decodeExportCursor(opts.Cursor) + if err != nil { + return ExportVersionsResult{}, err + } + if opts.MaxVersions <= 0 { + return ExportVersionsResult{Done: true}, nil + } + + s.mtx.RLock() + defer s.mtx.RUnlock() + + result := newExportVersionsResult(opts.MaxVersions) + it := s.tree.Iterator() + if !s.seekMemoryExportStart(&it, opts.StartKey, pos.key) { + result.Done = true + return result, nil + } + + for ok := true; ok; ok = it.Next() { + key, ok := it.Key().([]byte) + if err := checkExportKey(ctx, key, ok, opts.EndKey); err != nil { + if errors.Is(err, errExportReachedEnd) { + result.Done = true + result.NextCursor = nil + return result, nil + } + return ExportVersionsResult{}, err + } + if !ok { + continue + } + done, err := exportMemoryIteratorKey(ctx, opts, pos, key, it.Value(), &result) + if err != nil || !done { + return result, err + } + } + result.Done = true + result.NextCursor = nil + return result, nil +} + +var errExportReachedEnd = errors.New("export reached end") +var errExportChunkFull = errors.New("export chunk full") + +func checkExportKey(ctx context.Context, key []byte, keyOK bool, end []byte) error { + if err := ctx.Err(); err != nil { + return errors.WithStack(err) + } + if !keyOK { + return nil + } + if end != nil && bytes.Compare(key, end) >= 0 { + return errExportReachedEnd + } + return nil +} + +func newExportVersionsResult(maxVersions int) ExportVersionsResult { + return ExportVersionsResult{ + Versions: make([]MVCCVersion, 0, min(maxVersions, scanResultCapacityLimit)), + } +} + +func (s *mvccStore) seekMemoryExportStart(it *treemap.Iterator, startKey, cursorKey []byte) bool { + if len(cursorKey) > 0 { + return seekForwardIteratorStart(s.tree, it, cursorKey) + } + return seekForwardIteratorStart(s.tree, it, startKey) +} + +func exportMemoryIteratorKey( + ctx context.Context, + opts ExportVersionsOptions, + pos exportCursorPosition, + key []byte, + value any, + result *ExportVersionsResult, +) (bool, error) { + versions, _ := value.([]VersionedValue) + cursorCommitTS := uint64(0) + if len(pos.key) > 0 && bytes.Equal(key, pos.key) { + cursorCommitTS = pos.commitTS + } + return exportMemoryVersionsForKey(ctx, opts, cursorCommitTS, key, versions, result) +} + +func finishExportIfLimited(opts ExportVersionsOptions, result *ExportVersionsResult) bool { + return len(result.Versions) >= opts.MaxVersions || + (opts.MaxBytes > 0 && exportedVersionsSize(result.Versions) >= opts.MaxBytes) || + (opts.MaxScannedBytes > 0 && result.ScannedBytes >= opts.MaxScannedBytes) +} + +func appendMemoryExportVersion(opts ExportVersionsOptions, key []byte, version VersionedValue, result *ExportVersionsResult) byte { + if opts.AcceptKey != nil && !opts.AcceptKey(key) { + return exportCursorTagScanned + } + if opts.MaxCommitTSInclusive != 0 && version.TS > opts.MaxCommitTSInclusive { + return exportCursorTagScanned + } + result.Versions = append(result.Versions, MVCCVersion{ + Key: bytes.Clone(key), + CommitTS: version.TS, + Tombstone: version.Tombstone, + Value: bytes.Clone(version.Value), + KeyFamily: opts.KeyFamily, + ExpireAt: version.ExpireAt, + }) + result.AcceptedRows++ + return exportCursorTagEmitted +} + +func finishMemoryExportPosition(opts ExportVersionsOptions, key []byte, version VersionedValue, tag byte, result *ExportVersionsResult) bool { + result.ScannedBytes += versionExportSize(key, len(version.Value)) + result.NextCursor = encodeExportCursor(key, version.TS, tag) + if finishExportIfLimited(opts, result) { + result.Done = false + return false + } + return true +} + +func shouldSkipMemoryVersion(cursorCommitTS uint64, version VersionedValue) bool { + return cursorCommitTS != 0 && version.TS >= cursorCommitTS +} + +func exportMemoryVersion(opts ExportVersionsOptions, cursorCommitTS uint64, key []byte, version VersionedValue, result *ExportVersionsResult) bool { + if shouldSkipMemoryVersion(cursorCommitTS, version) { + return true + } + if version.TS <= opts.MinCommitTSExclusive { + return false + } + tag := appendMemoryExportVersion(opts, key, version, result) + return finishMemoryExportPosition(opts, key, version, tag, result) +} + +func exportMemoryVersionsForKey( + ctx context.Context, + opts ExportVersionsOptions, + cursorCommitTS uint64, + key []byte, + versions []VersionedValue, + result *ExportVersionsResult, +) (bool, error) { + for i := len(versions) - 1; i >= 0; i-- { + if err := ctx.Err(); err != nil { + return false, errors.WithStack(err) + } + if !exportMemoryVersion(opts, cursorCommitTS, key, versions[i], result) { + return !finishExportIfLimited(opts, result), nil + } + if !result.Done && finishExportIfLimited(opts, result) { + return false, nil + } + } + return true, nil +} + +func exportedVersionsSize(versions []MVCCVersion) uint64 { + var total uint64 + for _, version := range versions { + total += versionExportSize(version.Key, len(version.Value)) + } + return total +} + +func (s *mvccStore) ImportVersions(_ context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) { + s.mtx.Lock() + defer s.mtx.Unlock() + + key := string(migrationAckKey(opts.JobID, opts.BracketID)) + existing, hasExisting := s.migrationAcks[key] + duplicate, err := validateNextImportBatch(existing, hasExisting, opts.BatchSeq) + if err != nil { + return ImportVersionsResult{}, err + } + if duplicate { + return ImportVersionsResult{AckedCursor: bytes.Clone(existing.cursor), Duplicate: true}, nil + } + + for _, version := range opts.Versions { + if err := validateImportVersion(version); err != nil { + return ImportVersionsResult{}, err + } + } + for _, version := range opts.Versions { + if version.Tombstone { + s.deleteVersionLocked(version.Key, version.CommitTS) + continue + } + s.putVersionLocked(version.Key, version.Value, version.CommitTS, version.ExpireAt) + } + + batchMax := importBatchMaxTS(opts.Versions) + if batchMax > s.lastCommitTS { + s.lastCommitTS = batchMax + } + if batchMax > s.migrationHLCFloors[opts.JobID] { + s.migrationHLCFloors[opts.JobID] = batchMax + } + s.migrationAcks[key] = migrationImportAck{batchSeq: opts.BatchSeq, cursor: bytes.Clone(opts.Cursor)} + return ImportVersionsResult{AckedCursor: bytes.Clone(opts.Cursor), MaxImportedTS: batchMax}, nil +} + +func (s *mvccStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return s.migrationHLCFloors[jobID], nil +} diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go new file mode 100644 index 000000000..77badeedd --- /dev/null +++ b/store/migration_versions_test.go @@ -0,0 +1,217 @@ +package store + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func runMigrationStoreSuite(t *testing.T, test func(t *testing.T, st MVCCStore)) { + t.Helper() + t.Run("memory", func(t *testing.T) { + test(t, NewMVCCStore()) + }) + t.Run("pebble", func(t *testing.T) { + dir, err := os.MkdirTemp("", "migration-versions-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + test(t, st) + }) +} + +func TestExportVersionsPreservesRawVersionMetadata(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("k1"), []byte("v10"), 10, 0)) + require.NoError(t, st.PutWithTTLAt(ctx, []byte("k1"), []byte("v20"), 20, 55)) + require.NoError(t, st.DeleteAt(ctx, []byte("k1"), 30)) + require.NoError(t, st.PutAt(ctx, []byte("k2"), []byte("v15"), 15, 0)) + + result, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k1"), + EndKey: []byte("k3"), + MinCommitTSExclusive: 9, + MaxCommitTSInclusive: 30, + MaxVersions: 10, + KeyFamily: 7, + }) + require.NoError(t, err) + require.True(t, result.Done) + require.Empty(t, result.NextCursor) + require.Equal(t, []MVCCVersion{ + {Key: []byte("k1"), CommitTS: 30, Tombstone: true, KeyFamily: 7}, + {Key: []byte("k1"), CommitTS: 20, Value: []byte("v20"), KeyFamily: 7, ExpireAt: 55}, + {Key: []byte("k1"), CommitTS: 10, Value: []byte("v10"), KeyFamily: 7}, + {Key: []byte("k2"), CommitTS: 15, Value: []byte("v15"), KeyFamily: 7}, + }, result.Versions) + }) +} + +func TestExportVersionsCursorResumesWithinHotKey(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("hot"), []byte("v10"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("hot"), []byte("v20"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("hot"), []byte("v30"), 30, 0)) + require.NoError(t, st.PutAt(ctx, []byte("tail"), []byte("v15"), 15, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 2}) + require.NoError(t, err) + require.False(t, first.Done) + require.Len(t, first.Versions, 2) + require.Equal(t, uint64(30), first.Versions[0].CommitTS) + require.Equal(t, uint64(20), first.Versions[1].CommitTS) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: first.NextCursor, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{ + {Key: []byte("hot"), CommitTS: 10, Value: []byte("v10")}, + {Key: []byte("tail"), CommitTS: 15, Value: []byte("v15")}, + }, second.Versions) + }) +} + +func TestExportVersionsSparseScanBudgetAdvancesRejectedRows(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("drop-a"), []byte("a"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("keep"), []byte("b"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + MaxVersions: 10, + MaxScannedBytes: 1, + AcceptKey: func(key []byte) bool { + return string(key) == "keep" + }, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: first.NextCursor, + MaxVersions: 10, + AcceptKey: func(key []byte) bool { + return string(key) == "keep" + }, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("keep"), CommitTS: 20, Value: []byte("b")}}, second.Versions) + }) +} + +func TestImportVersionsIdempotencyAndMetadata(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + first := ImportVersionsOptions{ + JobID: 1, + BracketID: 2, + BatchSeq: 1, + Cursor: []byte("c1"), + Versions: []MVCCVersion{ + {Key: []byte("ttl"), CommitTS: 20, Value: []byte("v20"), ExpireAt: 50}, + {Key: []byte("gone"), CommitTS: 30, Tombstone: true}, + }, + } + res, err := st.ImportVersions(ctx, first) + require.NoError(t, err) + require.Equal(t, []byte("c1"), res.AckedCursor) + require.Equal(t, uint64(30), res.MaxImportedTS) + require.False(t, res.Duplicate) + require.Equal(t, uint64(30), st.LastCommitTS()) + floor, err := st.MigrationHLCFloor(ctx, 1) + require.NoError(t, err) + require.Equal(t, uint64(30), floor) + + val, err := st.GetAt(ctx, []byte("ttl"), 25) + require.NoError(t, err) + require.Equal(t, []byte("v20"), val) + _, err = st.GetAt(ctx, []byte("ttl"), 55) + require.ErrorIs(t, err, ErrKeyNotFound) + _, err = st.GetAt(ctx, []byte("gone"), 35) + require.ErrorIs(t, err, ErrKeyNotFound) + + dup := first + dup.Cursor = []byte("changed") + dup.Versions = []MVCCVersion{{Key: []byte("ttl"), CommitTS: 40, Value: []byte("bad")}} + res, err = st.ImportVersions(ctx, dup) + require.NoError(t, err) + require.True(t, res.Duplicate) + require.Equal(t, []byte("c1"), res.AckedCursor) + val, err = st.GetAt(ctx, []byte("ttl"), 45) + require.NoError(t, err) + require.Equal(t, []byte("v20"), val) + + _, err = st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 1, + BracketID: 2, + BatchSeq: 3, + Cursor: []byte("gap"), + Versions: []MVCCVersion{{Key: []byte("gap"), CommitTS: 60, Value: []byte("bad")}}, + }) + require.ErrorIs(t, err, ErrImportBatchGap) + _, err = st.GetAt(ctx, []byte("gap"), 60) + require.ErrorIs(t, err, ErrKeyNotFound) + + res, err = st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 1, + BracketID: 2, + BatchSeq: 2, + Cursor: []byte("c2"), + }) + require.NoError(t, err) + require.Equal(t, []byte("c2"), res.AckedCursor) + require.Zero(t, res.MaxImportedTS) + require.Equal(t, uint64(30), st.LastCommitTS()) + floor, err = st.MigrationHLCFloor(ctx, 1) + require.NoError(t, err) + require.Equal(t, uint64(30), floor) + }) +} + +func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-import-persist-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + st, err := NewPebbleStore(dir) + require.NoError(t, err) + _, err = st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 9, + BracketID: 4, + BatchSeq: 1, + Cursor: []byte("persisted"), + Versions: []MVCCVersion{{Key: []byte("k"), CommitTS: 99, Value: []byte("v")}}, + }) + require.NoError(t, err) + require.NoError(t, st.Close()) + + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + defer func() { require.NoError(t, reopened.Close()) }() + floor, err := reopened.MigrationHLCFloor(ctx, 9) + require.NoError(t, err) + require.Equal(t, uint64(99), floor) + res, err := reopened.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 9, + BracketID: 4, + BatchSeq: 1, + Cursor: []byte("different"), + }) + require.NoError(t, err) + require.True(t, res.Duplicate) + require.Equal(t, []byte("persisted"), res.AckedCursor) +} diff --git a/store/mvcc_store.go b/store/mvcc_store.go index abbda3240..50be08ba3 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -60,11 +60,13 @@ func byteSliceComparator(a, b any) int { // mvccStore is an in-memory MVCC implementation backed by a treemap for // deterministic iteration order and range scans. type mvccStore struct { - tree *treemap.Map // key []byte -> []VersionedValue - mtx sync.RWMutex - log *slog.Logger - lastCommitTS uint64 - minRetainedTS uint64 + tree *treemap.Map // key []byte -> []VersionedValue + mtx sync.RWMutex + log *slog.Logger + lastCommitTS uint64 + minRetainedTS uint64 + migrationAcks map[string]migrationImportAck + migrationHLCFloors map[uint64]uint64 // writeConflicts mirrors the per-(kind, key_prefix) counter from // the pebble-backed store so the in-memory implementation shows up // in the same Prometheus series (even if the counts are usually @@ -111,7 +113,9 @@ func NewMVCCStore(opts ...MVCCStoreOption) MVCCStore { log: slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ Level: slog.LevelWarn, })), - writeConflicts: newWriteConflictCounter(), + migrationAcks: make(map[string]migrationImportAck), + migrationHLCFloors: make(map[uint64]uint64), + writeConflicts: newWriteConflictCounter(), } for _, opt := range opts { opt(s) diff --git a/store/store.go b/store/store.go index 3991da19d..72294081a 100644 --- a/store/store.go +++ b/store/store.go @@ -25,6 +25,8 @@ var ErrReadTSCompacted = errors.New("read timestamp has been compacted") var ErrSnapshotKeyTooLarge = errors.New("mvcc snapshot key too large") var ErrSnapshotVersionCountTooLarge = errors.New("mvcc snapshot version count too large") var ErrValueTooLarge = errors.New("value too large") +var ErrInvalidExportCursor = errors.New("invalid export cursor") +var ErrImportBatchGap = errors.New("migration import batch gap") // validateValueSize returns ErrValueTooLarge when the value exceeds maxSnapshotValueSize. func validateValueSize(value []byte) error { @@ -63,6 +65,56 @@ type KVPair struct { Value []byte } +// MVCCVersion is a raw committed MVCC version for range migration. +// Unlike scan results, it preserves tombstones and TTL expiry metadata. +type MVCCVersion struct { + Key []byte + CommitTS uint64 + Tombstone bool + Value []byte + KeyFamily uint32 + ExpireAt uint64 +} + +// ExportVersionsOptions selects a raw MVCC-version export window. +type ExportVersionsOptions struct { + StartKey []byte + EndKey []byte + MinCommitTSExclusive uint64 + MaxCommitTSInclusive uint64 + Cursor []byte + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + KeyFamily uint32 + AcceptKey func([]byte) bool +} + +// ExportVersionsResult is one resumable chunk of raw MVCC versions. +type ExportVersionsResult struct { + Versions []MVCCVersion + NextCursor []byte + Done bool + ScannedBytes uint64 + AcceptedRows uint64 +} + +// ImportVersionsOptions applies one idempotent migration-import batch. +type ImportVersionsOptions struct { + JobID uint64 + BracketID uint64 + BatchSeq uint64 + Versions []MVCCVersion + Cursor []byte +} + +// ImportVersionsResult reports the cursor durably acknowledged by the target. +type ImportVersionsResult struct { + AckedCursor []byte + MaxImportedTS uint64 + Duplicate bool +} + // OpType describes a mutation kind. type OpType int @@ -207,6 +259,14 @@ type MVCCStore interface { WriteConflictCountsByPrefix() map[string]uint64 // Compact removes versions older than minTS that are no longer needed. Compact(ctx context.Context, minTS uint64) error + // ExportVersions exports raw committed MVCC versions for range migration. + ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) + // ImportVersions applies a migration import batch idempotently by + // (jobID, bracketID, batchSeq), preserving tombstones and expireAt. + ImportVersions(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) + // MigrationHLCFloor returns the full-HLC target-local migration floor + // persisted by ImportVersions for jobID. + MigrationHLCFloor(ctx context.Context, jobID uint64) (uint64, error) Snapshot() (Snapshot, error) Restore(buf io.Reader) error Close() error From edf74ffa0e81cd26fba13bdd92c3725fef73fc18 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 20:46:56 +0900 Subject: [PATCH 002/162] store: tighten migration export accounting --- store/lsm_migration.go | 22 +++++++++++----------- store/migration_versions.go | 11 ++--------- store/store.go | 11 ++++++----- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 88d56e9fe..3ae343afd 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -148,6 +148,7 @@ func (s *pebbleStore) exportPebbleVersion( return false, err } result.Versions = append(result.Versions, version) + result.ExportedBytes += versionExportSize(userKey, len(version.Value)) result.AcceptedRows++ tag = exportCursorTagEmitted } @@ -206,13 +207,9 @@ func (s *pebbleStore) ImportVersions(ctx context.Context, opts ImportVersionsOpt } batchMax := importBatchMaxTS(opts.Versions) - newLastTS, err := s.commitPebbleImportBatch(opts, batchMax) - if err != nil { + if err := s.commitPebbleImportBatch(opts, batchMax); err != nil { return ImportVersionsResult{}, errors.WithStack(err) } - if batchMax > 0 { - s.lastCommitTS = newLastTS - } s.log.InfoContext(ctx, "import_versions", "job_id", opts.JobID, "bracket_id", opts.BracketID, @@ -243,27 +240,30 @@ func (s *pebbleStore) validatePebbleImportBatch(opts ImportVersionsOptions) (boo return false, nil, nil } -func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchMax uint64) (uint64, error) { +func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchMax uint64) error { batch := s.db.NewBatch() defer batch.Close() if err := s.applyImportVersionsBatch(batch, opts.Versions); err != nil { - return 0, err + return err } if err := batch.Set(migrationAckKey(opts.JobID, opts.BracketID), encodeMigrationImportAck(migrationImportAck{ batchSeq: opts.BatchSeq, cursor: opts.Cursor, }), nil); err != nil { - return 0, errors.WithStack(err) + return errors.WithStack(err) } unlock, newLastTS, err := s.stageMigrationClockMetadataIfNeeded(batch, opts.JobID, batchMax) if err != nil { - return 0, err + return err } defer unlock() if err := batch.Commit(s.directApplyWriteOpts()); err != nil { - return 0, errors.WithStack(err) + return errors.WithStack(err) } - return newLastTS, nil + if batchMax > 0 { + s.lastCommitTS = newLastTS + } + return nil } func (s *pebbleStore) stageMigrationClockMetadataIfNeeded(batch *pebble.Batch, jobID, batchMax uint64) (func(), uint64, error) { diff --git a/store/migration_versions.go b/store/migration_versions.go index 45f221279..54b7dd19b 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -256,7 +256,7 @@ func exportMemoryIteratorKey( func finishExportIfLimited(opts ExportVersionsOptions, result *ExportVersionsResult) bool { return len(result.Versions) >= opts.MaxVersions || - (opts.MaxBytes > 0 && exportedVersionsSize(result.Versions) >= opts.MaxBytes) || + (opts.MaxBytes > 0 && result.ExportedBytes >= opts.MaxBytes) || (opts.MaxScannedBytes > 0 && result.ScannedBytes >= opts.MaxScannedBytes) } @@ -275,6 +275,7 @@ func appendMemoryExportVersion(opts ExportVersionsOptions, key []byte, version V KeyFamily: opts.KeyFamily, ExpireAt: version.ExpireAt, }) + result.ExportedBytes += versionExportSize(key, len(version.Value)) result.AcceptedRows++ return exportCursorTagEmitted } @@ -326,14 +327,6 @@ func exportMemoryVersionsForKey( return true, nil } -func exportedVersionsSize(versions []MVCCVersion) uint64 { - var total uint64 - for _, version := range versions { - total += versionExportSize(version.Key, len(version.Value)) - } - return total -} - func (s *mvccStore) ImportVersions(_ context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) { s.mtx.Lock() defer s.mtx.Unlock() diff --git a/store/store.go b/store/store.go index 72294081a..78cb59354 100644 --- a/store/store.go +++ b/store/store.go @@ -92,11 +92,12 @@ type ExportVersionsOptions struct { // ExportVersionsResult is one resumable chunk of raw MVCC versions. type ExportVersionsResult struct { - Versions []MVCCVersion - NextCursor []byte - Done bool - ScannedBytes uint64 - AcceptedRows uint64 + Versions []MVCCVersion + NextCursor []byte + Done bool + ScannedBytes uint64 + ExportedBytes uint64 + AcceptedRows uint64 } // ImportVersionsOptions applies one idempotent migration-import batch. From f95b557dbaf31b809d97f6ab396bbd0faf0aece3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 21:31:57 +0900 Subject: [PATCH 003/162] distribution: add migration bracket planner --- distribution/migrator.go | 425 ++++++++++++++++++++++ distribution/migrator_export_plan_test.go | 235 ++++++++++++ kv/migrator_filter.go | 21 ++ kv/shard_key.go | 144 ++++++-- kv/shard_key_test.go | 99 +++++ kv/txn_keys.go | 59 +++ 6 files changed, 957 insertions(+), 26 deletions(-) create mode 100644 distribution/migrator.go create mode 100644 distribution/migrator_export_plan_test.go create mode 100644 kv/migrator_filter.go diff --git a/distribution/migrator.go b/distribution/migrator.go new file mode 100644 index 000000000..472a09035 --- /dev/null +++ b/distribution/migrator.go @@ -0,0 +1,425 @@ +package distribution + +import ( + "bytes" + + "github.com/bootjp/elastickv/internal/s3keys" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +const ( + MigrationFamilyUser uint32 = iota + 1 + MigrationFamilyTxnIntent + MigrationFamilyTxnCommit + MigrationFamilyTxnRollback + MigrationFamilyTxnSuccess + MigrationFamilyTxnMeta + MigrationFamilyTxnLock + MigrationFamilyListMeta + MigrationFamilyListItem + MigrationFamilyListMetaDelta + MigrationFamilyListClaim + MigrationFamilyRedisLegacy + MigrationFamilyHash + MigrationFamilySet + MigrationFamilyZSet + MigrationFamilyStreamMeta + MigrationFamilyStreamEntry + MigrationFamilyDynamoTableMeta + MigrationFamilyDynamoTableGeneration + MigrationFamilyDynamoItem + MigrationFamilyDynamoGSI + MigrationFamilySQSQueueMeta + MigrationFamilySQSQueueGeneration + MigrationFamilySQSQueueSequence + MigrationFamilySQSQueueTombstone + MigrationFamilySQSMessageData + MigrationFamilySQSMessageVisibility + MigrationFamilySQSMessageDedup + MigrationFamilySQSMessageGroup + MigrationFamilySQSMessageByAge + MigrationFamilySQSPartitionedMessageData + MigrationFamilySQSPartitionedMessageVisibility + MigrationFamilySQSPartitionedMessageDedup + MigrationFamilySQSPartitionedMessageGroup + MigrationFamilySQSPartitionedMessageByAge + MigrationFamilyS3BucketMeta + MigrationFamilyS3BucketGeneration + MigrationFamilyS3ObjectManifest + MigrationFamilyS3UploadMeta + MigrationFamilyS3UploadPart + MigrationFamilyS3Blob + MigrationFamilyS3GCUpload +) + +const ( + migrationTxnIntentPrefix = "!txn|int|" + migrationTxnCommitPrefix = "!txn|cmt|" + migrationTxnRollbackPrefix = "!txn|rb|" + migrationTxnSuccessPrefix = "!txn|ok|" + migrationTxnMetaPrefix = "!txn|meta|" + migrationTxnLockPrefix = "!txn|lock|" + migrationRedisPrefix = "!redis|" + migrationHashPrefix = "!hs|" + migrationSetPrefix = "!st|" + migrationZSetPrefix = "!zs|" + migrationDynamoMetaPrefix = "!ddb|meta|table|" + migrationDynamoGenPrefix = "!ddb|meta|gen|" + migrationDynamoItemPrefix = "!ddb|item|" + migrationDynamoGSIPrefix = "!ddb|gsi|" +) + +const ( + migrationSQSQueueMetaPrefix = "!sqs|queue|meta|" + migrationSQSQueueGenPrefix = "!sqs|queue|gen|" + migrationSQSQueueSeqPrefix = "!sqs|queue|seq|" + migrationSQSQueueTombstonePrefix = "!sqs|queue|tombstone|" + migrationSQSMsgDataPrefix = "!sqs|msg|data|" + migrationSQSMsgVisPrefix = "!sqs|msg|vis|" + migrationSQSMsgDedupPrefix = "!sqs|msg|dedup|" + migrationSQSMsgGroupPrefix = "!sqs|msg|group|" + migrationSQSMsgByAgePrefix = "!sqs|msg|byage|" + migrationSQSPartitionedSuffix = "p|" +) + +var ( + ErrMigrationReservedRange = errors.New("migration range intersects reserved control prefix") + ErrMigrationInvalidRoute = errors.New("migration route is invalid") + ErrMigrationDataMoveRequired = errors.New("migration data move is not implemented") + ErrMigrationSourceRouteChanged = errors.New("migration source route does not match split job") +) + +var migrationReservedControlPrefixes = [][]byte{ + []byte("!dist|"), + []byte("!migstage|"), + []byte("!migwrite|"), + []byte("!migfence|"), +} + +var migrationInternalFamilyPrefixes = [][]byte{ + []byte(migrationTxnLockPrefix), + []byte(migrationTxnIntentPrefix), + []byte(migrationTxnCommitPrefix), + []byte(migrationTxnRollbackPrefix), + []byte(migrationTxnSuccessPrefix), + []byte(migrationTxnMetaPrefix), + []byte(store.ListMetaDeltaPrefix), + []byte(store.ListClaimPrefix), + []byte(store.ListMetaPrefix), + []byte(store.ListItemPrefix), + []byte(migrationRedisPrefix), + []byte(migrationHashPrefix), + []byte(migrationSetPrefix), + []byte(migrationZSetPrefix), + []byte(store.StreamMetaPrefix), + []byte(store.StreamEntryPrefix), + []byte(migrationDynamoMetaPrefix), + []byte(migrationDynamoGenPrefix), + []byte(migrationDynamoItemPrefix), + []byte(migrationDynamoGSIPrefix), + []byte(migrationSQSQueueMetaPrefix), + []byte(migrationSQSQueueGenPrefix), + []byte(migrationSQSQueueSeqPrefix), + []byte(migrationSQSQueueTombstonePrefix), + []byte(migrationSQSMsgDataPrefix), + []byte(migrationSQSMsgVisPrefix), + []byte(migrationSQSMsgDedupPrefix), + []byte(migrationSQSMsgGroupPrefix), + []byte(migrationSQSMsgByAgePrefix), + []byte(s3keys.BucketMetaPrefix), + []byte(s3keys.BucketGenerationPrefix), + []byte(s3keys.ObjectManifestPrefix), + []byte(s3keys.UploadMetaPrefix), + []byte(s3keys.UploadPartPrefix), + []byte(s3keys.BlobPrefix), + []byte(s3keys.GCUploadPrefix), +} + +// MigrationBracket is a raw MVCC export or drain slice used by the migrator. +type MigrationBracket struct { + BracketID uint64 + Family uint32 + Start []byte + End []byte + ExcludePrefixes [][]byte + ExcludeKnownInternal bool + DrainOnly bool + RequiresRouteKeyCheck bool +} + +// PlanMigrationBrackets returns the full M2 migration plan, including the +// drain-only transaction lock bracket. Data export callers should use +// PlanExportBrackets, which omits drain-only control state. +func PlanMigrationBrackets(routeStart, routeEnd []byte) ([]MigrationBracket, error) { + if err := ValidateMigrationRouteRange(routeStart, routeEnd); err != nil { + return nil, err + } + + brackets := []MigrationBracket{ + { + BracketID: uint64(MigrationFamilyUser), + Family: MigrationFamilyUser, + Start: CloneBytes(routeStart), + End: CloneBytes(routeEnd), + ExcludeKnownInternal: true, + RequiresRouteKeyCheck: true, + }, + } + brackets = append(brackets, migrationFamilyBrackets()...) + return brackets, nil +} + +// PlanExportBrackets returns the data-copy bracket plan. Intent locks are +// deliberately absent because the source drains them route-faithfully before +// cutover and the target must not materialize in-flight intents as data. +func PlanExportBrackets(routeStart, routeEnd []byte) ([]MigrationBracket, error) { + brackets, err := PlanMigrationBrackets(routeStart, routeEnd) + if err != nil { + return nil, err + } + out := make([]MigrationBracket, 0, len(brackets)) + for _, bracket := range brackets { + if bracket.DrainOnly { + continue + } + out = append(out, bracket) + } + return out, nil +} + +// SplitJobBracketProgressForPlan creates durable per-bracket resume state for +// a SplitJob phase. +func SplitJobBracketProgressForPlan(brackets []MigrationBracket, phase SplitJobExportPhase) []SplitJobBracketProgress { + out := make([]SplitJobBracketProgress, 0, len(brackets)) + for _, bracket := range brackets { + if bracket.DrainOnly { + continue + } + out = append(out, SplitJobBracketProgress{ + BracketID: bracket.BracketID, + Family: bracket.Family, + ExportPhase: phase, + }) + } + return out +} + +// ContainsRawKey reports whether rawKey is inside the bracket's raw scan +// interval after applying bracket-local exclusions. Route ownership still +// requires the caller's RouteKeyFilter for every bracket. +func (b MigrationBracket) ContainsRawKey(rawKey []byte) bool { + if bytes.Compare(rawKey, b.Start) < 0 { + return false + } + if len(b.End) > 0 && bytes.Compare(rawKey, b.End) >= 0 { + return false + } + if b.ExcludeKnownInternal && IsMigrationKnownInternalKey(rawKey) { + return false + } + return !hasAnyPrefix(rawKey, b.ExcludePrefixes) +} + +// InitializeSplitJobPlan validates the source route and seeds the job's +// bracket progress for the moving right child [SplitKey, source.End). +func InitializeSplitJobPlan(job SplitJob, source RouteDescriptor, nowMs int64) (SplitJob, error) { + if err := validateSplitJobSource(job, source); err != nil { + return SplitJob{}, err + } + routeStart := CloneBytes(job.SplitKey) + routeEnd := CloneBytes(source.End) + brackets, err := PlanExportBrackets(routeStart, routeEnd) + if err != nil { + return SplitJob{}, err + } + + out := CloneSplitJob(job) + if out.Phase == SplitJobPhaseNone { + out.Phase = SplitJobPhasePlanned + } + if len(out.BracketProgress) == 0 { + out.BracketProgress = SplitJobBracketProgressForPlan(brackets, SplitJobExportPhaseBackfill) + } + if out.StartedAtMs == 0 { + out.StartedAtMs = nowMs + } + out.UpdatedAtMs = nowMs + return out, nil +} + +// AdvanceSameGroupNoop completes the PR4 same-group path without attempting +// data movement. Cross-group data copy is added by later M2 PRs. +func AdvanceSameGroupNoop(job SplitJob, source RouteDescriptor, nowMs int64) (SplitJob, error) { + planned, err := InitializeSplitJobPlan(job, source, nowMs) + if err != nil { + return SplitJob{}, err + } + if planned.TargetGroupID != source.GroupID { + return SplitJob{}, errors.WithStack(ErrMigrationDataMoveRequired) + } + + planned.Phase = SplitJobPhaseDone + planned.TargetPromotionDone = true + planned.PromotionCompletedTS = migrationWallMillisToUint64(nowMs) + planned.UpdatedAtMs = nowMs + planned.TerminalAtMs = nowMs + for i := range planned.BracketProgress { + planned.BracketProgress[i].Done = true + } + return planned, nil +} + +// MigrationKnownInternalPrefixes returns the concrete internal data/control +// prefixes that the user bracket must exclude. It intentionally does not +// include broad umbrellas such as !txn|, !ddb|, !sqs|, !s3|, or !stream|. +func MigrationKnownInternalPrefixes() [][]byte { + return cloneByteSlices(migrationInternalFamilyPrefixes) +} + +// IsMigrationKnownInternalKey reports whether a raw key belongs to a concrete +// internal family owned by an explicit export bracket or by the txn-lock drain. +func IsMigrationKnownInternalKey(key []byte) bool { + return hasAnyPrefix(key, migrationInternalFamilyPrefixes) +} + +// ValidateMigrationRouteRange rejects route intervals that intersect reserved +// distribution/migration control namespaces. +func ValidateMigrationRouteRange(routeStart, routeEnd []byte) error { + if len(routeEnd) > 0 && bytes.Compare(routeStart, routeEnd) >= 0 { + return errors.WithStack(ErrMigrationInvalidRoute) + } + for _, prefix := range migrationReservedControlPrefixes { + if rangesIntersect(routeStart, routeEnd, prefix, prefixScanEnd(prefix)) { + return errors.WithStack(ErrMigrationReservedRange) + } + } + return nil +} + +func migrationFamilyBrackets() []MigrationBracket { + defs := []struct { + family uint32 + prefix string + drainOnly bool + excludePrefixes []string + }{ + {family: MigrationFamilyTxnLock, prefix: migrationTxnLockPrefix, drainOnly: true}, + {family: MigrationFamilyTxnIntent, prefix: migrationTxnIntentPrefix}, + {family: MigrationFamilyTxnCommit, prefix: migrationTxnCommitPrefix}, + {family: MigrationFamilyTxnRollback, prefix: migrationTxnRollbackPrefix}, + {family: MigrationFamilyTxnSuccess, prefix: migrationTxnSuccessPrefix}, + {family: MigrationFamilyTxnMeta, prefix: migrationTxnMetaPrefix}, + {family: MigrationFamilyListMetaDelta, prefix: store.ListMetaDeltaPrefix}, + {family: MigrationFamilyListClaim, prefix: store.ListClaimPrefix}, + {family: MigrationFamilyListMeta, prefix: store.ListMetaPrefix, excludePrefixes: []string{store.ListMetaDeltaPrefix}}, + {family: MigrationFamilyListItem, prefix: store.ListItemPrefix}, + {family: MigrationFamilyRedisLegacy, prefix: migrationRedisPrefix}, + {family: MigrationFamilyHash, prefix: migrationHashPrefix}, + {family: MigrationFamilySet, prefix: migrationSetPrefix}, + {family: MigrationFamilyZSet, prefix: migrationZSetPrefix}, + {family: MigrationFamilyStreamMeta, prefix: store.StreamMetaPrefix}, + {family: MigrationFamilyStreamEntry, prefix: store.StreamEntryPrefix}, + {family: MigrationFamilyDynamoTableMeta, prefix: migrationDynamoMetaPrefix}, + {family: MigrationFamilyDynamoTableGeneration, prefix: migrationDynamoGenPrefix}, + {family: MigrationFamilyDynamoItem, prefix: migrationDynamoItemPrefix}, + {family: MigrationFamilyDynamoGSI, prefix: migrationDynamoGSIPrefix}, + {family: MigrationFamilySQSQueueMeta, prefix: migrationSQSQueueMetaPrefix}, + {family: MigrationFamilySQSQueueGeneration, prefix: migrationSQSQueueGenPrefix}, + {family: MigrationFamilySQSQueueSequence, prefix: migrationSQSQueueSeqPrefix}, + {family: MigrationFamilySQSQueueTombstone, prefix: migrationSQSQueueTombstonePrefix}, + {family: MigrationFamilySQSMessageData, prefix: migrationSQSMsgDataPrefix, excludePrefixes: []string{migrationSQSMsgDataPrefix + migrationSQSPartitionedSuffix}}, + {family: MigrationFamilySQSMessageVisibility, prefix: migrationSQSMsgVisPrefix, excludePrefixes: []string{migrationSQSMsgVisPrefix + migrationSQSPartitionedSuffix}}, + {family: MigrationFamilySQSMessageDedup, prefix: migrationSQSMsgDedupPrefix, excludePrefixes: []string{migrationSQSMsgDedupPrefix + migrationSQSPartitionedSuffix}}, + {family: MigrationFamilySQSMessageGroup, prefix: migrationSQSMsgGroupPrefix, excludePrefixes: []string{migrationSQSMsgGroupPrefix + migrationSQSPartitionedSuffix}}, + {family: MigrationFamilySQSMessageByAge, prefix: migrationSQSMsgByAgePrefix, excludePrefixes: []string{migrationSQSMsgByAgePrefix + migrationSQSPartitionedSuffix}}, + {family: MigrationFamilySQSPartitionedMessageData, prefix: migrationSQSMsgDataPrefix + migrationSQSPartitionedSuffix}, + {family: MigrationFamilySQSPartitionedMessageVisibility, prefix: migrationSQSMsgVisPrefix + migrationSQSPartitionedSuffix}, + {family: MigrationFamilySQSPartitionedMessageDedup, prefix: migrationSQSMsgDedupPrefix + migrationSQSPartitionedSuffix}, + {family: MigrationFamilySQSPartitionedMessageGroup, prefix: migrationSQSMsgGroupPrefix + migrationSQSPartitionedSuffix}, + {family: MigrationFamilySQSPartitionedMessageByAge, prefix: migrationSQSMsgByAgePrefix + migrationSQSPartitionedSuffix}, + {family: MigrationFamilyS3BucketMeta, prefix: s3keys.BucketMetaPrefix}, + {family: MigrationFamilyS3BucketGeneration, prefix: s3keys.BucketGenerationPrefix}, + {family: MigrationFamilyS3ObjectManifest, prefix: s3keys.ObjectManifestPrefix}, + {family: MigrationFamilyS3UploadMeta, prefix: s3keys.UploadMetaPrefix}, + {family: MigrationFamilyS3UploadPart, prefix: s3keys.UploadPartPrefix}, + {family: MigrationFamilyS3Blob, prefix: s3keys.BlobPrefix}, + {family: MigrationFamilyS3GCUpload, prefix: s3keys.GCUploadPrefix}, + } + + out := make([]MigrationBracket, 0, len(defs)) + for _, def := range defs { + start := []byte(def.prefix) + out = append(out, MigrationBracket{ + BracketID: uint64(def.family), + Family: def.family, + Start: CloneBytes(start), + End: prefixScanEnd(start), + ExcludePrefixes: stringsToByteSlices(def.excludePrefixes), + DrainOnly: def.drainOnly, + RequiresRouteKeyCheck: true, + }) + } + return out +} + +func stringsToByteSlices(in []string) [][]byte { + if len(in) == 0 { + return nil + } + out := make([][]byte, len(in)) + for i := range in { + out[i] = []byte(in[i]) + } + return out +} + +func migrationWallMillisToUint64(ms int64) uint64 { + if ms <= 0 { + return 0 + } + return uint64(ms) +} + +func validateSplitJobSource(job SplitJob, source RouteDescriptor) error { + if job.SourceRouteID != source.RouteID { + return errors.WithStack(ErrMigrationSourceRouteChanged) + } + if len(job.SplitKey) == 0 { + return errors.WithStack(ErrMigrationInvalidRoute) + } + if bytes.Compare(job.SplitKey, source.Start) <= 0 { + return errors.WithStack(ErrMigrationInvalidRoute) + } + if len(source.End) > 0 && bytes.Compare(job.SplitKey, source.End) >= 0 { + return errors.WithStack(ErrMigrationInvalidRoute) + } + return nil +} + +func rangesIntersect(aStart, aEnd, bStart, bEnd []byte) bool { + if len(aEnd) > 0 && bytes.Compare(aEnd, bStart) <= 0 { + return false + } + if len(bEnd) > 0 && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true +} + +func hasAnyPrefix(key []byte, prefixes [][]byte) bool { + for _, prefix := range prefixes { + if bytes.HasPrefix(key, prefix) { + return true + } + } + return false +} + +func cloneByteSlices(in [][]byte) [][]byte { + out := make([][]byte, len(in)) + for i := range in { + out[i] = CloneBytes(in[i]) + } + return out +} diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go new file mode 100644 index 000000000..22e563cd2 --- /dev/null +++ b/distribution/migrator_export_plan_test.go @@ -0,0 +1,235 @@ +package distribution + +import ( + "bytes" + "testing" + + "github.com/bootjp/elastickv/internal/s3keys" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestPlanMigrationBracketsIncludesRequiredFamilies(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("m"), []byte("z")) + require.NoError(t, err) + + byFamily := bracketsByFamily(brackets) + required := map[uint32]string{ + MigrationFamilyUser: "user", + MigrationFamilyTxnIntent: migrationTxnIntentPrefix, + MigrationFamilyTxnCommit: migrationTxnCommitPrefix, + MigrationFamilyTxnRollback: migrationTxnRollbackPrefix, + MigrationFamilyTxnSuccess: migrationTxnSuccessPrefix, + MigrationFamilyTxnMeta: migrationTxnMetaPrefix, + MigrationFamilyTxnLock: migrationTxnLockPrefix, + MigrationFamilyListMeta: store.ListMetaPrefix, + MigrationFamilyListItem: store.ListItemPrefix, + MigrationFamilyListMetaDelta: store.ListMetaDeltaPrefix, + MigrationFamilyListClaim: store.ListClaimPrefix, + MigrationFamilyRedisLegacy: migrationRedisPrefix, + MigrationFamilyHash: migrationHashPrefix, + MigrationFamilySet: migrationSetPrefix, + MigrationFamilyZSet: migrationZSetPrefix, + MigrationFamilyStreamMeta: store.StreamMetaPrefix, + MigrationFamilyStreamEntry: store.StreamEntryPrefix, + MigrationFamilyDynamoTableMeta: migrationDynamoMetaPrefix, + MigrationFamilyDynamoTableGeneration: migrationDynamoGenPrefix, + MigrationFamilyDynamoItem: migrationDynamoItemPrefix, + MigrationFamilyDynamoGSI: migrationDynamoGSIPrefix, + MigrationFamilySQSQueueMeta: migrationSQSQueueMetaPrefix, + MigrationFamilySQSQueueGeneration: migrationSQSQueueGenPrefix, + MigrationFamilySQSQueueSequence: migrationSQSQueueSeqPrefix, + MigrationFamilySQSQueueTombstone: migrationSQSQueueTombstonePrefix, + MigrationFamilySQSMessageData: migrationSQSMsgDataPrefix, + MigrationFamilySQSMessageVisibility: migrationSQSMsgVisPrefix, + MigrationFamilySQSMessageDedup: migrationSQSMsgDedupPrefix, + MigrationFamilySQSMessageGroup: migrationSQSMsgGroupPrefix, + MigrationFamilySQSMessageByAge: migrationSQSMsgByAgePrefix, + MigrationFamilySQSPartitionedMessageData: migrationSQSMsgDataPrefix + migrationSQSPartitionedSuffix, + MigrationFamilySQSPartitionedMessageVisibility: migrationSQSMsgVisPrefix + migrationSQSPartitionedSuffix, + MigrationFamilySQSPartitionedMessageDedup: migrationSQSMsgDedupPrefix + migrationSQSPartitionedSuffix, + MigrationFamilySQSPartitionedMessageGroup: migrationSQSMsgGroupPrefix + migrationSQSPartitionedSuffix, + MigrationFamilySQSPartitionedMessageByAge: migrationSQSMsgByAgePrefix + migrationSQSPartitionedSuffix, + MigrationFamilyS3BucketMeta: s3keys.BucketMetaPrefix, + MigrationFamilyS3BucketGeneration: s3keys.BucketGenerationPrefix, + MigrationFamilyS3ObjectManifest: s3keys.ObjectManifestPrefix, + MigrationFamilyS3UploadMeta: s3keys.UploadMetaPrefix, + MigrationFamilyS3UploadPart: s3keys.UploadPartPrefix, + MigrationFamilyS3Blob: s3keys.BlobPrefix, + MigrationFamilyS3GCUpload: s3keys.GCUploadPrefix, + } + + for family, prefix := range required { + bracket, ok := byFamily[family] + require.True(t, ok, "missing family %d", family) + require.Equal(t, uint64(family), bracket.BracketID) + require.True(t, bracket.RequiresRouteKeyCheck) + if family == MigrationFamilyUser { + require.Equal(t, []byte("m"), bracket.Start) + require.Equal(t, []byte("z"), bracket.End) + require.True(t, bracket.ExcludeKnownInternal) + continue + } + require.Equal(t, []byte(prefix), bracket.Start, "family %d start", family) + require.Equal(t, prefixScanEnd([]byte(prefix)), bracket.End, "family %d end", family) + } + + require.True(t, byFamily[MigrationFamilyTxnLock].DrainOnly) + export, err := PlanExportBrackets([]byte("m"), []byte("z")) + require.NoError(t, err) + _, exportedLock := bracketsByFamily(export)[MigrationFamilyTxnLock] + require.False(t, exportedLock, "txn locks are drain-only and must not be exported as data") +} + +func TestPlanMigrationBracketsDisjointPrefixContainment(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("m"), []byte("z")) + require.NoError(t, err) + byFamily := bracketsByFamily(brackets) + + listDelta := store.ListMetaDeltaKey([]byte("list"), 1, 0) + require.True(t, byFamily[MigrationFamilyListMetaDelta].ContainsRawKey(listDelta)) + require.False(t, byFamily[MigrationFamilyListMeta].ContainsRawKey(listDelta)) + + partitionedSQS := []byte(migrationSQSMsgDataPrefix + migrationSQSPartitionedSuffix + "queue|0|1|msg") + require.True(t, byFamily[MigrationFamilySQSPartitionedMessageData].ContainsRawKey(partitionedSQS)) + require.False(t, byFamily[MigrationFamilySQSMessageData].ContainsRawKey(partitionedSQS)) + + user := byFamily[MigrationFamilyUser] + user.Start = nil + user.End = nil + for _, raw := range [][]byte{ + []byte("!txn|foo"), + []byte("!stream|foo"), + []byte("!ddb|foo"), + []byte("!sqs|foo"), + []byte("!s3|foo"), + []byte("ordinary-user-key"), + } { + require.True(t, user.ContainsRawKey(raw), "raw user key %q must stay in familyUser", raw) + } + for _, raw := range [][]byte{ + []byte(migrationTxnSuccessPrefix + "x"), + []byte(store.StreamMetaPrefix + "x"), + []byte(migrationDynamoItemPrefix + "x"), + []byte(migrationSQSMsgVisPrefix + "x"), + []byte(s3keys.ObjectManifestPrefix + "x"), + []byte(migrationRedisPrefix + "string|k"), + []byte(migrationHashPrefix + "meta|x"), + } { + require.False(t, user.ContainsRawKey(raw), "concrete internal key %q must be excluded from familyUser", raw) + } +} + +func TestMigrationKnownInternalPrefixesAreConcreteOnly(t *testing.T) { + t.Parallel() + + for _, raw := range [][]byte{ + []byte(migrationTxnIntentPrefix + "k"), + []byte(migrationTxnSuccessPrefix + "k"), + []byte(store.ListClaimPrefix + "k"), + []byte(store.HashFieldPrefix + "k"), + []byte(store.StreamEntryPrefix + "k"), + []byte(migrationDynamoMetaPrefix + "t"), + []byte(migrationSQSQueueMetaPrefix + "q"), + []byte(s3keys.BlobPrefix + "b"), + } { + require.True(t, IsMigrationKnownInternalKey(raw), "concrete internal key %q", raw) + } + + for _, raw := range [][]byte{ + []byte("!txn|foo"), + []byte("!stream|foo"), + []byte("!ddb|foo"), + []byte("!sqs|foo"), + []byte("!s3|foo"), + } { + require.False(t, IsMigrationKnownInternalKey(raw), "umbrella-looking user key %q", raw) + } + + prefixes := MigrationKnownInternalPrefixes() + require.NotEmpty(t, prefixes) + prefixes[0][0] ^= 0xff + require.False(t, bytes.Equal(prefixes[0], MigrationKnownInternalPrefixes()[0]), "prefix list must be cloned") +} + +func TestValidateMigrationRouteRangeRejectsReservedControlPrefixes(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + start []byte + end []byte + }{ + {name: "exact dist", start: []byte("!dist|"), end: prefixScanEnd([]byte("!dist|"))}, + {name: "migstage", start: []byte("!migstage|"), end: prefixScanEnd([]byte("!migstage|"))}, + {name: "broad intersection", start: []byte("!"), end: []byte("~")}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + err := ValidateMigrationRouteRange(tc.start, tc.end) + require.True(t, errors.Is(err, ErrMigrationReservedRange), "got %v", err) + }) + } + + require.NoError(t, ValidateMigrationRouteRange([]byte("m"), []byte("z"))) + err := ValidateMigrationRouteRange([]byte("z"), []byte("m")) + require.True(t, errors.Is(err, ErrMigrationInvalidRoute), "got %v", err) +} + +func TestSplitJobPlannerAndSameGroupNoop(t *testing.T) { + t.Parallel() + + source := RouteDescriptor{ + RouteID: 9, + Start: []byte("a"), + End: []byte("z"), + GroupID: 3, + State: RouteStateActive, + } + job := SplitJob{ + JobID: 1, + SourceRouteID: source.RouteID, + SplitKey: []byte("m"), + TargetGroupID: source.GroupID, + Phase: SplitJobPhasePlanned, + } + + planned, err := InitializeSplitJobPlan(job, source, 1000) + require.NoError(t, err) + require.Equal(t, SplitJobPhasePlanned, planned.Phase) + require.NotEmpty(t, planned.BracketProgress) + require.Equal(t, int64(1000), planned.StartedAtMs) + require.Equal(t, int64(1000), planned.UpdatedAtMs) + for _, progress := range planned.BracketProgress { + require.Equal(t, SplitJobExportPhaseBackfill, progress.ExportPhase) + require.NotEqual(t, MigrationFamilyTxnLock, progress.Family) + } + + done, err := AdvanceSameGroupNoop(job, source, 2000) + require.NoError(t, err) + require.Equal(t, SplitJobPhaseDone, done.Phase) + require.True(t, done.TargetPromotionDone) + require.Equal(t, uint64(2000), done.PromotionCompletedTS) + require.Equal(t, int64(2000), done.TerminalAtMs) + for _, progress := range done.BracketProgress { + require.True(t, progress.Done) + } + + crossGroup := job + crossGroup.TargetGroupID = source.GroupID + 1 + _, err = AdvanceSameGroupNoop(crossGroup, source, 3000) + require.True(t, errors.Is(err, ErrMigrationDataMoveRequired), "got %v", err) +} + +func bracketsByFamily(brackets []MigrationBracket) map[uint32]MigrationBracket { + out := make(map[uint32]MigrationBracket, len(brackets)) + for _, bracket := range brackets { + out[bracket.Family] = bracket + } + return out +} diff --git a/kv/migrator_filter.go b/kv/migrator_filter.go new file mode 100644 index 000000000..d1b4ddc19 --- /dev/null +++ b/kv/migrator_filter.go @@ -0,0 +1,21 @@ +package kv + +import "bytes" + +// RouteKeyFilter returns the migration export predicate for raw MVCC keys. +// rangeEnd nil or empty means +infinity, matching the route descriptor wire +// convention. +func RouteKeyFilter(rangeStart, rangeEnd []byte) func([]byte) bool { + start := bytes.Clone(rangeStart) + end := bytes.Clone(rangeEnd) + return func(rawKey []byte) bool { + rkey := routeKey(rawKey) + if bytes.Compare(rkey, start) < 0 { + return false + } + if len(end) > 0 && bytes.Compare(rkey, end) >= 0 { + return false + } + return true + } +} diff --git a/kv/shard_key.go b/kv/shard_key.go index 6871ce789..deb5fe468 100644 --- a/kv/shard_key.go +++ b/kv/shard_key.go @@ -36,6 +36,17 @@ const ( // family (!sqs|queue|meta|, !sqs|msg|vis|, etc.). Used by // sqsRouteKey to dispatch the routing decision. sqsInternalPrefix = "!sqs|" + + sqsQueueMetaPrefix = "!sqs|queue|meta|" + sqsQueueGenPrefix = "!sqs|queue|gen|" + sqsQueueSeqPrefix = "!sqs|queue|seq|" + sqsQueueTombstonePrefix = "!sqs|queue|tombstone|" + sqsMsgDataPrefix = "!sqs|msg|data|" + sqsMsgVisPrefix = "!sqs|msg|vis|" + sqsMsgDedupPrefix = "!sqs|msg|dedup|" + sqsMsgGroupPrefix = "!sqs|msg|group|" + sqsMsgByAgePrefix = "!sqs|msg|byage|" + sqsPartitionMarker = "p|" ) var ( @@ -44,8 +55,31 @@ var ( dynamoTableGenerationPrefixBytes = []byte(DynamoTableGenerationPrefix) dynamoItemPrefixBytes = []byte(DynamoItemPrefix) dynamoGSIPrefixBytes = []byte(DynamoGSIPrefix) - sqsRoutePrefixBytes = []byte(sqsRoutePrefix) sqsInternalPrefixBytes = []byte(sqsInternalPrefix) + sqsGlobalRouteKey = []byte(sqsRoutePrefix + "global") + sqsConcreteInternalPrefixBytes = [][]byte{ + []byte(sqsQueueMetaPrefix), + []byte(sqsQueueGenPrefix), + []byte(sqsQueueSeqPrefix), + []byte(sqsQueueTombstonePrefix), + []byte(sqsMsgDataPrefix), + []byte(sqsMsgVisPrefix), + []byte(sqsMsgDedupPrefix), + []byte(sqsMsgGroupPrefix), + []byte(sqsMsgByAgePrefix), + } + routeKeyExtractors = []func([]byte) []byte{ + redisRouteKey, + dynamoRouteKey, + sqsRouteKey, + s3keys.ExtractRouteKey, + listRouteKey, + hashRouteKey, + setRouteKey, + zsetRouteKey, + streamRouteKey, + store.ExtractListUserKey, + } ) // routeKey normalizes internal keys (e.g., list metadata/items) to the logical @@ -61,20 +95,10 @@ func routeKey(key []byte) []byte { } func normalizeRouteKey(key []byte) []byte { - if user := redisRouteKey(key); user != nil { - return user - } - if table := dynamoRouteKey(key); table != nil { - return table - } - if route := sqsRouteKey(key); route != nil { - return route - } - if user := s3keys.ExtractRouteKey(key); user != nil { - return user - } - if user := store.ExtractListUserKey(key); user != nil { - return user + for _, extract := range routeKeyExtractors { + if user := extract(key); user != nil { + return user + } } return key } @@ -124,19 +148,87 @@ func dynamoRouteTableKey(tableSegment []byte) []byte { return out } -// sqsRouteKey maps any !sqs|... internal key to a stable route key so -// multi-shard deployments that partition by user-key range still land -// every SQS mutation on a configured group. Milestone 1 collapses all -// SQS keys to a single !sqs|route|global route — this keeps the -// catalog and every queue's message keyspace on the same group, which -// is the minimum needed for FIFO group-lock semantics (landing later) -// to work. When per-queue sharding is implemented it will live here. +func listRouteKey(key []byte) []byte { + switch { + case store.IsListMetaDeltaKey(key): + return store.ExtractListUserKeyFromDelta(key) + case store.IsListClaimKey(key): + return store.ExtractListUserKeyFromClaim(key) + default: + return nil + } +} + +func hashRouteKey(key []byte) []byte { + switch { + case store.IsHashMetaDeltaKey(key): + return store.ExtractHashUserKeyFromDelta(key) + case store.IsHashMetaKey(key): + return store.ExtractHashUserKeyFromMeta(key) + case store.IsHashFieldKey(key): + return store.ExtractHashUserKeyFromField(key) + default: + return nil + } +} + +func setRouteKey(key []byte) []byte { + switch { + case store.IsSetMetaDeltaKey(key): + return store.ExtractSetUserKeyFromDelta(key) + case store.IsSetMetaKey(key): + return store.ExtractSetUserKeyFromMeta(key) + case store.IsSetMemberKey(key): + return store.ExtractSetUserKeyFromMember(key) + default: + return nil + } +} + +func zsetRouteKey(key []byte) []byte { + switch { + case store.IsZSetMetaDeltaKey(key): + return store.ExtractZSetUserKeyFromDelta(key) + case store.IsZSetMetaKey(key): + return store.ExtractZSetUserKeyFromMeta(key) + case store.IsZSetMemberKey(key): + return store.ExtractZSetUserKeyFromMember(key) + case store.IsZSetScoreKey(key): + return store.ExtractZSetUserKeyFromScore(key) + default: + return nil + } +} + +func streamRouteKey(key []byte) []byte { + switch { + case store.IsStreamMetaKey(key): + return store.ExtractStreamUserKeyFromMeta(key) + case store.IsStreamEntryKey(key): + return store.ExtractStreamUserKeyFromEntry(key) + default: + return nil + } +} + +// sqsRouteKey maps concrete persisted !sqs|... storage prefixes to a stable +// route key. Adapter-looking raw user keys such as !sqs|foo intentionally stay +// on their raw route and migrate through the user-key bracket. func sqsRouteKey(key []byte) []byte { if !bytes.HasPrefix(key, sqsInternalPrefixBytes) { return nil } - out := make([]byte, 0, len(sqsRoutePrefixBytes)+len("global")) - out = append(out, sqsRoutePrefixBytes...) - out = append(out, []byte("global")...) - return out + if !hasSQSConcreteInternalPrefix(key) { + return nil + } + return sqsGlobalRouteKey +} + +func hasSQSConcreteInternalPrefix(key []byte) bool { + for _, prefix := range sqsConcreteInternalPrefixBytes { + if bytes.HasPrefix(key, prefix) { + return true + } + } + return false } diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 8257fbb77..e7d6c55d7 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/bootjp/elastickv/internal/s3keys" + "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -95,3 +96,101 @@ func TestRouteKey_CollapsesDynamoGenerationsToSameTableRoute(t *testing.T) { require.Equal(t, want, routeKey(sourceGSIKey), "migration source generation GSI key must route to the same table group as the current generation") } + +func TestRouteKey_NormalizesCollectionMigrationFamilies(t *testing.T) { + t.Parallel() + + userKey := []byte("redis:user|with|separators") + cases := [][]byte{ + store.ListMetaDeltaKey(userKey, 10, 1), + store.ListClaimKey(userKey, -2), + store.HashMetaKey(userKey), + store.HashFieldKey(userKey, []byte("field")), + store.HashMetaDeltaKey(userKey, 11, 2), + store.SetMetaKey(userKey), + store.SetMemberKey(userKey, []byte("member")), + store.SetMetaDeltaKey(userKey, 12, 3), + store.ZSetMetaKey(userKey), + store.ZSetMemberKey(userKey, []byte("member")), + store.ZSetScoreKey(userKey, 1.25, []byte("member")), + store.ZSetMetaDeltaKey(userKey, 13, 4), + store.StreamMetaKey(userKey), + store.StreamEntryKey(userKey, 14, 5), + } + + for _, raw := range cases { + require.Equal(t, userKey, routeKey(raw), "raw key %q must route by its logical user key", raw) + } +} + +func TestRouteKey_NormalizesTxnSuccessMarkerByLockedKey(t *testing.T) { + t.Parallel() + + lockedKey := []byte("secondary|key\x00with|separators") + primaryKey := []byte("primary|key\x00with|separators") + marker := TxnSuccessMarkerKey(lockedKey, 100, 200, primaryKey) + + require.Equal(t, lockedKey, routeKey(marker)) + + malformed := append([]byte(nil), marker...) + malformed[len(txnSuccessPrefixBytes)] = 2 + require.Equal(t, malformed, routeKey(malformed), "malformed success markers must fall back to their raw key") +} + +func TestRouteKey_SQSDecoderIsConcreteOnly(t *testing.T) { + t.Parallel() + + want := []byte(sqsRoutePrefix + "global") + for _, raw := range [][]byte{ + []byte(sqsQueueMetaPrefix + "queue"), + []byte(sqsQueueGenPrefix + "queue"), + []byte(sqsQueueSeqPrefix + "queue"), + []byte(sqsQueueTombstonePrefix + "queue"), + []byte(sqsMsgDataPrefix + "queue|1|msg"), + []byte(sqsMsgVisPrefix + "queue|1|msg"), + []byte(sqsMsgDedupPrefix + "queue|1|dedup"), + []byte(sqsMsgGroupPrefix + "queue|1|group"), + []byte(sqsMsgByAgePrefix + "queue|1|ts"), + []byte(sqsMsgDataPrefix + sqsPartitionMarker + "queue|0|1|msg"), + []byte(sqsMsgVisPrefix + sqsPartitionMarker + "queue|0|1|msg"), + []byte(sqsMsgDedupPrefix + sqsPartitionMarker + "queue|0|1|dedup"), + []byte(sqsMsgGroupPrefix + sqsPartitionMarker + "queue|0|1|group"), + []byte(sqsMsgByAgePrefix + sqsPartitionMarker + "queue|0|1|ts"), + } { + require.Equal(t, want, routeKey(raw), "concrete SQS key %q must use the SQS route", raw) + } + + rawUser := []byte("!sqs|foo") + require.Equal(t, rawUser, routeKey(rawUser), "adapter-looking raw user key must stay on its raw route") +} + +func TestRouteKey_S3DecoderIsConcreteOnly(t *testing.T) { + t.Parallel() + + manifest := s3keys.ObjectManifestKey("bucket", 2, "obj") + require.Equal(t, s3keys.RouteKey("bucket", 2, "obj"), routeKey(manifest)) + + rawUser := []byte("!s3|foo") + require.Equal(t, rawUser, routeKey(rawUser), "adapter-looking raw user key must stay on its raw route") +} + +func TestRouteKeyFilterTreatsNilAndEmptyEndAsInfinity(t *testing.T) { + t.Parallel() + + start := []byte("m") + for _, tc := range []struct { + name string + end []byte + }{ + {name: "nil"}, + {name: "empty", end: []byte{}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + filter := RouteKeyFilter(start, tc.end) + require.False(t, filter([]byte("a"))) + require.True(t, filter([]byte("m"))) + require.True(t, filter([]byte("z"))) + }) + } +} diff --git a/kv/txn_keys.go b/kv/txn_keys.go index e997722ea..9ddfb3c9f 100644 --- a/kv/txn_keys.go +++ b/kv/txn_keys.go @@ -16,6 +16,7 @@ const ( txnIntentPrefix = TxnKeyPrefix + "int|" txnCommitPrefix = TxnKeyPrefix + "cmt|" txnRollbackPrefix = TxnKeyPrefix + "rb|" + txnSuccessPrefix = TxnKeyPrefix + "ok|" txnMetaPrefix = TxnKeyPrefix + "meta|" ) @@ -27,11 +28,14 @@ var ( txnIntentPrefixBytes = []byte(txnIntentPrefix) txnCommitPrefixBytes = []byte(txnCommitPrefix) txnRollbackPrefixBytes = []byte(txnRollbackPrefix) + txnSuccessPrefixBytes = []byte(txnSuccessPrefix) txnMetaPrefixBytes = []byte(txnMetaPrefix) txnCommonPrefix = []byte(TxnKeyPrefix) ) const txnStartTSSuffixLen = 8 +const txnSuccessMarkerVersion = byte(1) +const maxIntValue = int(^uint(0) >> 1) func txnLockKey(userKey []byte) []byte { k := make([]byte, 0, len(txnLockPrefixBytes)+len(userKey)) @@ -75,6 +79,7 @@ func isTxnInternalKey(key []byte) bool { bytes.HasPrefix(key, txnIntentPrefixBytes) || bytes.HasPrefix(key, txnCommitPrefixBytes) || bytes.HasPrefix(key, txnRollbackPrefixBytes) || + bytes.HasPrefix(key, txnSuccessPrefixBytes) || bytes.HasPrefix(key, txnMetaPrefixBytes) } @@ -104,6 +109,8 @@ func txnRouteKey(key []byte) ([]byte, bool) { return nil, false } return rest[:len(rest)-txnStartTSSuffixLen], true + case bytes.HasPrefix(key, txnSuccessPrefixBytes): + return txnSuccessLockedKey(key) default: return nil, false } @@ -119,3 +126,55 @@ func ExtractTxnUserKey(key []byte) []byte { } return userKey } + +// TxnSuccessMarkerKey builds the route-local transaction-success marker key +// used by the migration planner. Normal transaction traffic only writes this +// after the migration capability gate opens in a later PR. +func TxnSuccessMarkerKey(lockedKey []byte, startTS, commitTS uint64, primaryKey []byte) []byte { + out := make([]byte, 0, len(txnSuccessPrefixBytes)+1+binary.MaxVarintLen64+len(lockedKey)+2*txnStartTSSuffixLen+binary.MaxVarintLen64+len(primaryKey)) + out = append(out, txnSuccessPrefixBytes...) + out = append(out, txnSuccessMarkerVersion) + out = binary.AppendUvarint(out, uint64(len(lockedKey))) + out = append(out, lockedKey...) + var raw [txnStartTSSuffixLen]byte + binary.BigEndian.PutUint64(raw[:], startTS) + out = append(out, raw[:]...) + binary.BigEndian.PutUint64(raw[:], commitTS) + out = append(out, raw[:]...) + out = binary.AppendUvarint(out, uint64(len(primaryKey))) + out = append(out, primaryKey...) + return out +} + +func txnSuccessLockedKey(key []byte) ([]byte, bool) { + rest := key[len(txnSuccessPrefixBytes):] + if len(rest) == 0 || rest[0] != txnSuccessMarkerVersion { + return nil, false + } + rest = rest[1:] + lockedLenRaw, n := binary.Uvarint(rest) + lockedLen, ok := uvarintToInt(lockedLenRaw) + if n <= 0 || !ok || lockedLen > len(rest)-n { + return nil, false + } + lockedStart := n + lockedEnd := lockedStart + lockedLen + rest = rest[lockedEnd:] + if len(rest) < 2*txnStartTSSuffixLen { + return nil, false + } + rest = rest[2*txnStartTSSuffixLen:] + primaryLenRaw, n := binary.Uvarint(rest) + primaryLen, ok := uvarintToInt(primaryLenRaw) + if n <= 0 || !ok || primaryLen != len(rest)-n { + return nil, false + } + return key[len(txnSuccessPrefixBytes)+1+lockedStart : len(txnSuccessPrefixBytes)+1+lockedEnd], true +} + +func uvarintToInt(v uint64) (int, bool) { + if v > uint64(maxIntValue) { + return 0, false + } + return int(v), true +} From af847ef2631c31e6705de2baabe42a7eaa24bb5f Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 22:03:35 +0900 Subject: [PATCH 004/162] Fix migration bracket edge cases --- distribution/migrator.go | 38 +++++++++++++++++++++-- distribution/migrator_export_plan_test.go | 22 ++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/distribution/migrator.go b/distribution/migrator.go index 472a09035..c795f70f6 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -146,12 +146,14 @@ type MigrationBracket struct { ExcludeKnownInternal bool DrainOnly bool RequiresRouteKeyCheck bool + RequiresDecodedS3 bool } // PlanMigrationBrackets returns the full M2 migration plan, including the // drain-only transaction lock bracket. Data export callers should use // PlanExportBrackets, which omits drain-only control state. func PlanMigrationBrackets(routeStart, routeEnd []byte) ([]MigrationBracket, error) { + routeEnd = normalizeMigrationRouteEnd(routeEnd) if err := ValidateMigrationRouteRange(routeStart, routeEnd); err != nil { return nil, err } @@ -215,12 +217,26 @@ func (b MigrationBracket) ContainsRawKey(rawKey []byte) bool { if len(b.End) > 0 && bytes.Compare(rawKey, b.End) >= 0 { return false } + if !b.containsFamilyShape(rawKey) { + return false + } if b.ExcludeKnownInternal && IsMigrationKnownInternalKey(rawKey) { return false } return !hasAnyPrefix(rawKey, b.ExcludePrefixes) } +func (b MigrationBracket) containsFamilyShape(rawKey []byte) bool { + switch b.Family { + case MigrationFamilyListMetaDelta: + return store.ExtractListUserKeyFromDelta(rawKey) != nil + case MigrationFamilyListMeta: + return store.ExtractListUserKeyFromDelta(rawKey) == nil + default: + return true + } +} + // InitializeSplitJobPlan validates the source route and seeds the job's // bracket progress for the moving right child [SplitKey, source.End). func InitializeSplitJobPlan(job SplitJob, source RouteDescriptor, nowMs int64) (SplitJob, error) { @@ -312,7 +328,7 @@ func migrationFamilyBrackets() []MigrationBracket { {family: MigrationFamilyTxnMeta, prefix: migrationTxnMetaPrefix}, {family: MigrationFamilyListMetaDelta, prefix: store.ListMetaDeltaPrefix}, {family: MigrationFamilyListClaim, prefix: store.ListClaimPrefix}, - {family: MigrationFamilyListMeta, prefix: store.ListMetaPrefix, excludePrefixes: []string{store.ListMetaDeltaPrefix}}, + {family: MigrationFamilyListMeta, prefix: store.ListMetaPrefix}, {family: MigrationFamilyListItem, prefix: store.ListItemPrefix}, {family: MigrationFamilyRedisLegacy, prefix: migrationRedisPrefix}, {family: MigrationFamilyHash, prefix: migrationHashPrefix}, @@ -350,6 +366,7 @@ func migrationFamilyBrackets() []MigrationBracket { out := make([]MigrationBracket, 0, len(defs)) for _, def := range defs { start := []byte(def.prefix) + requiresRouteKeyCheck, requiresDecodedS3 := migrationBracketRouteCheck(def.family) out = append(out, MigrationBracket{ BracketID: uint64(def.family), Family: def.family, @@ -357,12 +374,29 @@ func migrationFamilyBrackets() []MigrationBracket { End: prefixScanEnd(start), ExcludePrefixes: stringsToByteSlices(def.excludePrefixes), DrainOnly: def.drainOnly, - RequiresRouteKeyCheck: true, + RequiresRouteKeyCheck: requiresRouteKeyCheck, + RequiresDecodedS3: requiresDecodedS3, }) } return out } +func migrationBracketRouteCheck(family uint32) (requiresRouteKeyCheck bool, requiresDecodedS3 bool) { + switch family { + case MigrationFamilyS3BucketMeta, MigrationFamilyS3BucketGeneration: + return false, true + default: + return true, false + } +} + +func normalizeMigrationRouteEnd(routeEnd []byte) []byte { + if len(routeEnd) == 0 { + return nil + } + return routeEnd +} + func stringsToByteSlices(in []string) [][]byte { if len(in) == 0 { return nil diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index 22e563cd2..6888aa3bf 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -66,7 +66,13 @@ func TestPlanMigrationBracketsIncludesRequiredFamilies(t *testing.T) { bracket, ok := byFamily[family] require.True(t, ok, "missing family %d", family) require.Equal(t, uint64(family), bracket.BracketID) - require.True(t, bracket.RequiresRouteKeyCheck) + if family == MigrationFamilyS3BucketMeta || family == MigrationFamilyS3BucketGeneration { + require.False(t, bracket.RequiresRouteKeyCheck) + require.True(t, bracket.RequiresDecodedS3) + } else { + require.True(t, bracket.RequiresRouteKeyCheck) + require.False(t, bracket.RequiresDecodedS3) + } if family == MigrationFamilyUser { require.Equal(t, []byte("m"), bracket.Start) require.Equal(t, []byte("z"), bracket.End) @@ -95,6 +101,10 @@ func TestPlanMigrationBracketsDisjointPrefixContainment(t *testing.T) { require.True(t, byFamily[MigrationFamilyListMetaDelta].ContainsRawKey(listDelta)) require.False(t, byFamily[MigrationFamilyListMeta].ContainsRawKey(listDelta)) + listMetaWithDeltaLookingUserKey := store.ListMetaKey([]byte("d|list")) + require.True(t, byFamily[MigrationFamilyListMeta].ContainsRawKey(listMetaWithDeltaLookingUserKey)) + require.False(t, byFamily[MigrationFamilyListMetaDelta].ContainsRawKey(listMetaWithDeltaLookingUserKey)) + partitionedSQS := []byte(migrationSQSMsgDataPrefix + migrationSQSPartitionedSuffix + "queue|0|1|msg") require.True(t, byFamily[MigrationFamilySQSPartitionedMessageData].ContainsRawKey(partitionedSQS)) require.False(t, byFamily[MigrationFamilySQSMessageData].ContainsRawKey(partitionedSQS)) @@ -125,6 +135,16 @@ func TestPlanMigrationBracketsDisjointPrefixContainment(t *testing.T) { } } +func TestPlanMigrationBracketsNormalizesEmptyRouteEnd(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("m"), []byte{}) + require.NoError(t, err) + user := bracketsByFamily(brackets)[MigrationFamilyUser] + require.Nil(t, user.End) + require.True(t, user.ContainsRawKey([]byte("z"))) +} + func TestMigrationKnownInternalPrefixesAreConcreteOnly(t *testing.T) { t.Parallel() From eab9622059690e60f87786e872a5da24f446bbd6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 22:15:41 +0900 Subject: [PATCH 005/162] Add migration fence drain guards --- adapter/distribution_server.go | 66 +++++++++++++++ adapter/distribution_server_test.go | 64 +++++++++++++++ distribution/engine.go | 40 +++++++++- distribution/split_job_catalog.go | 1 + kv/fsm.go | 83 ++++++++++++++++++- kv/fsm_migration_fence_test.go | 97 +++++++++++++++++++++++ kv/migrator_lock_drain.go | 85 ++++++++++++++++++++ kv/migrator_lock_drain_test.go | 45 +++++++++++ kv/route_history.go | 14 ++++ kv/sharded_coordinator.go | 59 ++++++++++++-- kv/sharded_coordinator_del_prefix_test.go | 79 ++++++++++++++++++ 11 files changed, 625 insertions(+), 8 deletions(-) create mode 100644 kv/fsm_migration_fence_test.go create mode 100644 kv/migrator_lock_drain.go create mode 100644 kv/migrator_lock_drain_test.go diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 30c111b6b..3001b1329 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -162,6 +162,9 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR if err := validateSplitKey(parent, splitKey); err != nil { return nil, err } + if err := s.rejectLiveSplitJobOverlap(ctx, snapshot, parent); err != nil { + return nil, err + } leftID, rightID, err := s.allocateChildRouteIDs(ctx, snapshot.ReadTS, snapshot.Routes) if err != nil { @@ -376,6 +379,69 @@ func validateSplitKey(parent distribution.RouteDescriptor, splitKey []byte) erro return nil } +func (s *DistributionServer) rejectLiveSplitJobOverlap(ctx context.Context, snapshot distribution.CatalogSnapshot, parent distribution.RouteDescriptor) error { + jobs, err := s.catalog.ListSplitJobsAt(ctx, snapshot.ReadTS) + if err != nil { + return grpcStatusErrorf(codes.Internal, "load split jobs: %v", err) + } + for _, job := range jobs { + if !splitJobIsLive(job) { + continue + } + for _, interval := range liveSplitJobIntervals(job, snapshot.Routes) { + if routeRangeIntersects(parent.Start, parent.End, interval.start, interval.end) { + return grpcStatusError(codes.Aborted, distribution.ErrSplitJobOverlap.Error()) + } + } + } + return nil +} + +func splitJobIsLive(job distribution.SplitJob) bool { + return job.Phase != distribution.SplitJobPhaseDone && job.Phase != distribution.SplitJobPhaseAbandoned +} + +type routeInterval struct { + start []byte + end []byte +} + +const initialLiveSplitJobIntervalCapacity = 2 + +func liveSplitJobIntervals(job distribution.SplitJob, routes []distribution.RouteDescriptor) []routeInterval { + out := make([]routeInterval, 0, initialLiveSplitJobIntervalCapacity) + for _, route := range routes { + switch { + case route.RouteID == job.SourceRouteID: + out = append(out, routeInterval{ + start: distribution.CloneBytes(job.SplitKey), + end: distribution.CloneBytes(route.End), + }) + case route.ParentRouteID == job.SourceRouteID && routeRangeIntersects(route.Start, route.End, job.SplitKey, nil): + out = append(out, routeInterval{ + start: distribution.CloneBytes(route.Start), + end: distribution.CloneBytes(route.End), + }) + case job.JobID != 0 && route.MigrationJobID == job.JobID: + out = append(out, routeInterval{ + start: distribution.CloneBytes(route.Start), + end: distribution.CloneBytes(route.End), + }) + } + } + return out +} + +func routeRangeIntersects(aStart, aEnd, bStart, bEnd []byte) bool { + if aEnd != nil && bytes.Compare(aEnd, bStart) <= 0 { + return false + } + if bEnd != nil && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true +} + func splitCatalogRoutes( parent distribution.RouteDescriptor, splitKey []byte, diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 8112872fc..7604d0e91 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -269,6 +269,70 @@ func TestDistributionServerSplitRange_VersionConflict(t *testing.T) { require.ErrorContains(t, err, errDistributionCatalogConflict.Error()) } +func TestDistributionServerSplitRange_RejectsLiveSplitJobOverlap(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, distribution.SplitJob{ + JobID: 10, + SourceRouteID: 1, + SplitKey: []byte("g"), + TargetGroupID: 8, + Phase: distribution.SplitJobPhaseBackfill, + })) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err = s.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("c"), + }) + require.Error(t, err) + require.Equal(t, codes.Aborted, status.Code(err)) + require.ErrorContains(t, err, distribution.ErrSplitJobOverlap.Error()) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerSplitRange_AllowsDisjointRouteWhileSplitJobLive(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 3, Start: []byte("a"), End: []byte("g"), GroupID: 1, State: distribution.RouteStateActive, ParentRouteID: 1}, + {RouteID: 4, Start: []byte("g"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateWriteFenced, ParentRouteID: 1}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, distribution.SplitJob{ + JobID: 10, + SourceRouteID: 1, + SplitKey: []byte("g"), + TargetGroupID: 8, + Phase: distribution.SplitJobPhaseFence, + })) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + resp, err := s.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 3, + SplitKey: []byte("c"), + }) + require.NoError(t, err) + require.Equal(t, uint64(2), resp.CatalogVersion) + require.Equal(t, 1, coordinator.dispatchCalls) +} + func TestDistributionServerSplitRange_UsesCoordinatorForCatalogWrites(t *testing.T) { t.Parallel() diff --git a/distribution/engine.go b/distribution/engine.go index bc6613894..7963d1282 100644 --- a/distribution/engine.go +++ b/distribution/engine.go @@ -158,6 +158,15 @@ func (s RouteHistorySnapshot) Version() uint64 { return s.version } // scan from O(N) to "first non-covering gap" without changing the // resolution semantics). func (s RouteHistorySnapshot) OwnerOf(key []byte) (uint64, bool) { + r, ok := s.RouteOf(key) + if !ok { + return 0, false + } + return r.GroupID, true +} + +// RouteOf returns the route that covered key at this snapshot's version. +func (s RouteHistorySnapshot) RouteOf(key []byte) (Route, bool) { for _, r := range s.routes { if bytes.Compare(key, r.Start) < 0 { break @@ -165,9 +174,25 @@ func (s RouteHistorySnapshot) OwnerOf(key []byte) (uint64, bool) { if r.End != nil && bytes.Compare(key, r.End) >= 0 { continue } - return r.GroupID, true + return cloneRoute(r), true } - return 0, false + return Route{}, false +} + +// IntersectingRoutes returns every route whose range intersects [start, end) +// in this snapshot. A nil end denotes +infinity. +func (s RouteHistorySnapshot) IntersectingRoutes(start, end []byte) []Route { + out := make([]Route, 0) + for _, r := range s.routes { + if r.End != nil && bytes.Compare(r.End, start) <= 0 { + continue + } + if end != nil && bytes.Compare(r.Start, end) >= 0 { + continue + } + out = append(out, cloneRoute(r)) + } + return out } // Current returns the route catalog snapshot at the engine's current @@ -385,6 +410,17 @@ func (e *Engine) GetIntersectingRoutes(start, end []byte) []Route { return result } +func cloneRoute(r Route) Route { + return Route{ + RouteID: r.RouteID, + Start: CloneBytes(r.Start), + End: CloneBytes(r.End), + GroupID: r.GroupID, + State: r.State, + Load: r.Load, + } +} + func (e *Engine) routeIndex(key []byte) int { if len(e.routes) == 0 { return -1 diff --git a/distribution/split_job_catalog.go b/distribution/split_job_catalog.go index d4d322d45..808ad8498 100644 --- a/distribution/split_job_catalog.go +++ b/distribution/split_job_catalog.go @@ -35,6 +35,7 @@ var ( ErrCatalogSplitJobKeyIDMismatch = errors.New("catalog split job key and record job id mismatch") ErrCatalogSplitJobConflict = errors.New("catalog split job conflict") ErrCatalogSplitJobTerminalRequired = errors.New("catalog split job terminal state is required") + ErrSplitJobOverlap = errors.New("split job overlaps requested route") ) // SplitJobPhase is the durable phase of a split migration job. diff --git a/kv/fsm.go b/kv/fsm.go index 421f5fc3b..92ef461a8 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -120,6 +120,12 @@ type RouteSnapshot interface { // OwnerOf returns the Raft group ID that owned key at this // snapshot's version. (0, false) when no route covered key. OwnerOf(key []byte) (uint64, bool) + // WriteFencedForKey reports whether key is currently inside a + // WriteFenced route in this snapshot. + WriteFencedForKey(key []byte) bool + // WriteFencedIntersects reports whether [start, end) intersects + // any WriteFenced route in this snapshot. + WriteFencedIntersects(start, end []byte) bool } // SetApplyIndex implements raftengine.ApplyIndexAware. The engine @@ -261,6 +267,11 @@ var _ raftengine.StateMachine = (*kvFSM)(nil) var ErrUnknownRequestType = errors.New("unknown request type") +// ErrRouteWriteFenced is returned when a mutation targets a route that is in +// WriteFenced state during split migration. Callers should retry after routing +// catches up to the promoted owner. +var ErrRouteWriteFenced = errors.New("route is write-fenced; retry after route migration") + // ErrComposed1Violation is returned by verifyComposed1 when the // transaction's commit cannot proceed on this Raft group because the // txn's read-set or write-set keys are not owned by this group at @@ -485,6 +496,9 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui if isTxnInternalKey(mut.Key) { return errors.WithStack(ErrInvalidRequest) } + if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { + return err + } if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { return err } @@ -517,6 +531,9 @@ func extractDelPrefix(muts []*pb.Mutation) (bool, []byte) { // handleDelPrefix delegates prefix deletion to the store. Transaction-internal // keys are always excluded to preserve transactional integrity. func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uint64) error { + if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { + return err + } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -524,6 +541,56 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin return nil } +func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { + if f.routes == nil { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + rkey := routeKey(key) + if !snap.WriteFencedForKey(rkey) { + return nil + } + return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", key, rkey) +} + +func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { + if f.routes == nil { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + start, end := routePrefixRange(prefix) + if !snap.WriteFencedIntersects(start, end) { + return nil + } + return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", prefix, start, end) +} + +func routePrefixRange(prefix []byte) ([]byte, []byte) { + if len(prefix) == 0 { + return []byte(""), nil + } + start := routeKey(prefix) + return start, prefixScanEnd(start) +} + var ErrNotImplemented = errors.New("not implemented") func (f *kvFSM) Snapshot() (raftengine.Snapshot, error) { @@ -836,6 +903,9 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } + if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { + return err + } if err := f.validateConflicts(ctx, uniq, startTS); err != nil { return errors.WithStack(err) } @@ -902,7 +972,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueMutationsNotFenced(muts) if err != nil { return err } @@ -918,6 +988,17 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } +func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation) ([]*pb.Mutation, error) { + uniq, err := uniqueMutations(muts) + if err != nil { + return nil, err + } + if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { + return nil, err + } + return uniq, nil +} + // dedupProbeOnePhase decides whether handleOnePhaseTxnRequest should no-op // because the entry is a retry whose prior attempt already landed. Extracted // to keep handleOnePhaseTxnRequest under the cyclop budget; the determinism diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go new file mode 100644 index 000000000..a9f4b998f --- /dev/null +++ b/kv/fsm_migration_fence_test.go @@ -0,0 +1,97 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func newWriteFencedFSM(t *testing.T) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, + }) + return newComposed1FSM(t, engine, 1) +} + +func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("z"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("z"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: nil}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFencedFSM(t) + prepare := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: 11, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 11})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + err := fsm.handleTxnRequest(ctx, abort, 11) + require.False(t, errors.Is(err, ErrRouteWriteFenced), "ABORT must keep the narrow cleanup lane open") +} diff --git a/kv/migrator_lock_drain.go b/kv/migrator_lock_drain.go new file mode 100644 index 000000000..67396fc14 --- /dev/null +++ b/kv/migrator_lock_drain.go @@ -0,0 +1,85 @@ +package kv + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +// TxnLockDrainEntry describes one prepared transaction lock that still belongs +// to a migration route. The lock key is cloned so callers can safely retain it +// across drain ticks. +type TxnLockDrainEntry struct { + LockKey []byte + UserKey []byte + StartTS uint64 + TTLExpireAt uint64 + PrimaryKey []byte + IsPrimaryKey bool +} + +// PendingTxnLocksInRoute scans the txn-lock namespace and filters each lock by +// routeKey(userKey). It intentionally does not bracket the scan by +// txnLockKey(routeStart)/txnLockKey(routeEnd): txn locks are sorted by raw user +// key while migration routes are defined in route-key space. +func PendingTxnLocksInRoute(ctx context.Context, st store.MVCCStore, routeStart, routeEnd []byte, ts uint64, limit int) ([]TxnLockDrainEntry, error) { + if st == nil { + return nil, nil + } + if limit <= 0 { + limit = maxTxnLockScanResults + } + start := txnLockKey(nil) + end := prefixScanEnd(start) + filter := RouteKeyFilter(routeStart, routeEnd) + cursor := start + out := make([]TxnLockDrainEntry, 0, min(limit, lockPageLimit)) + + for { + lockKVs, nextCursor, done, err := scanTxnLockPageAt(ctx, st, cursor, end, ts) + if err != nil { + return nil, err + } + for _, kvp := range lockKVs { + entry, ok, err := txnLockDrainEntry(kvp, filter) + if err != nil { + return nil, err + } + if !ok { + continue + } + out = append(out, entry) + if len(out) >= limit { + return out, nil + } + } + if done { + return out, nil + } + cursor = nextCursor + } +} + +func txnLockDrainEntry(kvp *store.KVPair, filter func([]byte) bool) (TxnLockDrainEntry, bool, error) { + if kvp == nil || !bytes.HasPrefix(kvp.Key, txnLockPrefixBytes) { + return TxnLockDrainEntry{}, false, nil + } + userKey := kvp.Key[len(txnLockPrefixBytes):] + if !filter(userKey) { + return TxnLockDrainEntry{}, false, nil + } + lock, err := decodeTxnLock(kvp.Value) + if err != nil { + return TxnLockDrainEntry{}, false, errors.Wrap(err, "decode txn lock during migration drain") + } + return TxnLockDrainEntry{ + LockKey: bytes.Clone(kvp.Key), + UserKey: bytes.Clone(userKey), + StartTS: lock.StartTS, + TTLExpireAt: lock.TTLExpireAt, + PrimaryKey: bytes.Clone(lock.PrimaryKey), + IsPrimaryKey: lock.IsPrimaryKey, + }, true, nil +} diff --git a/kv/migrator_lock_drain_test.go b/kv/migrator_lock_drain_test.go new file mode 100644 index 000000000..916586010 --- /dev/null +++ b/kv/migrator_lock_drain_test.go @@ -0,0 +1,45 @@ +package kv + +import ( + "bytes" + "context" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestPendingTxnLocksInRouteFiltersLocksByRouteKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + tableSegment := "tenant-a" + routeStart := []byte(dynamoRoutePrefix + tableSegment) + routeEnd := append(bytes.Clone(routeStart), 0xff) + itemKey := append([]byte(DynamoItemPrefix+tableSegment+"|7|"), []byte("pk\x00\x01")...) + outsideKey := []byte("outside") + + require.Less(t, bytes.Compare(itemKey, routeStart), 0, + "the red-control key must sort outside a txnLockKey(routeStart)/txnLockKey(routeEnd) bracket") + lock := encodeTxnLock(txnLock{ + StartTS: 11, + TTLExpireAt: 99, + PrimaryKey: itemKey, + IsPrimaryKey: true, + }) + require.NoError(t, st.PutAt(ctx, txnLockKey(itemKey), lock, 1, 0)) + require.NoError(t, st.PutAt(ctx, txnLockKey(outsideKey), encodeTxnLock(txnLock{ + StartTS: 12, + PrimaryKey: outsideKey, + }), 2, 0)) + + pending, err := PendingTxnLocksInRoute(ctx, st, routeStart, routeEnd, ^uint64(0), 10) + require.NoError(t, err) + require.Len(t, pending, 1) + require.Equal(t, itemKey, pending[0].UserKey) + require.Equal(t, uint64(11), pending[0].StartTS) + require.Equal(t, uint64(99), pending[0].TTLExpireAt) + require.True(t, pending[0].IsPrimaryKey) + require.Equal(t, txnLockKey(itemKey), pending[0].LockKey) +} diff --git a/kv/route_history.go b/kv/route_history.go index cb768b30b..c6ac85608 100644 --- a/kv/route_history.go +++ b/kv/route_history.go @@ -60,3 +60,17 @@ func (s distributionRouteSnapshot) Version() uint64 { func (s distributionRouteSnapshot) OwnerOf(key []byte) (uint64, bool) { return s.snap.OwnerOf(key) } + +func (s distributionRouteSnapshot) WriteFencedForKey(key []byte) bool { + route, ok := s.snap.RouteOf(key) + return ok && route.State == distribution.RouteStateWriteFenced +} + +func (s distributionRouteSnapshot) WriteFencedIntersects(start, end []byte) bool { + for _, route := range s.snap.IntersectingRoutes(start, end) { + if route.State == distribution.RouteStateWriteFenced { + return true + } + } + return false +} diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index b8b67dba5..7ed2ccb7c 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -700,11 +700,8 @@ func (c *ShardedCoordinator) Dispatch(ctx context.Context, reqs *OperationGroup[ return nil, err } - // DEL_PREFIX cannot be routed to a single shard because the prefix may - // span multiple shards (or be nil, meaning "all keys"). Broadcast the - // operation to every shard group so each FSM scans locally. - if hasDelPrefixElem(reqs.Elems) { - return c.dispatchDelPrefixBroadcast(ctx, reqs.IsTxn, reqs.Elems) + if resp, handled, err := c.dispatchBeforeShardRouting(ctx, reqs); handled { + return resp, err } // Capture whether the caller supplied a non-zero StartTS BEFORE @@ -744,6 +741,20 @@ func (c *ShardedCoordinator) Dispatch(ctx context.Context, reqs *OperationGroup[ return c.dispatchNonTxn(ctx, reqs) } +func (c *ShardedCoordinator) dispatchBeforeShardRouting(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, bool, error) { + // DEL_PREFIX cannot be routed to a single shard because the prefix may + // span multiple shards (or be nil, meaning "all keys"). Broadcast the + // operation to every shard group so each FSM scans locally. + if hasDelPrefixElem(reqs.Elems) { + resp, err := c.dispatchDelPrefixBroadcast(ctx, reqs.IsTxn, reqs.Elems) + return resp, true, err + } + if err := c.rejectWriteFencedPointElems(reqs.Elems); err != nil { + return nil, true, err + } + return nil, false, nil +} + // dispatchTxnWithComposed1Retry runs the M4 Composed-1 retry loop // (design doc // docs/design/2026_05_29_implemented_composed1_cross_group_commit_guard.md @@ -1017,6 +1028,9 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT if err := validateDelPrefixOnly(elems); err != nil { return nil, err } + if err := c.rejectWriteFencedDelPrefixes(elems); err != nil { + return nil, err + } ts, err := c.allocateTimestamp(ctx, "allocate DEL_PREFIX broadcast ts") if err != nil { @@ -1035,6 +1049,41 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT return c.broadcastToAllGroups(ctx, requests) } +func (c *ShardedCoordinator) rejectWriteFencedPointElems(elems []*Elem[OP]) error { + if c == nil || c.engine == nil { + return nil + } + for _, elem := range elems { + if elem == nil || len(elem.Key) == 0 { + continue + } + route, ok := c.engine.GetRoute(routeKey(elem.Key)) + if !ok || route.State != distribution.RouteStateWriteFenced { + continue + } + return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", elem.Key, routeKey(elem.Key)) + } + return nil +} + +func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) error { + if c == nil || c.engine == nil { + return nil + } + for _, elem := range elems { + if elem == nil { + continue + } + start, end := routePrefixRange(elem.Key) + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if route.State == distribution.RouteStateWriteFenced { + return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", elem.Key, start, end) + } + } + } + return nil +} + // broadcastToAllGroups sends the same set of requests to every shard group in // parallel and returns the maximum commit index. func (c *ShardedCoordinator) broadcastToAllGroups(ctx context.Context, requests []*pb.Request) (*CoordinateResponse, error) { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 11de9de78..01e63961a 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -121,6 +121,85 @@ func TestShardedCoordinator_DelPrefixBroadcastsToAllGroups(t *testing.T) { "same DEL_PREFIX element must use the same timestamp across shards") } +func TestShardedCoordinatorRejectsPointWriteOnWriteFencedRoute(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }, + })) + + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: &recordingTransactional{}}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: []byte("z"), Value: []byte("v")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteFenced) + require.Empty(t, g2Txn.requests, "coordinator must reject before proposing to the fenced shard") +} + +func TestShardedCoordinatorRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteFenced) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + +func TestShardedCoordinatorRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: nil}}, + }) + require.ErrorIs(t, err, ErrRouteWriteFenced) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + // TestShardedCoordinator_DelPrefixRejectsTxn verifies that DEL_PREFIX inside // a transactional group is rejected. func TestShardedCoordinator_DelPrefixRejectsTxn(t *testing.T) { From 07a428ee880738fe5aaa6c7306690e4e2d26dc21 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 22:23:16 +0900 Subject: [PATCH 006/162] Address migration guard review notes --- distribution/engine.go | 4 ++-- kv/migrator_lock_drain.go | 9 ++++++++- kv/migrator_lock_drain_test.go | 11 +++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/distribution/engine.go b/distribution/engine.go index 7963d1282..dc55ed74e 100644 --- a/distribution/engine.go +++ b/distribution/engine.go @@ -188,7 +188,7 @@ func (s RouteHistorySnapshot) IntersectingRoutes(start, end []byte) []Route { continue } if end != nil && bytes.Compare(r.Start, end) >= 0 { - continue + break } out = append(out, cloneRoute(r)) } @@ -395,7 +395,7 @@ func (e *Engine) GetIntersectingRoutes(start, end []byte) []Route { } // Route starts at or after scan ends: end != nil && rStart >= end if end != nil && bytes.Compare(r.Start, end) >= 0 { - continue + break } // Route intersects with scan range result = append(result, Route{ diff --git a/kv/migrator_lock_drain.go b/kv/migrator_lock_drain.go index 67396fc14..ef7a7e531 100644 --- a/kv/migrator_lock_drain.go +++ b/kv/migrator_lock_drain.go @@ -38,7 +38,7 @@ func PendingTxnLocksInRoute(ctx context.Context, st store.MVCCStore, routeStart, out := make([]TxnLockDrainEntry, 0, min(limit, lockPageLimit)) for { - lockKVs, nextCursor, done, err := scanTxnLockPageAt(ctx, st, cursor, end, ts) + lockKVs, nextCursor, done, err := scanTxnLockDrainPage(ctx, st, cursor, end, ts) if err != nil { return nil, err } @@ -62,6 +62,13 @@ func PendingTxnLocksInRoute(ctx context.Context, st store.MVCCStore, routeStart, } } +func scanTxnLockDrainPage(ctx context.Context, st store.MVCCStore, cursor, end []byte, ts uint64) ([]*store.KVPair, []byte, bool, error) { + if err := ctx.Err(); err != nil { + return nil, nil, false, errors.WithStack(err) + } + return scanTxnLockPageAt(ctx, st, cursor, end, ts) +} + func txnLockDrainEntry(kvp *store.KVPair, filter func([]byte) bool) (TxnLockDrainEntry, bool, error) { if kvp == nil || !bytes.HasPrefix(kvp.Key, txnLockPrefixBytes) { return TxnLockDrainEntry{}, false, nil diff --git a/kv/migrator_lock_drain_test.go b/kv/migrator_lock_drain_test.go index 916586010..d790e85c0 100644 --- a/kv/migrator_lock_drain_test.go +++ b/kv/migrator_lock_drain_test.go @@ -43,3 +43,14 @@ func TestPendingTxnLocksInRouteFiltersLocksByRouteKey(t *testing.T) { require.True(t, pending[0].IsPrimaryKey) require.Equal(t, txnLockKey(itemKey), pending[0].LockKey) } + +func TestPendingTxnLocksInRouteHonorsCanceledContext(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + pending, err := PendingTxnLocksInRoute(ctx, store.NewMVCCStore(), nil, nil, ^uint64(0), 10) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, pending) +} From 7c1780323eedc9d489ba55c5794b7c65c1d7e635 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 23:29:50 +0900 Subject: [PATCH 007/162] kv: fence broad mapped prefix deletes --- kv/fsm.go | 53 ++++++++++++++++++++ kv/fsm_migration_fence_test.go | 17 +++++++ kv/shard_key_test.go | 59 +++++++++++++++++++++++ kv/sharded_coordinator_del_prefix_test.go | 36 ++++++++++++++ 4 files changed, 165 insertions(+) diff --git a/kv/fsm.go b/kv/fsm.go index 92ef461a8..5cd95d05d 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -11,6 +11,7 @@ import ( "github.com/bootjp/elastickv/internal/encryption/fsmwire" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -587,10 +588,62 @@ func routePrefixRange(prefix []byte) ([]byte, []byte) { if len(prefix) == 0 { return []byte(""), nil } + if routeKeyspaceWideRawPrefix(prefix) { + return []byte(""), nil + } start := routeKey(prefix) return start, prefixScanEnd(start) } +func routeKeyspaceWideRawPrefix(prefix []byte) bool { + if !rawPrefixMayContainRouteMappedKeys(prefix) { + return false + } + return bytes.Equal(routeKey(prefix), prefix) +} + +func rawPrefixMayContainRouteMappedKeys(prefix []byte) bool { + for _, mappedPrefix := range routeMappedRawPrefixes { + if bytes.HasPrefix(prefix, mappedPrefix) || bytes.HasPrefix(mappedPrefix, prefix) { + return true + } + } + return false +} + +var routeMappedRawPrefixes = [][]byte{ + []byte(redisInternalRoutePrefix), + []byte(DynamoTableMetaPrefix), + []byte(DynamoTableGenerationPrefix), + []byte(DynamoItemPrefix), + []byte(DynamoGSIPrefix), + []byte(sqsInternalPrefix), + []byte(store.ListMetaPrefix), + []byte(store.ListItemPrefix), + []byte(store.ListMetaDeltaPrefix), + []byte(store.ListClaimPrefix), + []byte(store.HashMetaPrefix), + []byte(store.HashFieldPrefix), + []byte(store.HashMetaDeltaPrefix), + []byte(store.SetMetaPrefix), + []byte(store.SetMemberPrefix), + []byte(store.SetMetaDeltaPrefix), + []byte(store.ZSetMetaPrefix), + []byte(store.ZSetMemberPrefix), + []byte(store.ZSetScorePrefix), + []byte(store.ZSetMetaDeltaPrefix), + []byte(store.StreamMetaPrefix), + []byte(store.StreamEntryPrefix), + []byte(s3keys.BucketMetaPrefix), + []byte(s3keys.BucketGenerationPrefix), + []byte(s3keys.ObjectManifestPrefix), + []byte(s3keys.UploadMetaPrefix), + []byte(s3keys.UploadPartPrefix), + []byte(s3keys.BlobPrefix), + []byte(s3keys.GCUploadPrefix), + []byte(s3keys.RoutePrefix), +} + var ErrNotImplemented = errors.New("not implemented") func (f *kvFSM) Snapshot() (raftengine.Snapshot, error) { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index a9f4b998f..e47e71f95 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -67,6 +67,23 @@ func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + key := []byte("!redis|string|z") + require.NoError(t, fsm.store.PutAt(context.Background(), key, []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("!redis|")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), key, ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { t.Parallel() diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index e7d6c55d7..fb282236a 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -174,6 +174,65 @@ func TestRouteKey_S3DecoderIsConcreteOnly(t *testing.T) { require.Equal(t, rawUser, routeKey(rawUser), "adapter-looking raw user key must stay on its raw route") } +func TestRoutePrefixRangeTreatsBroadMappedPrefixesAsFullKeyspace(t *testing.T) { + t.Parallel() + + tableSegment := base64.RawURLEncoding.EncodeToString([]byte("users")) + dynamoTableRoute := dynamoRouteTableKey([]byte(tableSegment)) + + for _, tc := range []struct { + name string + prefix []byte + wantStart []byte + wantEnd []byte + }{ + { + name: "raw user prefix", + prefix: []byte("ab"), + wantStart: []byte("ab"), + wantEnd: prefixScanEnd([]byte("ab")), + }, + { + name: "concrete redis prefix", + prefix: []byte("!redis|string|ab"), + wantStart: []byte("ab"), + wantEnd: prefixScanEnd([]byte("ab")), + }, + { + name: "dynamo table cleanup prefix", + prefix: []byte(DynamoItemPrefix + tableSegment + "|7|"), + wantStart: dynamoTableRoute, + wantEnd: prefixScanEnd(dynamoTableRoute), + }, + { + name: "broad redis namespace", + prefix: []byte("!redis|"), + wantStart: []byte(""), + wantEnd: nil, + }, + { + name: "broad wide-column namespace", + prefix: []byte("!lst|"), + wantStart: []byte(""), + wantEnd: nil, + }, + { + name: "s3 bucket cleanup prefix", + prefix: s3keys.ObjectManifestPrefixForBucket("bucket", 2), + wantStart: []byte(""), + wantEnd: nil, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + start, end := routePrefixRange(tc.prefix) + require.Equal(t, tc.wantStart, start) + require.Equal(t, tc.wantEnd, end) + }) + } +} + func TestRouteKeyFilterTreatsNilAndEmptyEndAsInfinity(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 01e63961a..4b83dc131 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -200,6 +200,42 @@ func TestShardedCoordinatorRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *te require.Empty(t, g2Txn.requests) } +func TestShardedCoordinatorRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }, + })) + + for _, prefix := range [][]byte{ + []byte("!redis|"), + []byte("!lst|"), + } { + t.Run(string(prefix), func(t *testing.T) { + t.Parallel() + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: prefix}}, + }) + require.ErrorIs(t, err, ErrRouteWriteFenced) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) + }) + } +} + // TestShardedCoordinator_DelPrefixRejectsTxn verifies that DEL_PREFIX inside // a transactional group is rejected. func TestShardedCoordinator_DelPrefixRejectsTxn(t *testing.T) { From 2a8585283c15e2ab50007b24eb4ff1d3de2b1b10 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 22:51:40 +0900 Subject: [PATCH 008/162] migration: add range version RPC handlers --- adapter/internal.go | 138 +++++++++++++++++++++++++++++ adapter/internal_migration_test.go | 80 +++++++++++++++++ adapter/test_util.go | 8 +- distribution/engine.go | 55 ++++++------ distribution/engine_test.go | 24 ++++- main.go | 5 +- main_bootstrap_e2e_test.go | 8 +- 7 files changed, 286 insertions(+), 32 deletions(-) create mode 100644 adapter/internal_migration_test.go diff --git a/adapter/internal.go b/adapter/internal.go index aa98f16b9..ae9ac2e34 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -7,7 +7,10 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type InternalOption func(*Internal) @@ -18,6 +21,12 @@ func WithInternalTimestampAllocator(alloc kv.TimestampAllocator) InternalOption } } +func WithInternalStore(st store.MVCCStore) InternalOption { + return func(i *Internal) { + i.store = st + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ leader: leader, @@ -37,6 +46,7 @@ type Internal struct { clock *kv.HLC tsAllocator kv.TimestampAllocator relay *RedisPubSubRelay + store store.MVCCStore pb.UnimplementedInternalServer } @@ -47,6 +57,12 @@ var ErrNotLeader = errors.New("not leader") var ErrLeaderNotFound = errors.New("leader not found") var ErrTxnTimestampOverflow = errors.New("txn timestamp overflow") +const ( + defaultMigrationExportChunkBytes = 4 << 20 + defaultMigrationExportScanFactor = 4 + defaultMigrationExportMaxVersions = 1024 +) + func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.ForwardResponse, error) { if i.leader == nil || i.leader.State() != raftengine.StateLeader { return nil, errors.WithStack(ErrNotLeader) @@ -85,6 +101,128 @@ func (i *Internal) RelayPublish(_ context.Context, req *pb.RelayPublishRequest) }, nil } +func (i *Internal) ExportRangeVersions(req *pb.ExportRangeVersionsRequest, stream pb.Internal_ExportRangeVersionsServer) error { + if req == nil { + return errors.WithStack(status.Error(codes.InvalidArgument, "export range versions request is nil")) + } + if i.store == nil { + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration export store is not configured")) + } + if err := i.verifyInternalLeader(stream.Context()); err != nil { + return err + } + opts := exportRangeVersionsOptions(req) + for { + result, err := i.store.ExportVersions(stream.Context(), opts) + if err != nil { + return errors.WithStack(err) + } + if err := stream.Send(&pb.ExportRangeVersionsResponse{ + Versions: protoMVCCVersionsFromStore(result.Versions), + NextCursor: result.NextCursor, + Done: result.Done, + }); err != nil { + return errors.WithStack(err) + } + if result.Done { + return nil + } + opts.Cursor = result.NextCursor + } +} + +func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeVersionsRequest) (*pb.ImportRangeVersionsResponse, error) { + if req == nil { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "import range versions request is nil")) + } + if i.store == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration import store is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + result, err := i.store.ImportVersions(ctx, store.ImportVersionsOptions{ + JobID: req.GetJobId(), + BracketID: req.GetBracketId(), + BatchSeq: req.GetBatchSeq(), + Cursor: req.GetCursor(), + Versions: storeMVCCVersionsFromProto(req.GetVersions()), + }) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.ImportRangeVersionsResponse{AckedCursor: result.AckedCursor}, nil +} + +func (i *Internal) verifyInternalLeader(ctx context.Context) error { + if i.leader == nil { + return nil + } + if i.leader.State() != raftengine.StateLeader { + return errors.WithStack(ErrNotLeader) + } + if err := i.leader.VerifyLeader(ctx); err != nil { + return errors.WithStack(ErrNotLeader) + } + return nil +} + +func exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.ExportVersionsOptions { + chunkBytes := uint64(req.GetChunkBytes()) + if chunkBytes == 0 { + chunkBytes = defaultMigrationExportChunkBytes + } + maxScannedBytes := req.GetMaxScannedBytes() + if maxScannedBytes == 0 { + maxScannedBytes = chunkBytes * defaultMigrationExportScanFactor + } + opts := store.ExportVersionsOptions{ + StartKey: req.GetRangeStart(), + EndKey: req.GetRangeEnd(), + MinCommitTSExclusive: req.GetMinCommitTs(), + MaxCommitTSInclusive: req.GetMaxCommitTs(), + Cursor: req.GetCursor(), + MaxVersions: defaultMigrationExportMaxVersions, + MaxBytes: chunkBytes, + MaxScannedBytes: maxScannedBytes, + AcceptKey: kv.RouteKeyFilter(req.GetRouteStart(), req.GetRouteEnd()), + } + return opts +} + +func protoMVCCVersionsFromStore(in []store.MVCCVersion) []*pb.MVCCVersion { + out := make([]*pb.MVCCVersion, 0, len(in)) + for _, version := range in { + out = append(out, &pb.MVCCVersion{ + Key: bytes.Clone(version.Key), + CommitTs: version.CommitTS, + Tombstone: version.Tombstone, + Value: bytes.Clone(version.Value), + KeyFamily: version.KeyFamily, + ExpireAt: version.ExpireAt, + }) + } + return out +} + +func storeMVCCVersionsFromProto(in []*pb.MVCCVersion) []store.MVCCVersion { + out := make([]store.MVCCVersion, 0, len(in)) + for _, version := range in { + if version == nil { + continue + } + out = append(out, store.MVCCVersion{ + Key: bytes.Clone(version.GetKey()), + CommitTS: version.GetCommitTs(), + Tombstone: version.GetTombstone(), + Value: bytes.Clone(version.GetValue()), + KeyFamily: version.GetKeyFamily(), + ExpireAt: version.GetExpireAt(), + }) + } + return out +} + func (i *Internal) stampTimestamps(ctx context.Context, req *pb.ForwardRequest) error { if req == nil { return nil diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go new file mode 100644 index 000000000..5d2e30179 --- /dev/null +++ b/adapter/internal_migration_test.go @@ -0,0 +1,80 @@ +package adapter + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type captureExportRangeVersionsStream struct { + grpc.ServerStream + ctx context.Context + responses []*pb.ExportRangeVersionsResponse +} + +func (s *captureExportRangeVersionsStream) Context() context.Context { + if s.ctx != nil { + return s.ctx + } + return context.Background() +} + +func (s *captureExportRangeVersionsStream) Send(resp *pb.ExportRangeVersionsResponse) error { + s.responses = append(s.responses, resp) + return nil +} + +func TestInternalExportRangeVersionsUsesStoreAndRouteFilter(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("va"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("z"), []byte("vz"), 10, 0)) + internal := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(st)) + stream := &captureExportRangeVersionsStream{ctx: ctx} + + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + RouteStart: []byte("a"), + RouteEnd: []byte("b"), + }, stream) + require.NoError(t, err) + require.Len(t, stream.responses, 1) + require.True(t, stream.responses[0].GetDone()) + require.Empty(t, stream.responses[0].GetNextCursor()) + require.Equal(t, []*pb.MVCCVersion{ + {Key: []byte("a"), CommitTs: 10, Value: []byte("va")}, + }, stream.responses[0].GetVersions()) +} + +func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + internal := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(st)) + + resp, err := internal.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ + JobId: 7, + BracketId: 3, + BatchSeq: 1, + Cursor: []byte("cursor-1"), + Versions: []*pb.MVCCVersion{ + {Key: []byte("k"), CommitTs: 30, Value: []byte("v"), ExpireAt: 100}, + }, + }) + require.NoError(t, err) + require.Equal(t, []byte("cursor-1"), resp.GetAckedCursor()) + + got, err := st.GetAt(ctx, []byte("k"), 30) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) + floor, err := st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(30), floor) +} diff --git a/adapter/test_util.go b/adapter/test_util.go index 8fe5646fa..6804657c5 100644 --- a/adapter/test_util.go +++ b/adapter/test_util.go @@ -641,7 +641,13 @@ func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ( } pb.RegisterRawKVServer(s, gs) pb.RegisterTransactionalKVServer(s, gs) - pb.RegisterInternalServer(s, NewInternalWithEngine(trx, result.Engine, coordinator.Clock(), relay)) + pb.RegisterInternalServer(s, NewInternalWithEngine( + trx, + result.Engine, + coordinator.Clock(), + relay, + WithInternalStore(st), + )) internalraftadmin.RegisterOperationalServices(opsCtx, s, result.Engine, []string{"Example"}) grpcAdders = append(grpcAdders, port.grpcAddress) diff --git a/distribution/engine.go b/distribution/engine.go index dc55ed74e..8f6868021 100644 --- a/distribution/engine.go +++ b/distribution/engine.go @@ -25,6 +25,13 @@ type Route struct { GroupID uint64 // State tracks control-plane state for this route. State RouteState + // StagedVisibilityActive makes migrated versions visible through the + // staged/live merge path after cross-group CUTOVER. + StagedVisibilityActive bool + // MigrationJobID identifies the migration job that owns staged visibility. + MigrationJobID uint64 + // MinWriteTSExclusive is the post-migration write timestamp floor. + MinWriteTSExclusive uint64 // Load tracks the number of accesses served by this range. Load uint64 } @@ -365,14 +372,7 @@ func (e *Engine) Stats() []Route { defer e.mu.RUnlock() stats := make([]Route, len(e.routes)) for i, r := range e.routes { - stats[i] = Route{ - RouteID: r.RouteID, - Start: CloneBytes(r.Start), - End: CloneBytes(r.End), - GroupID: r.GroupID, - State: r.State, - Load: r.Load, - } + stats[i] = cloneRoute(r) } return stats } @@ -398,26 +398,22 @@ func (e *Engine) GetIntersectingRoutes(start, end []byte) []Route { break } // Route intersects with scan range - result = append(result, Route{ - RouteID: r.RouteID, - Start: CloneBytes(r.Start), - End: CloneBytes(r.End), - GroupID: r.GroupID, - State: r.State, - Load: r.Load, - }) + result = append(result, cloneRoute(*r)) } return result } func cloneRoute(r Route) Route { return Route{ - RouteID: r.RouteID, - Start: CloneBytes(r.Start), - End: CloneBytes(r.End), - GroupID: r.GroupID, - State: r.State, - Load: r.Load, + RouteID: r.RouteID, + Start: CloneBytes(r.Start), + End: CloneBytes(r.End), + GroupID: r.GroupID, + State: r.State, + StagedVisibilityActive: r.StagedVisibilityActive, + MigrationJobID: r.MigrationJobID, + MinWriteTSExclusive: r.MinWriteTSExclusive, + Load: r.Load, } } @@ -454,12 +450,15 @@ func routesFromCatalog(routes []RouteDescriptor) ([]Route, error) { } seen[rd.RouteID] = struct{}{} out[i] = Route{ - RouteID: rd.RouteID, - Start: CloneBytes(rd.Start), - End: CloneBytes(rd.End), - GroupID: rd.GroupID, - State: rd.State, - Load: 0, + RouteID: rd.RouteID, + Start: CloneBytes(rd.Start), + End: CloneBytes(rd.End), + GroupID: rd.GroupID, + State: rd.State, + StagedVisibilityActive: rd.StagedVisibilityActive, + MigrationJobID: rd.MigrationJobID, + MinWriteTSExclusive: rd.MinWriteTSExclusive, + Load: 0, } } diff --git a/distribution/engine_test.go b/distribution/engine_test.go index 346e4fa2d..1794608c5 100644 --- a/distribution/engine_test.go +++ b/distribution/engine_test.go @@ -169,7 +169,16 @@ func TestEngineApplySnapshot_ReplacesRoutesAndVersion(t *testing.T) { Version: 1, Routes: []RouteDescriptor{ {RouteID: 10, Start: []byte(""), End: []byte("m"), GroupID: 1, State: RouteStateActive}, - {RouteID: 11, Start: []byte("m"), End: nil, GroupID: 2, State: RouteStateWriteFenced}, + { + RouteID: 11, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: RouteStateWriteFenced, + StagedVisibilityActive: true, + MigrationJobID: 42, + MinWriteTSExclusive: 99, + }, }, }) if err != nil { @@ -190,6 +199,19 @@ func TestEngineApplySnapshot_ReplacesRoutesAndVersion(t *testing.T) { if stats[1].RouteID != 11 || stats[1].State != RouteStateWriteFenced { t.Fatalf("unexpected second route metadata: %+v", stats[1]) } + assertStagedRouteMetadata(t, stats[1]) + route, ok := e.GetRoute([]byte("m")) + if !ok { + t.Fatalf("expected route for m") + } + assertStagedRouteMetadata(t, route) +} + +func assertStagedRouteMetadata(t *testing.T, route Route) { + t.Helper() + if !route.StagedVisibilityActive || route.MigrationJobID != 42 || route.MinWriteTSExclusive != 99 { + t.Fatalf("staged route metadata was not preserved: %+v", route) + } } func TestEngineApplySnapshot_RejectsOldVersion(t *testing.T) { diff --git a/main.go b/main.go index 530dbf4b7..bd111ea59 100644 --- a/main.go +++ b/main.go @@ -2093,7 +2093,10 @@ func startRaftServers( rt.engine, coordinate.Clock(), relay, - internalTimestampOptions(coordinate)..., + append( + internalTimestampOptions(coordinate), + adapter.WithInternalStore(rt.store), + )..., )) pb.RegisterDistributionServer(gs, distServer) if adminServer != nil { diff --git a/main_bootstrap_e2e_test.go b/main_bootstrap_e2e_test.go index d962c8105..c8434716a 100644 --- a/main_bootstrap_e2e_test.go +++ b/main_bootstrap_e2e_test.go @@ -762,7 +762,13 @@ func startBoundGRPCServer( grpcSvc := adapter.NewGRPCServer(shardStore, coordinate) pb.RegisterRawKVServer(gs, grpcSvc) pb.RegisterTransactionalKVServer(gs, grpcSvc) - pb.RegisterInternalServer(gs, adapter.NewInternalWithEngine(trx, rt.engine, coordinate.Clock(), relay)) + pb.RegisterInternalServer(gs, adapter.NewInternalWithEngine( + trx, + rt.engine, + coordinate.Clock(), + relay, + adapter.WithInternalStore(rt.store), + )) pb.RegisterDistributionServer(gs, distServer) rt.registerGRPC(gs) internalraftadmin.RegisterOperationalServices(ctx, gs, rt.engine, []string{"RawKV"}) From fbd7f563b168cc03a183f27bf57ad4f767f9ccda Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 23:20:52 +0900 Subject: [PATCH 009/162] migration: raft-apply range imports --- adapter/internal.go | 124 +++++++++++++++++++++-------- adapter/internal_migration_test.go | 83 +++++++++++++++++-- adapter/test_util.go | 1 + kv/fsm.go | 13 ++- kv/fsm_migration_import.go | 83 +++++++++++++++++++ main.go | 1 + main_bootstrap_e2e_test.go | 1 + proto/internal.pb.go | 39 ++++++++- proto/internal.proto | 9 +++ 9 files changed, 309 insertions(+), 45 deletions(-) create mode 100644 kv/fsm_migration_import.go diff --git a/adapter/internal.go b/adapter/internal.go index ae9ac2e34..de61f5485 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -4,6 +4,7 @@ import ( "bytes" "context" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" @@ -27,6 +28,12 @@ func WithInternalStore(st store.MVCCStore) InternalOption { } } +func WithInternalMigrationProposer(proposer raftengine.Proposer) InternalOption { + return func(i *Internal) { + i.migrationProposer = proposer + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ leader: leader, @@ -47,6 +54,7 @@ type Internal struct { tsAllocator kv.TimestampAllocator relay *RedisPubSubRelay store store.MVCCStore + migrationProposer raftengine.Proposer pb.UnimplementedInternalServer } @@ -102,21 +110,41 @@ func (i *Internal) RelayPublish(_ context.Context, req *pb.RelayPublishRequest) } func (i *Internal) ExportRangeVersions(req *pb.ExportRangeVersionsRequest, stream pb.Internal_ExportRangeVersionsServer) error { + if err := i.validateExportRangeVersionsRequest(req); err != nil { + return err + } + if err := i.verifyInternalLeader(stream.Context()); err != nil { + return err + } + return i.streamExportRangeVersions(req, stream) +} + +func (i *Internal) validateExportRangeVersionsRequest(req *pb.ExportRangeVersionsRequest) error { if req == nil { return errors.WithStack(status.Error(codes.InvalidArgument, "export range versions request is nil")) } if i.store == nil { return errors.WithStack(status.Error(codes.FailedPrecondition, "migration export store is not configured")) } - if err := i.verifyInternalLeader(stream.Context()); err != nil { - return err + if req.GetMaxCommitTs() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "migration export max_commit_ts is required")) } + if req.GetKeyFamily() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "migration export key_family is required")) + } + return nil +} + +func (i *Internal) streamExportRangeVersions(req *pb.ExportRangeVersionsRequest, stream pb.Internal_ExportRangeVersionsServer) error { opts := exportRangeVersionsOptions(req) for { result, err := i.store.ExportVersions(stream.Context(), opts) if err != nil { return errors.WithStack(err) } + if !result.Done && bytes.Equal(opts.Cursor, result.NextCursor) { + return errors.WithStack(status.Error(codes.Internal, "migration export cursor did not progress")) + } if err := stream.Send(&pb.ExportRangeVersionsResponse{ Versions: protoMVCCVersionsFromStore(result.Versions), NextCursor: result.NextCursor, @@ -135,29 +163,23 @@ func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeV if req == nil { return nil, errors.WithStack(status.Error(codes.InvalidArgument, "import range versions request is nil")) } - if i.store == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration import store is not configured")) + if i.migrationProposer == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration import proposer is not configured")) } if err := i.verifyInternalLeader(ctx); err != nil { return nil, err } - result, err := i.store.ImportVersions(ctx, store.ImportVersionsOptions{ - JobID: req.GetJobId(), - BracketID: req.GetBracketId(), - BatchSeq: req.GetBatchSeq(), - Cursor: req.GetCursor(), - Versions: storeMVCCVersionsFromProto(req.GetVersions()), - }) + result, err := i.proposeMigrationImport(ctx, req) if err != nil { return nil, errors.WithStack(err) } + if i.clock != nil && result.MaxImportedTS > 0 { + i.clock.Observe(result.MaxImportedTS) + } return &pb.ImportRangeVersionsResponse{AckedCursor: result.AckedCursor}, nil } func (i *Internal) verifyInternalLeader(ctx context.Context) error { - if i.leader == nil { - return nil - } if i.leader.State() != raftengine.StateLeader { return errors.WithStack(ErrNotLeader) } @@ -167,6 +189,33 @@ func (i *Internal) verifyInternalLeader(ctx context.Context) error { return nil } +func (i *Internal) proposeMigrationImport(ctx context.Context, req *pb.ImportRangeVersionsRequest) (store.ImportVersionsResult, error) { + cmd, err := kv.MarshalMigrationImportCommand(req) + if err != nil { + return store.ImportVersionsResult{}, errors.WithStack(err) + } + result, err := i.migrationProposer.Propose(ctx, cmd) + if err != nil { + return store.ImportVersionsResult{}, errors.WithStack(err) + } + if result == nil { + return store.ImportVersionsResult{}, errors.New("migration import proposal returned nil result") + } + switch resp := result.Response.(type) { + case store.ImportVersionsResult: + return resp, nil + case *store.ImportVersionsResult: + if resp == nil { + return store.ImportVersionsResult{}, errors.New("migration import apply returned nil result") + } + return *resp, nil + case error: + return store.ImportVersionsResult{}, errors.WithStack(resp) + default: + return store.ImportVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration import apply response type %T", resp)) + } +} + func exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.ExportVersionsOptions { chunkBytes := uint64(req.GetChunkBytes()) if chunkBytes == 0 { @@ -185,11 +234,38 @@ func exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.Export MaxVersions: defaultMigrationExportMaxVersions, MaxBytes: chunkBytes, MaxScannedBytes: maxScannedBytes, - AcceptKey: kv.RouteKeyFilter(req.GetRouteStart(), req.GetRouteEnd()), + KeyFamily: req.GetKeyFamily(), + AcceptKey: migrationExportFilter(req), } return opts } +func migrationExportFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool { + routeFilter := kv.RouteKeyFilter(req.GetRouteStart(), req.GetRouteEnd()) + excludeKnownInternal := req.GetExcludeKnownInternal() || req.GetKeyFamily() == distribution.MigrationFamilyUser + bracket := distribution.MigrationBracket{ + Family: req.GetKeyFamily(), + Start: bytes.Clone(req.GetRangeStart()), + End: bytes.Clone(req.GetRangeEnd()), + ExcludeKnownInternal: excludeKnownInternal, + ExcludePrefixes: cloneByteSlices(req.GetExcludePrefixes()), + } + return func(rawKey []byte) bool { + return bracket.ContainsRawKey(rawKey) && routeFilter(rawKey) + } +} + +func cloneByteSlices(in [][]byte) [][]byte { + if len(in) == 0 { + return nil + } + out := make([][]byte, len(in)) + for i := range in { + out[i] = bytes.Clone(in[i]) + } + return out +} + func protoMVCCVersionsFromStore(in []store.MVCCVersion) []*pb.MVCCVersion { out := make([]*pb.MVCCVersion, 0, len(in)) for _, version := range in { @@ -205,24 +281,6 @@ func protoMVCCVersionsFromStore(in []store.MVCCVersion) []*pb.MVCCVersion { return out } -func storeMVCCVersionsFromProto(in []*pb.MVCCVersion) []store.MVCCVersion { - out := make([]store.MVCCVersion, 0, len(in)) - for _, version := range in { - if version == nil { - continue - } - out = append(out, store.MVCCVersion{ - Key: bytes.Clone(version.GetKey()), - CommitTS: version.GetCommitTs(), - Tombstone: version.GetTombstone(), - Value: bytes.Clone(version.GetValue()), - KeyFamily: version.GetKeyFamily(), - ExpireAt: version.GetExpireAt(), - }) - } - return out -} - func (i *Internal) stampTimestamps(ctx context.Context, req *pb.ForwardRequest) error { if req == nil { return nil diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 5d2e30179..56d1b1350 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -4,12 +4,54 @@ import ( "context" "testing" + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) +type mockInternalLeader struct { + raftengine.LeaderView +} + +func (mockInternalLeader) State() raftengine.State { + return raftengine.StateLeader +} + +func (mockInternalLeader) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "n1", Address: "127.0.0.1:50051"} +} + +func (mockInternalLeader) VerifyLeader(context.Context) error { + return nil +} + +func (mockInternalLeader) LinearizableRead(context.Context) (uint64, error) { + return 1, nil +} + +type applyingMigrationProposer struct { + fsm raftengine.StateMachine + calls uint64 +} + +func (p *applyingMigrationProposer) Propose(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { + p.calls++ + return &raftengine.ProposalResult{ + CommitIndex: p.calls, + Response: p.fsm.Apply(data), + }, nil +} + +func (p *applyingMigrationProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + return p.Propose(ctx, data) +} + type captureExportRangeVersionsStream struct { grpc.ServerStream ctx context.Context @@ -35,29 +77,56 @@ func TestInternalExportRangeVersionsUsesStoreAndRouteFilter(t *testing.T) { st := store.NewMVCCStore() require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("va"), 10, 0)) require.NoError(t, st.PutAt(ctx, []byte("z"), []byte("vz"), 10, 0)) - internal := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(st)) + require.NoError(t, st.PutAt(ctx, []byte("!txn|int|a"), []byte("intent"), 10, 0)) + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) stream := &captureExportRangeVersionsStream{ctx: ctx} err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ - MaxCommitTs: 20, - RouteStart: []byte("a"), - RouteEnd: []byte("b"), + MaxCommitTs: 20, + RouteStart: []byte("a"), + RouteEnd: []byte("b"), + KeyFamily: distribution.MigrationFamilyUser, + ExcludeKnownInternal: true, + RangeStart: []byte(""), + RangeEnd: []byte("z"), + MaxScannedBytes: 1 << 20, + ExcludePrefixes: [][]byte{[]byte("!custom|")}, }, stream) require.NoError(t, err) require.Len(t, stream.responses, 1) require.True(t, stream.responses[0].GetDone()) require.Empty(t, stream.responses[0].GetNextCursor()) require.Equal(t, []*pb.MVCCVersion{ - {Key: []byte("a"), CommitTs: 10, Value: []byte("va")}, + {Key: []byte("a"), CommitTs: 10, Value: []byte("va"), KeyFamily: distribution.MigrationFamilyUser}, }, stream.responses[0].GetVersions()) } +func TestInternalExportRangeVersionsRejectsUnboundedExport(t *testing.T) { + t.Parallel() + + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(store.NewMVCCStore())) + stream := &captureExportRangeVersionsStream{ctx: context.Background()} + + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + KeyFamily: distribution.MigrationFamilyUser, + }, stream) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() ctx := context.Background() st := store.NewMVCCStore() - internal := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(st)) + clock := kv.NewHLC() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, clock), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + ) resp, err := internal.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ JobId: 7, @@ -70,6 +139,7 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { }) require.NoError(t, err) require.Equal(t, []byte("cursor-1"), resp.GetAckedCursor()) + require.Equal(t, uint64(1), proposer.calls) got, err := st.GetAt(ctx, []byte("k"), 30) require.NoError(t, err) @@ -77,4 +147,5 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { floor, err := st.MigrationHLCFloor(ctx, 7) require.NoError(t, err) require.Equal(t, uint64(30), floor) + require.GreaterOrEqual(t, clock.Current(), uint64(30)) } diff --git a/adapter/test_util.go b/adapter/test_util.go index 6804657c5..65fd07de8 100644 --- a/adapter/test_util.go +++ b/adapter/test_util.go @@ -647,6 +647,7 @@ func setupNodes(t *testing.T, ctx context.Context, n int, ports []portsAdress) ( coordinator.Clock(), relay, WithInternalStore(st), + WithInternalMigrationProposer(result.Engine), )) internalraftadmin.RegisterOperationalServices(opsCtx, s, result.Engine, []string{"Example"}) diff --git a/kv/fsm.go b/kv/fsm.go index 5cd95d05d..00e4a5d3c 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -310,12 +310,11 @@ type fsmApplyResponse struct { } func (f *kvFSM) Apply(data []byte) any { - if resp, handled := f.applyReservedOpcode(data); handled { + ctx := context.TODO() + if resp, handled := f.applyReservedOpcode(ctx, data); handled { return resp } - ctx := context.TODO() - reqs, err := decodeRaftRequests(data) if err != nil { return errors.WithStack(err) @@ -356,13 +355,15 @@ func (f *kvFSM) Apply(data []byte) any { // opcode with ErrEncryptionApply, which the engine's HaltApply seam // recognises as a halt — same fail-closed shape as the Stage 3 // raft-envelope unwrap path. -func (f *kvFSM) applyReservedOpcode(data []byte) (any, bool) { +func (f *kvFSM) applyReservedOpcode(ctx context.Context, data []byte) (any, bool) { if len(data) == 0 { return nil, false } switch { case data[0] == raftEncodeHLCLease: return f.applyHLCLease(data[1:]), true + case data[0] == raftEncodeMigrationImport: + return f.applyMigrationImport(ctx, data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true default: @@ -394,6 +395,10 @@ const ( // These entries do not touch the MVCC store; they only advance the shared HLC // physicalCeiling so the logical counter can continue to increment in memory. raftEncodeHLCLease byte = 0x02 + // raftEncodeMigrationImport carries a target-group range-migration import + // batch. Every target voter applies the raw MVCC versions, import ack, and + // migration HLC floor before the RPC handler returns success. + raftEncodeMigrationImport byte = 0x09 ) func decodeRaftRequests(data []byte) ([]*pb.Request, error) { diff --git a/kv/fsm_migration_import.go b/kv/fsm_migration_import.go new file mode 100644 index 000000000..d6a048130 --- /dev/null +++ b/kv/fsm_migration_import.go @@ -0,0 +1,83 @@ +package kv + +import ( + "bytes" + "context" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// MarshalMigrationImportCommand encodes a target-group migration import batch +// as a Raft FSM command. The target Internal RPC handler uses this instead of +// mutating its local store directly so an acknowledged batch has been applied +// by the target group's voters. +func MarshalMigrationImportCommand(req *pb.ImportRangeVersionsRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled migration import request too large") + } + return prependByte(raftEncodeMigrationImport, b), nil +} + +func (f *kvFSM) applyMigrationImport(ctx context.Context, data []byte) any { + req := &pb.ImportRangeVersionsRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + result, err := f.store.ImportVersions(ctx, store.ImportVersionsOptions{ + JobID: req.GetJobId(), + BracketID: req.GetBracketId(), + BatchSeq: req.GetBatchSeq(), + Cursor: req.GetCursor(), + Versions: migrationStoreVersionsFromProto(req.GetVersions()), + }) + if err != nil { + return errors.WithStack(err) + } + result.MaxImportedTS, err = f.migrationHLCFloorForApply(ctx, req, result) + if err != nil { + return errors.WithStack(err) + } + if f.hlc != nil && result.MaxImportedTS > 0 { + f.hlc.Observe(result.MaxImportedTS) + } + return result +} + +func (f *kvFSM) migrationHLCFloorForApply(ctx context.Context, req *pb.ImportRangeVersionsRequest, result store.ImportVersionsResult) (uint64, error) { + if result.MaxImportedTS > 0 || len(req.GetVersions()) == 0 { + return result.MaxImportedTS, nil + } + floor, err := f.store.MigrationHLCFloor(ctx, req.GetJobId()) + if err != nil { + return 0, errors.WithStack(err) + } + return floor, nil +} + +func migrationStoreVersionsFromProto(in []*pb.MVCCVersion) []store.MVCCVersion { + out := make([]store.MVCCVersion, 0, len(in)) + for _, version := range in { + if version == nil { + continue + } + out = append(out, store.MVCCVersion{ + Key: bytes.Clone(version.GetKey()), + CommitTS: version.GetCommitTs(), + Tombstone: version.GetTombstone(), + Value: bytes.Clone(version.GetValue()), + KeyFamily: version.GetKeyFamily(), + ExpireAt: version.GetExpireAt(), + }) + } + return out +} diff --git a/main.go b/main.go index bd111ea59..c236598d8 100644 --- a/main.go +++ b/main.go @@ -2096,6 +2096,7 @@ func startRaftServers( append( internalTimestampOptions(coordinate), adapter.WithInternalStore(rt.store), + adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), )..., )) pb.RegisterDistributionServer(gs, distServer) diff --git a/main_bootstrap_e2e_test.go b/main_bootstrap_e2e_test.go index c8434716a..81ca49dce 100644 --- a/main_bootstrap_e2e_test.go +++ b/main_bootstrap_e2e_test.go @@ -768,6 +768,7 @@ func startBoundGRPCServer( coordinate.Clock(), relay, adapter.WithInternalStore(rt.store), + adapter.WithInternalMigrationProposer(rt.engine), )) pb.RegisterDistributionServer(gs, distServer) rt.registerGRPC(gs) diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 0e2bd4664..a7a9e30ce 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -540,6 +540,15 @@ type ExportRangeVersionsRequest struct { RouteStart []byte `protobuf:"bytes,7,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` RouteEnd []byte `protobuf:"bytes,8,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` MaxScannedBytes uint64 `protobuf:"varint,9,opt,name=max_scanned_bytes,json=maxScannedBytes,proto3" json:"max_scanned_bytes,omitempty"` + // Migration bracket family tag copied into exported MVCCVersion.key_family. + // Zero is invalid on the RPC path: callers must pass the bracket family they + // are exporting so target promotion can keep family-specific metadata. + KeyFamily uint32 `protobuf:"varint,10,opt,name=key_family,json=keyFamily,proto3" json:"key_family,omitempty"` + // Applies the user-bracket exclusion list for known internal families. + ExcludeKnownInternal bool `protobuf:"varint,11,opt,name=exclude_known_internal,json=excludeKnownInternal,proto3" json:"exclude_known_internal,omitempty"` + // Bracket-local raw-prefix exclusions, e.g. non-partitioned SQS brackets + // excluding their partitioned subprefixes. + ExcludePrefixes [][]byte `protobuf:"bytes,12,rep,name=exclude_prefixes,json=excludePrefixes,proto3" json:"exclude_prefixes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -637,6 +646,27 @@ func (x *ExportRangeVersionsRequest) GetMaxScannedBytes() uint64 { return 0 } +func (x *ExportRangeVersionsRequest) GetKeyFamily() uint32 { + if x != nil { + return x.KeyFamily + } + return 0 +} + +func (x *ExportRangeVersionsRequest) GetExcludeKnownInternal() bool { + if x != nil { + return x.ExcludeKnownInternal + } + return false +} + +func (x *ExportRangeVersionsRequest) GetExcludePrefixes() [][]byte { + if x != nil { + return x.ExcludePrefixes + } + return nil +} + type ExportRangeVersionsResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Versions []*MVCCVersion `protobuf:"bytes,1,rep,name=versions,proto3" json:"versions,omitempty"` @@ -929,7 +959,7 @@ const file_internal_proto_rawDesc = "" + "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + "\x14RelayPublishResponse\x12 \n" + - "\vsubscribers\x18\x01 \x01(\x03R\vsubscribers\"\xc5\x02\n" + + "\vsubscribers\x18\x01 \x01(\x03R\vsubscribers\"\xc5\x03\n" + "\x1aExportRangeVersionsRequest\x12\x1f\n" + "\vrange_start\x18\x01 \x01(\fR\n" + "rangeStart\x12\x1b\n" + @@ -942,7 +972,12 @@ const file_internal_proto_rawDesc = "" + "\vroute_start\x18\a \x01(\fR\n" + "routeStart\x12\x1b\n" + "\troute_end\x18\b \x01(\fR\brouteEnd\x12*\n" + - "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\"|\n" + + "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + + "\n" + + "key_family\x18\n" + + " \x01(\rR\tkeyFamily\x124\n" + + "\x16exclude_known_internal\x18\v \x01(\bR\x14excludeKnownInternal\x12)\n" + + "\x10exclude_prefixes\x18\f \x03(\fR\x0fexcludePrefixes\"|\n" + "\x1bExportRangeVersionsResponse\x12(\n" + "\bversions\x18\x01 \x03(\v2\f.MVCCVersionR\bversions\x12\x1f\n" + "\vnext_cursor\x18\x02 \x01(\fR\n" + diff --git a/proto/internal.proto b/proto/internal.proto index 1f7c77712..b60c83793 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -91,6 +91,15 @@ message ExportRangeVersionsRequest { bytes route_start = 7; bytes route_end = 8; uint64 max_scanned_bytes = 9; + // Migration bracket family tag copied into exported MVCCVersion.key_family. + // Zero is invalid on the RPC path: callers must pass the bracket family they + // are exporting so target promotion can keep family-specific metadata. + uint32 key_family = 10; + // Applies the user-bracket exclusion list for known internal families. + bool exclude_known_internal = 11; + // Bracket-local raw-prefix exclusions, e.g. non-partitioned SQS brackets + // excluding their partitioned subprefixes. + repeated bytes exclude_prefixes = 12; } message ExportRangeVersionsResponse { From 0fe341daea7fa1747c8e7e4a17e77608291747b0 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 23:43:10 +0900 Subject: [PATCH 010/162] distribution: serve versioned ownership lookups --- adapter/distribution_server.go | 82 ++++++++++++++++-- adapter/distribution_server_test.go | 124 ++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 9 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 3001b1329..9703c828b 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -68,15 +68,16 @@ var ( defaultCatalogReloadRetryAttempts = 20 defaultCatalogReloadRetryInterval = 10 * time.Millisecond - errDistributionCatalogNotConfigured = errors.New("route catalog is not configured") - errDistributionUnknownRoute = errors.New("unknown route") - errDistributionInvalidSplitKey = errors.New("invalid split key") - errDistributionSplitKeyAtBoundary = errors.New("split key at route boundary") - errDistributionCatalogConflict = errors.New("catalog version conflict") - errDistributionRouteIDOverflow = errors.New("route id overflow") - errDistributionNotLeader = errors.New("not leader for distribution catalog") - errDistributionCoordinatorRequired = errors.New("distribution coordinator is not configured") - errDistributionEngineNotConfigured = errors.New("distribution engine is not configured") + errDistributionCatalogNotConfigured = errors.New("route catalog is not configured") + errDistributionUnknownRoute = errors.New("unknown route") + errDistributionInvalidSplitKey = errors.New("invalid split key") + errDistributionSplitKeyAtBoundary = errors.New("split key at route boundary") + errDistributionCatalogConflict = errors.New("catalog version conflict") + errDistributionRouteIDOverflow = errors.New("route id overflow") + errDistributionNotLeader = errors.New("not leader for distribution catalog") + errDistributionCoordinatorRequired = errors.New("distribution coordinator is not configured") + errDistributionEngineNotConfigured = errors.New("distribution engine is not configured") + errDistributionCatalogVersionNotFound = errors.New("route catalog version not found") ) // NewDistributionServer creates a new server. @@ -132,6 +133,56 @@ func (s *DistributionServer) ListRoutes(ctx context.Context, req *pb.ListRoutesR }, nil } +func (s *DistributionServer) GetRouteOwnership(ctx context.Context, req *pb.GetRouteOwnershipRequest) (*pb.GetRouteOwnershipResponse, error) { + snapshot, err := s.routeSnapshotAt(req.GetCatalogVersion()) + if err != nil { + return nil, err + } + route, ok := snapshot.RouteOf(req.GetKey()) + if !ok { + return &pb.GetRouteOwnershipResponse{ + CatalogVersion: snapshot.Version(), + Found: false, + }, nil + } + return &pb.GetRouteOwnershipResponse{ + Route: toProtoRoute(route), + CatalogVersion: snapshot.Version(), + Found: true, + }, nil +} + +func (s *DistributionServer) GetIntersectingRoutes(ctx context.Context, req *pb.GetIntersectingRoutesRequest) (*pb.GetIntersectingRoutesResponse, error) { + snapshot, err := s.routeSnapshotAt(req.GetCatalogVersion()) + if err != nil { + return nil, err + } + end := req.GetEnd() + if len(end) == 0 { + end = nil + } + routes := snapshot.IntersectingRoutes(req.GetStart(), end) + out := make([]*pb.RouteDescriptor, 0, len(routes)) + for _, route := range routes { + out = append(out, toProtoRoute(route)) + } + return &pb.GetIntersectingRoutesResponse{ + Routes: out, + CatalogVersion: snapshot.Version(), + }, nil +} + +func (s *DistributionServer) routeSnapshotAt(version uint64) (distribution.RouteHistorySnapshot, error) { + if s.engine == nil { + return distribution.RouteHistorySnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionEngineNotConfigured.Error()) + } + snapshot, ok := s.engine.SnapshotAt(version) + if !ok { + return distribution.RouteHistorySnapshot{}, grpcStatusErrorf(codes.NotFound, "%s: %d", errDistributionCatalogVersionNotFound, version) + } + return snapshot, nil +} + // SplitRange splits a route into two child routes in the same raft group. func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeRequest) (*pb.SplitRangeResponse, error) { // SplitRange performs a read-modify-write cycle across catalog and engine. @@ -525,6 +576,19 @@ func toProtoRouteDescriptor(route distribution.RouteDescriptor) *pb.RouteDescrip } } +func toProtoRoute(route distribution.Route) *pb.RouteDescriptor { + return &pb.RouteDescriptor{ + RouteId: route.RouteID, + Start: distribution.CloneBytes(route.Start), + End: distribution.CloneBytes(route.End), + RaftGroupId: route.GroupID, + State: toProtoRouteState(route.State), + StagedVisibilityActive: route.StagedVisibilityActive, + MigrationJobId: route.MigrationJobID, + MinWriteTsExclusive: route.MinWriteTSExclusive, + } +} + func toProtoRouteState(state distribution.RouteState) pb.RouteState { switch state { case distribution.RouteStateActive: diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 7604d0e91..f717dfb56 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -129,6 +129,130 @@ func TestDistributionServerListRoutes_RequiresCatalog(t *testing.T) { require.ErrorContains(t, err, errDistributionCatalogNotConfigured.Error()) } +func TestDistributionServerGetRouteOwnership_UsesExactVersionSnapshot(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateMigratingTarget, + StagedVisibilityActive: true, + MigrationJobID: 44, + MinWriteTSExclusive: 55, + }, + }, + })) + + s := NewDistributionServer(engine, nil) + resp, err := s.GetRouteOwnership(context.Background(), &pb.GetRouteOwnershipRequest{ + Key: []byte("t"), + CatalogVersion: 7, + }) + require.NoError(t, err) + require.True(t, resp.Found) + require.Equal(t, uint64(7), resp.CatalogVersion) + require.Equal(t, uint64(2), resp.Route.RouteId) + require.Equal(t, uint64(2), resp.Route.RaftGroupId) + require.Equal(t, pb.RouteState_ROUTE_STATE_MIGRATING_TARGET, resp.Route.State) + require.True(t, resp.Route.StagedVisibilityActive) + require.Equal(t, uint64(44), resp.Route.MigrationJobId) + require.Equal(t, uint64(55), resp.Route.MinWriteTsExclusive) + + miss, err := s.GetRouteOwnership(context.Background(), &pb.GetRouteOwnershipRequest{ + Key: []byte("0"), + CatalogVersion: 7, + }) + require.NoError(t, err) + require.False(t, miss.Found) + require.Equal(t, uint64(7), miss.CatalogVersion) + require.Nil(t, miss.Route) +} + +func TestDistributionServerGetIntersectingRoutes_UsesExactVersionSnapshot(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 9, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("g"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("g"), End: []byte("m"), GroupID: 2, State: distribution.RouteStateWriteFenced}, + {RouteID: 3, Start: []byte("m"), End: nil, GroupID: 3, State: distribution.RouteStateActive}, + }, + })) + + s := NewDistributionServer(engine, nil) + resp, err := s.GetIntersectingRoutes(context.Background(), &pb.GetIntersectingRoutesRequest{ + Start: []byte("f"), + End: []byte("z"), + CatalogVersion: 9, + }) + require.NoError(t, err) + require.Equal(t, uint64(9), resp.CatalogVersion) + require.Len(t, resp.Routes, 3) + require.Equal(t, []uint64{1, 2, 3}, []uint64{resp.Routes[0].RouteId, resp.Routes[1].RouteId, resp.Routes[2].RouteId}) + + rightOpen, err := s.GetIntersectingRoutes(context.Background(), &pb.GetIntersectingRoutesRequest{ + Start: []byte("m"), + End: nil, + CatalogVersion: 9, + }) + require.NoError(t, err) + require.Len(t, rightOpen.Routes, 1) + require.Equal(t, uint64(3), rightOpen.Routes[0].RouteId) +} + +func TestDistributionServerOwnershipRPCs_RejectUnknownCatalogVersion(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + + s := NewDistributionServer(engine, nil) + _, err := s.GetRouteOwnership(context.Background(), &pb.GetRouteOwnershipRequest{ + Key: []byte("a"), + CatalogVersion: 2, + }) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) + require.ErrorContains(t, err, errDistributionCatalogVersionNotFound.Error()) + + _, err = s.GetIntersectingRoutes(context.Background(), &pb.GetIntersectingRoutesRequest{ + Start: []byte(""), + CatalogVersion: 2, + }) + require.Error(t, err) + require.Equal(t, codes.NotFound, status.Code(err)) + require.ErrorContains(t, err, errDistributionCatalogVersionNotFound.Error()) +} + +func TestDistributionServerOwnershipRPCs_RequireEngine(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(nil, nil) + _, err := s.GetRouteOwnership(context.Background(), &pb.GetRouteOwnershipRequest{CatalogVersion: 1}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionEngineNotConfigured.Error()) + + _, err = s.GetIntersectingRoutes(context.Background(), &pb.GetIntersectingRoutesRequest{CatalogVersion: 1}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionEngineNotConfigured.Error()) +} + func TestDistributionServerSplitRange_Success(t *testing.T) { t.Parallel() From 7d3b01cd74c6221cc05da177e93a36134321ec4d Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 13 Jul 2026 23:52:48 +0900 Subject: [PATCH 011/162] migration: stage imported versions --- adapter/internal_migration_test.go | 5 +- distribution/migrator.go | 36 +++++++++++ distribution/migrator_export_plan_test.go | 22 +++++++ kv/fsm_migration_import.go | 7 ++- kv/fsm_migration_import_test.go | 75 +++++++++++++++++++++++ 5 files changed, 141 insertions(+), 4 deletions(-) create mode 100644 kv/fsm_migration_import_test.go diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 56d1b1350..dd8b48baf 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -141,9 +141,12 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { require.Equal(t, []byte("cursor-1"), resp.GetAckedCursor()) require.Equal(t, uint64(1), proposer.calls) - got, err := st.GetAt(ctx, []byte("k"), 30) + staged := distribution.MigrationStagedDataKey(7, []byte("k")) + got, err := st.GetAt(ctx, staged, 30) require.NoError(t, err) require.Equal(t, []byte("v"), got) + _, err = st.GetAt(ctx, []byte("k"), 30) + require.ErrorIs(t, err, store.ErrKeyNotFound) floor, err := st.MigrationHLCFloor(ctx, 7) require.NoError(t, err) require.Equal(t, uint64(30), floor) diff --git a/distribution/migrator.go b/distribution/migrator.go index c795f70f6..3b8df4876 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -2,6 +2,7 @@ package distribution import ( "bytes" + "encoding/binary" "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/store" @@ -68,6 +69,7 @@ const ( migrationDynamoGenPrefix = "!ddb|meta|gen|" migrationDynamoItemPrefix = "!ddb|item|" migrationDynamoGSIPrefix = "!ddb|gsi|" + migrationStagedDataPrefix = "!dist|migstage|" ) const ( @@ -81,6 +83,8 @@ const ( migrationSQSMsgGroupPrefix = "!sqs|msg|group|" migrationSQSMsgByAgePrefix = "!sqs|msg|byage|" migrationSQSPartitionedSuffix = "p|" + migrationStagedDataJobIDBytes = 8 + migrationStagedDataSeparator = byte('|') ) var ( @@ -237,6 +241,38 @@ func (b MigrationBracket) containsFamilyShape(rawKey []byte) bool { } } +// MigrationStagedDataKey returns the target-local shadow key used while a +// cross-group split imports data before CUTOVER/promotion. +func MigrationStagedDataKey(jobID uint64, rawKey []byte) []byte { + key := make([]byte, len(migrationStagedDataPrefix)+migrationStagedDataJobIDBytes+1+len(rawKey)) + copy(key, migrationStagedDataPrefix) + binary.BigEndian.PutUint64(key[len(migrationStagedDataPrefix):], jobID) + key[len(migrationStagedDataPrefix)+migrationStagedDataJobIDBytes] = migrationStagedDataSeparator + copy(key[len(migrationStagedDataPrefix)+migrationStagedDataJobIDBytes+1:], rawKey) + return key +} + +// MigrationStagedDataKeyPrefix returns the prefix covering all staged data for +// one migration job. +func MigrationStagedDataKeyPrefix(jobID uint64) []byte { + return MigrationStagedDataKey(jobID, nil) +} + +func IsMigrationStagedDataKey(key []byte) bool { + return len(key) >= len(migrationStagedDataPrefix)+migrationStagedDataJobIDBytes+1 && + bytes.HasPrefix(key, []byte(migrationStagedDataPrefix)) && + key[len(migrationStagedDataPrefix)+migrationStagedDataJobIDBytes] == migrationStagedDataSeparator +} + +func MigrationStagedDataKeyParts(key []byte) (uint64, []byte, bool) { + if !IsMigrationStagedDataKey(key) { + return 0, nil, false + } + jobID := binary.BigEndian.Uint64(key[len(migrationStagedDataPrefix):]) + rawKey := bytes.Clone(key[len(migrationStagedDataPrefix)+migrationStagedDataJobIDBytes+1:]) + return jobID, rawKey, true +} + // InitializeSplitJobPlan validates the source route and seeds the job's // bracket progress for the moving right child [SplitKey, source.End). func InitializeSplitJobPlan(job SplitJob, source RouteDescriptor, nowMs int64) (SplitJob, error) { diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index 6888aa3bf..d445876e4 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -177,6 +177,28 @@ func TestMigrationKnownInternalPrefixesAreConcreteOnly(t *testing.T) { require.False(t, bytes.Equal(prefixes[0], MigrationKnownInternalPrefixes()[0]), "prefix list must be cloned") } +func TestMigrationStagedDataKeyRoundTrip(t *testing.T) { + t.Parallel() + + raw := []byte("user|raw") + key := MigrationStagedDataKey(42, raw) + require.True(t, IsMigrationStagedDataKey(key)) + require.True(t, bytes.HasPrefix(key, MigrationStagedDataKeyPrefix(42))) + require.False(t, IsMigrationStagedDataKey([]byte("!dist|migstage|short"))) + + jobID, original, ok := MigrationStagedDataKeyParts(key) + require.True(t, ok) + require.Equal(t, uint64(42), jobID) + require.Equal(t, []byte("user|raw"), original) + + raw[0] = 'X' + original[0] = 'Y' + jobID, original, ok = MigrationStagedDataKeyParts(key) + require.True(t, ok) + require.Equal(t, uint64(42), jobID) + require.Equal(t, []byte("user|raw"), original) +} + func TestValidateMigrationRouteRangeRejectsReservedControlPrefixes(t *testing.T) { t.Parallel() diff --git a/kv/fsm_migration_import.go b/kv/fsm_migration_import.go index d6a048130..9de47d897 100644 --- a/kv/fsm_migration_import.go +++ b/kv/fsm_migration_import.go @@ -4,6 +4,7 @@ import ( "bytes" "context" + "github.com/bootjp/elastickv/distribution" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -38,7 +39,7 @@ func (f *kvFSM) applyMigrationImport(ctx context.Context, data []byte) any { BracketID: req.GetBracketId(), BatchSeq: req.GetBatchSeq(), Cursor: req.GetCursor(), - Versions: migrationStoreVersionsFromProto(req.GetVersions()), + Versions: migrationStoreVersionsFromProto(req.GetJobId(), req.GetVersions()), }) if err != nil { return errors.WithStack(err) @@ -64,14 +65,14 @@ func (f *kvFSM) migrationHLCFloorForApply(ctx context.Context, req *pb.ImportRan return floor, nil } -func migrationStoreVersionsFromProto(in []*pb.MVCCVersion) []store.MVCCVersion { +func migrationStoreVersionsFromProto(jobID uint64, in []*pb.MVCCVersion) []store.MVCCVersion { out := make([]store.MVCCVersion, 0, len(in)) for _, version := range in { if version == nil { continue } out = append(out, store.MVCCVersion{ - Key: bytes.Clone(version.GetKey()), + Key: distribution.MigrationStagedDataKey(jobID, version.GetKey()), CommitTS: version.GetCommitTs(), Tombstone: version.GetTombstone(), Value: bytes.Clone(version.GetValue()), diff --git a/kv/fsm_migration_import_test.go b/kv/fsm_migration_import_test.go new file mode 100644 index 000000000..589b6e1b1 --- /dev/null +++ b/kv/fsm_migration_import_test.go @@ -0,0 +1,75 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestMigrationStoreVersionsFromProtoStagesKeys(t *testing.T) { + t.Parallel() + + rawKey := []byte("user|k") + value := []byte("value") + got := migrationStoreVersionsFromProto(7, []*pb.MVCCVersion{ + nil, + { + Key: rawKey, + CommitTs: 11, + Value: value, + KeyFamily: distribution.MigrationFamilyUser, + ExpireAt: 123, + }, + }) + + require.Len(t, got, 1) + require.Equal(t, distribution.MigrationStagedDataKey(7, []byte("user|k")), got[0].Key) + require.Equal(t, uint64(11), got[0].CommitTS) + require.Equal(t, []byte("value"), got[0].Value) + require.Equal(t, distribution.MigrationFamilyUser, got[0].KeyFamily) + require.Equal(t, uint64(123), got[0].ExpireAt) + + rawKey[0] = 'X' + value[0] = 'X' + require.Equal(t, distribution.MigrationStagedDataKey(7, []byte("user|k")), got[0].Key) + require.Equal(t, []byte("value"), got[0].Value) +} + +func TestApplyMigrationImportWritesOnlyStagedKeys(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + hlc := NewHLC() + fsm := &kvFSM{store: st, hlc: hlc} + req := &pb.ImportRangeVersionsRequest{ + JobId: 9, + BracketId: 1, + BatchSeq: 1, + Cursor: []byte("cursor"), + Versions: []*pb.MVCCVersion{ + {Key: []byte("user|k"), CommitTs: 10, Value: []byte("v")}, + }, + } + data, err := proto.Marshal(req) + require.NoError(t, err) + + applied := fsm.applyMigrationImport(ctx, data) + result, ok := applied.(store.ImportVersionsResult) + require.True(t, ok, "got %T: %v", applied, applied) + require.Equal(t, []byte("cursor"), result.AckedCursor) + require.Equal(t, uint64(10), result.MaxImportedTS) + require.GreaterOrEqual(t, hlc.Current(), uint64(10)) + + staged := distribution.MigrationStagedDataKey(9, []byte("user|k")) + got, err := st.GetAt(ctx, staged, 10) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) + _, err = st.GetAt(ctx, []byte("user|k"), 10) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} From e7f69efd9e822849426047307f30c1c349050201 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 00:02:32 +0900 Subject: [PATCH 012/162] migration: merge staged visibility reads --- kv/shard_store.go | 304 +++++++++++++++++++++++++++++++++++++---- kv/shard_store_test.go | 92 +++++++++++++ 2 files changed, 373 insertions(+), 23 deletions(-) diff --git a/kv/shard_store.go b/kv/shard_store.go index 175c81224..ba5ef6ac9 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -36,19 +36,19 @@ func NewShardStore(engine *distribution.Engine, groups map[uint64]*ShardGroup) * } func (s *ShardStore) GetAt(ctx context.Context, key []byte, ts uint64) ([]byte, error) { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return nil, store.ErrKeyNotFound } // Some tests use ShardStore without raft; in that case serve reads locally. if engineForGroup(g) == nil { - return s.localGetAt(ctx, g, key, ts) + return s.localGetAt(ctx, g, route, key, ts) } // Wait for a leader read fence before serving from local state. if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.leaderGetAt(ctx, g, key, ts) + return s.leaderGetAt(ctx, g, route, key, ts) } return s.proxyRawGet(ctx, g, key, ts, 0) } @@ -63,10 +63,10 @@ func (s *ShardStore) GetGroupAt(ctx context.Context, groupID uint64, key []byte, } if engineForGroup(g) == nil { - return s.localGetAt(ctx, g, key, ts) + return s.localGetAt(ctx, g, distribution.Route{}, key, ts) } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.leaderGetAt(ctx, g, key, ts) + return s.leaderGetAt(ctx, g, distribution.Route{}, key, ts) } return s.proxyRawGet(ctx, g, key, ts, groupID) } @@ -88,16 +88,19 @@ func isLinearizableRaftLeader(ctx context.Context, engine raftengine.LeaderView) return err == nil } -func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, key []byte, ts uint64) ([]byte, error) { +func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { if !isTxnInternalKey(key) { if err := s.maybeResolveTxnLock(ctx, g, key, ts); err != nil { return nil, err } } - return s.localGetAt(ctx, g, key, ts) + return s.localGetAt(ctx, g, route, key, ts) } -func (s *ShardStore) localGetAt(ctx context.Context, g *ShardGroup, key []byte, ts uint64) ([]byte, error) { +func (s *ShardStore) localGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + if routeHasStagedVisibility(route) { + return s.getAtWithStagedVisibility(ctx, g, route, key, ts) + } val, err := g.Store.GetAt(ctx, key, ts) if err != nil { return nil, errors.WithStack(err) @@ -105,6 +108,70 @@ func (s *ShardStore) localGetAt(ctx context.Context, g *ShardGroup, key []byte, return val, nil } +func routeHasStagedVisibility(route distribution.Route) bool { + return route.StagedVisibilityActive && route.MigrationJobID != 0 +} + +func (s *ShardStore) getAtWithStagedVisibility(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + live, liveOK, err := latestMVCCVersionAt(ctx, g.Store, key, ts) + if err != nil { + return nil, err + } + stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, key) + staged, stagedOK, err := latestMVCCVersionAt(ctx, g.Store, stagedKey, ts) + if err != nil { + return nil, err + } + if stagedOK { + staged.Key = bytes.Clone(key) + } + winner, ok := newerMigrationVersion(live, liveOK, staged, stagedOK) + if !ok || !migrationVersionVisible(winner, ts) { + return nil, store.ErrKeyNotFound + } + return bytes.Clone(winner.Value), nil +} + +func latestMVCCVersionAt(ctx context.Context, st store.MVCCStore, key []byte, ts uint64) (store.MVCCVersion, bool, error) { + result, err := st.ExportVersions(ctx, store.ExportVersionsOptions{ + StartKey: key, + EndKey: nextScanCursor(key), + MaxCommitTSInclusive: ts, + MaxVersions: 1, + MaxScannedBytes: 0, + MinCommitTSExclusive: 0, + MaxBytes: 0, + KeyFamily: 0, + AcceptKey: nil, + }) + if err != nil { + return store.MVCCVersion{}, false, errors.WithStack(err) + } + for _, version := range result.Versions { + if bytes.Equal(version.Key, key) { + return version, true, nil + } + } + return store.MVCCVersion{}, false, nil +} + +func newerMigrationVersion(a store.MVCCVersion, aOK bool, b store.MVCCVersion, bOK bool) (store.MVCCVersion, bool) { + switch { + case !aOK: + return b, bOK + case !bOK: + return a, true + case b.CommitTS >= a.CommitTS: + return b, true + default: + return a, true + } +} + +func migrationVersionVisible(version store.MVCCVersion, ts uint64) bool { + return !version.Tombstone && (version.ExpireAt == 0 || version.ExpireAt > ts) +} + func (s *ShardStore) ExistsAt(ctx context.Context, key []byte, ts uint64) (bool, error) { v, err := s.GetAt(ctx, key, ts) if err != nil { @@ -390,7 +457,7 @@ func (s *ShardStore) scanRouteAtDirection( } if engineForGroup(g) == nil { - kvs, err := s.scanRouteLocal(ctx, g, start, end, limit, ts, reverse) + kvs, err := s.scanRouteLocal(ctx, g, route, start, end, limit, ts, reverse) if err != nil { return nil, errors.WithStack(err) } @@ -398,7 +465,7 @@ func (s *ShardStore) scanRouteAtDirection( } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.scanRouteAtLeader(ctx, g, start, end, limit, ts, reverse) + return s.scanRouteAtLeader(ctx, g, route, start, end, limit, ts, reverse) } var groupID uint64 @@ -435,6 +502,9 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( } if engineForGroup(g) == nil { + if routeHasStagedVisibility(route) { + return nil, true, nil + } kvs, limitReached, err := scanLocalPhysicalLimit(ctx, g.Store, start, end, visibleLimit, physicalLimit, ts, reverse) if err != nil { return nil, limitReached, errors.WithStack(err) @@ -443,6 +513,9 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { + if routeHasStagedVisibility(route) { + return nil, true, nil + } return s.scanRouteAtLeaderPhysicalLimit(ctx, g, start, end, visibleLimit, physicalLimit, ts, reverse) } @@ -494,12 +567,16 @@ func scanPhysicalLimitLocal( func (s *ShardStore) scanRouteLocal( ctx context.Context, g *ShardGroup, + route distribution.Route, start []byte, end []byte, limit int, ts uint64, reverse bool, ) ([]*store.KVPair, error) { + if routeHasStagedVisibility(route) { + return s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) + } if reverse { kvs, err := g.Store.ReverseScanAt(ctx, start, end, limit, ts) return kvs, errors.WithStack(err) @@ -527,13 +604,14 @@ func (s *ShardStore) scanRouteAtLeaderPhysicalLimit( if err != nil { return nil, limitReached, err } - resolved, err := s.resolveScanLocks(ctx, g, kvs, lockKVs, ts) + resolved, err := s.resolveScanLocks(ctx, g, distribution.Route{}, kvs, lockKVs, ts) return resolved, limitReached, err } func (s *ShardStore) scanRouteAtLeader( ctx context.Context, g *ShardGroup, + route distribution.Route, start []byte, end []byte, limit int, @@ -544,9 +622,12 @@ func (s *ShardStore) scanRouteAtLeader( kvs []*store.KVPair err error ) - if reverse { + switch { + case routeHasStagedVisibility(route): + kvs, err = s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) + case reverse: kvs, err = g.Store.ReverseScanAt(ctx, start, end, limit, ts) - } else { + default: kvs, err = g.Store.ScanAt(ctx, start, end, limit, ts) } if err != nil { @@ -557,7 +638,155 @@ func (s *ShardStore) scanRouteAtLeader( if err != nil { return nil, err } - return s.resolveScanLocks(ctx, g, kvs, lockKVs, ts) + return s.resolveScanLocks(ctx, g, route, kvs, lockKVs, ts) +} + +const stagedVisibilityExportPageSize = 1024 + +func (s *ShardStore) scanRouteWithStagedVisibility( + ctx context.Context, + g *ShardGroup, + route distribution.Route, + start []byte, + end []byte, + limit int, + ts uint64, + reverse bool, +) ([]*store.KVPair, error) { + live, err := collectLatestLogicalVersions(ctx, g.Store, start, end, start, end, ts, liveLogicalVersionKey) + if err != nil { + return nil, err + } + stagedStart, stagedEnd := stagedVisibilityScanBounds(route.MigrationJobID, start, end) + staged, err := collectLatestLogicalVersions(ctx, g.Store, stagedStart, stagedEnd, start, end, ts, stagedLogicalVersionKey) + if err != nil { + return nil, err + } + merged := mergeLogicalVersionMaps(live, staged) + out := visibleLogicalKVs(merged, ts, reverse) + if len(out) > limit { + clear(out[limit:]) + out = out[:limit] + } + return out, nil +} + +func stagedVisibilityScanBounds(jobID uint64, start []byte, end []byte) ([]byte, []byte) { + prefix := distribution.MigrationStagedDataKeyPrefix(jobID) + scanStart := prefix + if start != nil { + scanStart = distribution.MigrationStagedDataKey(jobID, start) + } + scanEnd := prefixScanEnd(prefix) + if end != nil { + scanEnd = distribution.MigrationStagedDataKey(jobID, end) + } + return scanStart, scanEnd +} + +type logicalVersionKeyFunc func([]byte) ([]byte, bool) + +func liveLogicalVersionKey(key []byte) ([]byte, bool) { + if distribution.IsMigrationStagedDataKey(key) { + return nil, false + } + return bytes.Clone(key), true +} + +func stagedLogicalVersionKey(key []byte) ([]byte, bool) { + _, rawKey, ok := distribution.MigrationStagedDataKeyParts(key) + return rawKey, ok +} + +func collectLatestLogicalVersions( + ctx context.Context, + st store.MVCCStore, + scanStart []byte, + scanEnd []byte, + logicalStart []byte, + logicalEnd []byte, + ts uint64, + logicalKey logicalVersionKeyFunc, +) (map[string]store.MVCCVersion, error) { + out := make(map[string]store.MVCCVersion) + var cursor []byte + for { + result, err := st.ExportVersions(ctx, store.ExportVersionsOptions{ + StartKey: scanStart, + EndKey: scanEnd, + MaxCommitTSInclusive: ts, + Cursor: cursor, + MaxVersions: stagedVisibilityExportPageSize, + }) + if err != nil { + return nil, errors.WithStack(err) + } + for _, version := range result.Versions { + key, ok := logicalKey(version.Key) + if !ok || !keyInRange(key, logicalStart, logicalEnd) { + continue + } + version.Key = key + recordLatestLogicalVersion(out, version) + } + if result.Done { + return out, nil + } + if bytes.Equal(cursor, result.NextCursor) { + return nil, errors.New("staged visibility export cursor did not progress") + } + cursor = bytes.Clone(result.NextCursor) + } +} + +func recordLatestLogicalVersion(out map[string]store.MVCCVersion, version store.MVCCVersion) { + key := string(version.Key) + if existing, ok := out[key]; ok && existing.CommitTS >= version.CommitTS { + return + } + out[key] = version +} + +func mergeLogicalVersionMaps(live map[string]store.MVCCVersion, staged map[string]store.MVCCVersion) map[string]store.MVCCVersion { + out := make(map[string]store.MVCCVersion, len(live)+len(staged)) + for key, version := range live { + out[key] = version + } + for key, stagedVersion := range staged { + liveVersion, liveOK := out[key] + if winner, ok := newerMigrationVersion(liveVersion, liveOK, stagedVersion, true); ok { + out[key] = winner + } + } + return out +} + +func visibleLogicalKVs(versions map[string]store.MVCCVersion, ts uint64, reverse bool) []*store.KVPair { + out := make([]*store.KVPair, 0, len(versions)) + for _, version := range versions { + if !migrationVersionVisible(version, ts) { + continue + } + out = append(out, &store.KVPair{ + Key: bytes.Clone(version.Key), + Value: bytes.Clone(version.Value), + }) + } + sort.Slice(out, func(i, j int) bool { + cmp := bytes.Compare(out[i].Key, out[j].Key) + if reverse { + return cmp > 0 + } + return cmp < 0 + }) + return out +} + +func keyInRange(key []byte, start []byte, end []byte) bool { + if start != nil && bytes.Compare(key, start) < 0 { + return false + } + return end == nil || bytes.Compare(key, end) < 0 } func scanLockBoundsForKVs(kvs []*store.KVPair, scanStart []byte, scanEnd []byte, limit int) ([]byte, []byte) { @@ -708,13 +937,13 @@ func (s *ShardStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, } func (s *ShardStore) LatestCommitTS(ctx context.Context, key []byte) (uint64, bool, error) { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return 0, false, nil } if engineForGroup(g) == nil { - ts, exists, err := g.Store.LatestCommitTS(ctx, key) + ts, exists, err := s.localLatestCommitTS(ctx, g, route, key) if err != nil { return 0, false, errors.WithStack(err) } @@ -726,7 +955,7 @@ func (s *ShardStore) LatestCommitTS(ctx context.Context, key []byte) (uint64, bo // round-trip (same rationale as isLinearizableRaftLeader). if engine := engineForGroup(g); isLeaderEngine(engine) { if _, err := leaseReadEngineCtx(ctx, engine); err == nil { - ts, exists, err := g.Store.LatestCommitTS(ctx, key) + ts, exists, err := s.localLatestCommitTS(ctx, g, route, key) if err != nil { return 0, false, errors.WithStack(err) } @@ -737,6 +966,30 @@ func (s *ShardStore) LatestCommitTS(ctx context.Context, key []byte) (uint64, bo return s.proxyLatestCommitTS(ctx, g, key) } +func (s *ShardStore) localLatestCommitTS(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte) (uint64, bool, error) { + liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) + if err != nil { + return 0, false, errors.WithStack(err) + } + if !routeHasStagedVisibility(route) { + return liveTS, liveExists, nil + } + stagedTS, stagedExists, err := g.Store.LatestCommitTS(ctx, distribution.MigrationStagedDataKey(route.MigrationJobID, key)) + if err != nil { + return 0, false, errors.WithStack(err) + } + switch { + case !liveExists: + return stagedTS, stagedExists, nil + case !stagedExists: + return liveTS, true, nil + case stagedTS >= liveTS: + return stagedTS, true, nil + default: + return liveTS, true, nil + } +} + func (s *ShardStore) proxyLatestCommitTS(ctx context.Context, g *ShardGroup, key []byte) (uint64, bool, error) { engine := engineForGroup(g) if engine == nil { @@ -862,7 +1115,7 @@ func newScanLockPlan(size int) *scanLockPlan { } } -func (s *ShardStore) resolveScanLocks(ctx context.Context, g *ShardGroup, kvs []*store.KVPair, lockKVs []*store.KVPair, ts uint64) ([]*store.KVPair, error) { +func (s *ShardStore) resolveScanLocks(ctx context.Context, g *ShardGroup, route distribution.Route, kvs []*store.KVPair, lockKVs []*store.KVPair, ts uint64) ([]*store.KVPair, error) { if len(kvs) == 0 && len(lockKVs) == 0 { return kvs, nil } @@ -877,7 +1130,7 @@ func (s *ShardStore) resolveScanLocks(ctx context.Context, g *ShardGroup, kvs [] if err := applyScanLockResolutions(ctx, g, plan); err != nil { return nil, err } - return s.materializeScanLockResults(ctx, g, ts, plan.items) + return s.materializeScanLockResults(ctx, g, route, ts, plan.items) } func (s *ShardStore) planScanLockResolutions(ctx context.Context, g *ShardGroup, kvs []*store.KVPair, lockKVs []*store.KVPair, ts uint64) (*scanLockPlan, error) { @@ -1134,7 +1387,7 @@ func applyScanLockResolutions(ctx context.Context, g *ShardGroup, plan *scanLock return nil } -func (s *ShardStore) materializeScanLockResults(ctx context.Context, g *ShardGroup, ts uint64, items []scanItem) ([]*store.KVPair, error) { +func (s *ShardStore) materializeScanLockResults(ctx context.Context, g *ShardGroup, route distribution.Route, ts uint64, items []scanItem) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0, len(items)) for _, item := range items { if item.skip { @@ -1144,7 +1397,7 @@ func (s *ShardStore) materializeScanLockResults(ctx context.Context, g *ShardGro out = append(out, item.kvp) continue } - v, err := s.localGetAt(ctx, g, item.kvp.Key, ts) + v, err := s.localGetAt(ctx, g, route, item.kvp.Key, ts) if err != nil { if errors.Is(err, store.ErrKeyNotFound) { continue @@ -1640,12 +1893,17 @@ func (s *ShardStore) closeGroup(g *ShardGroup) error { } func (s *ShardStore) groupForKey(key []byte) (*ShardGroup, bool) { + _, g, ok := s.routeAndGroupForKey(key) + return g, ok +} + +func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *ShardGroup, bool) { route, ok := s.engine.GetRoute(routeKey(key)) if !ok { - return nil, false + return distribution.Route{}, nil, false } g, ok := s.groups[route.GroupID] - return g, ok + return route, g, ok } func (s *ShardStore) proxyRawGet(ctx context.Context, g *ShardGroup, key []byte, ts uint64, groupID uint64) ([]byte, error) { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 4f0da38b7..100b70e27 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -10,6 +10,98 @@ import ( "github.com/stretchr/testify/require" ) +func newStagedVisibilityShardStore(t *testing.T) (*ShardStore, *ShardGroup) { + t.Helper() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + return NewShardStore(engine, map[uint64]*ShardGroup{1: group}), group +} + +func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("k") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + + require.NoError(t, group.Store.PutAt(ctx, rawKey, []byte("live-old"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged-new"), 20, 0)) + got, err := st.GetAt(ctx, rawKey, 25) + require.NoError(t, err) + require.Equal(t, []byte("staged-new"), got) + + require.NoError(t, group.Store.PutAt(ctx, rawKey, []byte("live-new"), 30, 0)) + got, err = st.GetAt(ctx, rawKey, 35) + require.NoError(t, err) + require.Equal(t, []byte("live-new"), got) + + require.NoError(t, group.Store.DeleteAt(ctx, stagedKey, 40)) + _, err = st.GetAt(ctx, rawKey, 45) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live-b"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("c"), []byte("live-c"), 30, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged-b"), 20, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("d")), []byte("staged-d"), 15, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("e")), []byte("staged-e"), 40, 0)) + require.NoError(t, group.Store.DeleteAt(ctx, []byte("d"), 25)) + + kvs, err := st.ScanAt(ctx, []byte("a"), []byte("z"), 10, 50) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b"), Value: []byte("staged-b")}, + {Key: []byte("c"), Value: []byte("live-c")}, + {Key: []byte("e"), Value: []byte("staged-e")}, + }, kvs) + + kvs, err = st.ReverseScanAt(ctx, []byte("a"), []byte("z"), 10, 50) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("e"), Value: []byte("staged-e")}, + {Key: []byte("c"), Value: []byte("live-c")}, + {Key: []byte("b"), Value: []byte("staged-b")}, + }, kvs) + + ts, exists, err := st.LatestCommitTS(ctx, []byte("b")) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, uint64(20), ts) + + ts, exists, err = st.LatestCommitTS(ctx, []byte("d")) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, uint64(25), ts) + + ts, exists, err = st.LatestCommitTS(ctx, []byte("e")) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, uint64(40), ts) +} + func TestShardStoreScanAt_IncludesListKeysAcrossShards(t *testing.T) { t.Parallel() From b4f247751bd017bd0381256db704a7f92a97fcfa Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 00:44:05 +0900 Subject: [PATCH 013/162] migration: promote staged versions --- adapter/internal.go | 70 ++++- adapter/internal_migration_test.go | 35 +++ kv/fsm.go | 6 + kv/fsm_migration_promote.go | 88 +++++++ kv/fsm_migration_promote_test.go | 69 +++++ proto/internal.pb.go | 211 +++++++++++++-- proto/internal.proto | 16 ++ proto/internal_grpc.pb.go | 46 +++- store/lsm_migration.go | 10 +- store/migration_promote.go | 397 +++++++++++++++++++++++++++++ store/migration_versions.go | 11 +- store/migration_versions_test.go | 98 +++++++ store/mvcc_store.go | 22 +- store/store.go | 43 ++++ 14 files changed, 1074 insertions(+), 48 deletions(-) create mode 100644 kv/fsm_migration_promote.go create mode 100644 kv/fsm_migration_promote_test.go create mode 100644 store/migration_promote.go diff --git a/adapter/internal.go b/adapter/internal.go index de61f5485..efacdab71 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -179,6 +179,34 @@ func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeV return &pb.ImportRangeVersionsResponse{AckedCursor: result.AckedCursor}, nil } +func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteStagedVersionsRequest) (*pb.PromoteStagedVersionsResponse, error) { + if req == nil { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "promote staged versions request is nil")) + } + if req.GetJobId() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "promote staged versions job_id is required")) + } + if i.migrationProposer == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote proposer is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + result, err := i.proposeMigrationPromote(ctx, req) + if err != nil { + return nil, errors.WithStack(err) + } + if i.clock != nil && result.MaxPromotedTS > 0 { + i.clock.Observe(result.MaxPromotedTS) + } + return &pb.PromoteStagedVersionsResponse{ + NextCursor: result.NextCursor, + Done: result.Done, + PromotedRows: result.PromotedRows, + MaxPromotedTs: result.MaxPromotedTS, + }, nil +} + func (i *Internal) verifyInternalLeader(ctx context.Context) error { if i.leader.State() != raftengine.StateLeader { return errors.WithStack(ErrNotLeader) @@ -194,14 +222,11 @@ func (i *Internal) proposeMigrationImport(ctx context.Context, req *pb.ImportRan if err != nil { return store.ImportVersionsResult{}, errors.WithStack(err) } - result, err := i.migrationProposer.Propose(ctx, cmd) + resp, err := i.proposeMigrationCommand(ctx, cmd, "migration import") if err != nil { return store.ImportVersionsResult{}, errors.WithStack(err) } - if result == nil { - return store.ImportVersionsResult{}, errors.New("migration import proposal returned nil result") - } - switch resp := result.Response.(type) { + switch resp := resp.(type) { case store.ImportVersionsResult: return resp, nil case *store.ImportVersionsResult: @@ -216,6 +241,41 @@ func (i *Internal) proposeMigrationImport(ctx context.Context, req *pb.ImportRan } } +func (i *Internal) proposeMigrationPromote(ctx context.Context, req *pb.PromoteStagedVersionsRequest) (store.PromoteVersionsResult, error) { + cmd, err := kv.MarshalMigrationPromoteCommand(req) + if err != nil { + return store.PromoteVersionsResult{}, errors.WithStack(err) + } + resp, err := i.proposeMigrationCommand(ctx, cmd, "migration promote") + if err != nil { + return store.PromoteVersionsResult{}, errors.WithStack(err) + } + switch resp := resp.(type) { + case store.PromoteVersionsResult: + return resp, nil + case *store.PromoteVersionsResult: + if resp == nil { + return store.PromoteVersionsResult{}, errors.New("migration promote apply returned nil result") + } + return *resp, nil + case error: + return store.PromoteVersionsResult{}, errors.WithStack(resp) + default: + return store.PromoteVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration promote apply response type %T", resp)) + } +} + +func (i *Internal) proposeMigrationCommand(ctx context.Context, cmd []byte, label string) (any, error) { + result, err := i.migrationProposer.Propose(ctx, cmd) + if err != nil { + return nil, errors.WithStack(err) + } + if result == nil { + return nil, errors.WithStack(errors.Newf("%s proposal returned nil result", label)) + } + return result.Response, nil +} + func exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.ExportVersionsOptions { chunkBytes := uint64(req.GetChunkBytes()) if chunkBytes == 0 { diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index dd8b48baf..54227d418 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -152,3 +152,38 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { require.Equal(t, uint64(30), floor) require.GreaterOrEqual(t, clock.Current(), uint64(30)) } + +func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + clock := kv.NewHLC() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, clock), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + ) + + staged := distribution.MigrationStagedDataKey(7, []byte("k")) + require.NoError(t, st.PutAt(ctx, staged, []byte("v"), 30, 0)) + + resp, err := internal.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{ + JobId: 7, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, resp.GetDone()) + require.Equal(t, uint64(1), resp.GetPromotedRows()) + require.Equal(t, uint64(30), resp.GetMaxPromotedTs()) + require.Equal(t, uint64(1), proposer.calls) + + got, err := st.GetAt(ctx, []byte("k"), 30) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) + _, err = st.GetAt(ctx, staged, 30) + require.ErrorIs(t, err, store.ErrKeyNotFound) + require.GreaterOrEqual(t, clock.Current(), uint64(30)) +} diff --git a/kv/fsm.go b/kv/fsm.go index 00e4a5d3c..269b13954 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -364,6 +364,8 @@ func (f *kvFSM) applyReservedOpcode(ctx context.Context, data []byte) (any, bool return f.applyHLCLease(data[1:]), true case data[0] == raftEncodeMigrationImport: return f.applyMigrationImport(ctx, data[1:]), true + case data[0] == raftEncodeMigrationPromote: + return f.applyMigrationPromote(ctx, data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true default: @@ -399,6 +401,10 @@ const ( // batch. Every target voter applies the raw MVCC versions, import ack, and // migration HLC floor before the RPC handler returns success. raftEncodeMigrationImport byte = 0x09 + // raftEncodeMigrationPromote carries a target-group range-migration staged + // data promotion chunk. Every target voter atomically copies staged MVCC + // versions into the live keyspace and removes the promoted staged rows. + raftEncodeMigrationPromote byte = 0x0b ) func decodeRaftRequests(data []byte) ([]*pb.Request, error) { diff --git a/kv/fsm_migration_promote.go b/kv/fsm_migration_promote.go new file mode 100644 index 000000000..0d9d1ed86 --- /dev/null +++ b/kv/fsm_migration_promote.go @@ -0,0 +1,88 @@ +package kv + +import ( + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +const ( + defaultMigrationPromoteMaxVersions = 1024 + defaultMigrationPromoteMaxBytes = 4 << 20 + defaultMigrationPromoteMaxScannedBytes = defaultMigrationPromoteMaxBytes * 4 +) + +// MarshalMigrationPromoteCommand encodes a target-group staged-data promotion +// chunk as a Raft FSM command. +func MarshalMigrationPromoteCommand(req *pb.PromoteStagedVersionsRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled migration promote request too large") + } + return prependByte(raftEncodeMigrationPromote, b), nil +} + +func (f *kvFSM) applyMigrationPromote(ctx context.Context, data []byte) any { + req := &pb.PromoteStagedVersionsRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + promoter, ok := f.store.(store.MigrationPromoter) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + result, err := promoter.PromoteVersions(ctx, migrationPromoteOptionsFromProto(req)) + if err != nil { + return errors.WithStack(err) + } + if f.hlc != nil && result.MaxPromotedTS > 0 { + f.hlc.Observe(result.MaxPromotedTS) + } + return result +} + +func migrationPromoteOptionsFromProto(req *pb.PromoteStagedVersionsRequest) store.PromoteVersionsOptions { + maxVersions := int(req.GetMaxVersions()) + if maxVersions <= 0 { + maxVersions = defaultMigrationPromoteMaxVersions + } + maxBytes := req.GetMaxBytes() + if maxBytes == 0 { + maxBytes = defaultMigrationPromoteMaxBytes + } + maxScannedBytes := req.GetMaxScannedBytes() + if maxScannedBytes == 0 { + maxScannedBytes = defaultMigrationPromoteMaxScannedBytes + } + prefix := distribution.MigrationStagedDataKeyPrefix(req.GetJobId()) + return store.PromoteVersionsOptions{ + JobID: req.GetJobId(), + StartKey: prefix, + EndKey: store.PrefixScanEnd(prefix), + Cursor: req.GetCursor(), + MaxVersions: maxVersions, + MaxBytes: maxBytes, + MaxScannedBytes: maxScannedBytes, + TargetKey: migrationPromoteTargetKey(req.GetJobId()), + } +} + +func migrationPromoteTargetKey(jobID uint64) func([]byte) ([]byte, bool) { + return func(stagedKey []byte) ([]byte, bool) { + gotJobID, rawKey, ok := distribution.MigrationStagedDataKeyParts(stagedKey) + if !ok || gotJobID != jobID { + return nil, false + } + return rawKey, true + } +} diff --git a/kv/fsm_migration_promote_test.go b/kv/fsm_migration_promote_test.go new file mode 100644 index 000000000..2470c485d --- /dev/null +++ b/kv/fsm_migration_promote_test.go @@ -0,0 +1,69 @@ +package kv + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestMigrationPromoteTargetKeyRestoresRawKey(t *testing.T) { + t.Parallel() + + targetKey := migrationPromoteTargetKey(9) + raw, ok := targetKey(distribution.MigrationStagedDataKey(9, []byte("user|k"))) + require.True(t, ok) + require.Equal(t, []byte("user|k"), raw) + + _, ok = targetKey(distribution.MigrationStagedDataKey(10, []byte("user|k"))) + require.False(t, ok) +} + +func TestApplyMigrationPromoteMovesStagedVersions(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + hlc := NewHLC() + fsm := &kvFSM{store: st, hlc: hlc} + staged := distribution.MigrationStagedDataKey(9, []byte("user|k")) + require.NoError(t, st.PutAt(ctx, staged, []byte("v10"), 10, 0)) + require.NoError(t, st.DeleteAt(ctx, staged, 20)) + + cmd, err := MarshalMigrationPromoteCommand(&pb.PromoteStagedVersionsRequest{ + JobId: 9, + MaxVersions: 10, + }) + require.NoError(t, err) + applied := fsm.Apply(cmd) + result, ok := applied.(store.PromoteVersionsResult) + require.True(t, ok, "got %T: %v", applied, applied) + require.True(t, result.Done) + require.Equal(t, uint64(2), result.PromotedRows) + require.Equal(t, uint64(2), result.TotalPromotedRows) + require.Equal(t, uint64(20), result.MaxPromotedTS) + require.GreaterOrEqual(t, hlc.Current(), uint64(20)) + stateReader, ok := st.(store.MigrationPromotionStateReader) + require.True(t, ok) + state, ok, err := stateReader.MigrationPromotionState(ctx, 9) + require.NoError(t, err) + require.True(t, ok) + require.True(t, state.Done) + require.Equal(t, uint64(2), state.PromotedRows) + + got, err := st.GetAt(ctx, []byte("user|k"), 10) + require.NoError(t, err) + require.Equal(t, []byte("v10"), got) + _, err = st.GetAt(ctx, []byte("user|k"), 20) + require.ErrorIs(t, err, store.ErrKeyNotFound) + left, err := st.ExportVersions(ctx, store.ExportVersionsOptions{ + StartKey: distribution.MigrationStagedDataKeyPrefix(9), + EndKey: store.PrefixScanEnd(distribution.MigrationStagedDataKeyPrefix(9)), + MaxVersions: 10, + }) + require.NoError(t, err) + require.Empty(t, left.Versions) +} diff --git a/proto/internal.pb.go b/proto/internal.pb.go index a7a9e30ce..91bcff36b 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -931,6 +931,150 @@ func (x *ImportRangeVersionsResponse) GetAckedCursor() []byte { return nil } +type PromoteStagedVersionsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Cursor []byte `protobuf:"bytes,2,opt,name=cursor,proto3" json:"cursor,omitempty"` + MaxVersions uint32 `protobuf:"varint,3,opt,name=max_versions,json=maxVersions,proto3" json:"max_versions,omitempty"` + MaxBytes uint64 `protobuf:"varint,4,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + MaxScannedBytes uint64 `protobuf:"varint,5,opt,name=max_scanned_bytes,json=maxScannedBytes,proto3" json:"max_scanned_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PromoteStagedVersionsRequest) Reset() { + *x = PromoteStagedVersionsRequest{} + mi := &file_internal_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PromoteStagedVersionsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PromoteStagedVersionsRequest) ProtoMessage() {} + +func (x *PromoteStagedVersionsRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PromoteStagedVersionsRequest.ProtoReflect.Descriptor instead. +func (*PromoteStagedVersionsRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{12} +} + +func (x *PromoteStagedVersionsRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *PromoteStagedVersionsRequest) GetCursor() []byte { + if x != nil { + return x.Cursor + } + return nil +} + +func (x *PromoteStagedVersionsRequest) GetMaxVersions() uint32 { + if x != nil { + return x.MaxVersions + } + return 0 +} + +func (x *PromoteStagedVersionsRequest) GetMaxBytes() uint64 { + if x != nil { + return x.MaxBytes + } + return 0 +} + +func (x *PromoteStagedVersionsRequest) GetMaxScannedBytes() uint64 { + if x != nil { + return x.MaxScannedBytes + } + return 0 +} + +type PromoteStagedVersionsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + NextCursor []byte `protobuf:"bytes,1,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` + Done bool `protobuf:"varint,2,opt,name=done,proto3" json:"done,omitempty"` + PromotedRows uint64 `protobuf:"varint,3,opt,name=promoted_rows,json=promotedRows,proto3" json:"promoted_rows,omitempty"` + MaxPromotedTs uint64 `protobuf:"varint,4,opt,name=max_promoted_ts,json=maxPromotedTs,proto3" json:"max_promoted_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PromoteStagedVersionsResponse) Reset() { + *x = PromoteStagedVersionsResponse{} + mi := &file_internal_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PromoteStagedVersionsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PromoteStagedVersionsResponse) ProtoMessage() {} + +func (x *PromoteStagedVersionsResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PromoteStagedVersionsResponse.ProtoReflect.Descriptor instead. +func (*PromoteStagedVersionsResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{13} +} + +func (x *PromoteStagedVersionsResponse) GetNextCursor() []byte { + if x != nil { + return x.NextCursor + } + return nil +} + +func (x *PromoteStagedVersionsResponse) GetDone() bool { + if x != nil { + return x.Done + } + return false +} + +func (x *PromoteStagedVersionsResponse) GetPromotedRows() uint64 { + if x != nil { + return x.PromotedRows + } + return 0 +} + +func (x *PromoteStagedVersionsResponse) GetMaxPromotedTs() uint64 { + if x != nil { + return x.MaxPromotedTs + } + return 0 +} + var File_internal_proto protoreflect.FileDescriptor const file_internal_proto_rawDesc = "" + @@ -999,7 +1143,19 @@ const file_internal_proto_rawDesc = "" + "bracket_id\x18\x04 \x01(\x04R\tbracketId\x12\x1b\n" + "\tbatch_seq\x18\x05 \x01(\x04R\bbatchSeq\"@\n" + "\x1bImportRangeVersionsResponse\x12!\n" + - "\facked_cursor\x18\x01 \x01(\fR\vackedCursor*&\n" + + "\facked_cursor\x18\x01 \x01(\fR\vackedCursor\"\xb9\x01\n" + + "\x1cPromoteStagedVersionsRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x16\n" + + "\x06cursor\x18\x02 \x01(\fR\x06cursor\x12!\n" + + "\fmax_versions\x18\x03 \x01(\rR\vmaxVersions\x12\x1b\n" + + "\tmax_bytes\x18\x04 \x01(\x04R\bmaxBytes\x12*\n" + + "\x11max_scanned_bytes\x18\x05 \x01(\x04R\x0fmaxScannedBytes\"\xa1\x01\n" + + "\x1dPromoteStagedVersionsResponse\x12\x1f\n" + + "\vnext_cursor\x18\x01 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x02 \x01(\bR\x04done\x12#\n" + + "\rpromoted_rows\x18\x03 \x01(\x04R\fpromotedRows\x12&\n" + + "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs*&\n" + "\x02Op\x12\a\n" + "\x03PUT\x10\x00\x12\a\n" + "\x03DEL\x10\x01\x12\x0e\n" + @@ -1010,12 +1166,13 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\xa3\x02\n" + + "\x05ABORT\x10\x032\xfd\x02\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00\x12T\n" + "\x13ExportRangeVersions\x12\x1b.ExportRangeVersionsRequest\x1a\x1c.ExportRangeVersionsResponse\"\x000\x01\x12R\n" + - "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00\x12X\n" + + "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_internal_proto_rawDescOnce sync.Once @@ -1030,22 +1187,24 @@ func file_internal_proto_rawDescGZIP() []byte { } var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_internal_proto_goTypes = []any{ - (Op)(0), // 0: Op - (Phase)(0), // 1: Phase - (*Mutation)(nil), // 2: Mutation - (*Request)(nil), // 3: Request - (*RaftCommand)(nil), // 4: RaftCommand - (*ForwardRequest)(nil), // 5: ForwardRequest - (*ForwardResponse)(nil), // 6: ForwardResponse - (*RelayPublishRequest)(nil), // 7: RelayPublishRequest - (*RelayPublishResponse)(nil), // 8: RelayPublishResponse - (*ExportRangeVersionsRequest)(nil), // 9: ExportRangeVersionsRequest - (*ExportRangeVersionsResponse)(nil), // 10: ExportRangeVersionsResponse - (*MVCCVersion)(nil), // 11: MVCCVersion - (*ImportRangeVersionsRequest)(nil), // 12: ImportRangeVersionsRequest - (*ImportRangeVersionsResponse)(nil), // 13: ImportRangeVersionsResponse + (Op)(0), // 0: Op + (Phase)(0), // 1: Phase + (*Mutation)(nil), // 2: Mutation + (*Request)(nil), // 3: Request + (*RaftCommand)(nil), // 4: RaftCommand + (*ForwardRequest)(nil), // 5: ForwardRequest + (*ForwardResponse)(nil), // 6: ForwardResponse + (*RelayPublishRequest)(nil), // 7: RelayPublishRequest + (*RelayPublishResponse)(nil), // 8: RelayPublishResponse + (*ExportRangeVersionsRequest)(nil), // 9: ExportRangeVersionsRequest + (*ExportRangeVersionsResponse)(nil), // 10: ExportRangeVersionsResponse + (*MVCCVersion)(nil), // 11: MVCCVersion + (*ImportRangeVersionsRequest)(nil), // 12: ImportRangeVersionsRequest + (*ImportRangeVersionsResponse)(nil), // 13: ImportRangeVersionsResponse + (*PromoteStagedVersionsRequest)(nil), // 14: PromoteStagedVersionsRequest + (*PromoteStagedVersionsResponse)(nil), // 15: PromoteStagedVersionsResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op @@ -1059,12 +1218,14 @@ var file_internal_proto_depIdxs = []int32{ 7, // 8: Internal.RelayPublish:input_type -> RelayPublishRequest 9, // 9: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest 12, // 10: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest - 6, // 11: Internal.Forward:output_type -> ForwardResponse - 8, // 12: Internal.RelayPublish:output_type -> RelayPublishResponse - 10, // 13: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse - 13, // 14: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse - 11, // [11:15] is the sub-list for method output_type - 7, // [7:11] is the sub-list for method input_type + 14, // 11: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest + 6, // 12: Internal.Forward:output_type -> ForwardResponse + 8, // 13: Internal.RelayPublish:output_type -> RelayPublishResponse + 10, // 14: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse + 13, // 15: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse + 15, // 16: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse + 12, // [12:17] is the sub-list for method output_type + 7, // [7:12] is the sub-list for method input_type 7, // [7:7] is the sub-list for extension type_name 7, // [7:7] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name @@ -1081,7 +1242,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), NumEnums: 2, - NumMessages: 12, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index b60c83793..b45bf8fb5 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -9,6 +9,7 @@ service Internal { rpc RelayPublish(RelayPublishRequest) returns (RelayPublishResponse) {} rpc ExportRangeVersions(ExportRangeVersionsRequest) returns (stream ExportRangeVersionsResponse) {} rpc ImportRangeVersions(ImportRangeVersionsRequest) returns (ImportRangeVersionsResponse) {} + rpc PromoteStagedVersions(PromoteStagedVersionsRequest) returns (PromoteStagedVersionsResponse) {} } // internal.proto is node to node communication message in raft replication. @@ -128,3 +129,18 @@ message ImportRangeVersionsRequest { message ImportRangeVersionsResponse { bytes acked_cursor = 1; } + +message PromoteStagedVersionsRequest { + uint64 job_id = 1; + bytes cursor = 2; + uint32 max_versions = 3; + uint64 max_bytes = 4; + uint64 max_scanned_bytes = 5; +} + +message PromoteStagedVersionsResponse { + bytes next_cursor = 1; + bool done = 2; + uint64 promoted_rows = 3; + uint64 max_promoted_ts = 4; +} diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index 6a21b9eba..ba679ed80 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -19,10 +19,11 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Internal_Forward_FullMethodName = "/Internal/Forward" - Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" - Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" - Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" + Internal_Forward_FullMethodName = "/Internal/Forward" + Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" + Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" + Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" + Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" ) // InternalClient is the client API for Internal service. @@ -34,6 +35,7 @@ type InternalClient interface { RelayPublish(ctx context.Context, in *RelayPublishRequest, opts ...grpc.CallOption) (*RelayPublishResponse, error) ExportRangeVersions(ctx context.Context, in *ExportRangeVersionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ExportRangeVersionsResponse], error) ImportRangeVersions(ctx context.Context, in *ImportRangeVersionsRequest, opts ...grpc.CallOption) (*ImportRangeVersionsResponse, error) + PromoteStagedVersions(ctx context.Context, in *PromoteStagedVersionsRequest, opts ...grpc.CallOption) (*PromoteStagedVersionsResponse, error) } type internalClient struct { @@ -93,6 +95,16 @@ func (c *internalClient) ImportRangeVersions(ctx context.Context, in *ImportRang return out, nil } +func (c *internalClient) PromoteStagedVersions(ctx context.Context, in *PromoteStagedVersionsRequest, opts ...grpc.CallOption) (*PromoteStagedVersionsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(PromoteStagedVersionsResponse) + err := c.cc.Invoke(ctx, Internal_PromoteStagedVersions_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // InternalServer is the server API for Internal service. // All implementations must embed UnimplementedInternalServer // for forward compatibility. @@ -102,6 +114,7 @@ type InternalServer interface { RelayPublish(context.Context, *RelayPublishRequest) (*RelayPublishResponse, error) ExportRangeVersions(*ExportRangeVersionsRequest, grpc.ServerStreamingServer[ExportRangeVersionsResponse]) error ImportRangeVersions(context.Context, *ImportRangeVersionsRequest) (*ImportRangeVersionsResponse, error) + PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) mustEmbedUnimplementedInternalServer() } @@ -124,6 +137,9 @@ func (UnimplementedInternalServer) ExportRangeVersions(*ExportRangeVersionsReque func (UnimplementedInternalServer) ImportRangeVersions(context.Context, *ImportRangeVersionsRequest) (*ImportRangeVersionsResponse, error) { return nil, status.Error(codes.Unimplemented, "method ImportRangeVersions not implemented") } +func (UnimplementedInternalServer) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method PromoteStagedVersions not implemented") +} func (UnimplementedInternalServer) mustEmbedUnimplementedInternalServer() {} func (UnimplementedInternalServer) testEmbeddedByValue() {} @@ -210,6 +226,24 @@ func _Internal_ImportRangeVersions_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _Internal_PromoteStagedVersions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PromoteStagedVersionsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).PromoteStagedVersions(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_PromoteStagedVersions_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).PromoteStagedVersions(ctx, req.(*PromoteStagedVersionsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Internal_ServiceDesc is the grpc.ServiceDesc for Internal service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -229,6 +263,10 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "ImportRangeVersions", Handler: _Internal_ImportRangeVersions_Handler, }, + { + MethodName: "PromoteStagedVersions", + Handler: _Internal_PromoteStagedVersions_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 3ae343afd..e0796cbf6 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -10,6 +10,13 @@ import ( ) func (s *pebbleStore) ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + + return s.exportVersionsLocked(ctx, opts) +} + +func (s *pebbleStore) exportVersionsLocked(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { pos, err := decodeExportCursor(opts.Cursor) if err != nil { return ExportVersionsResult{}, err @@ -18,9 +25,6 @@ func (s *pebbleStore) ExportVersions(ctx context.Context, opts ExportVersionsOpt return ExportVersionsResult{Done: true}, nil } - s.dbMu.RLock() - defer s.dbMu.RUnlock() - iter, err := s.db.NewIter(pebbleExportIterOptions(opts)) if err != nil { return ExportVersionsResult{}, errors.WithStack(err) diff --git a/store/migration_promote.go b/store/migration_promote.go new file mode 100644 index 000000000..cde302a1a --- /dev/null +++ b/store/migration_promote.go @@ -0,0 +1,397 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +const migrationPromotionDoneFlag byte = 1 + +type promotedVersion struct { + staged MVCCVersion + target MVCCVersion +} + +func validatePromoteVersionsOptions(opts PromoteVersionsOptions) error { + if opts.TargetKey == nil { + return errors.New("migration promote target key mapper is required") + } + return nil +} + +func promotedVersionsFromStaged(opts PromoteVersionsOptions, versions []MVCCVersion) ([]promotedVersion, PromoteVersionsResult, error) { + out := make([]promotedVersion, 0, len(versions)) + result := PromoteVersionsResult{PromotedRows: uint64(len(versions))} //nolint:gosec // len is bounded by MaxVersions. + for _, staged := range versions { + targetKey, ok := opts.TargetKey(staged.Key) + if !ok { + return nil, PromoteVersionsResult{}, errors.WithStack(errors.Newf("migration promote target key rejected staged key %q", string(staged.Key))) + } + target := MVCCVersion{ + Key: bytes.Clone(targetKey), + CommitTS: staged.CommitTS, + Tombstone: staged.Tombstone, + Value: bytes.Clone(staged.Value), + KeyFamily: staged.KeyFamily, + ExpireAt: staged.ExpireAt, + } + if err := validateImportVersion(target); err != nil { + return nil, PromoteVersionsResult{}, err + } + result.PromotedBytes += versionExportSize(target.Key, len(target.Value)) + if target.CommitTS > result.MaxPromotedTS { + result.MaxPromotedTS = target.CommitTS + } + out = append(out, promotedVersion{ + staged: staged, + target: target, + }) + } + return out, result, nil +} + +func (s *mvccStore) PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) { + if err := validatePromoteVersionsOptions(opts); err != nil { + return PromoteVersionsResult{}, err + } + if opts.MaxVersions <= 0 { + return PromoteVersionsResult{Done: true}, nil + } + + s.mtx.Lock() + defer s.mtx.Unlock() + + state, cursor := s.promotionStateAndCursorLocked(opts) + if opts.JobID != 0 && state.Done { + return PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows}, nil + } + exported, toPromote, promoted, err := s.planMemoryPromotionLocked(ctx, opts, cursor) + if err != nil { + return PromoteVersionsResult{}, err + } + s.applyMemoryPromotionLocked(toPromote) + if promoted.MaxPromotedTS > s.lastCommitTS { + s.lastCommitTS = promoted.MaxPromotedTS + } + result, updatedState := finishPromotionResult(opts, state, exported, promoted) + if updatedState != nil { + s.migrationPromotions[opts.JobID] = clonePromotionState(*updatedState) + } + return result, nil +} + +func (s *mvccStore) planMemoryPromotionLocked( + ctx context.Context, + opts PromoteVersionsOptions, + cursor []byte, +) (ExportVersionsResult, []promotedVersion, PromoteVersionsResult, error) { + pos, err := decodeExportCursor(cursor) + if err != nil { + return ExportVersionsResult{}, nil, PromoteVersionsResult{}, err + } + opts.Cursor = cursor + exported, err := s.exportMemoryVersionsLocked(ctx, opts, pos) + if err != nil { + return ExportVersionsResult{}, nil, PromoteVersionsResult{}, err + } + toPromote, promoted, err := promotedVersionsFromStaged(opts, exported.Versions) + return exported, toPromote, promoted, err +} + +func (s *mvccStore) applyMemoryPromotionLocked(toPromote []promotedVersion) { + for _, version := range toPromote { + if version.target.Tombstone { + s.deleteVersionLocked(version.target.Key, version.target.CommitTS) + } else { + s.putVersionLocked(version.target.Key, version.target.Value, version.target.CommitTS, version.target.ExpireAt) + } + s.removeVersionLocked(version.staged.Key, version.staged.CommitTS) + } +} + +func (s *mvccStore) promotionStateAndCursorLocked(opts PromoteVersionsOptions) (PromotionState, []byte) { + if opts.JobID == 0 { + return PromotionState{}, opts.Cursor + } + state := clonePromotionState(s.migrationPromotions[opts.JobID]) + if len(state.Cursor) == 0 { + return state, opts.Cursor + } + return state, state.Cursor +} + +func (s *mvccStore) MigrationPromotionState(_ context.Context, jobID uint64) (PromotionState, bool, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + state, ok := s.migrationPromotions[jobID] + return clonePromotionState(state), ok, nil +} + +func (s *mvccStore) exportMemoryVersionsLocked(ctx context.Context, opts PromoteVersionsOptions, pos exportCursorPosition) (ExportVersionsResult, error) { + exportOpts := ExportVersionsOptions{ + StartKey: opts.StartKey, + EndKey: opts.EndKey, + Cursor: opts.Cursor, + MaxVersions: opts.MaxVersions, + MaxBytes: opts.MaxBytes, + MaxScannedBytes: opts.MaxScannedBytes, + } + result := newExportVersionsResult(opts.MaxVersions) + it := s.tree.Iterator() + if !s.seekMemoryExportStart(&it, opts.StartKey, pos.key) { + result.Done = true + return result, nil + } + for ok := true; ok; ok = it.Next() { + key, keyOK := it.Key().([]byte) + if err := checkExportKey(ctx, key, keyOK, opts.EndKey); err != nil { + if errors.Is(err, errExportReachedEnd) { + result.Done = true + result.NextCursor = nil + return result, nil + } + return ExportVersionsResult{}, err + } + if !keyOK { + continue + } + done, err := exportMemoryIteratorKey(ctx, exportOpts, pos, key, it.Value(), &result) + if err != nil || !done { + return result, err + } + } + result.Done = true + result.NextCursor = nil + return result, nil +} + +func (s *mvccStore) removeVersionLocked(key []byte, commitTS uint64) bool { + existing, ok := s.tree.Get(key) + if !ok { + return false + } + versions, _ := existing.([]VersionedValue) + idx := findVersionIndex(versions, commitTS) + if idx < 0 { + return false + } + next := make([]VersionedValue, len(versions)-1) + copy(next, versions[:idx]) + copy(next[idx:], versions[idx+1:]) + if len(next) == 0 { + s.tree.Remove(key) + return true + } + s.tree.Put(bytes.Clone(key), next) + return true +} + +func findVersionIndex(versions []VersionedValue, commitTS uint64) int { + for i := range versions { + if versions[i].TS == commitTS { + return i + } + } + return -1 +} + +func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) { + if err := validatePromoteVersionsOptions(opts); err != nil { + return PromoteVersionsResult{}, err + } + if opts.MaxVersions <= 0 { + return PromoteVersionsResult{Done: true}, nil + } + + s.dbMu.RLock() + defer s.dbMu.RUnlock() + + s.applyMu.Lock() + defer s.applyMu.Unlock() + + state, cursor, err := s.pebblePromotionStateAndCursor(opts) + if err != nil { + return PromoteVersionsResult{}, err + } + if opts.JobID != 0 && state.Done { + return PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows}, nil + } + opts.Cursor = cursor + exported, toPromote, promoted, err := s.planPebblePromotionLocked(ctx, opts) + if err != nil { + return PromoteVersionsResult{}, err + } + result, stateToWrite := finishPromotionResult(opts, state, exported, promoted) + return s.finishPebblePromotion(toPromote, opts.JobID, stateToWrite, result) +} + +func (s *pebbleStore) finishPebblePromotion( + toPromote []promotedVersion, + jobID uint64, + stateToWrite *PromotionState, + result PromoteVersionsResult, +) (PromoteVersionsResult, error) { + if len(toPromote) == 0 && stateToWrite == nil { + return result, nil + } + if err := s.commitPebblePromoteVersions(toPromote, jobID, stateToWrite); err != nil { + return PromoteVersionsResult{}, err + } + return result, nil +} + +func (s *pebbleStore) planPebblePromotionLocked( + ctx context.Context, + opts PromoteVersionsOptions, +) (ExportVersionsResult, []promotedVersion, PromoteVersionsResult, error) { + exported, err := s.exportVersionsLocked(ctx, ExportVersionsOptions{ + StartKey: opts.StartKey, + EndKey: opts.EndKey, + Cursor: opts.Cursor, + MaxVersions: opts.MaxVersions, + MaxBytes: opts.MaxBytes, + MaxScannedBytes: opts.MaxScannedBytes, + }) + if err != nil { + return ExportVersionsResult{}, nil, PromoteVersionsResult{}, err + } + toPromote, promoted, err := promotedVersionsFromStaged(opts, exported.Versions) + return exported, toPromote, promoted, err +} + +func finishPromotionResult( + opts PromoteVersionsOptions, + state PromotionState, + exported ExportVersionsResult, + promoted PromoteVersionsResult, +) (PromoteVersionsResult, *PromotionState) { + promoted.NextCursor = exported.NextCursor + promoted.Done = exported.Done + promoted.ScannedBytes = exported.ScannedBytes + promoted.TotalPromotedRows = promoted.PromotedRows + if opts.JobID == 0 { + return promoted, nil + } + state.Cursor = bytes.Clone(exported.NextCursor) + state.Done = exported.Done + state.PromotedRows += promoted.PromotedRows + state.LastError = "" + promoted.TotalPromotedRows = state.PromotedRows + return promoted, &state +} + +func (s *pebbleStore) pebblePromotionStateAndCursor(opts PromoteVersionsOptions) (PromotionState, []byte, error) { + if opts.JobID == 0 { + return PromotionState{}, opts.Cursor, nil + } + state, ok, err := s.readPebblePromotionState(opts.JobID) + if err != nil { + return PromotionState{}, nil, err + } + if !ok || len(state.Cursor) == 0 { + return state, opts.Cursor, nil + } + return state, state.Cursor, nil +} + +func (s *pebbleStore) MigrationPromotionState(_ context.Context, jobID uint64) (PromotionState, bool, error) { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + return s.readPebblePromotionState(jobID) +} + +func (s *pebbleStore) readPebblePromotionState(jobID uint64) (PromotionState, bool, error) { + val, closer, err := s.db.Get(migrationPromoteKey(jobID)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return PromotionState{}, false, nil + } + return PromotionState{}, false, errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + state, ok := decodePromotionState(val) + if !ok { + return PromotionState{}, false, errors.New("corrupt migration promotion state") + } + return state, true, nil +} + +func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jobID uint64, state *PromotionState) error { + batch := s.db.NewBatch() + defer batch.Close() + targets := make([]MVCCVersion, 0, len(versions)) + for _, version := range versions { + targets = append(targets, version.target) + } + if err := s.applyImportVersionsBatch(batch, targets); err != nil { + return err + } + for _, version := range versions { + if err := batch.Delete(encodeKey(version.staged.Key, version.staged.CommitTS), nil); err != nil { + return errors.WithStack(err) + } + } + if state != nil { + if err := batch.Set(migrationPromoteKey(jobID), encodePromotionState(*state), nil); err != nil { + return errors.WithStack(err) + } + } + if err := batch.Commit(s.directApplyWriteOpts()); err != nil { + return errors.WithStack(err) + } + return nil +} + +func encodePromotionState(state PromotionState) []byte { + buf := make([]byte, 0, 1+migrationUint64Bytes+binary.MaxVarintLen64*2+len(state.Cursor)+len(state.LastError)) + if state.Done { + buf = append(buf, migrationPromotionDoneFlag) + } else { + buf = append(buf, 0) + } + buf = binary.BigEndian.AppendUint64(buf, state.PromotedRows) + buf = binary.AppendUvarint(buf, lenAsUint64(len(state.Cursor))) + buf = append(buf, state.Cursor...) + buf = binary.AppendUvarint(buf, lenAsUint64(len(state.LastError))) + buf = append(buf, state.LastError...) + return buf +} + +func decodePromotionState(data []byte) (PromotionState, bool) { + if len(data) < 1+migrationUint64Bytes { + return PromotionState{}, false + } + state := PromotionState{ + Done: data[0]&migrationPromotionDoneFlag != 0, + PromotedRows: binary.BigEndian.Uint64(data[1 : 1+migrationUint64Bytes]), + } + rest := data[1+migrationUint64Bytes:] + cursorLen, n := binary.Uvarint(rest) + if n <= 0 || cursorLen > lenAsUint64(len(rest[n:])) { + return PromotionState{}, false + } + rest = rest[n:] + cursorEnd := int(cursorLen) //nolint:gosec // bounded by len(rest) above. + state.Cursor = bytes.Clone(rest[:cursorEnd]) + rest = rest[cursorEnd:] + errLen, n := binary.Uvarint(rest) + if n <= 0 || errLen != lenAsUint64(len(rest[n:])) { + return PromotionState{}, false + } + state.LastError = string(rest[n:]) + return state, true +} + +func clonePromotionState(state PromotionState) PromotionState { + state.Cursor = bytes.Clone(state.Cursor) + return state +} + +var _ MigrationPromoter = (*mvccStore)(nil) +var _ MigrationPromoter = (*pebbleStore)(nil) +var _ MigrationPromotionStateReader = (*mvccStore)(nil) +var _ MigrationPromotionStateReader = (*pebbleStore)(nil) diff --git a/store/migration_versions.go b/store/migration_versions.go index 54b7dd19b..28fd2f73d 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -15,6 +15,7 @@ const ( migrationAckPrefix = "!migstage|ack|" migrationHLCFloorPrefix = "!migstage|hlc_floor|" + migrationPromotePrefix = "!migstage|promote|" migrationUint64Bytes = 8 migrationAckKeyIDBytes = 2 * migrationUint64Bytes exportVersionSizeOverhead = 24 @@ -84,9 +85,17 @@ func migrationHLCFloorKey(jobID uint64) []byte { return key } +func migrationPromoteKey(jobID uint64) []byte { + key := make([]byte, len(migrationPromotePrefix)+migrationUint64Bytes) + copy(key, migrationPromotePrefix) + binary.BigEndian.PutUint64(key[len(migrationPromotePrefix):], jobID) + return key +} + func isMigrationMetadataKey(rawKey []byte) bool { return (len(rawKey) == len(migrationAckPrefix)+migrationAckKeyIDBytes && bytes.HasPrefix(rawKey, []byte(migrationAckPrefix))) || - (len(rawKey) == len(migrationHLCFloorPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationHLCFloorPrefix))) + (len(rawKey) == len(migrationHLCFloorPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationHLCFloorPrefix))) || + (len(rawKey) == len(migrationPromotePrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationPromotePrefix))) } func encodeMigrationImportAck(ack migrationImportAck) []byte { diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 77badeedd..91548bb00 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "os" "testing" @@ -181,6 +182,103 @@ func TestImportVersionsIdempotencyAndMetadata(t *testing.T) { }) } +func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + promoter, ok := st.(MigrationPromoter) + require.True(t, ok) + stateReader, ok := st.(MigrationPromotionStateReader) + require.True(t, ok) + + stage := func(raw string) []byte { + return append([]byte("stage|"), []byte(raw)...) + } + targetKey := func(staged []byte) ([]byte, bool) { + return bytes.TrimPrefix(staged, []byte("stage|")), bytes.HasPrefix(staged, []byte("stage|")) + } + prefix := []byte("stage|") + + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("old"), 5, 0)) + require.NoError(t, st.PutAt(ctx, stage("k"), []byte("v10"), 10, 0)) + require.NoError(t, st.PutWithTTLAt(ctx, stage("k"), []byte("v20"), 20, 55)) + require.NoError(t, st.DeleteAt(ctx, stage("k"), 30)) + require.NoError(t, st.PutAt(ctx, stage("z"), []byte("z15"), 15, 0)) + + first, err := promoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 99, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 2, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Equal(t, uint64(2), first.PromotedRows) + require.Equal(t, uint64(2), first.TotalPromotedRows) + require.Equal(t, uint64(30), first.MaxPromotedTS) + require.NotEmpty(t, first.NextCursor) + state, ok, err := stateReader.MigrationPromotionState(ctx, 99) + require.NoError(t, err) + require.True(t, ok) + require.False(t, state.Done) + require.Equal(t, first.NextCursor, state.Cursor) + require.Equal(t, uint64(2), state.PromotedRows) + + got, err := st.GetAt(ctx, []byte("k"), 25) + require.NoError(t, err) + require.Equal(t, []byte("v20"), got) + _, err = st.GetAt(ctx, []byte("k"), 35) + require.ErrorIs(t, err, ErrKeyNotFound) + + stagedLeft, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + }) + require.NoError(t, err) + require.Equal(t, []MVCCVersion{ + {Key: stage("k"), CommitTS: 10, Value: []byte("v10")}, + {Key: stage("z"), CommitTS: 15, Value: []byte("z15")}, + }, stagedLeft.Versions) + + second, err := promoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 99, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Empty(t, second.NextCursor) + require.Equal(t, uint64(2), second.PromotedRows) + require.Equal(t, uint64(4), second.TotalPromotedRows) + require.Equal(t, uint64(15), second.MaxPromotedTS) + state, ok, err = stateReader.MigrationPromotionState(ctx, 99) + require.NoError(t, err) + require.True(t, ok) + require.True(t, state.Done) + require.Empty(t, state.Cursor) + require.Equal(t, uint64(4), state.PromotedRows) + + got, err = st.GetAt(ctx, []byte("k"), 10) + require.NoError(t, err) + require.Equal(t, []byte("v10"), got) + got, err = st.GetAt(ctx, []byte("z"), 15) + require.NoError(t, err) + require.Equal(t, []byte("z15"), got) + + stagedLeft, err = st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, stagedLeft.Done) + require.Empty(t, stagedLeft.Versions) + }) +} + func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-import-persist-*") diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 50be08ba3..b20c35254 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -60,13 +60,14 @@ func byteSliceComparator(a, b any) int { // mvccStore is an in-memory MVCC implementation backed by a treemap for // deterministic iteration order and range scans. type mvccStore struct { - tree *treemap.Map // key []byte -> []VersionedValue - mtx sync.RWMutex - log *slog.Logger - lastCommitTS uint64 - minRetainedTS uint64 - migrationAcks map[string]migrationImportAck - migrationHLCFloors map[uint64]uint64 + tree *treemap.Map // key []byte -> []VersionedValue + mtx sync.RWMutex + log *slog.Logger + lastCommitTS uint64 + minRetainedTS uint64 + migrationAcks map[string]migrationImportAck + migrationHLCFloors map[uint64]uint64 + migrationPromotions map[uint64]PromotionState // writeConflicts mirrors the per-(kind, key_prefix) counter from // the pebble-backed store so the in-memory implementation shows up // in the same Prometheus series (even if the counts are usually @@ -113,9 +114,10 @@ func NewMVCCStore(opts ...MVCCStoreOption) MVCCStore { log: slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ Level: slog.LevelWarn, })), - migrationAcks: make(map[string]migrationImportAck), - migrationHLCFloors: make(map[uint64]uint64), - writeConflicts: newWriteConflictCounter(), + migrationAcks: make(map[string]migrationImportAck), + migrationHLCFloors: make(map[uint64]uint64), + migrationPromotions: make(map[uint64]PromotionState), + writeConflicts: newWriteConflictCounter(), } for _, opt := range opts { opt(s) diff --git a/store/store.go b/store/store.go index 78cb59354..a645a95c5 100644 --- a/store/store.go +++ b/store/store.go @@ -116,6 +116,49 @@ type ImportVersionsResult struct { Duplicate bool } +// PromoteVersionsOptions atomically copies staged MVCC versions to their +// target keys and physically removes the staged versions. +type PromoteVersionsOptions struct { + JobID uint64 + StartKey []byte + EndKey []byte + Cursor []byte + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + TargetKey func(stagedKey []byte) ([]byte, bool) +} + +// PromoteVersionsResult reports one resumable staged-version promotion chunk. +type PromoteVersionsResult struct { + NextCursor []byte + Done bool + PromotedRows uint64 + TotalPromotedRows uint64 + PromotedBytes uint64 + MaxPromotedTS uint64 + ScannedBytes uint64 +} + +// PromotionState is the target-local durable cursor for staged data promotion. +type PromotionState struct { + Cursor []byte + Done bool + PromotedRows uint64 + LastError string +} + +// MigrationPromoter is implemented by stores that can promote staged range +// migration data into the live keyspace. +type MigrationPromoter interface { + PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) +} + +// MigrationPromotionStateReader reads target-local staged promotion state. +type MigrationPromotionStateReader interface { + MigrationPromotionState(ctx context.Context, jobID uint64) (PromotionState, bool, error) +} + // OpType describes a mutation kind. type OpType int From 2edefb8553ef1c406e0e94406bb6262f0110d8da Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 01:11:25 +0900 Subject: [PATCH 014/162] migration: complete target promotion catalog state --- distribution/migration_promotion_complete.go | 230 ++++++++++++++++++ .../migration_promotion_complete_test.go | 186 ++++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 distribution/migration_promotion_complete.go create mode 100644 distribution/migration_promotion_complete_test.go diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go new file mode 100644 index 000000000..23473d6e5 --- /dev/null +++ b/distribution/migration_promotion_complete.go @@ -0,0 +1,230 @@ +package distribution + +import ( + "bytes" + "context" + "math" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var ( + ErrMigrationPromotionNotReady = errors.New("migration target promotion is not ready") + ErrMigrationPromotionTargetAbsent = errors.New("migration target promotion route is missing") +) + +// TargetPromotionCompletion is the result of applying the default-group +// promotion-complete transition to a SplitJob and its catalog routes. +type TargetPromotionCompletion struct { + Job SplitJob + Routes []RouteDescriptor + Changed bool + ClearedRouteIDs []uint64 +} + +// CompleteTargetPromotionState clears the target route's staged visibility +// fields after target-local promotion has completed. It deliberately preserves +// MinWriteTSExclusive; the timestamp floor remains a durable route invariant +// after the staged/live merge is no longer needed. +func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs int64) (TargetPromotionCompletion, error) { + normalized, err := normalizePromotionCompletionInput(job, routes) + if err != nil { + return TargetPromotionCompletion{}, err + } + + out := TargetPromotionCompletion{ + Job: CloneSplitJob(job), + Routes: normalized, + } + if out.Job.TargetPromotionDone { + if targetClearedDescriptorPresent(out.Job, out.Routes) { + return out, nil + } + return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + + cleared, err := clearTargetPromotionRoutes(job, out.Routes) + if err != nil { + return TargetPromotionCompletion{}, err + } + if len(cleared) == 0 { + return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + + out.Changed = true + out.ClearedRouteIDs = cleared + out.Job.TargetPromotionDone = true + out.Job.UpdatedAtMs = nowMs + return out, nil +} + +func normalizePromotionCompletionInput(job SplitJob, routes []RouteDescriptor) ([]RouteDescriptor, error) { + if err := validateSplitJob(job); err != nil { + return nil, err + } + if job.Phase != SplitJobPhaseCleanup { + return nil, errors.WithStack(ErrMigrationPromotionNotReady) + } + return normalizeRoutes(routes) +} + +func clearTargetPromotionRoutes(job SplitJob, routes []RouteDescriptor) ([]uint64, error) { + cleared := make([]uint64, 0, 1) + for i := range routes { + route := &routes[i] + if route.MigrationJobID != job.JobID { + continue + } + if !route.StagedVisibilityActive || + route.GroupID != job.TargetGroupID || + route.ParentRouteID != job.SourceRouteID || + !bytes.Equal(route.Start, job.SplitKey) { + return nil, errors.WithStack(ErrMigrationInvalidRoute) + } + route.StagedVisibilityActive = false + route.MigrationJobID = 0 + cleared = append(cleared, route.RouteID) + } + return cleared, nil +} + +// CompleteSplitJobTargetPromotion applies the promotion-complete catalog CAS: +// route descriptor staged fields are cleared, catalog version is bumped, and +// the SplitJob witness is updated in the same MVCC batch. +func (s *CatalogStore) CompleteSplitJobTargetPromotion( + ctx context.Context, + expectedVersion uint64, + expected SplitJob, + nowMs int64, +) (CatalogSnapshot, SplitJob, error) { + if err := ensureCatalogStore(s); err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + ctx = contextOrBackground(ctx) + + readTS, currentVersion, routes, err := s.loadPromotionCompleteInputs(ctx, expectedVersion, expected) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + completion, err := CompleteTargetPromotionState(expected, routes, nowMs) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if !completion.Changed { + return CatalogSnapshot{ + Version: currentVersion, + Routes: cloneRouteDescriptors(completion.Routes), + ReadTS: readTS, + }, completion.Job, nil + } + + plan, mutations, commitTS, err := s.buildPromotionCompleteMutations(ctx, readTS, expectedVersion, expected.JobID, &completion) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if err := s.applyPromotionCompleteMutations(ctx, plan, mutations, expected.JobID, commitTS); err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + + return CatalogSnapshot{ + Version: plan.nextVersion, + Routes: cloneRouteDescriptors(completion.Routes), + }, completion.Job, nil +} + +func (s *CatalogStore) loadPromotionCompleteInputs(ctx context.Context, expectedVersion uint64, expected SplitJob) (uint64, uint64, []RouteDescriptor, error) { + expectedRaw, err := EncodeSplitJob(expected) + if err != nil { + return 0, 0, nil, err + } + readTS := s.store.LastCommitTS() + currentVersion, err := s.versionAt(ctx, readTS) + if err != nil { + return 0, 0, nil, err + } + if currentVersion != expectedVersion { + return 0, 0, nil, errors.WithStack(ErrCatalogVersionMismatch) + } + if err := s.expectLiveSplitJobAt(ctx, expected.JobID, expectedRaw, readTS); err != nil { + return 0, 0, nil, err + } + routes, err := s.routesAt(ctx, readTS) + if err != nil { + return 0, 0, nil, err + } + return readTS, currentVersion, routes, nil +} + +func (s *CatalogStore) buildPromotionCompleteMutations( + ctx context.Context, + readTS uint64, + expectedVersion uint64, + jobID uint64, + completion *TargetPromotionCompletion, +) (savePlan, []*store.KVPairMutation, uint64, error) { + if expectedVersion == math.MaxUint64 { + return savePlan{}, nil, 0, errors.WithStack(ErrCatalogVersionOverflow) + } + minCommitTS := readTS + 1 + if minCommitTS == 0 { + return savePlan{}, nil, 0, errors.WithStack(ErrCatalogVersionOverflow) + } + + plan := savePlan{ + readTS: readTS, + minCommitTS: minCommitTS, + nextVersion: expectedVersion + 1, + routes: completion.Routes, + } + mutations, err := s.buildSaveMutations(ctx, plan) + if err != nil { + return savePlan{}, nil, 0, err + } + commitTS, err := s.commitTSForApply(plan.minCommitTS) + if err != nil { + return savePlan{}, nil, 0, err + } + completion.Job.PromotionCompletedTS = commitTS + encodedJob, err := EncodeSplitJob(completion.Job) + if err != nil { + return savePlan{}, nil, 0, err + } + jobMutations, err := s.buildSplitJobPutMutations(ctx, readTS, CatalogSplitJobKey(jobID), encodedJob, jobID) + if err != nil { + return savePlan{}, nil, 0, err + } + return plan, append(mutations, jobMutations...), commitTS, nil +} + +func (s *CatalogStore) applyPromotionCompleteMutations(ctx context.Context, plan savePlan, mutations []*store.KVPairMutation, jobID uint64, commitTS uint64) error { + readKeys := [][]byte{ + CatalogVersionKey(), + CatalogSplitJobKey(jobID), + } + if err := s.store.ApplyMutations(ctx, mutations, readKeys, plan.readTS, commitTS); err != nil { + if errors.Is(err, store.ErrWriteConflict) { + return errors.WithStack(ErrCatalogSplitJobConflict) + } + return errors.WithStack(err) + } + return nil +} + +func targetClearedDescriptorPresent(job SplitJob, routes []RouteDescriptor) bool { + for _, route := range routes { + if route.GroupID != job.TargetGroupID { + continue + } + if route.StagedVisibilityActive || route.MigrationJobID != 0 { + continue + } + if route.ParentRouteID != job.SourceRouteID { + continue + } + if bytes.Equal(route.Start, job.SplitKey) { + return true + } + } + return false +} diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go new file mode 100644 index 000000000..f10870a51 --- /dev/null +++ b/distribution/migration_promotion_complete_test.go @@ -0,0 +1,186 @@ +package distribution + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestCompleteTargetPromotionStateClearsStagedFieldsAndRetainsFloor(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + routes := promotionCompleteTestRoutes() + + result, err := CompleteTargetPromotionState(job, routes, 1000) + require.NoError(t, err) + require.True(t, result.Changed) + require.Equal(t, []uint64{3}, result.ClearedRouteIDs) + require.True(t, result.Job.TargetPromotionDone) + require.Zero(t, result.Job.PromotionCompletedTS) + require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) + + target := routeByID(t, result.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) + require.Equal(t, uint64(42), target.ParentRouteID) + + badRoutes := promotionCompleteTestRoutes() + badRoutes[1].ParentRouteID = 777 + _, err = CompleteTargetPromotionState(job, badRoutes, 1000) + require.ErrorIs(t, err, ErrMigrationInvalidRoute) +} + +func TestCompleteTargetPromotionStateAcceptsAlreadyClearedDescriptor(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.PromotionCompletedTS = 900 + job.UpdatedAtMs = 1000 + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + + result, err := CompleteTargetPromotionState(job, routes, 1100) + require.NoError(t, err) + require.False(t, result.Changed) + require.Equal(t, uint64(900), result.Job.PromotionCompletedTS) + require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + + target := routeByID(t, result.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionCommitsRouteAndJobTogether(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore()) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + before, err := cs.Snapshot(ctx) + require.NoError(t, err) + + snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.True(t, completed.TargetPromotionDone) + require.Greater(t, completed.PromotionCompletedTS, before.ReadTS) + require.Equal(t, int64(1000), completed.UpdatedAtMs) + + loadedJob, found, err := cs.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + assertSplitJobEqual(t, completed, loadedJob) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version+1, loaded.Version) + target := routeByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDescriptor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore()) + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := cs.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.PromotionCompletedTS = 900 + job.UpdatedAtMs = 1000 + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1100) + require.NoError(t, err) + require.Equal(t, saved.Version, snapshot.Version) + assertSplitJobEqual(t, job, completed) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version, loaded.Version) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionRejectsStaleInputs(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore()) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + _, _, err = cs.CompleteSplitJobTargetPromotion(ctx, saved.Version+1, job, 1000) + require.True(t, errors.Is(err, ErrCatalogVersionMismatch), "got %v", err) + + advanced := job + advanced.Cursor = []byte("advanced") + advanced.UpdatedAtMs++ + require.NoError(t, cs.SaveSplitJob(ctx, job, advanced)) + + _, _, err = cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.True(t, errors.Is(err, ErrCatalogSplitJobConflict), "got %v", err) +} + +func promotionCompleteTestJob() SplitJob { + return SplitJob{ + JobID: 99, + SourceRouteID: 42, + SplitKey: []byte("m"), + TargetGroupID: 2, + Phase: SplitJobPhaseCleanup, + } +} + +func promotionCompleteTestRoutes() []RouteDescriptor { + return []RouteDescriptor{ + { + RouteID: 2, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: RouteStateActive, + ParentRouteID: 42, + }, + { + RouteID: 3, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: RouteStateActive, + ParentRouteID: 42, + StagedVisibilityActive: true, + MigrationJobID: 99, + MinWriteTSExclusive: 777, + }, + } +} + +func routeByID(t *testing.T, routes []RouteDescriptor, routeID uint64) RouteDescriptor { + t.Helper() + for _, route := range routes { + if route.RouteID == routeID { + return route + } + } + t.Fatalf("route %d not found", routeID) + return RouteDescriptor{} +} From 2ea0d381f212cd9dabe51080f87e155ea119aa7f Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 01:27:31 +0900 Subject: [PATCH 015/162] migration: add target readiness guard --- kv/shard_store.go | 139 ++++++++++++++++-- kv/shard_store_test.go | 103 +++++++++++++ store/lsm_store.go | 37 +++-- store/migration_readiness.go | 238 +++++++++++++++++++++++++++++++ store/migration_versions.go | 11 +- store/migration_versions_test.go | 79 ++++++++++ store/mvcc_store.go | 19 +-- store/store.go | 24 ++++ 8 files changed, 621 insertions(+), 29 deletions(-) create mode 100644 store/migration_readiness.go diff --git a/kv/shard_store.go b/kv/shard_store.go index ba5ef6ac9..aaf18b57c 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -26,6 +26,8 @@ type ShardStore struct { } var ErrCrossShardMutationBatchNotSupported = errors.New("cross-shard mutation batches are not supported") +var ErrRouteCutoverPending = errors.New("route cutover pending") +var ErrRouteWriteBelowFloor = errors.New("route write timestamp is below route floor") // NewShardStore creates a sharded MVCC store wrapper. func NewShardStore(engine *distribution.Engine, groups map[uint64]*ShardGroup) *ShardStore { @@ -98,6 +100,9 @@ func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distr } func (s *ShardStore) localGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return nil, err + } if routeHasStagedVisibility(route) { return s.getAtWithStagedVisibility(ctx, g, route, key, ts) } @@ -112,6 +117,59 @@ func routeHasStagedVisibility(route distribution.Route) bool { return route.StagedVisibilityActive && route.MigrationJobID != 0 } +func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetStagedReadinessState) bool { + if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + return false + } + if route.MinWriteTSExclusive < ready.MinWriteTSExclusive { + return false + } + if route.StagedVisibilityActive { + return route.MigrationJobID == ready.MigrationJobID + } + return route.MigrationJobID == 0 +} + +func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { + if g == nil || g.Store == nil { + return nil + } + reader, ok := g.Store.(store.MigrationTargetReadinessReader) + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + for _, ready := range states { + if !ready.Armed || !routeRangeIntersects(start, end, ready.RouteStart, ready.RouteEnd) { + continue + } + if !routeSatisfiesTargetReadiness(route, ready) { + return errors.WithStack(ErrRouteCutoverPending) + } + } + return nil +} + +func verifyRouteWriteFloor(route distribution.Route, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d", commitTS, route.MinWriteTSExclusive) +} + +func routeRangeIntersects(aStart, aEnd, bStart, bEnd []byte) bool { + if aEnd != nil && bytes.Compare(aEnd, bStart) <= 0 { + return false + } + if bEnd != nil && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true +} + func (s *ShardStore) getAtWithStagedVisibility(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { live, liveOK, err := latestMVCCVersionAt(ctx, g.Store, key, ts) if err != nil { @@ -574,6 +632,9 @@ func (s *ShardStore) scanRouteLocal( ts uint64, reverse bool, ) ([]*store.KVPair, error) { + if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + return nil, err + } if routeHasStagedVisibility(route) { return s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) } @@ -618,6 +679,9 @@ func (s *ShardStore) scanRouteAtLeader( ts uint64, reverse bool, ) ([]*store.KVPair, error) { + if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + return nil, err + } var ( kvs []*store.KVPair err error @@ -905,34 +969,58 @@ func clampScanEnd(end []byte, routeEnd []byte) []byte { } func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { - g, ok := s.groupForKey(key) - if !ok || g.Store == nil { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + if err := verifyRouteWriteFloor(route, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.PutAt(ctx, key, value, commitTS, expireAt)) } func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) error { - g, ok := s.groupForKey(key) - if !ok || g.Store == nil { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + if err := verifyRouteWriteFloor(route, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.DeleteAt(ctx, key, commitTS)) } func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { - g, ok := s.groupForKey(key) - if !ok || g.Store == nil { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + if err := verifyRouteWriteFloor(route, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.PutWithTTLAt(ctx, key, value, commitTS, expireAt)) } func (s *ShardStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, commitTS uint64) error { - g, ok := s.groupForKey(key) - if !ok || g.Store == nil { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + if err := verifyRouteWriteFloor(route, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.ExpireAt(ctx, key, expireAt, commitTS)) } @@ -967,6 +1055,9 @@ func (s *ShardStore) LatestCommitTS(ctx context.Context, key []byte) (uint64, bo } func (s *ShardStore) localLatestCommitTS(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte) (uint64, bool, error) { + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return 0, false, err + } liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) if err != nil { return 0, false, errors.WithStack(err) @@ -1606,9 +1697,41 @@ func (s *ShardStore) ApplyMutations(ctx context.Context, mutations []*store.KVPa if err != nil || group == nil { return err } + if err := s.verifyMutationRoutes(ctx, mutations, readKeys, commitTS); err != nil { + return err + } return errors.WithStack(group.Store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS)) } +func (s *ShardStore) verifyMutationRoutes(ctx context.Context, mutations []*store.KVPairMutation, readKeys [][]byte, commitTS uint64) error { + for _, mut := range mutations { + if err := s.verifyMutationWriteRoute(ctx, mut.Key, commitTS); err != nil { + return err + } + } + for _, key := range readKeys { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + } + return nil +} + +func (s *ShardStore) verifyMutationWriteRoute(ctx context.Context, key []byte, commitTS uint64) error { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + return verifyRouteWriteFloor(route, commitTS) +} + // ApplyMutationsRaft is the raft-apply variant; see store.MVCCStore for the // durability contract. Only the FSM may call this method. func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store.KVPairMutation, readKeys [][]byte, startTS, commitTS uint64) error { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 100b70e27..cfe9aec59 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -33,6 +33,32 @@ func newStagedVisibilityShardStore(t *testing.T) (*ShardStore, *ShardGroup) { return NewShardStore(engine, map[uint64]*ShardGroup{1: group}), group } +func applyTargetReadiness(t *testing.T, group *ShardGroup, floor uint64) { + t.Helper() + writer, ok := group.Store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: floor, + Armed: true, + })) +} + +func newReadinessShardStore(t *testing.T, route distribution.RouteDescriptor) (*ShardStore, *ShardGroup) { + t.Helper() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{route}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + return NewShardStore(engine, map[uint64]*ShardGroup{route.GroupID: group}), group +} + func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { t.Parallel() @@ -57,6 +83,83 @@ func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) } +func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group, 100) + + _, err := st.GetAt(ctx, []byte("k"), 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + err = st.PutAt(ctx, []byte("k"), []byte("v"), 120, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessAcceptsStagedDescriptor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + applyTargetReadiness(t, group, 100) + rawKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 120, 0)) + + got, err := st.GetAt(ctx, rawKey, 130) + require.NoError(t, err) + require.Equal(t, []byte("staged"), got) +} + +func TestShardStoreTargetReadinessAcceptsClearedDescriptorAndRetainsFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + applyTargetReadiness(t, group, 100) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 120, 0)) + + got, err := st.GetAt(ctx, []byte("k"), 130) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) + + err = st.PutAt(ctx, []byte("k"), []byte("low"), 100, 0) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.ExpireAt(ctx, []byte("k"), 200, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.ApplyMutations(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("low"), + }}, nil, 0, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0) + require.NoError(t, err) + + err = st.ApplyMutations(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("ok"), + }}, nil, 0, 101) + require.NoError(t, err) +} + func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { t.Parallel() diff --git a/store/lsm_store.go b/store/lsm_store.go index 18913cddc..a1a2ac350 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -182,19 +182,20 @@ var metaAppliedIndexBytes = []byte(metaAppliedIndex) // (PR #915 round-3) so the snapshot-persist checkpoint cannot rewind // metaAppliedIndex below a concurrent per-Apply value. // 4. mtx – guards the in-memory metadata fields -// (lastCommitTS, minRetainedTS, pendingMinRetainedTS). +// (lastCommitTS, minRetainedTS, pendingMinRetainedTS, migrationReadinessCache). type pebbleStore struct { - db *pebble.DB - cache *pebble.Cache // owns one ref; Unref on Close / reopen. - dbMu sync.RWMutex // guards s.db pointer – see lock ordering above - log *slog.Logger - lastCommitTS uint64 - minRetainedTS uint64 - pendingMinRetainedTS uint64 - mtx sync.RWMutex - applyMu sync.Mutex // serializes ApplyMutations: conflict check → commit - maintenanceMu sync.Mutex - dir string + db *pebble.DB + cache *pebble.Cache // owns one ref; Unref on Close / reopen. + dbMu sync.RWMutex // guards s.db pointer – see lock ordering above + log *slog.Logger + lastCommitTS uint64 + minRetainedTS uint64 + pendingMinRetainedTS uint64 + migrationReadinessCache []TargetStagedReadinessState + mtx sync.RWMutex + applyMu sync.Mutex // serializes ApplyMutations: conflict check → commit + maintenanceMu sync.Mutex + dir string // writeConflicts tracks per-(kind, key_prefix) OCC conflict counts // detected inside ApplyMutations. Polled by the monitoring // WriteConflictCollector; not part of the authoritative OCC path. @@ -340,9 +341,15 @@ func NewPebbleStore(dir string, opts ...PebbleStoreOption) (MVCCStore, error) { cleanupOnInitFail() return nil, err } + readinessStates, err := s.loadPebbleTargetReadinessStatesFromDB() + if err != nil { + cleanupOnInitFail() + return nil, err + } s.lastCommitTS = maxTS s.minRetainedTS = minRetainedTS s.pendingMinRetainedTS = pendingMinRetainedTS + s.migrationReadinessCache = readinessStates return s, nil } @@ -2583,9 +2590,15 @@ func (s *pebbleStore) swapInTempDB(tmpDir string) error { cleanupOnSwapFail() return err } + readinessStates, err := s.loadPebbleTargetReadinessStatesFromDB() + if err != nil { + cleanupOnSwapFail() + return err + } s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS s.pendingMinRetainedTS = pendingMinRetainedTS + s.migrationReadinessCache = readinessStates return nil } diff --git a/store/migration_readiness.go b/store/migration_readiness.go new file mode 100644 index 000000000..7aa160ca3 --- /dev/null +++ b/store/migration_readiness.go @@ -0,0 +1,238 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + "sort" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +const ( + targetReadinessCodecVersion byte = 1 + targetReadinessArmedFlag byte = 1 +) + +func validateTargetStagedReadinessState(state TargetStagedReadinessState) error { + switch { + case state.JobID == 0: + return errors.New("target staged readiness job_id is required") + case state.MigrationJobID == 0: + return errors.New("target staged readiness migration_job_id is required") + case state.RouteEnd != nil && bytes.Compare(state.RouteStart, state.RouteEnd) >= 0: + return errors.New("target staged readiness route range is invalid") + default: + return nil + } +} + +func (s *mvccStore) ApplyTargetStagedReadiness(_ context.Context, state TargetStagedReadinessState) error { + if err := validateTargetStagedReadinessState(state); err != nil { + return err + } + s.mtx.Lock() + defer s.mtx.Unlock() + cloned := cloneTargetStagedReadinessState(state) + s.migrationReadiness[state.JobID] = cloned + s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, cloned) + return nil +} + +func (s *mvccStore) MigrationTargetReadinessStates(_ context.Context) ([]TargetStagedReadinessState, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return cloneTargetStagedReadinessStates(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) ApplyTargetStagedReadiness(_ context.Context, state TargetStagedReadinessState) error { + if err := validateTargetStagedReadinessState(state); err != nil { + return err + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + if err := s.db.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), s.directApplyWriteOpts()); err != nil { + return errors.WithStack(err) + } + s.mtx.Lock() + defer s.mtx.Unlock() + s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, state) + return nil +} + +func (s *pebbleStore) MigrationTargetReadinessStates(_ context.Context) ([]TargetStagedReadinessState, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return cloneTargetStagedReadinessStates(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) loadPebbleTargetReadinessStatesFromDB() ([]TargetStagedReadinessState, error) { + iter, err := s.db.NewIter(&pebble.IterOptions{ + LowerBound: []byte(migrationReadyPrefix), + UpperBound: PrefixScanEnd([]byte(migrationReadyPrefix)), + }) + if err != nil { + return nil, errors.WithStack(err) + } + defer iter.Close() + + var out []TargetStagedReadinessState + for valid := iter.First(); valid; valid = iter.Next() { + state, ok := decodeTargetStagedReadinessState(iter.Value()) + if !ok { + return nil, errors.New("corrupt target staged readiness state") + } + out = append(out, state) + } + if err := iter.Error(); err != nil { + return nil, errors.WithStack(err) + } + sortTargetStagedReadinessStates(out) + return out, nil +} + +func encodeTargetStagedReadinessState(state TargetStagedReadinessState) []byte { + buf := make([]byte, 0, targetReadinessEncodedSize(state)) + buf = append(buf, targetReadinessCodecVersion) + if state.Armed { + buf = append(buf, targetReadinessArmedFlag) + } else { + buf = append(buf, 0) + } + buf = binary.BigEndian.AppendUint64(buf, state.JobID) + buf = binary.BigEndian.AppendUint64(buf, state.ExpectedCutoverVersion) + buf = binary.BigEndian.AppendUint64(buf, state.MigrationJobID) + buf = binary.BigEndian.AppendUint64(buf, state.MinWriteTSExclusive) + buf = appendVarBytes(buf, state.RouteStart) + if state.RouteEnd == nil { + buf = append(buf, 0) + return buf + } + buf = append(buf, 1) + return appendVarBytes(buf, state.RouteEnd) +} + +func decodeTargetStagedReadinessState(data []byte) (TargetStagedReadinessState, bool) { + state, rest, ok := decodeTargetReadinessHeader(data) + if !ok { + return TargetStagedReadinessState{}, false + } + if !decodeTargetReadinessRange(&state, rest) { + return TargetStagedReadinessState{}, false + } + if err := validateTargetStagedReadinessState(state); err != nil { + return TargetStagedReadinessState{}, false + } + return state, true +} + +func decodeTargetReadinessHeader(data []byte) (TargetStagedReadinessState, []byte, bool) { + if len(data) < 2+4*migrationUint64Bytes || data[0] != targetReadinessCodecVersion { + return TargetStagedReadinessState{}, nil, false + } + if data[1] != 0 && data[1] != targetReadinessArmedFlag { + return TargetStagedReadinessState{}, nil, false + } + state := TargetStagedReadinessState{Armed: data[1] == targetReadinessArmedFlag} + rest := data[2:] + state.JobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.ExpectedCutoverVersion = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MigrationJobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinWriteTSExclusive = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + return state, rest, true +} + +func decodeTargetReadinessRange(state *TargetStagedReadinessState, data []byte) bool { + routeStart, rest, ok := readVarBytes(data) + if !ok || len(rest) == 0 { + return false + } + state.RouteStart = routeStart + endPresent := rest[0] + rest = rest[1:] + if endPresent == 0 { + return len(rest) == 0 + } + if endPresent != 1 { + return false + } + state.RouteEnd, rest, ok = readVarBytes(rest) + return ok && len(rest) == 0 +} + +func targetReadinessEncodedSize(state TargetStagedReadinessState) int { + size := 2 + 4*migrationUint64Bytes + binary.MaxVarintLen64 + len(state.RouteStart) + 1 + if state.RouteEnd != nil { + size += binary.MaxVarintLen64 + len(state.RouteEnd) + } + return size +} + +func appendVarBytes(dst []byte, value []byte) []byte { + dst = binary.AppendUvarint(dst, lenAsUint64(len(value))) + return append(dst, value...) +} + +func readVarBytes(data []byte) ([]byte, []byte, bool) { + l, n := binary.Uvarint(data) + if n <= 0 { + return nil, nil, false + } + rest := data[n:] + if l > lenAsUint64(len(rest)) { + return nil, nil, false + } + return bytes.Clone(rest[:l]), rest[l:], true +} + +func cloneTargetStagedReadinessState(state TargetStagedReadinessState) TargetStagedReadinessState { + return TargetStagedReadinessState{ + JobID: state.JobID, + RouteStart: bytes.Clone(state.RouteStart), + RouteEnd: bytes.Clone(state.RouteEnd), + ExpectedCutoverVersion: state.ExpectedCutoverVersion, + MigrationJobID: state.MigrationJobID, + MinWriteTSExclusive: state.MinWriteTSExclusive, + Armed: state.Armed, + } +} + +func cloneTargetStagedReadinessStates(states []TargetStagedReadinessState) []TargetStagedReadinessState { + if len(states) == 0 { + return nil + } + out := make([]TargetStagedReadinessState, 0, len(states)) + for _, state := range states { + out = append(out, cloneTargetStagedReadinessState(state)) + } + return out +} + +func upsertTargetStagedReadinessStateCache(cache []TargetStagedReadinessState, state TargetStagedReadinessState) []TargetStagedReadinessState { + cloned := cloneTargetStagedReadinessState(state) + for i := range cache { + if cache[i].JobID == cloned.JobID { + cache[i] = cloned + return cache + } + } + cache = append(cache, cloned) + sortTargetStagedReadinessStates(cache) + return cache +} + +func sortTargetStagedReadinessStates(states []TargetStagedReadinessState) { + sort.Slice(states, func(i, j int) bool { + return states[i].JobID < states[j].JobID + }) +} + +var _ MigrationTargetReadinessReader = (*mvccStore)(nil) +var _ MigrationTargetReadinessReader = (*pebbleStore)(nil) +var _ MigrationTargetReadinessWriter = (*mvccStore)(nil) +var _ MigrationTargetReadinessWriter = (*pebbleStore)(nil) diff --git a/store/migration_versions.go b/store/migration_versions.go index 28fd2f73d..b1e275ecb 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -16,6 +16,7 @@ const ( migrationAckPrefix = "!migstage|ack|" migrationHLCFloorPrefix = "!migstage|hlc_floor|" migrationPromotePrefix = "!migstage|promote|" + migrationReadyPrefix = "!migstage|ready|" migrationUint64Bytes = 8 migrationAckKeyIDBytes = 2 * migrationUint64Bytes exportVersionSizeOverhead = 24 @@ -92,10 +93,18 @@ func migrationPromoteKey(jobID uint64) []byte { return key } +func migrationReadyKey(jobID uint64) []byte { + key := make([]byte, len(migrationReadyPrefix)+migrationUint64Bytes) + copy(key, migrationReadyPrefix) + binary.BigEndian.PutUint64(key[len(migrationReadyPrefix):], jobID) + return key +} + func isMigrationMetadataKey(rawKey []byte) bool { return (len(rawKey) == len(migrationAckPrefix)+migrationAckKeyIDBytes && bytes.HasPrefix(rawKey, []byte(migrationAckPrefix))) || (len(rawKey) == len(migrationHLCFloorPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationHLCFloorPrefix))) || - (len(rawKey) == len(migrationPromotePrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationPromotePrefix))) + (len(rawKey) == len(migrationPromotePrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationPromotePrefix))) || + (len(rawKey) == len(migrationReadyPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationReadyPrefix))) } func encodeMigrationImportAck(ack migrationImportAck) []byte { diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 91548bb00..7332f318b 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -279,6 +279,85 @@ func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { }) } +func TestTargetStagedReadinessStatePersistsAndIsCloned(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + reader, ok := st.(MigrationTargetReadinessReader) + require.True(t, ok) + + state := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) + + states[0].RouteStart[0] = 'x' + states, err = reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []byte("a"), states[0].RouteStart) + + updated := state + updated.MinWriteTSExclusive = 101 + updated.RouteStart = []byte("b") + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, updated)) + + states, err = reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{updated}, states) + + exported, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("!"), + EndKey: []byte("~"), + MaxVersions: 100, + }) + require.NoError(t, err) + require.Empty(t, exported.Versions) + }) +} + +func TestPebbleTargetStagedReadinessPersistsAcrossReopen(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-ready-persist-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + st, err := NewPebbleStore(dir) + require.NoError(t, err) + state := TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 22, + MigrationJobID: 11, + MinWriteTSExclusive: 333, + Armed: true, + } + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + require.NoError(t, st.Close()) + + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + reader, ok := reopened.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) +} + func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-import-persist-*") diff --git a/store/mvcc_store.go b/store/mvcc_store.go index b20c35254..1bea2a9f7 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -60,14 +60,16 @@ func byteSliceComparator(a, b any) int { // mvccStore is an in-memory MVCC implementation backed by a treemap for // deterministic iteration order and range scans. type mvccStore struct { - tree *treemap.Map // key []byte -> []VersionedValue - mtx sync.RWMutex - log *slog.Logger - lastCommitTS uint64 - minRetainedTS uint64 - migrationAcks map[string]migrationImportAck - migrationHLCFloors map[uint64]uint64 - migrationPromotions map[uint64]PromotionState + tree *treemap.Map // key []byte -> []VersionedValue + mtx sync.RWMutex + log *slog.Logger + lastCommitTS uint64 + minRetainedTS uint64 + migrationAcks map[string]migrationImportAck + migrationHLCFloors map[uint64]uint64 + migrationPromotions map[uint64]PromotionState + migrationReadiness map[uint64]TargetStagedReadinessState + migrationReadinessCache []TargetStagedReadinessState // writeConflicts mirrors the per-(kind, key_prefix) counter from // the pebble-backed store so the in-memory implementation shows up // in the same Prometheus series (even if the counts are usually @@ -117,6 +119,7 @@ func NewMVCCStore(opts ...MVCCStoreOption) MVCCStore { migrationAcks: make(map[string]migrationImportAck), migrationHLCFloors: make(map[uint64]uint64), migrationPromotions: make(map[uint64]PromotionState), + migrationReadiness: make(map[uint64]TargetStagedReadinessState), writeConflicts: newWriteConflictCounter(), } for _, opt := range opts { diff --git a/store/store.go b/store/store.go index a645a95c5..ae53203f9 100644 --- a/store/store.go +++ b/store/store.go @@ -148,6 +148,19 @@ type PromotionState struct { LastError string } +// TargetStagedReadinessState is a target-local guard that makes a target voter +// fail closed for a moving route until it has either the staged descriptor or +// the cleared descriptor with the retained write timestamp floor. +type TargetStagedReadinessState struct { + JobID uint64 + RouteStart []byte + RouteEnd []byte + ExpectedCutoverVersion uint64 + MigrationJobID uint64 + MinWriteTSExclusive uint64 + Armed bool +} + // MigrationPromoter is implemented by stores that can promote staged range // migration data into the live keyspace. type MigrationPromoter interface { @@ -159,6 +172,17 @@ type MigrationPromotionStateReader interface { MigrationPromotionState(ctx context.Context, jobID uint64) (PromotionState, bool, error) } +// MigrationTargetReadinessWriter persists a target-local staged-readiness +// guard through the target Raft group. +type MigrationTargetReadinessWriter interface { + ApplyTargetStagedReadiness(ctx context.Context, state TargetStagedReadinessState) error +} + +// MigrationTargetReadinessReader reads retained target-local readiness guards. +type MigrationTargetReadinessReader interface { + MigrationTargetReadinessStates(ctx context.Context) ([]TargetStagedReadinessState, error) +} + // OpType describes a mutation kind. type OpType int From b93f5b9d473a3b88d1069794c665f479911d543b Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 02:03:50 +0900 Subject: [PATCH 016/162] distribution: add split job management RPCs --- adapter/distribution_server.go | 195 +++++++++++++++++++++++++ adapter/distribution_server_test.go | 152 ++++++++++++++++++- distribution/split_job_catalog.go | 5 + distribution/split_job_catalog_test.go | 92 ++++++++++++ distribution/split_job_lifecycle.go | 145 ++++++++++++++++++ 5 files changed, 586 insertions(+), 3 deletions(-) create mode 100644 distribution/split_job_lifecycle.go diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 9703c828b..7052a8561 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "math" + "strings" "sync" "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -78,6 +80,7 @@ var ( errDistributionCoordinatorRequired = errors.New("distribution coordinator is not configured") errDistributionEngineNotConfigured = errors.New("distribution engine is not configured") errDistributionCatalogVersionNotFound = errors.New("route catalog version not found") + errDistributionClusterNotReady = errors.New("cluster is not ready for split migration") ) // NewDistributionServer creates a new server. @@ -172,6 +175,97 @@ func (s *DistributionServer) GetIntersectingRoutes(ctx context.Context, req *pb. }, nil } +func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.StartSplitMigrationRequest) (*pb.StartSplitMigrationResponse, error) { + if req.GetTargetGroupId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) +} + +func (s *DistributionServer) GetSplitJob(ctx context.Context, req *pb.GetSplitJobRequest) (*pb.GetSplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + job, found, err := s.catalog.SplitJob(ctx, req.GetJobId()) + if err != nil { + return nil, splitJobCatalogStatusError(err) + } + if !found { + return nil, grpcStatusError(codes.NotFound, distribution.ErrCatalogSplitJobNotFound.Error()) + } + return &pb.GetSplitJobResponse{Job: distribution.SplitJobToProto(job)}, nil +} + +func (s *DistributionServer) ListSplitJobs(ctx context.Context, req *pb.ListSplitJobsRequest) (*pb.ListSplitJobsResponse, error) { + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if req == nil { + req = &pb.ListSplitJobsRequest{} + } + phaseFilter := normalizeSplitJobPhaseFilter(req.GetPhase()) + if phaseFilter != "" && !validSplitJobPhaseFilter(phaseFilter) { + return nil, grpcStatusErrorf(codes.InvalidArgument, "unknown split job phase %q", req.GetPhase()) + } + jobs, err := s.catalog.ListSplitJobs(ctx) + if err != nil { + return nil, splitJobCatalogStatusError(err) + } + resp := &pb.ListSplitJobsResponse{} + for _, job := range jobs { + if !splitJobPassesListFilter(job, req.GetSinceTerminalAtMs(), phaseFilter) { + continue + } + resp.Jobs = append(resp.Jobs, distribution.SplitJobToProto(job)) + } + return resp, nil +} + +func (s *DistributionServer) AbandonSplitJob(ctx context.Context, req *pb.AbandonSplitJobRequest) (*pb.AbandonSplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if err := s.updateSplitJobViaCoordinator(ctx, req.GetJobId(), func(job distribution.SplitJob) (distribution.SplitJob, error) { + return distribution.BeginSplitJobAbandon(job, time.Now().UnixMilli()) + }); err != nil { + return nil, err + } + return &pb.AbandonSplitJobResponse{}, nil +} + +func (s *DistributionServer) RetrySplitJob(ctx context.Context, req *pb.RetrySplitJobRequest) (*pb.RetrySplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if err := s.updateSplitJobViaCoordinator(ctx, req.GetJobId(), func(job distribution.SplitJob) (distribution.SplitJob, error) { + return distribution.RetrySplitJobState(job, time.Now().UnixMilli()) + }); err != nil { + return nil, err + } + return &pb.RetrySplitJobResponse{}, nil +} + func (s *DistributionServer) routeSnapshotAt(version uint64) (distribution.RouteHistorySnapshot, error) { if s.engine == nil { return distribution.RouteHistorySnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionEngineNotConfigured.Error()) @@ -401,6 +495,42 @@ func (s *DistributionServer) applyEngineSnapshot(snapshot distribution.CatalogSn return nil } +func (s *DistributionServer) updateSplitJobViaCoordinator( + ctx context.Context, + jobID uint64, + transition func(distribution.SplitJob) (distribution.SplitJob, error), +) error { + expected, readTS, err := s.catalog.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return splitJobCatalogStatusError(err) + } + next, err := transition(expected) + if err != nil { + return splitJobCatalogStatusError(err) + } + if splitJobsEqualByEncoding(expected, next) { + return nil + } + encoded, err := distribution.EncodeSplitJob(next) + if err != nil { + return splitJobCatalogStatusError(err) + } + key := distribution.CatalogSplitJobKey(jobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{{ + Op: kv.Put, + Key: key, + Value: encoded, + }}, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{key}, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + func validateExpectedCatalogVersion(currentVersion, expectedVersion uint64) error { if currentVersion != expectedVersion { return grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) @@ -493,6 +623,71 @@ func routeRangeIntersects(aStart, aEnd, bStart, bEnd []byte) bool { return true } +func splitJobPassesListFilter(job distribution.SplitJob, sinceTerminalAtMs uint64, phaseFilter string) bool { + if sinceTerminalAtMs > 0 && job.TerminalAtMs > 0 && uint64(job.TerminalAtMs) < sinceTerminalAtMs { + return false + } + if phaseFilter == "" { + return true + } + return normalizeSplitJobPhaseFilter(pb.SplitJobPhase(job.Phase).String()) == phaseFilter || + normalizeSplitJobPhaseFilter(strings.TrimPrefix(pb.SplitJobPhase(job.Phase).String(), "SPLIT_JOB_PHASE_")) == phaseFilter +} + +func normalizeSplitJobPhaseFilter(phase string) string { + phase = strings.TrimSpace(phase) + if phase == "" { + return "" + } + phase = strings.ReplaceAll(phase, "-", "_") + phase = strings.ReplaceAll(phase, " ", "_") + return strings.ToUpper(phase) +} + +func validSplitJobPhaseFilter(phase string) bool { + for _, candidate := range pb.SplitJobPhase_name { + full := normalizeSplitJobPhaseFilter(candidate) + short := normalizeSplitJobPhaseFilter(strings.TrimPrefix(candidate, "SPLIT_JOB_PHASE_")) + if phase == full || phase == short { + return true + } + } + return false +} + +func splitJobsEqualByEncoding(left, right distribution.SplitJob) bool { + leftRaw, leftErr := distribution.EncodeSplitJob(left) + rightRaw, rightErr := distribution.EncodeSplitJob(right) + return leftErr == nil && rightErr == nil && bytes.Equal(leftRaw, rightRaw) +} + +func splitJobCatalogStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrCatalogSplitJobIDRequired): + return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobNotFound): + return grpcStatusError(codes.NotFound, distribution.ErrCatalogSplitJobNotFound.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobCannotRetry), + errors.Is(err, distribution.ErrCatalogSplitJobCannotAbandon): + return grpcStatusError(codes.FailedPrecondition, err.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobConflict): + return grpcStatusError(codes.Aborted, distribution.ErrCatalogSplitJobConflict.Error()) + default: + return grpcStatusErrorf(codes.Internal, "split job catalog: %v", err) + } +} + +func splitJobCoordinatorStatusError(err error) error { + switch { + case errors.Is(err, store.ErrWriteConflict): + return grpcStatusError(codes.Aborted, distribution.ErrCatalogSplitJobConflict.Error()) + case errors.Is(err, kv.ErrLeaderNotFound): + return grpcStatusError(codes.FailedPrecondition, errDistributionNotLeader.Error()) + default: + return grpcStatusErrorf(codes.Internal, "commit split job mutation: %v", err) + } +} + func splitCatalogRoutes( parent distribution.RouteDescriptor, splitKey []byte, diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index f717dfb56..434180c0a 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -129,6 +129,137 @@ func TestDistributionServerListRoutes_RequiresCatalog(t *testing.T) { require.ErrorContains(t, err, errDistributionCatalogNotConfigured.Error()) } +func TestDistributionServerStartSplitMigration_FailsClosedUntilCapabilityGate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: 1, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionClusterNotReady.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) +} + +func TestDistributionServerSplitJobRPCs_ReadAndListCatalogJobs(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + live := sampleDistributionSplitJob(10) + live.Phase = distribution.SplitJobPhaseDeltaCopy + require.NoError(t, catalog.CreateSplitJob(ctx, live)) + history := sampleDistributionSplitJob(11) + history.Phase = distribution.SplitJobPhaseDone + history.TerminalAtMs = 2000 + require.NoError(t, catalog.CreateSplitJob(ctx, history)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, history, history)) + + s := NewDistributionServer(distribution.NewEngine(), catalog) + got, err := s.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: live.JobID}) + require.NoError(t, err) + require.Equal(t, live.JobID, got.Job.JobId) + require.Equal(t, pb.SplitJobPhase_SPLIT_JOB_PHASE_DELTA_COPY, got.Job.Phase) + + listAll, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.NoError(t, err) + require.Len(t, listAll.Jobs, 2) + require.Equal(t, []uint64{live.JobID, history.JobID}, []uint64{listAll.Jobs[0].JobId, listAll.Jobs[1].JobId}) + + listDone, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{Phase: "done"}) + require.NoError(t, err) + require.Len(t, listDone.Jobs, 1) + require.Equal(t, history.JobID, listDone.Jobs[0].JobId) + + _, err = s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{Phase: "not-a-phase"}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestDistributionServerRetrySplitJob_UsesCoordinatorCAS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(12) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseFence + job.LastError = "retry me" + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + require.Equal(t, 1, coordinator.dispatchCalls) + + got, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseFence, got.Phase) + require.Equal(t, distribution.SplitJobPhaseNone, got.RetryPhase) + require.Empty(t, got.LastError) +} + +func TestDistributionServerAbandonSplitJob_RecordsAbandoningViaCoordinator(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(13) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseBackfill + job.AbandonFromPhase = distribution.SplitJobPhaseNone + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.AbandonSplitJob(ctx, &pb.AbandonSplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + require.Equal(t, 1, coordinator.dispatchCalls) + + got, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoning, got.Phase) + require.Equal(t, distribution.SplitJobPhaseNone, got.RetryPhase) + require.Equal(t, distribution.SplitJobPhaseBackfill, got.AbandonFromPhase) +} + +func TestDistributionServerRetrySplitJob_RequiresCatalogLeader(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(14) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseBackfill + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, false) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Zero(t, coordinator.dispatchCalls) +} + func TestDistributionServerGetRouteOwnership_UsesExactVersionSnapshot(t *testing.T) { t.Parallel() @@ -781,6 +912,19 @@ func seededDistributionServerWithoutCoordinator(t *testing.T) (*DistributionServ return NewDistributionServer(distribution.NewEngine(), catalog), saved.Version } +func sampleDistributionSplitJob(jobID uint64) distribution.SplitJob { + return distribution.SplitJob{ + JobID: jobID, + SourceRouteID: 1, + SplitKey: []byte("g"), + TargetGroupID: 2, + Phase: distribution.SplitJobPhaseBackfill, + RetryPhase: distribution.SplitJobPhaseNone, + StartedAtMs: 1000, + UpdatedAtMs: 1000, + } +} + type distributionCoordinatorStub struct { store store.MVCCStore leader bool @@ -814,16 +958,17 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope if s.asyncApplyDelay > 0 { done := s.asyncApplyDone delay := s.asyncApplyDelay + readKeys := cloneByteSlices(reqs.ReadKeys) go func() { time.Sleep(delay) - err := s.applyDispatch(ctx, mutations, startTS, commitTS) + err := s.applyDispatch(ctx, mutations, readKeys, startTS, commitTS) if done != nil { done <- err } }() return &kv.CoordinateResponse{CommitIndex: commitTS}, nil } - if err := s.applyDispatch(ctx, mutations, startTS, commitTS); err != nil { + if err := s.applyDispatch(ctx, mutations, reqs.ReadKeys, startTS, commitTS); err != nil { return nil, err } return &kv.CoordinateResponse{CommitIndex: commitTS}, nil @@ -854,10 +999,11 @@ func (s *distributionCoordinatorStub) nextTimestamps(startTS uint64) (uint64, ui func (s *distributionCoordinatorStub) applyDispatch( ctx context.Context, mutations []*store.KVPairMutation, + readKeys [][]byte, startTS uint64, commitTS uint64, ) error { - if err := s.store.ApplyMutations(ctx, mutations, nil, startTS, commitTS); err != nil { + if err := s.store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS); err != nil { return err } if s.afterDispatch != nil { diff --git a/distribution/split_job_catalog.go b/distribution/split_job_catalog.go index 808ad8498..082f9aaf6 100644 --- a/distribution/split_job_catalog.go +++ b/distribution/split_job_catalog.go @@ -844,6 +844,11 @@ func CloneSplitJob(job SplitJob) SplitJob { } } +// SplitJobToProto converts a catalog SplitJob into its wire representation. +func SplitJobToProto(job SplitJob) *pb.SplitJob { + return splitJobToProto(job) +} + func splitJobToProto(job SplitJob) *pb.SplitJob { job = CloneSplitJob(job) return &pb.SplitJob{ diff --git a/distribution/split_job_catalog_test.go b/distribution/split_job_catalog_test.go index 0ac91a59b..cdeef3c3a 100644 --- a/distribution/split_job_catalog_test.go +++ b/distribution/split_job_catalog_test.go @@ -242,6 +242,98 @@ func TestCatalogStoreSaveSplitJobRejectsStaleExpectedJob(t *testing.T) { assertSplitJobEqual(t, advanced, got) } +func TestCatalogStoreRetrySplitJobUsesDurableRetryPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(18) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseDeltaCopy + job.LastError = "transient" + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + retried, err := cs.RetrySplitJob(ctx, job.JobID, 1200) + if err != nil { + t.Fatalf("retry split job: %v", err) + } + if retried.Phase != SplitJobPhaseDeltaCopy || retried.RetryPhase != SplitJobPhaseNone { + t.Fatalf("unexpected retry transition: %+v", retried) + } + if retried.AbandonFromPhase != SplitJobPhaseNone || retried.LastError != "" || retried.UpdatedAtMs != 1200 { + t.Fatalf("unexpected retry witness cleanup: %+v", retried) + } + got, found, err := cs.SplitJob(ctx, job.JobID) + if err != nil { + t.Fatalf("load retried split job: %v", err) + } + if !found { + t.Fatal("expected retried split job") + } + assertSplitJobEqual(t, retried, got) +} + +func TestCatalogStoreRetrySplitJobRejectsMissingRetryWitness(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(19) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + if _, err := cs.RetrySplitJob(ctx, job.JobID, 1200); !errors.Is(err, ErrCatalogSplitJobCannotRetry) { + t.Fatalf("expected ErrCatalogSplitJobCannotRetry, got %v", err) + } +} + +func TestCatalogStoreBeginSplitJobAbandonRecordsPreCutoverPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(20) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseFence + job.AbandonFromPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + abandoning, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1300) + if err != nil { + t.Fatalf("begin split job abandon: %v", err) + } + if abandoning.Phase != SplitJobPhaseAbandoning || + abandoning.RetryPhase != SplitJobPhaseNone || + abandoning.AbandonFromPhase != SplitJobPhaseFence || + abandoning.UpdatedAtMs != 1300 { + t.Fatalf("unexpected abandon transition: %+v", abandoning) + } + got, found, err := cs.SplitJob(ctx, job.JobID) + if err != nil { + t.Fatalf("load abandoning split job: %v", err) + } + if !found { + t.Fatal("expected abandoning split job") + } + assertSplitJobEqual(t, abandoning, got) +} + +func TestCatalogStoreBeginSplitJobAbandonRejectsPostCutoverPhases(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(21) + job.Phase = SplitJobPhaseCleanup + job.RetryPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + if _, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1300); !errors.Is(err, ErrCatalogSplitJobCannotAbandon) { + t.Fatalf("expected ErrCatalogSplitJobCannotAbandon, got %v", err) + } +} + func TestCatalogStoreListSplitJobsIncludesLiveAndHistory(t *testing.T) { cs := NewCatalogStore(store.NewMVCCStore()) ctx := context.Background() diff --git a/distribution/split_job_lifecycle.go b/distribution/split_job_lifecycle.go new file mode 100644 index 000000000..2f6170c1c --- /dev/null +++ b/distribution/split_job_lifecycle.go @@ -0,0 +1,145 @@ +package distribution + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" +) + +var ( + ErrCatalogSplitJobNotFound = errors.New("catalog split job is not found") + ErrCatalogSplitJobCannotRetry = errors.New("catalog split job cannot be retried") + ErrCatalogSplitJobCannotAbandon = errors.New("catalog split job cannot be abandoned") +) + +// RetrySplitJobState returns the durable state transition for RetrySplitJob. +// The retry target must come from the recorded RetryPhase witness; callers must +// not infer a phase from cursors, timestamps, or diagnostics after restart. +func RetrySplitJobState(job SplitJob, nowMs int64) (SplitJob, error) { + out := CloneSplitJob(job) + if out.Phase != SplitJobPhaseFailed || !splitJobRetryPhaseAllowed(out.RetryPhase) { + return SplitJob{}, errors.WithStack(ErrCatalogSplitJobCannotRetry) + } + out.Phase = out.RetryPhase + out.RetryPhase = SplitJobPhaseNone + out.AbandonFromPhase = SplitJobPhaseNone + out.LastError = "" + out.UpdatedAtMs = nowMs + return out, nil +} + +// BeginSplitJobAbandon records the durable ABANDONING witness before any +// pre-CUTOVER cleanup side effect is removed. +func BeginSplitJobAbandon(job SplitJob, nowMs int64) (SplitJob, error) { + out := CloneSplitJob(job) + from, ok := splitJobAbandonFromPhase(out) + if !ok { + return SplitJob{}, errors.WithStack(ErrCatalogSplitJobCannotAbandon) + } + if out.Phase == SplitJobPhaseAbandoning { + return out, nil + } + out.Phase = SplitJobPhaseAbandoning + out.RetryPhase = SplitJobPhaseNone + out.AbandonFromPhase = from + out.UpdatedAtMs = nowMs + return out, nil +} + +// RetrySplitJob CASes a FAILED live job back to its recorded retry phase. +func (s *CatalogStore) RetrySplitJob(ctx context.Context, jobID uint64, nowMs int64) (SplitJob, error) { + expected, _, err := s.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return SplitJob{}, err + } + next, err := RetrySplitJobState(expected, nowMs) + if err != nil { + return SplitJob{}, err + } + return next, s.SaveSplitJob(ctx, expected, next) +} + +// BeginSplitJobAbandon CASes a live pre-CUTOVER job into ABANDONING. +func (s *CatalogStore) BeginSplitJobAbandon(ctx context.Context, jobID uint64, nowMs int64) (SplitJob, error) { + expected, _, err := s.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return SplitJob{}, err + } + next, err := BeginSplitJobAbandon(expected, nowMs) + if err != nil { + return SplitJob{}, err + } + if splitJobsEquivalent(expected, next) { + return next, nil + } + return next, s.SaveSplitJob(ctx, expected, next) +} + +// LiveSplitJobForUpdate reads the latest live split job and the snapshot +// timestamp used for the read. RPC callers use readTS as a CAS StartTS when +// proposing the update through Raft. +func (s *CatalogStore) LiveSplitJobForUpdate(ctx context.Context, jobID uint64) (SplitJob, uint64, error) { + if err := ensureCatalogStore(s); err != nil { + return SplitJob{}, 0, err + } + if jobID == 0 { + return SplitJob{}, 0, errors.WithStack(ErrCatalogSplitJobIDRequired) + } + ctx = contextOrBackground(ctx) + readTS := s.store.LastCommitTS() + job, found, err := s.liveSplitJobAt(ctx, jobID, readTS) + if err != nil { + return SplitJob{}, 0, err + } + if !found { + return SplitJob{}, 0, errors.WithStack(ErrCatalogSplitJobNotFound) + } + return job, readTS, nil +} + +func splitJobRetryPhaseAllowed(phase SplitJobPhase) bool { + switch phase { + case SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, SplitJobPhaseCutover, SplitJobPhaseCleanup: + return true + case SplitJobPhaseNone, SplitJobPhasePlanned, SplitJobPhaseDone, SplitJobPhaseFailed, SplitJobPhaseAbandoning, SplitJobPhaseAbandoned: + return false + } + return false +} + +func splitJobAbandonFromPhase(job SplitJob) (SplitJobPhase, bool) { + switch job.Phase { + case SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: + return job.Phase, true + case SplitJobPhaseFailed: + if splitJobAbandonRetryPhaseAllowed(job.RetryPhase) { + return job.RetryPhase, true + } + case SplitJobPhaseAbandoning: + if splitJobAbandonRetryPhaseAllowed(job.AbandonFromPhase) { + return job.AbandonFromPhase, true + } + case SplitJobPhaseNone, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, SplitJobPhaseAbandoned: + } + return SplitJobPhaseNone, false +} + +func splitJobAbandonRetryPhaseAllowed(phase SplitJobPhase) bool { + switch phase { + case SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: + return true + case SplitJobPhaseNone, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, SplitJobPhaseFailed, SplitJobPhaseAbandoning, SplitJobPhaseAbandoned: + return false + } + return false +} + +func splitJobsEquivalent(left, right SplitJob) bool { + leftRaw, leftErr := EncodeSplitJob(left) + rightRaw, rightErr := EncodeSplitJob(right) + if leftErr != nil || rightErr != nil { + return false + } + return bytes.Equal(leftRaw, rightRaw) +} From c1afe2706097644945bfefaf21e155dd7a6c1431 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 02:08:26 +0900 Subject: [PATCH 017/162] store: persist promote applied index --- kv/fsm_migration_promote.go | 5 ++-- store/lsm_store_applied_index_test.go | 41 +++++++++++++++++++++++++++ store/migration_promote.go | 12 ++++++-- store/store.go | 5 +++- 4 files changed, 57 insertions(+), 6 deletions(-) diff --git a/kv/fsm_migration_promote.go b/kv/fsm_migration_promote.go index 0d9d1ed86..0551b9216 100644 --- a/kv/fsm_migration_promote.go +++ b/kv/fsm_migration_promote.go @@ -41,7 +41,7 @@ func (f *kvFSM) applyMigrationPromote(ctx context.Context, data []byte) any { if !ok { return errors.WithStack(store.ErrNotSupported) } - result, err := promoter.PromoteVersions(ctx, migrationPromoteOptionsFromProto(req)) + result, err := promoter.PromoteVersions(ctx, migrationPromoteOptionsFromProto(req, f.pendingApplyIdx)) if err != nil { return errors.WithStack(err) } @@ -51,7 +51,7 @@ func (f *kvFSM) applyMigrationPromote(ctx context.Context, data []byte) any { return result } -func migrationPromoteOptionsFromProto(req *pb.PromoteStagedVersionsRequest) store.PromoteVersionsOptions { +func migrationPromoteOptionsFromProto(req *pb.PromoteStagedVersionsRequest, appliedIndex uint64) store.PromoteVersionsOptions { maxVersions := int(req.GetMaxVersions()) if maxVersions <= 0 { maxVersions = defaultMigrationPromoteMaxVersions @@ -67,6 +67,7 @@ func migrationPromoteOptionsFromProto(req *pb.PromoteStagedVersionsRequest) stor prefix := distribution.MigrationStagedDataKeyPrefix(req.GetJobId()) return store.PromoteVersionsOptions{ JobID: req.GetJobId(), + AppliedIndex: appliedIndex, StartKey: prefix, EndKey: store.PrefixScanEnd(prefix), Cursor: req.GetCursor(), diff --git a/store/lsm_store_applied_index_test.go b/store/lsm_store_applied_index_test.go index ff00ea950..02c77b1d3 100644 --- a/store/lsm_store_applied_index_test.go +++ b/store/lsm_store_applied_index_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "os" "testing" @@ -96,6 +97,46 @@ func TestApplyMutationsRaftAt_BundlesMetaAppliedIndex(t *testing.T) { require.Equal(t, []byte("v1"), val) } +func TestPromoteVersions_BundlesMetaAppliedIndex(t *testing.T) { + ctx := context.Background() + st := newApplyIndexPebbleStore(t) + ps := pebbleStoreApplied(t, st) + + stage := func(raw string) []byte { + return append([]byte("stage|"), []byte(raw)...) + } + targetKey := func(staged []byte) ([]byte, bool) { + return bytes.TrimPrefix(staged, []byte("stage|")), bytes.HasPrefix(staged, []byte("stage|")) + } + prefix := []byte("stage|") + + require.NoError(t, ps.PutAt(ctx, stage("k"), []byte("v10"), 10, 0)) + + const entryIdx uint64 = 77 + result, err := ps.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 9, + AppliedIndex: entryIdx, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, result.Done) + require.Equal(t, uint64(1), result.PromotedRows) + + got, present, err := ps.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present, "PromoteVersions must persist metaAppliedIndex") + require.Equal(t, entryIdx, got) + + val, err := ps.GetAt(ctx, []byte("k"), 10) + require.NoError(t, err) + require.Equal(t, []byte("v10"), val) + _, err = ps.GetAt(ctx, stage("k"), 10) + require.ErrorIs(t, err, ErrKeyNotFound) +} + // TestApplyMutationsRaftAt_ZeroIndexLeavesMetaKey covers the // appliedIndex==0 escape hatch — callers without a raft entry index // (test fakes, legacy ApplyMutationsRaft) MUST NOT bump the meta key. diff --git a/store/migration_promote.go b/store/migration_promote.go index cde302a1a..91af77c7e 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -226,7 +226,7 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO return PromoteVersionsResult{}, err } result, stateToWrite := finishPromotionResult(opts, state, exported, promoted) - return s.finishPebblePromotion(toPromote, opts.JobID, stateToWrite, result) + return s.finishPebblePromotion(toPromote, opts.JobID, stateToWrite, result, opts.AppliedIndex) } func (s *pebbleStore) finishPebblePromotion( @@ -234,11 +234,12 @@ func (s *pebbleStore) finishPebblePromotion( jobID uint64, stateToWrite *PromotionState, result PromoteVersionsResult, + appliedIndex uint64, ) (PromoteVersionsResult, error) { if len(toPromote) == 0 && stateToWrite == nil { return result, nil } - if err := s.commitPebblePromoteVersions(toPromote, jobID, stateToWrite); err != nil { + if err := s.commitPebblePromoteVersions(toPromote, jobID, stateToWrite, appliedIndex); err != nil { return PromoteVersionsResult{}, err } return result, nil @@ -320,7 +321,7 @@ func (s *pebbleStore) readPebblePromotionState(jobID uint64) (PromotionState, bo return state, true, nil } -func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jobID uint64, state *PromotionState) error { +func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jobID uint64, state *PromotionState, appliedIndex uint64) error { batch := s.db.NewBatch() defer batch.Close() targets := make([]MVCCVersion, 0, len(versions)) @@ -340,6 +341,11 @@ func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jo return errors.WithStack(err) } } + if appliedIndex > 0 { + if err := setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex); err != nil { + return err + } + } if err := batch.Commit(s.directApplyWriteOpts()); err != nil { return errors.WithStack(err) } diff --git a/store/store.go b/store/store.go index a645a95c5..b381ca789 100644 --- a/store/store.go +++ b/store/store.go @@ -119,7 +119,10 @@ type ImportVersionsResult struct { // PromoteVersionsOptions atomically copies staged MVCC versions to their // target keys and physically removes the staged versions. type PromoteVersionsOptions struct { - JobID uint64 + JobID uint64 + // AppliedIndex is the optional Raft entry index to bundle with Pebble + // promotion batches as metaAppliedIndex. Zero leaves the meta key unchanged. + AppliedIndex uint64 StartKey []byte EndKey []byte Cursor []byte From ca1a05070c7d18bff6741fa11e72e6b622116ee7 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 02:40:05 +0900 Subject: [PATCH 018/162] migration: harden staged route writes --- adapter/internal.go | 34 +++ adapter/internal_migration_test.go | 74 ++++++ internal/s3keys/keys.go | 11 + internal/s3keys/keys_test.go | 19 ++ kv/fsm.go | 129 ++++++++-- kv/fsm_migration_fence_test.go | 118 +++++++++ kv/route_history.go | 8 + kv/shard_store.go | 300 +++++++++++++++------- kv/shard_store_test.go | 85 ++++++ kv/sharded_coordinator.go | 63 +++++ kv/sharded_coordinator_del_prefix_test.go | 50 ++++ kv/sharded_coordinator_txn_test.go | 20 ++ 12 files changed, 807 insertions(+), 104 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index de61f5485..f72d116f2 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -6,6 +6,7 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" @@ -242,6 +243,9 @@ func exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.Export func migrationExportFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool { routeFilter := kv.RouteKeyFilter(req.GetRouteStart(), req.GetRouteEnd()) + if migrationFamilyRequiresDecodedS3(req.GetKeyFamily()) { + routeFilter = decodedS3BucketRouteFilter(req.GetKeyFamily(), req.GetRouteStart(), req.GetRouteEnd()) + } excludeKnownInternal := req.GetExcludeKnownInternal() || req.GetKeyFamily() == distribution.MigrationFamilyUser bracket := distribution.MigrationBracket{ Family: req.GetKeyFamily(), @@ -255,6 +259,36 @@ func migrationExportFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool } } +func migrationFamilyRequiresDecodedS3(family uint32) bool { + return family == distribution.MigrationFamilyS3BucketMeta || + family == distribution.MigrationFamilyS3BucketGeneration +} + +func decodedS3BucketRouteFilter(family uint32, routeStart, routeEnd []byte) func([]byte) bool { + return func(rawKey []byte) bool { + bucket, ok := decodedS3BucketName(family, rawKey) + if !ok { + return false + } + routeKey := s3keys.RouteKey(bucket, 0, "") + if routeStart != nil && bytes.Compare(routeKey, routeStart) < 0 { + return false + } + return routeEnd == nil || bytes.Compare(routeKey, routeEnd) < 0 + } +} + +func decodedS3BucketName(family uint32, rawKey []byte) (string, bool) { + switch family { + case distribution.MigrationFamilyS3BucketMeta: + return s3keys.ParseBucketMetaKey(rawKey) + case distribution.MigrationFamilyS3BucketGeneration: + return s3keys.ParseBucketGenerationKey(rawKey) + default: + return "", false + } +} + func cloneByteSlices(in [][]byte) [][]byte { if len(in) == 0 { return nil diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index dd8b48baf..b43a8f4b9 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -6,6 +6,7 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" @@ -70,6 +71,17 @@ func (s *captureExportRangeVersionsStream) Send(resp *pb.ExportRangeVersionsResp return nil } +func testPrefixScanEnd(prefix []byte) []byte { + out := append([]byte(nil), prefix...) + for i := len(out) - 1; i >= 0; i-- { + if out[i] != 0xFF { + out[i]++ + return out[:i+1] + } + } + return nil +} + func TestInternalExportRangeVersionsUsesStoreAndRouteFilter(t *testing.T) { t.Parallel() @@ -114,6 +126,68 @@ func TestInternalExportRangeVersionsRejectsUnboundedExport(t *testing.T) { require.Equal(t, codes.InvalidArgument, status.Code(err)) } +func TestInternalExportRangeVersionsUsesDecodedS3BucketRouteFilter(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + + for _, tc := range []struct { + name string + family uint32 + prefix string + keyFor func(string) []byte + value []byte + routeStart []byte + routeEnd []byte + }{ + { + name: "bucket meta", + family: distribution.MigrationFamilyS3BucketMeta, + prefix: s3keys.BucketMetaPrefix, + keyFor: s3keys.BucketMetaKey, + value: []byte("meta"), + routeStart: s3keys.RouteKey("bucket-b", 0, ""), + routeEnd: s3keys.RouteKey("bucket-c", 0, ""), + }, + { + name: "bucket generation", + family: distribution.MigrationFamilyS3BucketGeneration, + prefix: s3keys.BucketGenerationPrefix, + keyFor: s3keys.BucketGenerationKey, + value: []byte("generation"), + routeStart: s3keys.RouteKey("bucket-b", 0, ""), + routeEnd: s3keys.RouteKey("bucket-c", 0, ""), + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + inRouteKey := tc.keyFor("bucket-b") + outRouteKey := tc.keyFor("bucket-a") + require.NoError(t, st.PutAt(ctx, inRouteKey, tc.value, 10, 0)) + require.NoError(t, st.PutAt(ctx, outRouteKey, []byte("skip"), 10, 0)) + + stream := &captureExportRangeVersionsStream{ctx: ctx} + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + RouteStart: tc.routeStart, + RouteEnd: tc.routeEnd, + KeyFamily: tc.family, + RangeStart: []byte(tc.prefix), + RangeEnd: testPrefixScanEnd([]byte(tc.prefix)), + MaxScannedBytes: 1 << 20, + }, stream) + require.NoError(t, err) + require.Len(t, stream.responses, 1) + require.Equal(t, []*pb.MVCCVersion{ + {Key: inRouteKey, CommitTs: 10, Value: tc.value, KeyFamily: tc.family}, + }, stream.responses[0].GetVersions()) + }) + } +} + func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() diff --git a/internal/s3keys/keys.go b/internal/s3keys/keys.go index 682105219..21326fb5e 100644 --- a/internal/s3keys/keys.go +++ b/internal/s3keys/keys.go @@ -80,6 +80,17 @@ func ParseBucketMetaKey(key []byte) (string, bool) { return string(segment), true } +func ParseBucketGenerationKey(key []byte) (string, bool) { + if !bytes.HasPrefix(key, bucketGenerationPrefixBytes) { + return "", false + } + segment, next, ok := decodeSegment(key, len(bucketGenerationPrefixBytes)) + if !ok || next != len(key) { + return "", false + } + return string(segment), true +} + func ObjectManifestKey(bucket string, generation uint64, object string) []byte { return buildObjectKey(objectManifestPrefixBytes, bucket, generation, object, "", 0, 0) } diff --git a/internal/s3keys/keys_test.go b/internal/s3keys/keys_test.go index e5e5fca5a..34449227a 100644 --- a/internal/s3keys/keys_test.go +++ b/internal/s3keys/keys_test.go @@ -18,6 +18,25 @@ func TestBucketMetaKey_RoundTripsZeroByteSegments(t *testing.T) { require.Equal(t, bucket, parsed) } +func TestBucketGenerationKey_RoundTripsZeroByteSegments(t *testing.T) { + t.Parallel() + + bucket := string([]byte{'b', 'u', 0x00, 'c', 'k', 'e', 't'}) + key := BucketGenerationKey(bucket) + + parsed, ok := ParseBucketGenerationKey(key) + require.True(t, ok) + require.Equal(t, bucket, parsed) +} + +func TestParseBucketGenerationKey_RejectsNonGenerationKey(t *testing.T) { + t.Parallel() + + parsed, ok := ParseBucketGenerationKey(BucketMetaKey("bucket")) + require.False(t, ok) + require.Empty(t, parsed) +} + func TestObjectManifestKey_RoundTripsZeroByteSegments(t *testing.T) { t.Parallel() diff --git a/kv/fsm.go b/kv/fsm.go index 00e4a5d3c..1693d5eaa 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -9,6 +9,7 @@ import ( "log/slog" "os" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/encryption/fsmwire" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/internal/s3keys" @@ -121,6 +122,10 @@ type RouteSnapshot interface { // OwnerOf returns the Raft group ID that owned key at this // snapshot's version. (0, false) when no route covered key. OwnerOf(key []byte) (uint64, bool) + // RouteOf returns the complete route descriptor covering key. + RouteOf(key []byte) (distribution.Route, bool) + // IntersectingRoutes returns every route intersecting [start, end). + IntersectingRoutes(start, end []byte) []distribution.Route // WriteFencedForKey reports whether key is currently inside a // WriteFenced route in this snapshot. WriteFencedForKey(key []byte) bool @@ -273,6 +278,8 @@ var ErrUnknownRequestType = errors.New("unknown request type") // catches up to the promoted owner. var ErrRouteWriteFenced = errors.New("route is write-fenced; retry after route migration") +var ErrRouteWriteTimestampTooLow = errors.New("route write timestamp is below migration floor") + // ErrComposed1Violation is returned by verifyComposed1 when the // transaction's commit cannot proceed on this Raft group because the // txn's read-set or write-set keys are not owned by this group at @@ -494,20 +501,8 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui return f.handleDelPrefix(ctx, prefix, commitTS) } - for _, mut := range r.Mutations { - if mut == nil || len(mut.Key) == 0 { - return errors.WithStack(ErrInvalidRequest) - } - // Raw requests should not mutate txn-internal keys. - if isTxnInternalKey(mut.Key) { - return errors.WithStack(ErrInvalidRequest) - } - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { - return err - } - if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { - return err - } + if err := f.validateRawMutationsForApply(ctx, r.Mutations, commitTS); err != nil { + return err } muts, err := toStoreMutations(r.Mutations) @@ -523,6 +518,35 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui return nil } +func (f *kvFSM) validateRawMutationsForApply(ctx context.Context, muts []*pb.Mutation, commitTS uint64) error { + for _, mut := range muts { + if err := f.validateRawMutationForApply(ctx, mut, commitTS); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutation, commitTS uint64) error { + if mut == nil || len(mut.Key) == 0 { + return errors.WithStack(ErrInvalidRequest) + } + // Raw requests should not mutate txn-internal keys. + if isTxnInternalKey(mut.Key) { + return errors.WithStack(ErrInvalidRequest) + } + if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { + return err + } + if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { + return err + } + if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { + return err + } + return nil +} + // extractDelPrefix checks if the mutations contain a DEL_PREFIX operation. // If found, it validates that no other operation types are mixed in. func extractDelPrefix(muts []*pb.Mutation) (bool, []byte) { @@ -540,6 +564,9 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { return err } + if err := f.verifyRouteWriteTimestampFloorForPrefix(prefix, commitTS); err != nil { + return err + } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -547,6 +574,51 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin return nil } +func (f *kvFSM) verifyRouteWriteTimestampFloorForKey(key []byte, commitTS uint64) error { + if f.routes == nil || commitTS == 0 { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + rkey := routeKey(key) + route, ok := snap.RouteOf(rkey) + if !ok || route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, rkey, commitTS, route.MinWriteTSExclusive) +} + +func (f *kvFSM) verifyRouteWriteTimestampFloorForPrefix(prefix []byte, commitTS uint64) error { + if f.routes == nil || commitTS == 0 { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + start, end := routePrefixRange(prefix) + for _, route := range snap.IntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", prefix, start, end, commitTS, route.MinWriteTSExclusive) + } + } + return nil +} + +func (f *kvFSM) verifyRouteWriteTimestampFloorForMutations(muts []*pb.Mutation, commitTS uint64) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { + return err + } + } + return nil +} + func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation) error { for _, mut := range muts { if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { @@ -964,6 +1036,9 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { return err } + if err := f.verifyRouteWriteTimestampFloorForMutations(uniq, meta.CommitTS); err != nil { + return err + } if err := f.validateConflicts(ctx, uniq, startTS); err != nil { return errors.WithStack(err) } @@ -1030,7 +1105,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsNotFenced(muts) + uniq, err := f.uniqueMutationsNotFencedAboveWriteFloor(muts, commitTS) if err != nil { return err } @@ -1057,6 +1132,28 @@ func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation) ([]*pb.Mutation, e return uniq, nil } +func (f *kvFSM) uniqueMutationsNotFencedAboveWriteFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + uniq, err := f.uniqueMutationsNotFenced(muts) + if err != nil { + return nil, err + } + if err := f.verifyRouteWriteTimestampFloorForMutations(uniq, commitTS); err != nil { + return nil, err + } + return uniq, nil +} + +func (f *kvFSM) uniqueMutationsAboveWriteFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + uniq, err := uniqueMutations(muts) + if err != nil { + return nil, err + } + if err := f.verifyRouteWriteTimestampFloorForMutations(uniq, commitTS); err != nil { + return nil, err + } + return uniq, nil +} + // dedupProbeOnePhase decides whether handleOnePhaseTxnRequest should no-op // because the entry is a retry whose prior attempt already landed. Extracted // to keep handleOnePhaseTxnRequest under the cyclop budget; the determinism @@ -1096,7 +1193,7 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueMutationsAboveWriteFloor(muts, commitTS) if err != nil { return err } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index e47e71f95..045a9d04d 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -22,6 +22,16 @@ func newWriteFencedFSM(t *testing.T) *kvFSM { return newComposed1FSM(t, engine, 1) } +func newWriteFloorFSM(t *testing.T) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive, MinWriteTSExclusive: 100}, + }) + return newComposed1FSM(t, engine, 1) +} + func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() @@ -112,3 +122,111 @@ func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { err := fsm.handleTxnRequest(ctx, abort, 11) require.False(t, errors.Is(err, ErrRouteWriteFenced), "ABORT must keep the narrow cleanup lane open") } + +func TestFSMRejectsRawPointWriteAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("low")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + + _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) + + err = fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("ok")}}, + }, 101) + require.NoError(t, err) + got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("ok"), got) +} + +func TestFSMRejectsDelPrefixAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + require.NoError(t, fsm.store.PutAt(ctx, []byte("z"), []byte("v"), 10, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + + got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsOnePhaseTxnAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + req := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_NONE, + Ts: 90, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 100})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("low")}, + }, + } + err := fsm.handleTxnRequest(ctx, req, 100) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + + _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) + + req.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 101}) + req.Mutations[1].Value = []byte("ok") + err = fsm.handleTxnRequest(ctx, req, 101) + require.NoError(t, err) + got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("ok"), got) +} + +func TestFSMRejectsCommitButNotPrepareAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + prepare := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 90, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + require.NoError(t, fsm.handleTxnRequest(ctx, prepare, 90)) + + commit := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: 90, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 100})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("low")}, + }, + } + err := fsm.handleTxnRequest(ctx, commit, 100) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + + _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) + + commit.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 101}) + commit.Mutations[1].Value = []byte("ignored") + err = fsm.handleTxnRequest(ctx, commit, 101) + require.NoError(t, err) + got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} diff --git a/kv/route_history.go b/kv/route_history.go index c6ac85608..8bc50c58e 100644 --- a/kv/route_history.go +++ b/kv/route_history.go @@ -61,6 +61,14 @@ func (s distributionRouteSnapshot) OwnerOf(key []byte) (uint64, bool) { return s.snap.OwnerOf(key) } +func (s distributionRouteSnapshot) RouteOf(key []byte) (distribution.Route, bool) { + return s.snap.RouteOf(key) +} + +func (s distributionRouteSnapshot) IntersectingRoutes(start, end []byte) []distribution.Route { + return s.snap.IntersectingRoutes(start, end) +} + func (s distributionRouteSnapshot) WriteFencedForKey(key []byte) bool { route, ok := s.snap.RouteOf(key) return ok && route.State == distribution.RouteStateWriteFenced diff --git a/kv/shard_store.go b/kv/shard_store.go index ba5ef6ac9..fcf47f15c 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -113,6 +113,9 @@ func routeHasStagedVisibility(route distribution.Route) bool { } func (s *ShardStore) getAtWithStagedVisibility(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + if err := ensureReadTSRetained(g.Store, ts); err != nil { + return nil, err + } live, liveOK, err := latestMVCCVersionAt(ctx, g.Store, key, ts) if err != nil { return nil, err @@ -135,14 +138,16 @@ func (s *ShardStore) getAtWithStagedVisibility(ctx context.Context, g *ShardGrou func latestMVCCVersionAt(ctx context.Context, st store.MVCCStore, key []byte, ts uint64) (store.MVCCVersion, bool, error) { result, err := st.ExportVersions(ctx, store.ExportVersionsOptions{ StartKey: key, - EndKey: nextScanCursor(key), + EndKey: prefixScanEnd(key), MaxCommitTSInclusive: ts, MaxVersions: 1, MaxScannedBytes: 0, MinCommitTSExclusive: 0, MaxBytes: 0, KeyFamily: 0, - AcceptKey: nil, + AcceptKey: func(rawKey []byte) bool { + return bytes.Equal(rawKey, key) + }, }) if err != nil { return store.MVCCVersion{}, false, errors.WithStack(err) @@ -172,6 +177,18 @@ func migrationVersionVisible(version store.MVCCVersion, ts uint64) bool { return !version.Tombstone && (version.ExpireAt == 0 || version.ExpireAt > ts) } +func ensureReadTSRetained(st store.MVCCStore, ts uint64) error { + retention, ok := st.(store.RetentionController) + if !ok { + return nil + } + minRetainedTS := retention.MinRetainedTS() + if minRetainedTS != 0 && ts != 0 && ts != ^uint64(0) && ts < minRetainedTS { + return errors.WithStack(store.ErrReadTSCompacted) + } + return nil +} + func (s *ShardStore) ExistsAt(ctx context.Context, key []byte, ts uint64) (bool, error) { v, err := s.GetAt(ctx, key, ts) if err != nil { @@ -276,6 +293,9 @@ func (s *ShardStore) ScanAtPhysicalLimit(ctx context.Context, start []byte, end return []*store.KVPair{}, false, nil } routes, clampToRoutes := s.routesForScan(start, end) + if routesContainStagedVisibility(routes) { + return nil, true, nil + } if len(routes) != 1 || clampToRoutes { kvs, err := s.ScanAt(ctx, start, end, visibleLimit, ts) return kvs, false, err @@ -316,6 +336,9 @@ func (s *ShardStore) ReverseScanAtPhysicalLimit(ctx context.Context, start []byt return []*store.KVPair{}, false, nil } routes, clampToRoutes := s.routesForScan(start, end) + if routesContainStagedVisibility(routes) { + return nil, true, nil + } if len(routes) != 1 || clampToRoutes { kvs, err := s.ReverseScanAt(ctx, start, end, visibleLimit, ts) return kvs, false, err @@ -348,6 +371,15 @@ func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Rou return routes, true } +func routesContainStagedVisibility(routes []distribution.Route) bool { + for _, route := range routes { + if routeHasStagedVisibility(route) { + return true + } + } + return false +} + func (s *ShardStore) scanRoutesAt(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) for _, route := range routes { @@ -641,7 +673,10 @@ func (s *ShardStore) scanRouteAtLeader( return s.resolveScanLocks(ctx, g, route, kvs, lockKVs, ts) } -const stagedVisibilityExportPageSize = 1024 +const ( + stagedVisibilityMaxCandidateWindow = 8192 + stagedVisibilityWindowGrowthFactor = 2 +) func (s *ShardStore) scanRouteWithStagedVisibility( ctx context.Context, @@ -653,112 +688,139 @@ func (s *ShardStore) scanRouteWithStagedVisibility( ts uint64, reverse bool, ) ([]*store.KVPair, error) { - live, err := collectLatestLogicalVersions(ctx, g.Store, start, end, start, end, ts, liveLogicalVersionKey) - if err != nil { + if err := ensureReadTSRetained(g.Store, ts); err != nil { return nil, err } stagedStart, stagedEnd := stagedVisibilityScanBounds(route.MigrationJobID, start, end) - staged, err := collectLatestLogicalVersions(ctx, g.Store, stagedStart, stagedEnd, start, end, ts, stagedLogicalVersionKey) - if err != nil { - return nil, err - } - merged := mergeLogicalVersionMaps(live, staged) - out := visibleLogicalKVs(merged, ts, reverse) - if len(out) > limit { - clear(out[limit:]) - out = out[:limit] + window := stagedVisibilityCandidateWindow(limit) + for { + liveKVs, err := scanVisibleCandidates(ctx, g.Store, start, end, window, ts, reverse) + if err != nil { + return nil, err + } + stagedKVs, err := scanVisibleCandidates(ctx, g.Store, stagedStart, stagedEnd, window, ts, reverse) + if err != nil { + return nil, err + } + versions, err := s.latestStagedVisibilityCandidates(ctx, g.Store, route, liveKVs, stagedKVs, ts) + if err != nil { + return nil, err + } + out := visibleLogicalKVs(versions, ts, reverse) + if len(out) >= limit { + clear(out[limit:]) + return out[:limit], nil + } + if len(liveKVs) < window && len(stagedKVs) < window { + return out, nil + } + nextWindow := nextStagedVisibilityCandidateWindow(window) + if nextWindow == window { + return out, nil + } + window = nextWindow } - return out, nil } -func stagedVisibilityScanBounds(jobID uint64, start []byte, end []byte) ([]byte, []byte) { - prefix := distribution.MigrationStagedDataKeyPrefix(jobID) - scanStart := prefix - if start != nil { - scanStart = distribution.MigrationStagedDataKey(jobID, start) +func stagedVisibilityCandidateWindow(limit int) int { + if limit <= 0 { + return 0 } - scanEnd := prefixScanEnd(prefix) - if end != nil { - scanEnd = distribution.MigrationStagedDataKey(jobID, end) + if limit > stagedVisibilityMaxCandidateWindow { + return stagedVisibilityMaxCandidateWindow } - return scanStart, scanEnd + return limit } -type logicalVersionKeyFunc func([]byte) ([]byte, bool) - -func liveLogicalVersionKey(key []byte) ([]byte, bool) { - if distribution.IsMigrationStagedDataKey(key) { - return nil, false +func nextStagedVisibilityCandidateWindow(window int) int { + if window >= stagedVisibilityMaxCandidateWindow { + return window + } + next := window * stagedVisibilityWindowGrowthFactor + if next < window || next > stagedVisibilityMaxCandidateWindow { + return stagedVisibilityMaxCandidateWindow } - return bytes.Clone(key), true + return next } -func stagedLogicalVersionKey(key []byte) ([]byte, bool) { - _, rawKey, ok := distribution.MigrationStagedDataKeyParts(key) - return rawKey, ok +func scanVisibleCandidates(ctx context.Context, st store.MVCCStore, start, end []byte, limit int, ts uint64, reverse bool) ([]*store.KVPair, error) { + if limit <= 0 { + return []*store.KVPair{}, nil + } + if reverse { + kvs, err := st.ReverseScanAt(ctx, start, end, limit, ts) + return kvs, errors.WithStack(err) + } + kvs, err := st.ScanAt(ctx, start, end, limit, ts) + return kvs, errors.WithStack(err) } -func collectLatestLogicalVersions( +func (s *ShardStore) latestStagedVisibilityCandidates( ctx context.Context, st store.MVCCStore, - scanStart []byte, - scanEnd []byte, - logicalStart []byte, - logicalEnd []byte, + route distribution.Route, + liveKVs []*store.KVPair, + stagedKVs []*store.KVPair, ts uint64, - logicalKey logicalVersionKeyFunc, ) (map[string]store.MVCCVersion, error) { - out := make(map[string]store.MVCCVersion) - var cursor []byte - for { - result, err := st.ExportVersions(ctx, store.ExportVersionsOptions{ - StartKey: scanStart, - EndKey: scanEnd, - MaxCommitTSInclusive: ts, - Cursor: cursor, - MaxVersions: stagedVisibilityExportPageSize, - }) + keys := stagedVisibilityCandidateKeys(liveKVs, stagedKVs) + out := make(map[string]store.MVCCVersion, len(keys)) + for _, key := range keys { + live, liveOK, err := latestMVCCVersionAt(ctx, st, key, ts) if err != nil { - return nil, errors.WithStack(err) + return nil, err } - for _, version := range result.Versions { - key, ok := logicalKey(version.Key) - if !ok || !keyInRange(key, logicalStart, logicalEnd) { - continue - } - version.Key = key - recordLatestLogicalVersion(out, version) + stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, key) + staged, stagedOK, err := latestMVCCVersionAt(ctx, st, stagedKey, ts) + if err != nil { + return nil, err } - if result.Done { - return out, nil + if stagedOK { + staged.Key = bytes.Clone(key) } - if bytes.Equal(cursor, result.NextCursor) { - return nil, errors.New("staged visibility export cursor did not progress") + if winner, ok := newerMigrationVersion(live, liveOK, staged, stagedOK); ok { + out[string(key)] = winner } - cursor = bytes.Clone(result.NextCursor) } + return out, nil } -func recordLatestLogicalVersion(out map[string]store.MVCCVersion, version store.MVCCVersion) { - key := string(version.Key) - if existing, ok := out[key]; ok && existing.CommitTS >= version.CommitTS { - return +func stagedVisibilityCandidateKeys(liveKVs []*store.KVPair, stagedKVs []*store.KVPair) [][]byte { + seen := make(map[string][]byte, len(liveKVs)+len(stagedKVs)) + for _, kvp := range liveKVs { + if kvp == nil { + continue + } + seen[string(kvp.Key)] = bytes.Clone(kvp.Key) + } + for _, kvp := range stagedKVs { + if kvp == nil { + continue + } + _, rawKey, ok := distribution.MigrationStagedDataKeyParts(kvp.Key) + if !ok { + continue + } + seen[string(rawKey)] = bytes.Clone(rawKey) + } + out := make([][]byte, 0, len(seen)) + for _, key := range seen { + out = append(out, key) } - out[key] = version + return out } -func mergeLogicalVersionMaps(live map[string]store.MVCCVersion, staged map[string]store.MVCCVersion) map[string]store.MVCCVersion { - out := make(map[string]store.MVCCVersion, len(live)+len(staged)) - for key, version := range live { - out[key] = version +func stagedVisibilityScanBounds(jobID uint64, start []byte, end []byte) ([]byte, []byte) { + prefix := distribution.MigrationStagedDataKeyPrefix(jobID) + scanStart := prefix + if start != nil { + scanStart = distribution.MigrationStagedDataKey(jobID, start) } - for key, stagedVersion := range staged { - liveVersion, liveOK := out[key] - if winner, ok := newerMigrationVersion(liveVersion, liveOK, stagedVersion, true); ok { - out[key] = winner - } + scanEnd := prefixScanEnd(prefix) + if end != nil { + scanEnd = distribution.MigrationStagedDataKey(jobID, end) } - return out + return scanStart, scanEnd } func visibleLogicalKVs(versions map[string]store.MVCCVersion, ts uint64, reverse bool) []*store.KVPair { @@ -782,13 +844,6 @@ func visibleLogicalKVs(versions map[string]store.MVCCVersion, ts uint64, reverse return out } -func keyInRange(key []byte, start []byte, end []byte) bool { - if start != nil && bytes.Compare(key, start) < 0 { - return false - } - return end == nil || bytes.Compare(key, end) < 0 -} - func scanLockBoundsForKVs(kvs []*store.KVPair, scanStart []byte, scanEnd []byte, limit int) ([]byte, []byte) { if countNonInternalKVs(kvs) < limit { return scanStart, scanEnd @@ -905,34 +960,46 @@ func clampScanEnd(end []byte, routeEnd []byte) []byte { } func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return store.ErrNotSupported } + if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.PutAt(ctx, key, value, commitTS, expireAt)) } func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) error { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return store.ErrNotSupported } + if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.DeleteAt(ctx, key, commitTS)) } func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return store.ErrNotSupported } + if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.PutWithTTLAt(ctx, key, value, commitTS, expireAt)) } func (s *ShardStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, commitTS uint64) error { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return store.ErrNotSupported } + if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.ExpireAt(ctx, key, expireAt, commitTS)) } @@ -1606,6 +1673,9 @@ func (s *ShardStore) ApplyMutations(ctx context.Context, mutations []*store.KVPa if err != nil || group == nil { return err } + if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { + return err + } return errors.WithStack(group.Store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS)) } @@ -1616,6 +1686,9 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. if err != nil || group == nil { return err } + if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { + return err + } return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) } @@ -1627,9 +1700,38 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor if err != nil || group == nil { return err } + if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { + return err + } return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) } +func ensureRouteWriteTimestampFloor(route distribution.Route, key []byte, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, routeKey(key), commitTS, route.MinWriteTSExclusive) +} + +func (s *ShardStore) ensureMutationWriteTimestampFloors(mutations []*store.KVPairMutation, commitTS uint64) error { + if commitTS == 0 { + return nil + } + for _, mut := range mutations { + if mut == nil || len(mut.Key) == 0 { + continue + } + route, _, ok := s.routeAndGroupForKey(mut.Key) + if !ok { + return store.ErrNotSupported + } + if err := ensureRouteWriteTimestampFloor(route, mut.Key, commitTS); err != nil { + return err + } + } + return nil +} + // resolveSingleShardGroup returns the shard group that owns every // mutation in the batch, or an error if the batch is cross-shard or // references an unknown group. A nil group with nil error means "empty @@ -1656,6 +1758,9 @@ func (s *ShardStore) resolveSingleShardGroup(mutations []*store.KVPairMutation) // DeletePrefixAt applies a prefix delete to every shard in the store. func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { + if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue @@ -1667,8 +1772,24 @@ func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludeP return nil } +func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS uint64) error { + if s == nil || s.engine == nil || commitTS == 0 { + return nil + } + start, end := routePrefixRange(prefix) + for _, route := range s.engine.GetIntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", prefix, start, end, commitTS, route.MinWriteTSExclusive) + } + } + return nil +} + // DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { + if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue @@ -1695,6 +1816,9 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl // is the receiver only when an aggregate (admin / coordinator) path // is replaying a global FLUSHALL, which is not raft-applied. func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { + if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 100b70e27..e3a3ea8d5 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -33,6 +33,32 @@ func newStagedVisibilityShardStore(t *testing.T) (*ShardStore, *ShardGroup) { return NewShardStore(engine, map[uint64]*ShardGroup{1: group}), group } +func newStagedVisibilityPebbleShardStore(t *testing.T) (*ShardStore, *ShardGroup) { + t.Helper() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }, + }, + })) + st, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + group := &ShardGroup{Store: st} + return NewShardStore(engine, map[uint64]*ShardGroup{1: group}), group +} + func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { t.Parallel() @@ -57,6 +83,37 @@ func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) } +func TestShardStoreGetAt_MergesStagedVisibilityPebbleExactKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityPebbleShardStore(t) + rawKey := []byte("k") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + + require.NoError(t, group.Store.PutAt(ctx, rawKey, []byte("live-old"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged-new"), 20, 0)) + + got, err := st.GetAt(ctx, rawKey, 25) + require.NoError(t, err) + require.Equal(t, []byte("staged-new"), got) +} + +func TestShardStoreStagedVisibilityReadTSCompacted(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + retention, ok := group.Store.(store.RetentionController) + require.True(t, ok) + retention.SetMinRetainedTS(15) + + _, err := st.GetAt(ctx, []byte("k"), 10) + require.ErrorIs(t, err, store.ErrReadTSCompacted) + _, err = st.ScanAt(ctx, []byte("a"), []byte("z"), 10, 10) + require.ErrorIs(t, err, store.ErrReadTSCompacted) +} + func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { t.Parallel() @@ -102,6 +159,34 @@ func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { require.Equal(t, uint64(40), ts) } +func TestShardStorePhysicalLimitFailsClosedBeforeStagedVisibilityFallback(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, _ := newStagedVisibilityShardStore(t) + + kvs, limitReached, err := st.ScanAtPhysicalLimit(ctx, []byte("b"), []byte("c"), 10, 10, 50) + require.NoError(t, err) + require.True(t, limitReached) + require.Nil(t, kvs) + + kvs, limitReached, err = st.ReverseScanAtPhysicalLimit(ctx, []byte("b"), []byte("c"), 10, 10, 50) + require.NoError(t, err) + require.True(t, limitReached) + require.Nil(t, kvs) +} + +func TestShardStoreRejectsWritesAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, _ := newStagedVisibilityShardStore(t) + + err := st.PutAt(ctx, []byte("k"), []byte("low"), 100, 0) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0)) +} + func TestShardStoreScanAt_IncludesListKeysAcrossShards(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 7ed2ccb7c..a071a7e86 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1036,6 +1036,9 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT if err != nil { return nil, err } + if err := c.rejectWriteTimestampFloorDelPrefixes(elems, ts); err != nil { + return nil, err + } requests := make([]*pb.Request, 0, len(elems)) for _, elem := range elems { requests = append(requests, &pb.Request{ @@ -1084,6 +1087,42 @@ func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) err return nil } +func (c *ShardedCoordinator) rejectWriteTimestampFloorPointElems(elems []*Elem[OP], commitTS uint64) error { + if c == nil || c.engine == nil || commitTS == 0 { + return nil + } + for _, elem := range elems { + if elem == nil || len(elem.Key) == 0 { + continue + } + rkey := routeKey(elem.Key) + route, ok := c.engine.GetRoute(rkey) + if !ok || route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + continue + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", elem.Key, rkey, commitTS, route.MinWriteTSExclusive) + } + return nil +} + +func (c *ShardedCoordinator) rejectWriteTimestampFloorDelPrefixes(elems []*Elem[OP], commitTS uint64) error { + if c == nil || c.engine == nil || commitTS == 0 { + return nil + } + for _, elem := range elems { + if elem == nil { + continue + } + start, end := routePrefixRange(elem.Key) + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", elem.Key, start, end, commitTS, route.MinWriteTSExclusive) + } + } + } + return nil +} + // broadcastToAllGroups sends the same set of requests to every shard group in // parallel and returns the maximum commit index. func (c *ShardedCoordinator) broadcastToAllGroups(ctx context.Context, requests []*pb.Request) (*CoordinateResponse, error) { @@ -1141,6 +1180,9 @@ func (c *ShardedCoordinator) dispatchTxn(ctx context.Context, startTS uint64, co if err != nil { return nil, err } + if err := c.rejectWriteTimestampFloorPointElems(elems, commitTS); err != nil { + return nil, err + } if len(gids) == 1 && c.allReadKeysInShard(readKeys, gids[0]) { // Fast path: all mutations and read keys are in a single shard. @@ -1984,6 +2026,9 @@ func (c *ShardedCoordinator) rawLogs(ctx context.Context, reqs *OperationGroup[O if err != nil { return nil, err } + if err := c.rejectWriteTimestampFloorMutations(grouped[gid], ts); err != nil { + return nil, err + } logs = append(logs, &pb.Request{ IsTxn: false, Phase: pb.Phase_NONE, @@ -1994,6 +2039,24 @@ func (c *ShardedCoordinator) rawLogs(ctx context.Context, reqs *OperationGroup[O return logs, nil } +func (c *ShardedCoordinator) rejectWriteTimestampFloorMutations(muts []*pb.Mutation, commitTS uint64) error { + if c == nil || c.engine == nil || commitTS == 0 { + return nil + } + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 { + continue + } + rkey := routeKey(mut.Key) + route, ok := c.engine.GetRoute(rkey) + if !ok || route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + continue + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", mut.Key, rkey, commitTS, route.MinWriteTSExclusive) + } + return nil +} + func (c *ShardedCoordinator) rawLogTimestamp(ctx context.Context) (uint64, error) { if c.tsAllocator != nil { return 0, nil diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 4b83dc131..a83a5ce95 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -121,6 +121,56 @@ func TestShardedCoordinator_DelPrefixBroadcastsToAllGroups(t *testing.T) { "same DEL_PREFIX element must use the same timestamp across shards") } +func newMigrationFloorEngine(t *testing.T, floor uint64) *distribution.Engine { + t.Helper() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: floor, + }, + }, + })) + return engine +} + +func TestShardedCoordinatorRejectsPointWriteAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + g1Txn := &recordingTransactional{} + coord := NewShardedCoordinator(newMigrationFloorEngine(t, ^uint64(0)), map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: []byte("z"), Value: []byte("v")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + require.Empty(t, g1Txn.requests, "coordinator must reject before proposing a floor-violating point write") +} + +func TestShardedCoordinatorRejectsDelPrefixAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + g1Txn := &recordingTransactional{} + coord := NewShardedCoordinator(newMigrationFloorEngine(t, ^uint64(0)), map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + require.Empty(t, g1Txn.requests, "coordinator must reject before broadcasting a floor-violating prefix delete") +} + func TestShardedCoordinatorRejectsPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 964f2713f..c07733d16 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -246,6 +246,26 @@ func TestShardedCoordinatorDispatchTxn_UsesProvidedCommitTS(t *testing.T) { require.Equal(t, commitTS, commitMeta2.CommitTS) } +func TestShardedCoordinatorDispatchTxn_RejectsMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + g1Txn := &recordingTransactional{} + coord := NewShardedCoordinator(newMigrationFloorEngine(t, 100), map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + IsTxn: true, + StartTS: 90, + CommitTS: 100, + Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("z"), Value: []byte("v")}, + }, + }) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + require.Empty(t, g1Txn.requests, "coordinator must reject before preparing a floor-violating txn") +} + func TestCommitSecondaryWithRetry_RetriesAndSucceeds(t *testing.T) { t.Parallel() From 58c212801aca3dcc87478ca51707fdf2a3f58818 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 02:57:33 +0900 Subject: [PATCH 019/162] distribution: page split job listing --- adapter/distribution_server.go | 157 +++++++++++++++++++++++++--- adapter/distribution_server_test.go | 102 ++++++++++++++++++ distribution/split_job_lifecycle.go | 6 +- 3 files changed, 247 insertions(+), 18 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 7052a8561..78106c6bb 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -3,12 +3,15 @@ package adapter import ( "bytes" "context" + "encoding/binary" "math" + "sort" "strings" "sync" "time" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" @@ -62,8 +65,13 @@ func WithCatalogReloadRetryPolicy(attempts int, interval time.Duration) Distribu } const ( - childRouteCount = 2 - splitMutationOpCount = childRouteCount + 3 + childRouteCount = 2 + splitMutationOpCount = childRouteCount + 3 + listSplitJobsDefaultPageSize = 200 + splitJobListCursorVersion = byte(1) + splitJobListCursorTerminalOff = 1 + splitJobListCursorJobIDOff = splitJobListCursorTerminalOff + 8 + splitJobListCursorEncodedBytes = splitJobListCursorJobIDOff + 8 ) var ( @@ -220,12 +228,9 @@ func (s *DistributionServer) ListSplitJobs(ctx context.Context, req *pb.ListSpli if err != nil { return nil, splitJobCatalogStatusError(err) } - resp := &pb.ListSplitJobsResponse{} - for _, job := range jobs { - if !splitJobPassesListFilter(job, req.GetSinceTerminalAtMs(), phaseFilter) { - continue - } - resp.Jobs = append(resp.Jobs, distribution.SplitJobToProto(job)) + resp, err := splitJobListPage(jobs, req, phaseFilter) + if err != nil { + return nil, err } return resp, nil } @@ -508,7 +513,7 @@ func (s *DistributionServer) updateSplitJobViaCoordinator( if err != nil { return splitJobCatalogStatusError(err) } - if splitJobsEqualByEncoding(expected, next) { + if distribution.SplitJobsEquivalent(expected, next) { return nil } encoded, err := distribution.EncodeSplitJob(next) @@ -655,12 +660,6 @@ func validSplitJobPhaseFilter(phase string) bool { return false } -func splitJobsEqualByEncoding(left, right distribution.SplitJob) bool { - leftRaw, leftErr := distribution.EncodeSplitJob(left) - rightRaw, rightErr := distribution.EncodeSplitJob(right) - return leftErr == nil && rightErr == nil && bytes.Equal(leftRaw, rightRaw) -} - func splitJobCatalogStatusError(err error) error { switch { case errors.Is(err, distribution.ErrCatalogSplitJobIDRequired): @@ -681,13 +680,139 @@ func splitJobCoordinatorStatusError(err error) error { switch { case errors.Is(err, store.ErrWriteConflict): return grpcStatusError(codes.Aborted, distribution.ErrCatalogSplitJobConflict.Error()) - case errors.Is(err, kv.ErrLeaderNotFound): + case splitJobCoordinatorLeadershipError(err): return grpcStatusError(codes.FailedPrecondition, errDistributionNotLeader.Error()) default: return grpcStatusErrorf(codes.Internal, "commit split job mutation: %v", err) } } +func splitJobListPage(jobs []distribution.SplitJob, req *pb.ListSplitJobsRequest, phaseFilter string) (*pb.ListSplitJobsResponse, error) { + filtered := make([]distribution.SplitJob, 0, len(jobs)) + for _, job := range jobs { + if splitJobPassesListFilter(job, req.GetSinceTerminalAtMs(), phaseFilter) { + filtered = append(filtered, job) + } + } + sortSplitJobsForList(filtered) + + start, err := splitJobListStartIndex(filtered, req.GetPageCursor()) + if err != nil { + return nil, err + } + end := start + listSplitJobsDefaultPageSize + if end > len(filtered) { + end = len(filtered) + } + + resp := &pb.ListSplitJobsResponse{ + Jobs: make([]*pb.SplitJob, 0, end-start), + } + for _, job := range filtered[start:end] { + resp.Jobs = append(resp.Jobs, distribution.SplitJobToProto(job)) + } + if end < len(filtered) { + resp.NextPageCursor = encodeSplitJobListCursor(filtered[end-1]) + } + return resp, nil +} + +func sortSplitJobsForList(jobs []distribution.SplitJob) { + sort.Slice(jobs, func(i, j int) bool { + left, right := jobs[i], jobs[j] + leftLive := left.TerminalAtMs <= 0 + rightLive := right.TerminalAtMs <= 0 + if leftLive != rightLive { + return leftLive + } + if leftLive { + if left.UpdatedAtMs != right.UpdatedAtMs { + return left.UpdatedAtMs > right.UpdatedAtMs + } + return left.JobID > right.JobID + } + if left.TerminalAtMs != right.TerminalAtMs { + return left.TerminalAtMs > right.TerminalAtMs + } + return left.JobID > right.JobID + }) +} + +func splitJobListStartIndex(jobs []distribution.SplitJob, cursor []byte) (int, error) { + if len(cursor) == 0 { + return 0, nil + } + terminalAtMs, jobID, err := decodeSplitJobListCursor(cursor) + if err != nil { + return 0, err + } + for i, job := range jobs { + if job.JobID == jobID && splitJobListCursorTerminalAtMs(job) == terminalAtMs { + return i + 1, nil + } + } + return 0, grpcStatusError(codes.InvalidArgument, "split job page cursor does not match the filtered result set") +} + +func encodeSplitJobListCursor(job distribution.SplitJob) []byte { + cursor := make([]byte, splitJobListCursorEncodedBytes) + cursor[0] = splitJobListCursorVersion + binary.BigEndian.PutUint64( + cursor[splitJobListCursorTerminalOff:splitJobListCursorJobIDOff], + splitJobListCursorTerminalAtMs(job), + ) + binary.BigEndian.PutUint64(cursor[splitJobListCursorJobIDOff:], job.JobID) + return cursor +} + +func decodeSplitJobListCursor(cursor []byte) (uint64, uint64, error) { + if len(cursor) != splitJobListCursorEncodedBytes || cursor[0] != splitJobListCursorVersion { + return 0, 0, grpcStatusError(codes.InvalidArgument, "invalid split job page cursor") + } + terminalAtMs := binary.BigEndian.Uint64(cursor[splitJobListCursorTerminalOff:splitJobListCursorJobIDOff]) + jobID := binary.BigEndian.Uint64(cursor[splitJobListCursorJobIDOff:]) + if jobID == 0 { + return 0, 0, grpcStatusError(codes.InvalidArgument, "invalid split job page cursor") + } + return terminalAtMs, jobID, nil +} + +func splitJobListCursorTerminalAtMs(job distribution.SplitJob) uint64 { + if job.TerminalAtMs <= 0 { + return 0 + } + return uint64(job.TerminalAtMs) +} + +func splitJobCoordinatorLeadershipError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, kv.ErrLeaderNotFound) || + errors.Is(err, raftengine.ErrNotLeader) || + errors.Is(err, raftengine.ErrLeadershipLost) || + errors.Is(err, raftengine.ErrLeadershipTransferInProgress) { + return true + } + return hasSplitJobLeaderErrorSuffix(err.Error()) +} + +var splitJobLeaderErrorPhrases = []string{ + "not leader", + "leader not found", + "leadership lost", + "leadership transfer in progress", +} + +func hasSplitJobLeaderErrorSuffix(msg string) bool { + for _, phrase := range splitJobLeaderErrorPhrases { + if len(msg) >= len(phrase) && strings.EqualFold(msg[len(msg)-len(phrase):], phrase) { + return true + } + } + return false +} + func splitCatalogRoutes( parent distribution.RouteDescriptor, splitKey []byte, diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 434180c0a..109f21d62 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -2,10 +2,12 @@ package adapter import ( "context" + "errors" "testing" "time" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" @@ -189,6 +191,57 @@ func TestDistributionServerSplitJobRPCs_ReadAndListCatalogJobs(t *testing.T) { require.Equal(t, codes.InvalidArgument, status.Code(err)) } +func TestDistributionServerListSplitJobs_PaginatesNewestHistory(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + for jobID, terminalAtMs := uint64(1), int64(1001); jobID <= 205; jobID, terminalAtMs = jobID+1, terminalAtMs+1 { + job := sampleDistributionSplitJob(jobID) + job.Phase = distribution.SplitJobPhaseDone + job.TerminalAtMs = terminalAtMs + job.UpdatedAtMs = job.TerminalAtMs + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, job, job)) + } + + s := NewDistributionServer(distribution.NewEngine(), catalog) + first, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.NoError(t, err) + require.Len(t, first.Jobs, listSplitJobsDefaultPageSize) + require.NotEmpty(t, first.NextPageCursor) + require.Equal(t, uint64(205), first.Jobs[0].JobId) + require.Equal(t, uint64(6), first.Jobs[len(first.Jobs)-1].JobId) + + second, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: first.NextPageCursor}) + require.NoError(t, err) + require.Empty(t, second.NextPageCursor) + require.Equal(t, []uint64{5, 4, 3, 2, 1}, splitJobIDs(second.Jobs)) +} + +func TestDistributionServerListSplitJobs_RejectsInvalidCursor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + job := sampleDistributionSplitJob(1) + job.Phase = distribution.SplitJobPhaseDone + job.TerminalAtMs = 1000 + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, job, job)) + + s := NewDistributionServer(distribution.NewEngine(), catalog) + _, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: []byte("bad")}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + + missing := sampleDistributionSplitJob(999) + missing.TerminalAtMs = 9999 + _, err = s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: encodeSplitJobListCursor(missing)}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + func TestDistributionServerRetrySplitJob_UsesCoordinatorCAS(t *testing.T) { t.Parallel() @@ -215,6 +268,43 @@ func TestDistributionServerRetrySplitJob_UsesCoordinatorCAS(t *testing.T) { require.Empty(t, got.LastError) } +func TestDistributionServerRetrySplitJob_MapsDispatchLeadershipLoss(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + }{ + {name: "leader not found", err: kv.ErrLeaderNotFound}, + {name: "raft not leader", err: raftengine.ErrNotLeader}, + {name: "leadership lost", err: raftengine.ErrLeadershipLost}, + {name: "transfer in progress", err: raftengine.ErrLeadershipTransferInProgress}, + {name: "wrapped grpc detail", err: errors.New("rpc error: code = Unknown desc = raft engine: leadership lost")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(15) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseFence + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + coordinator.dispatchErr = tc.err + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionNotLeader.Error()) + require.Equal(t, 1, coordinator.dispatchCalls) + }) + } +} + func TestDistributionServerAbandonSplitJob_RecordsAbandoningViaCoordinator(t *testing.T) { t.Parallel() @@ -925,9 +1015,18 @@ func sampleDistributionSplitJob(jobID uint64) distribution.SplitJob { } } +func splitJobIDs(jobs []*pb.SplitJob) []uint64 { + ids := make([]uint64, 0, len(jobs)) + for _, job := range jobs { + ids = append(ids, job.GetJobId()) + } + return ids +} + type distributionCoordinatorStub struct { store store.MVCCStore leader bool + dispatchErr error nextTS uint64 lastStartTS uint64 afterDispatch func(context.Context, store.MVCCStore, uint64) error @@ -948,6 +1047,9 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope return nil, err } s.dispatchCalls++ + if s.dispatchErr != nil { + return nil, s.dispatchErr + } startTS, commitTS := s.nextTimestamps(reqs.StartTS) s.lastStartTS = startTS diff --git a/distribution/split_job_lifecycle.go b/distribution/split_job_lifecycle.go index 2f6170c1c..f2427a4e0 100644 --- a/distribution/split_job_lifecycle.go +++ b/distribution/split_job_lifecycle.go @@ -70,7 +70,7 @@ func (s *CatalogStore) BeginSplitJobAbandon(ctx context.Context, jobID uint64, n if err != nil { return SplitJob{}, err } - if splitJobsEquivalent(expected, next) { + if SplitJobsEquivalent(expected, next) { return next, nil } return next, s.SaveSplitJob(ctx, expected, next) @@ -135,7 +135,9 @@ func splitJobAbandonRetryPhaseAllowed(phase SplitJobPhase) bool { return false } -func splitJobsEquivalent(left, right SplitJob) bool { +// SplitJobsEquivalent reports whether two split jobs encode to the same +// durable catalog value. +func SplitJobsEquivalent(left, right SplitJob) bool { leftRaw, leftErr := EncodeSplitJob(left) rightRaw, rightErr := EncodeSplitJob(right) if leftErr != nil || rightErr != nil { From e3079c09fdc0f44be6c3fe59a9f81b6415897e2c Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 03:07:58 +0900 Subject: [PATCH 020/162] store: fix migration export blockers --- store/lsm_migration.go | 20 +++---- store/migration_versions.go | 32 ++++++++---- store/migration_versions_test.go | 87 +++++++++++++++++++++++++++++++ store/mvcc_store.go | 2 + store/mvcc_store_snapshot_test.go | 43 +++++++++++++++ 5 files changed, 163 insertions(+), 21 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 3ae343afd..1f6749e18 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -10,6 +10,7 @@ import ( ) func (s *pebbleStore) ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { + opts = normalizeExportVersionsOptions(opts) pos, err := decodeExportCursor(opts.Cursor) if err != nil { return ExportVersionsResult{}, err @@ -77,17 +78,11 @@ func (s *pebbleStore) runPebbleExportLoop( } func pebbleExportIterOptions(opts ExportVersionsOptions) *pebble.IterOptions { - iterOpts := &pebble.IterOptions{ - LowerBound: encodeKey(opts.StartKey, math.MaxUint64), - } - if opts.EndKey != nil { - iterOpts.UpperBound = encodeKey(opts.EndKey, math.MaxUint64) - } - return iterOpts + return &pebble.IterOptions{} } func pebbleExportSeekKey(opts ExportVersionsOptions, pos exportCursorPosition) ([]byte, error) { - if len(pos.key) == 0 { + if !pos.hasKey { return encodeKey(opts.StartKey, math.MaxUint64), nil } if opts.StartKey != nil && bytes.Compare(pos.key, opts.StartKey) < 0 { @@ -117,8 +112,13 @@ func (s *pebbleStore) exportPebbleIteratorPosition( if userKey == nil || pebbleExportCursorEqual(pos, userKey, commitTS) { return true, true, nil } + if opts.StartKey != nil && bytes.Compare(userKey, opts.StartKey) < 0 { + _ = s.skipToNextUserKey(iter, userKey) + return false, true, nil + } if opts.EndKey != nil && bytes.Compare(userKey, opts.EndKey) >= 0 { - return false, true, errExportReachedEnd + _ = s.skipToNextUserKey(iter, userKey) + return false, true, nil } if commitTS <= opts.MinCommitTSExclusive { _ = s.skipToNextUserKey(iter, userKey) @@ -129,7 +129,7 @@ func (s *pebbleStore) exportPebbleIteratorPosition( } func pebbleExportCursorEqual(pos exportCursorPosition, userKey []byte, commitTS uint64) bool { - return len(pos.key) > 0 && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS + return pos.hasKey && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS } func (s *pebbleStore) exportPebbleVersion( diff --git a/store/migration_versions.go b/store/migration_versions.go index 54b7dd19b..e9aa56546 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -13,17 +13,19 @@ const ( exportCursorTagEmitted byte = iota exportCursorTagScanned - migrationAckPrefix = "!migstage|ack|" - migrationHLCFloorPrefix = "!migstage|hlc_floor|" - migrationUint64Bytes = 8 - migrationAckKeyIDBytes = 2 * migrationUint64Bytes - exportVersionSizeOverhead = 24 + migrationAckPrefix = "!migstage|ack|" + migrationHLCFloorPrefix = "!migstage|hlc_floor|" + migrationUint64Bytes = 8 + migrationAckKeyIDBytes = 2 * migrationUint64Bytes + exportVersionSizeOverhead = 24 + defaultSparseExportMaxScannedBytes = 1 << 20 ) type exportCursorPosition struct { key []byte commitTS uint64 tag byte + hasKey bool } type migrationImportAck struct { @@ -66,7 +68,14 @@ func decodeExportCursor(cursor []byte) (exportCursorPosition, error) { if tag != exportCursorTagEmitted && tag != exportCursorTagScanned { return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) } - return exportCursorPosition{key: key, commitTS: commitTS, tag: tag}, nil + return exportCursorPosition{key: key, commitTS: commitTS, tag: tag, hasKey: true}, nil +} + +func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOptions { + if opts.AcceptKey != nil && opts.MaxScannedBytes == 0 { + opts.MaxScannedBytes = defaultSparseExportMaxScannedBytes + } + return opts } func migrationAckKey(jobID, bracketID uint64) []byte { @@ -168,6 +177,7 @@ func validateNextImportBatch(existing migrationImportAck, hasExisting bool, batc } func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { + opts = normalizeExportVersionsOptions(opts) pos, err := decodeExportCursor(opts.Cursor) if err != nil { return ExportVersionsResult{}, err @@ -181,7 +191,7 @@ func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptio result := newExportVersionsResult(opts.MaxVersions) it := s.tree.Iterator() - if !s.seekMemoryExportStart(&it, opts.StartKey, pos.key) { + if !s.seekMemoryExportStart(&it, opts.StartKey, pos) { result.Done = true return result, nil } @@ -231,9 +241,9 @@ func newExportVersionsResult(maxVersions int) ExportVersionsResult { } } -func (s *mvccStore) seekMemoryExportStart(it *treemap.Iterator, startKey, cursorKey []byte) bool { - if len(cursorKey) > 0 { - return seekForwardIteratorStart(s.tree, it, cursorKey) +func (s *mvccStore) seekMemoryExportStart(it *treemap.Iterator, startKey []byte, pos exportCursorPosition) bool { + if pos.hasKey { + return seekForwardIteratorStart(s.tree, it, pos.key) } return seekForwardIteratorStart(s.tree, it, startKey) } @@ -248,7 +258,7 @@ func exportMemoryIteratorKey( ) (bool, error) { versions, _ := value.([]VersionedValue) cursorCommitTS := uint64(0) - if len(pos.key) > 0 && bytes.Equal(key, pos.key) { + if pos.hasKey && bytes.Equal(key, pos.key) { cursorCommitTS = pos.commitTS } return exportMemoryVersionsForKey(ctx, opts, cursorCommitTS, key, versions, result) diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 77badeedd..900afedb4 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "os" "testing" @@ -112,6 +113,92 @@ func TestExportVersionsSparseScanBudgetAdvancesRejectedRows(t *testing.T) { }) } +func TestExportVersionsPreservesEmptyKeyCursor(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, nil, []byte("v10"), 10, 0)) + require.NoError(t, st.PutAt(ctx, nil, []byte("v20"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 1}) + require.NoError(t, err) + require.False(t, first.Done) + require.Len(t, first.Versions, 1) + require.Empty(t, first.Versions[0].Key) + require.Equal(t, uint64(20), first.Versions[0].CommitTS) + require.Equal(t, []byte("v20"), first.Versions[0].Value) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: first.NextCursor, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Len(t, second.Versions, 1) + require.Empty(t, second.Versions[0].Key) + require.Equal(t, uint64(10), second.Versions[0].CommitTS) + require.Equal(t, []byte("v10"), second.Versions[0].Value) + }) +} + +func TestExportVersionsAppliesDefaultSparseScanBudget(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + value := bytes.Repeat([]byte("x"), defaultSparseExportMaxScannedBytes) + require.NoError(t, st.PutAt(ctx, []byte("drop"), value, 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("keep"), []byte("v"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + MaxVersions: 10, + AcceptKey: func(key []byte) bool { + return bytes.Equal(key, []byte("keep")) + }, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.GreaterOrEqual(t, first.ScannedBytes, uint64(defaultSparseExportMaxScannedBytes)) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: first.NextCursor, + MaxVersions: 10, + AcceptKey: func(key []byte) bool { + return bytes.Equal(key, []byte("keep")) + }, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("keep"), CommitTS: 20, Value: []byte("v")}}, second.Versions) + }) +} + +func TestExportVersionsUsesUserKeyRangeBounds(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("a-value"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("aa"), []byte("aa-value"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("b-value"), 30, 0)) + + mid, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("aa"), + EndKey: []byte("b"), + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, mid.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("aa"), CommitTS: 20, Value: []byte("aa-value")}}, mid.Versions) + + before, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("aa"), + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, before.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("a"), CommitTS: 10, Value: []byte("a-value")}}, before.Versions) + }) +} + func TestImportVersionsIdempotencyAndMetadata(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 50be08ba3..46ded171e 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -934,6 +934,8 @@ func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { s.tree = tree s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS + s.migrationAcks = make(map[string]migrationImportAck) + s.migrationHLCFloors = make(map[uint64]uint64) return nil } diff --git a/store/mvcc_store_snapshot_test.go b/store/mvcc_store_snapshot_test.go index 3ebdbcac8..b16cbe68b 100644 --- a/store/mvcc_store_snapshot_test.go +++ b/store/mvcc_store_snapshot_test.go @@ -56,6 +56,49 @@ func TestMVCCStore_RestoreRejectsInvalidChecksum(t *testing.T) { require.ErrorIs(t, st.Restore(bytes.NewReader(raw)), ErrInvalidChecksum) } +func TestMVCCStore_RestoreClearsMigrationMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := newTestMVCCStore(t) + require.NoError(t, st.PutAt(ctx, []byte("base"), []byte("v1"), 10, 0)) + + snap, err := st.Snapshot() + require.NoError(t, err) + defer snap.Close() + raw := snapshotBytes(t, snap) + + _, err = st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("stale"), + Versions: []MVCCVersion{{Key: []byte("imported"), CommitTS: 50, Value: []byte("v50")}}, + }) + require.NoError(t, err) + floor, err := st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(50), floor) + + require.NoError(t, st.Restore(bytes.NewReader(raw))) + floor, err = st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Zero(t, floor) + _, err = st.GetAt(ctx, []byte("imported"), 50) + require.ErrorIs(t, err, ErrKeyNotFound) + + res, err := st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("fresh"), + Versions: []MVCCVersion{{Key: []byte("imported"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.False(t, res.Duplicate) + require.Equal(t, []byte("fresh"), res.AckedCursor) +} + func TestMVCCStore_ApplyMutations_WriteConflict(t *testing.T) { t.Parallel() From dd52db81988f75ecc1c7718935ac897dc2b52ac4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 03:09:48 +0900 Subject: [PATCH 021/162] Gate migration promotion replay safety --- adapter/internal.go | 61 ++++++++++++++++++++++----- adapter/internal_migration_test.go | 28 ++++++++++++ kv/fsm_migration_promote.go | 8 ++-- kv/fsm_migration_promote_test.go | 9 ++++ store/lsm_store_applied_index_test.go | 18 ++++++++ store/migration_promote.go | 5 ++- 6 files changed, 113 insertions(+), 16 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index efacdab71..a3e5024d2 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -3,6 +3,8 @@ package adapter import ( "bytes" "context" + "os" + "strings" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" @@ -34,12 +36,19 @@ func WithInternalMigrationProposer(proposer raftengine.Proposer) InternalOption } } +func WithInternalMigrationPromoteGate(gate func(context.Context) error) InternalOption { + return func(i *Internal) { + i.migrationPromoteGate = gate + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ - leader: leader, - transactionManager: txm, - clock: clock, - relay: relay, + leader: leader, + transactionManager: txm, + clock: clock, + relay: relay, + migrationPromoteGate: defaultMigrationPromoteGate, } for _, opt := range opts { opt(i) @@ -48,13 +57,14 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c } type Internal struct { - leader raftengine.LeaderView - transactionManager kv.Transactional - clock *kv.HLC - tsAllocator kv.TimestampAllocator - relay *RedisPubSubRelay - store store.MVCCStore - migrationProposer raftengine.Proposer + leader raftengine.LeaderView + transactionManager kv.Transactional + clock *kv.HLC + tsAllocator kv.TimestampAllocator + relay *RedisPubSubRelay + store store.MVCCStore + migrationProposer raftengine.Proposer + migrationPromoteGate func(context.Context) error pb.UnimplementedInternalServer } @@ -192,6 +202,9 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta if err := i.verifyInternalLeader(ctx); err != nil { return nil, err } + if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { + return nil, err + } result, err := i.proposeMigrationPromote(ctx, req) if err != nil { return nil, errors.WithStack(err) @@ -207,6 +220,32 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta }, nil } +func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { + if i.migrationPromoteGate == nil { + return nil + } + if err := i.migrationPromoteGate(ctx); err != nil { + return errors.WithStack(err) + } + return nil +} + +func defaultMigrationPromoteGate(context.Context) error { + if migrationPromoteOpcodeEnabledFromEnv() { + return nil + } + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote opcode is disabled; enable after every voter is running a build that supports migration promotion")) +} + +func migrationPromoteOpcodeEnabledFromEnv() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv("ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE"))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + func (i *Internal) verifyInternalLeader(ctx context.Context) error { if i.leader.State() != raftengine.StateLeader { return errors.WithStack(ErrNotLeader) diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 54227d418..e4cde4997 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -153,6 +153,33 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { require.GreaterOrEqual(t, clock.Current(), uint64(30)) } +func TestInternalPromoteStagedVersionsRejectsWhenOpcodeGateClosed(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + clock := kv.NewHLC() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, clock), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration promote disabled for test") + }), + ) + + resp, err := internal.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{ + JobId: 7, + MaxVersions: 10, + }) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, uint64(0), proposer.calls) +} + func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() @@ -165,6 +192,7 @@ func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, WithInternalStore(st), WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), ) staged := distribution.MigrationStagedDataKey(7, []byte("k")) diff --git a/kv/fsm_migration_promote.go b/kv/fsm_migration_promote.go index 0551b9216..c4001b634 100644 --- a/kv/fsm_migration_promote.go +++ b/kv/fsm_migration_promote.go @@ -16,6 +16,8 @@ const ( defaultMigrationPromoteMaxScannedBytes = defaultMigrationPromoteMaxBytes * 4 ) +var ErrMigrationPromoteApply = errors.New("migration promote: FSM apply failed; halting apply") + // MarshalMigrationPromoteCommand encodes a target-group staged-data promotion // chunk as a Raft FSM command. func MarshalMigrationPromoteCommand(req *pb.PromoteStagedVersionsRequest) ([]byte, error) { @@ -35,15 +37,15 @@ func MarshalMigrationPromoteCommand(req *pb.PromoteStagedVersionsRequest) ([]byt func (f *kvFSM) applyMigrationPromote(ctx context.Context, data []byte) any { req := &pb.PromoteStagedVersionsRequest{} if err := proto.Unmarshal(data, req); err != nil { - return errors.WithStack(err) + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationPromoteApply), "kv/fsm: decode migration promote")) } promoter, ok := f.store.(store.MigrationPromoter) if !ok { - return errors.WithStack(store.ErrNotSupported) + return haltErr(errors.Wrap(errors.Mark(store.ErrNotSupported, ErrMigrationPromoteApply), "kv/fsm: migration promote store")) } result, err := promoter.PromoteVersions(ctx, migrationPromoteOptionsFromProto(req, f.pendingApplyIdx)) if err != nil { - return errors.WithStack(err) + return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationPromoteApply), "kv/fsm: apply migration promote")) } if f.hlc != nil && result.MaxPromotedTS > 0 { f.hlc.Observe(result.MaxPromotedTS) diff --git a/kv/fsm_migration_promote_test.go b/kv/fsm_migration_promote_test.go index 2470c485d..2397243e2 100644 --- a/kv/fsm_migration_promote_test.go +++ b/kv/fsm_migration_promote_test.go @@ -7,6 +7,7 @@ import ( "github.com/bootjp/elastickv/distribution" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) @@ -67,3 +68,11 @@ func TestApplyMigrationPromoteMovesStagedVersions(t *testing.T) { require.NoError(t, err) require.Empty(t, left.Versions) } + +func TestApplyMigrationPromoteMalformedPayloadHalts(t *testing.T) { + t.Parallel() + + fsm := &kvFSM{store: store.NewMVCCStore()} + err := haltApplyOf(fsm.Apply([]byte{raftEncodeMigrationPromote, 0xff, 0xff})) + require.True(t, errors.Is(err, ErrMigrationPromoteApply), "got %v", err) +} diff --git a/store/lsm_store_applied_index_test.go b/store/lsm_store_applied_index_test.go index 02c77b1d3..4d4ecd4da 100644 --- a/store/lsm_store_applied_index_test.go +++ b/store/lsm_store_applied_index_test.go @@ -135,6 +135,24 @@ func TestPromoteVersions_BundlesMetaAppliedIndex(t *testing.T) { require.Equal(t, []byte("v10"), val) _, err = ps.GetAt(ctx, stage("k"), 10) require.ErrorIs(t, err, ErrKeyNotFound) + + const retryEntryIdx uint64 = 78 + retry, err := ps.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 9, + AppliedIndex: retryEntryIdx, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, retry.Done) + require.Equal(t, uint64(1), retry.TotalPromotedRows) + + got, present, err = ps.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present, "completed PromoteVersions retry must persist metaAppliedIndex") + require.Equal(t, retryEntryIdx, got) } // TestApplyMutationsRaftAt_ZeroIndexLeavesMetaKey covers the diff --git a/store/migration_promote.go b/store/migration_promote.go index 91af77c7e..a0b8174a5 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -218,7 +218,8 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO return PromoteVersionsResult{}, err } if opts.JobID != 0 && state.Done { - return PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows}, nil + result := PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows} + return s.finishPebblePromotion(nil, opts.JobID, nil, result, opts.AppliedIndex) } opts.Cursor = cursor exported, toPromote, promoted, err := s.planPebblePromotionLocked(ctx, opts) @@ -236,7 +237,7 @@ func (s *pebbleStore) finishPebblePromotion( result PromoteVersionsResult, appliedIndex uint64, ) (PromoteVersionsResult, error) { - if len(toPromote) == 0 && stateToWrite == nil { + if len(toPromote) == 0 && stateToWrite == nil && appliedIndex == 0 { return result, nil } if err := s.commitPebblePromoteVersions(toPromote, jobID, stateToWrite, appliedIndex); err != nil { From 5ba3e82db274ca12a076e70d4092048e811d565e Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 03:18:10 +0900 Subject: [PATCH 022/162] kv: enforce target readiness guards --- kv/fsm.go | 110 ++++++++++++++++++++++++++---- kv/fsm_migration_fence_test.go | 57 ++++++++++++++++ kv/route_history.go | 18 +++++ kv/shard_store.go | 57 ++++++++++++++-- kv/shard_store_test.go | 98 ++++++++++++++++++++++++-- store/lsm_store.go | 42 +++++++++--- store/lsm_store_test.go | 33 +++++++++ store/mvcc_store.go | 105 +++++++++++++++++++++++----- store/mvcc_store_snapshot_test.go | 55 +++++++++++++++ 9 files changed, 526 insertions(+), 49 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 269b13954..a4bc98dfe 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -127,6 +127,12 @@ type RouteSnapshot interface { // WriteFencedIntersects reports whether [start, end) intersects // any WriteFenced route in this snapshot. WriteFencedIntersects(start, end []byte) bool + // WriteFloorForKey returns the post-migration write timestamp floor + // for key, when the current route retains one. + WriteFloorForKey(key []byte) (uint64, bool) + // WriteFloorIntersects returns the maximum post-migration write + // timestamp floor across routes intersecting [start, end). + WriteFloorIntersects(start, end []byte) (uint64, bool) } // SetApplyIndex implements raftengine.ApplyIndexAware. The engine @@ -501,17 +507,7 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui } for _, mut := range r.Mutations { - if mut == nil || len(mut.Key) == 0 { - return errors.WithStack(ErrInvalidRequest) - } - // Raw requests should not mutate txn-internal keys. - if isTxnInternalKey(mut.Key) { - return errors.WithStack(ErrInvalidRequest) - } - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { - return err - } - if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { + if err := f.verifyRawMutationCanApply(ctx, mut, commitTS); err != nil { return err } } @@ -529,6 +525,26 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui return nil } +func (f *kvFSM) verifyRawMutationCanApply(ctx context.Context, mut *pb.Mutation, commitTS uint64) error { + if mut == nil || len(mut.Key) == 0 { + return errors.WithStack(ErrInvalidRequest) + } + // Raw requests should not mutate txn-internal keys. + if isTxnInternalKey(mut.Key) { + return errors.WithStack(ErrInvalidRequest) + } + if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { + return err + } + if err := f.verifyRouteWriteFloorForKey(mut.Key, commitTS); err != nil { + return err + } + if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { + return err + } + return nil +} + // extractDelPrefix checks if the mutations contain a DEL_PREFIX operation. // If found, it validates that no other operation types are mixed in. func extractDelPrefix(muts []*pb.Mutation) (bool, []byte) { @@ -546,6 +562,9 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { return err } + if err := f.verifyRouteWriteFloorForPrefix(prefix, commitTS); err != nil { + return err + } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -595,6 +614,50 @@ func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", prefix, start, end) } +func (f *kvFSM) verifyRouteWriteFloorForMutations(muts []*pb.Mutation, commitTS uint64) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyRouteWriteFloorForKey(mut.Key, commitTS); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyRouteWriteFloorForKey(key []byte, commitTS uint64) error { + if f.routes == nil { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + rkey := routeKey(key) + floor, ok := snap.WriteFloorForKey(rkey) + if !ok || commitTS > floor { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q routeKey %q", commitTS, floor, key, rkey) +} + +func (f *kvFSM) verifyRouteWriteFloorForPrefix(prefix []byte, commitTS uint64) error { + if f.routes == nil { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + start, end := routePrefixRange(prefix) + floor, ok := snap.WriteFloorIntersects(start, end) + if !ok || commitTS > floor { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for prefix %q route range [%q,%q)", commitTS, floor, prefix, start, end) +} + func routePrefixRange(prefix []byte) ([]byte, []byte) { if len(prefix) == 0 { return []byte(""), nil @@ -970,6 +1033,9 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { return err } + if err := f.verifyRouteWriteFloorForMutations(uniq, startTS); err != nil { + return err + } if err := f.validateConflicts(ctx, uniq, startTS); err != nil { return errors.WithStack(err) } @@ -1036,7 +1102,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsNotFenced(muts) + uniq, err := f.uniqueMutationsNotFenced(muts, commitTS) if err != nil { return err } @@ -1052,7 +1118,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } -func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation) ([]*pb.Mutation, error) { +func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { return nil, err @@ -1060,6 +1126,9 @@ func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation) ([]*pb.Mutation, e if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { return nil, err } + if err := f.verifyRouteWriteFloorForMutations(uniq, commitTS); err != nil { + return nil, err + } return uniq, nil } @@ -1102,7 +1171,7 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueMutationsAboveWriteFloor(muts, commitTS) if err != nil { return err } @@ -1219,7 +1288,7 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u // shouldClearAbortKey (lock-missing ⇒ nothing to do) and for the // rollback-marker Put in appendRollbackRecord. - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueMutationsAboveWriteFloor(muts, abortTS) if err != nil { return err } @@ -1239,6 +1308,17 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u return errors.WithStack(f.store.ApplyMutationsRaftAt(ctx, storeMuts, nil, startTS, abortTS, f.pendingApplyIdx)) } +func (f *kvFSM) uniqueMutationsAboveWriteFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + uniq, err := uniqueMutations(muts) + if err != nil { + return nil, err + } + if err := f.verifyRouteWriteFloorForMutations(uniq, commitTS); err != nil { + return nil, err + } + return uniq, nil +} + func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt uint64) ([]*store.KVPairMutation, error) { storeMuts := make([]*store.KVPairMutation, 0, len(muts)*txnPrepareStoreMutationFactor) for _, mut := range muts { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index e47e71f95..00b02e31b 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -22,6 +22,23 @@ func newWriteFencedFSM(t *testing.T) *kvFSM { return newComposed1FSM(t, engine, 1) } +func newWriteFloorFSM(t *testing.T) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + }) + return newComposed1FSM(t, engine, 1) +} + func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() @@ -67,6 +84,35 @@ func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMRejectsRawPointWriteBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMRejectsDelPrefixBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("b"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("b")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + func TestFSMRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() @@ -84,6 +130,17 @@ func TestFSMRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMRejectsOnePhaseTxnBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleTxnRequest(context.Background(), onePhaseReq(10, 100, 0, []byte("b"), []byte("v")), 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { t.Parallel() diff --git a/kv/route_history.go b/kv/route_history.go index c6ac85608..bd6989513 100644 --- a/kv/route_history.go +++ b/kv/route_history.go @@ -74,3 +74,21 @@ func (s distributionRouteSnapshot) WriteFencedIntersects(start, end []byte) bool } return false } + +func (s distributionRouteSnapshot) WriteFloorForKey(key []byte) (uint64, bool) { + route, ok := s.snap.RouteOf(key) + if !ok || route.MinWriteTSExclusive == 0 { + return 0, false + } + return route.MinWriteTSExclusive, true +} + +func (s distributionRouteSnapshot) WriteFloorIntersects(start, end []byte) (uint64, bool) { + var maxFloor uint64 + for _, route := range s.snap.IntersectingRoutes(start, end) { + if route.MinWriteTSExclusive > maxFloor { + maxFloor = route.MinWriteTSExclusive + } + } + return maxFloor, maxFloor > 0 +} diff --git a/kv/shard_store.go b/kv/shard_store.go index aaf18b57c..f2e1beba6 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -63,12 +63,13 @@ func (s *ShardStore) GetGroupAt(ctx context.Context, groupID uint64, key []byte, if !ok || g.Store == nil { return nil, store.ErrKeyNotFound } + route := s.explicitGroupRouteForKey(groupID, key) if engineForGroup(g) == nil { - return s.localGetAt(ctx, g, distribution.Route{}, key, ts) + return s.localGetAt(ctx, g, route, key, ts) } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.leaderGetAt(ctx, g, distribution.Route{}, key, ts) + return s.leaderGetAt(ctx, g, route, key, ts) } return s.proxyRawGet(ctx, g, key, ts, groupID) } @@ -142,8 +143,9 @@ func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *Shard if err != nil { return errors.WithStack(err) } + routeStart, routeEnd := readinessRouteRange(start, end) for _, ready := range states { - if !ready.Armed || !routeRangeIntersects(start, end, ready.RouteStart, ready.RouteEnd) { + if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { continue } if !routeSatisfiesTargetReadiness(route, ready) { @@ -153,6 +155,18 @@ func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *Shard return nil } +func readinessRouteRange(start []byte, end []byte) ([]byte, []byte) { + routeStart := routeKey(start) + if end == nil { + return routeStart, nil + } + routeEnd := routeKey(end) + if bytes.Compare(routeEnd, routeStart) <= 0 { + routeEnd = nextScanCursor(routeStart) + } + return routeStart, routeEnd +} + func verifyRouteWriteFloor(route distribution.Route, commitTS uint64) error { if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { return nil @@ -350,7 +364,7 @@ func (s *ShardStore) ScanGroupAt(ctx context.Context, groupID uint64, start []by if limit <= 0 { return []*store.KVPair{}, nil } - return s.scanRouteAtDirection(ctx, distribution.Route{GroupID: groupID}, start, end, limit, ts, false, true) + return s.scanRouteAtDirection(ctx, s.explicitGroupRouteForRange(groupID, start, end), start, end, limit, ts, false, true) } func (s *ShardStore) ReverseScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { @@ -558,6 +572,9 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( if !ok || g == nil || g.Store == nil { return nil, false, nil } + if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + return nil, false, err + } if engineForGroup(g) == nil { if routeHasStagedVisibility(route) { @@ -1739,6 +1756,9 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. if err != nil || group == nil { return err } + if err := s.verifyMutationRoutes(ctx, mutations, readKeys, commitTS); err != nil { + return err + } return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) } @@ -1750,6 +1770,9 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor if err != nil || group == nil { return err } + if err := s.verifyMutationRoutes(ctx, mutations, readKeys, commitTS); err != nil { + return err + } return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) } @@ -2020,6 +2043,32 @@ func (s *ShardStore) groupForKey(key []byte) (*ShardGroup, bool) { return g, ok } +func (s *ShardStore) explicitGroupRouteForKey(groupID uint64, key []byte) distribution.Route { + fallback := distribution.Route{GroupID: groupID} + if s == nil || s.engine == nil { + return fallback + } + route, ok := s.engine.GetRoute(routeKey(key)) + if !ok || route.GroupID != groupID { + return fallback + } + return route +} + +func (s *ShardStore) explicitGroupRouteForRange(groupID uint64, start []byte, end []byte) distribution.Route { + fallback := distribution.Route{GroupID: groupID} + if s == nil || s.engine == nil { + return fallback + } + routeStart, routeEnd := readinessRouteRange(start, end) + for _, route := range s.engine.GetIntersectingRoutes(routeStart, routeEnd) { + if route.GroupID == groupID { + return route + } + } + return fallback +} + func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *ShardGroup, bool) { route, ok := s.engine.GetRoute(routeKey(key)) if !ok { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index cfe9aec59..c1fe77814 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -33,7 +33,7 @@ func newStagedVisibilityShardStore(t *testing.T) (*ShardStore, *ShardGroup) { return NewShardStore(engine, map[uint64]*ShardGroup{1: group}), group } -func applyTargetReadiness(t *testing.T, group *ShardGroup, floor uint64) { +func applyTargetReadiness(t *testing.T, group *ShardGroup) { t.Helper() writer, ok := group.Store.(store.MigrationTargetReadinessWriter) require.True(t, ok) @@ -43,7 +43,7 @@ func applyTargetReadiness(t *testing.T, group *ShardGroup, floor uint64) { RouteEnd: []byte("z"), ExpectedCutoverVersion: 2, MigrationJobID: 9, - MinWriteTSExclusive: floor, + MinWriteTSExclusive: 100, Armed: true, })) } @@ -94,7 +94,7 @@ func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T GroupID: 1, State: distribution.RouteStateActive, }) - applyTargetReadiness(t, group, 100) + applyTargetReadiness(t, group) _, err := st.GetAt(ctx, []byte("k"), 120) require.ErrorIs(t, err, ErrRouteCutoverPending) @@ -103,12 +103,32 @@ func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreTargetReadinessNormalizesInternalKeyRange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + itemKey := store.ListItemKey([]byte("b"), 0) + require.NoError(t, group.Store.PutAt(ctx, itemKey, []byte("item"), 120, 0)) + + _, err := st.GetAt(ctx, itemKey, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestShardStoreTargetReadinessAcceptsStagedDescriptor(t *testing.T) { t.Parallel() ctx := context.Background() st, group := newStagedVisibilityShardStore(t) - applyTargetReadiness(t, group, 100) + applyTargetReadiness(t, group) rawKey := []byte("k") require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 120, 0)) @@ -129,7 +149,7 @@ func TestShardStoreTargetReadinessAcceptsClearedDescriptorAndRetainsFloor(t *tes State: distribution.RouteStateActive, MinWriteTSExclusive: 100, }) - applyTargetReadiness(t, group, 100) + applyTargetReadiness(t, group) require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 120, 0)) got, err := st.GetAt(ctx, []byte("k"), 130) @@ -160,6 +180,74 @@ func TestShardStoreTargetReadinessAcceptsClearedDescriptorAndRetainsFloor(t *tes require.NoError(t, err) } +func TestShardStoreExplicitGroupReadUsesRouteProofForReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live"), 120, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("c"), []byte("scan"), 121, 0)) + + got, err := st.GetGroupAt(ctx, 42, []byte("b"), 130) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.NoError(t, err) + require.Len(t, kvs, 2) +} + +func TestShardStorePhysicalLimitScanChecksTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + userKey := []byte("b") + start := store.ListItemKey(userKey, 0) + end := store.ListItemKey(userKey, 2) + require.NoError(t, group.Store.PutAt(ctx, start, []byte("v"), 120, 0)) + + _, _, err := st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreApplyMutationsRaftAtEnforcesRouteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, _ := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + + err := st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("b"), + Value: []byte("low"), + }}, nil, 0, 100, 7) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) +} + func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { t.Parallel() diff --git a/store/lsm_store.go b/store/lsm_store.go index a1a2ac350..dbd38e3a1 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -2273,6 +2273,7 @@ func (s *pebbleStore) handleRestorePeekError(err error, header []byte) error { s.lastCommitTS = 0 s.minRetainedTS = 0 s.pendingMinRetainedTS = 0 + s.migrationReadinessCache = nil if setErr := writePebbleUint64(s.db, metaLastCommitTSBytes, 0, pebble.NoSync); setErr != nil { return errors.WithStack(setErr) } @@ -2443,22 +2444,34 @@ func writeNativeSnapshotToTempDir(r io.Reader, tmpDir string, ts uint64) error { // Entries are written to a temporary Pebble directory and only swapped into // place after the CRC32 checksum is verified, preserving the existing store // on failure. -func readStreamingMVCCRestoreHeader(r io.Reader) (io.Reader, hash.Hash32, uint32, uint64, uint64, error) { - expectedChecksum, err := readMVCCSnapshotHeader(r) +func readStreamingMVCCRestoreHeader(r io.Reader) (io.Reader, hash.Hash32, uint32, uint64, uint64, []TargetStagedReadinessState, error) { + expectedChecksum, version, err := readMVCCSnapshotHeader(r) if err != nil { - return nil, nil, 0, 0, 0, err + return nil, nil, 0, 0, 0, nil, err } hash := crc32.NewIEEE() body := io.TeeReader(r, hash) lastCommitTS, minRetainedTS, err := readMVCCSnapshotMetadata(body) if err != nil { - return nil, nil, 0, 0, 0, err + return nil, nil, 0, 0, 0, nil, err } - return body, hash, expectedChecksum, lastCommitTS, minRetainedTS, nil + readinessStates, err := readMVCCSnapshotReadinessStates(body, version) + if err != nil { + return nil, nil, 0, 0, 0, nil, err + } + return body, hash, expectedChecksum, lastCommitTS, minRetainedTS, readinessStates, nil } -func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash32, expectedChecksum uint32, lastCommitTS uint64, minRetainedTS uint64) (string, error) { +func writeStreamingMVCCRestoreTempDB( + dir string, + body io.Reader, + hash hash.Hash32, + expectedChecksum uint32, + lastCommitTS uint64, + minRetainedTS uint64, + readinessStates []TargetStagedReadinessState, +) (string, error) { tmpDir := filepath.Clean(dir) + ".restore-tmp" if err := os.RemoveAll(tmpDir); err != nil { return "", errors.WithStack(err) @@ -2478,6 +2491,10 @@ func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash3 _ = os.RemoveAll(tmpDir) } + if err := writePebbleTargetReadinessStates(tmpDB, readinessStates); err != nil { + cleanupTmp() + return "", err + } if err := writeMVCCEntriesToDB(body, tmpDB); err != nil { cleanupTmp() return "", err @@ -2497,13 +2514,22 @@ func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash3 return tmpDir, nil } +func writePebbleTargetReadinessStates(db *pebble.DB, states []TargetStagedReadinessState) error { + for _, state := range states { + if err := db.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), pebble.NoSync); err != nil { + return errors.WithStack(err) + } + } + return nil +} + func (s *pebbleStore) restoreFromStreamingMVCC(r io.Reader) error { - body, hash, expectedChecksum, lastCommitTS, minRetainedTS, err := readStreamingMVCCRestoreHeader(r) + body, hash, expectedChecksum, lastCommitTS, minRetainedTS, readinessStates, err := readStreamingMVCCRestoreHeader(r) if err != nil { return err } - tmpDir, err := writeStreamingMVCCRestoreTempDB(s.dir, body, hash, expectedChecksum, lastCommitTS, minRetainedTS) + tmpDir, err := writeStreamingMVCCRestoreTempDB(s.dir, body, hash, expectedChecksum, lastCommitTS, minRetainedTS, readinessStates) if err != nil { return err } diff --git a/store/lsm_store_test.go b/store/lsm_store_test.go index 27b0fe306..5e5a8a366 100644 --- a/store/lsm_store_test.go +++ b/store/lsm_store_test.go @@ -690,6 +690,18 @@ func TestPebbleStore_RestoreFromStreamingMVCC(t *testing.T) { require.NoError(t, src.PutAt(ctx, []byte("key1"), []byte("val1-updated"), 20, 0)) require.NoError(t, src.DeleteAt(ctx, []byte("key2"), 15)) require.NoError(t, src.PutWithTTLAt(ctx, []byte("key3"), []byte("val3"), 30, 9999)) + readyState := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + readyWriter, ok := src.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, readyWriter.ApplyTargetStagedReadiness(ctx, readyState)) snap, err := src.Snapshot() require.NoError(t, err) @@ -727,6 +739,11 @@ func TestPebbleStore_RestoreFromStreamingMVCC(t *testing.T) { assert.Equal(t, []byte("val3"), val) assert.Equal(t, src.LastCommitTS(), dst.LastCommitTS()) + readyReader, ok := dst.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := readyReader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{readyState}, states) } func TestPebbleStore_RestoreFromStreamingMVCCPreservesMinRetainedTS(t *testing.T) { @@ -775,6 +792,17 @@ func TestPebbleStore_Restore_EmptySnapshot(t *testing.T) { // Pre-populate so we can verify the restore clears the data. require.NoError(t, s.PutAt(ctx, []byte("k"), []byte("v"), 42, 0)) + readyWriter, ok := s.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, readyWriter.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) requirePebbleRetentionController(t, s).SetMinRetainedTS(21) require.Equal(t, uint64(42), s.LastCommitTS()) @@ -786,6 +814,11 @@ func TestPebbleStore_Restore_EmptySnapshot(t *testing.T) { assert.ErrorIs(t, err, ErrKeyNotFound) assert.Equal(t, uint64(0), s.LastCommitTS()) assert.Equal(t, uint64(0), requirePebbleRetentionController(t, s).MinRetainedTS()) + readyReader, ok := s.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := readyReader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + assert.Empty(t, states) } // TestPebbleStore_Restore_TruncatedHeader verifies that a partial magic diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 1bea2a9f7..926032757 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -25,9 +25,12 @@ type VersionedValue struct { } const ( - mvccSnapshotVersion = uint32(1) - maxSnapshotKeySize = 1 << 20 // 1 MiB per key - maxSnapshotVersionCount = 1 << 20 // 1M versions per key + mvccSnapshotVersion = uint32(2) + mvccSnapshotLegacyVersion = uint32(1) + maxSnapshotKeySize = 1 << 20 // 1 MiB per key + maxSnapshotVersionCount = 1 << 20 // 1M versions per key + maxSnapshotReadinessStateCount = 1 << 20 + maxSnapshotReadinessEncodedBytes = 1 << 20 ) // maxSnapshotValueSize caps the allowed size of a single value during streaming @@ -843,6 +846,9 @@ func (s *mvccStore) writeSnapshotBody(f *os.File) (uint32, error) { if err := binary.Write(w, binary.LittleEndian, s.minRetainedTS); err != nil { return 0, errors.WithStack(err) } + if err := writeMVCCSnapshotReadinessStates(w, s.migrationReadinessCache); err != nil { + return 0, err + } iter := s.tree.Iterator() for iter.Next() { key, ok := iter.Key().([]byte) @@ -913,6 +919,22 @@ func writeMVCCSnapshotVersion(w io.Writer, version VersionedValue) error { return nil } +func writeMVCCSnapshotReadinessStates(w io.Writer, states []TargetStagedReadinessState) error { + if err := binary.Write(w, binary.LittleEndian, uint64(len(states))); err != nil { + return errors.WithStack(err) + } + for _, state := range states { + encoded := encodeTargetStagedReadinessState(state) + if err := binary.Write(w, binary.LittleEndian, uint64(len(encoded))); err != nil { + return errors.WithStack(err) + } + if _, err := w.Write(encoded); err != nil { + return errors.WithStack(err) + } + } + return nil +} + func mvccSnapshotTombstoneByte(tombstone bool) byte { if tombstone { return 1 @@ -921,12 +943,12 @@ func mvccSnapshotTombstoneByte(tombstone bool) byte { } func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { - expected, err := readMVCCSnapshotHeader(r) + expected, version, err := readMVCCSnapshotHeader(r) if err != nil { return err } - tree, lastCommitTS, minRetainedTS, actual, err := restoreStreamingMVCCSnapshotBody(r) + tree, lastCommitTS, minRetainedTS, readinessStates, actual, err := restoreStreamingMVCCSnapshotBody(r, version) if err != nil { return err } @@ -939,48 +961,54 @@ func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { s.tree = tree s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS + s.migrationReadiness = targetReadinessStateMap(readinessStates) + s.migrationReadinessCache = cloneTargetStagedReadinessStates(readinessStates) return nil } -func readMVCCSnapshotHeader(r io.Reader) (uint32, error) { +func readMVCCSnapshotHeader(r io.Reader) (uint32, uint32, error) { var magic [8]byte if _, err := io.ReadFull(r, magic[:]); err != nil { - return 0, errors.WithStack(err) + return 0, 0, errors.WithStack(err) } if magic != mvccSnapshotMagic { - return 0, errors.WithStack(ErrInvalidChecksum) + return 0, 0, errors.WithStack(ErrInvalidChecksum) } var version uint32 if err := binary.Read(r, binary.LittleEndian, &version); err != nil { - return 0, errors.WithStack(err) + return 0, 0, errors.WithStack(err) } - if version != mvccSnapshotVersion { - return 0, errors.WithStack(errors.Newf("unsupported mvcc snapshot version %d", version)) + if version != mvccSnapshotLegacyVersion && version != mvccSnapshotVersion { + return 0, 0, errors.WithStack(errors.Newf("unsupported mvcc snapshot version %d", version)) } var expected uint32 if err := binary.Read(r, binary.LittleEndian, &expected); err != nil { - return 0, errors.WithStack(err) + return 0, 0, errors.WithStack(err) } - return expected, nil + return expected, version, nil } -func restoreStreamingMVCCSnapshotBody(r io.Reader) (*treemap.Map, uint64, uint64, uint32, error) { +func restoreStreamingMVCCSnapshotBody(r io.Reader, version uint32) (*treemap.Map, uint64, uint64, []TargetStagedReadinessState, uint32, error) { hash := crc32.NewIEEE() body := io.TeeReader(r, hash) lastCommitTS, minRetainedTS, err := readMVCCSnapshotMetadata(body) if err != nil { - return nil, 0, 0, 0, err + return nil, 0, 0, nil, 0, err + } + readinessStates, err := readMVCCSnapshotReadinessStates(body, version) + if err != nil { + return nil, 0, 0, nil, 0, err } tree, err := readMVCCSnapshotTree(body) if err != nil { - return nil, 0, 0, 0, err + return nil, 0, 0, nil, 0, err } - return tree, lastCommitTS, minRetainedTS, hash.Sum32(), nil + return tree, lastCommitTS, minRetainedTS, readinessStates, hash.Sum32(), nil } func readMVCCSnapshotMetadata(r io.Reader) (uint64, uint64, error) { @@ -995,6 +1023,49 @@ func readMVCCSnapshotMetadata(r io.Reader) (uint64, uint64, error) { return lastCommitTS, minRetainedTS, nil } +func readMVCCSnapshotReadinessStates(r io.Reader, version uint32) ([]TargetStagedReadinessState, error) { + if version < mvccSnapshotVersion { + return nil, nil + } + var count uint64 + if err := binary.Read(r, binary.LittleEndian, &count); err != nil { + return nil, errors.WithStack(err) + } + if count > maxSnapshotReadinessStateCount { + return nil, errors.Wrapf(ErrSnapshotVersionCountTooLarge, "readiness states %d > %d", count, maxSnapshotReadinessStateCount) + } + states := make([]TargetStagedReadinessState, 0, count) + for range count { + var encodedLen uint64 + if err := binary.Read(r, binary.LittleEndian, &encodedLen); err != nil { + return nil, errors.WithStack(err) + } + if encodedLen > maxSnapshotReadinessEncodedBytes { + return nil, errors.Wrapf(ErrValueTooLarge, "readiness state %d > %d", encodedLen, maxSnapshotReadinessEncodedBytes) + } + encoded := make([]byte, encodedLen) + if _, err := io.ReadFull(r, encoded); err != nil { + return nil, errors.WithStack(err) + } + state, ok := decodeTargetStagedReadinessState(encoded) + if !ok { + return nil, errors.New("corrupt target staged readiness state") + } + states = append(states, state) + } + sortTargetStagedReadinessStates(states) + return states, nil +} + +func targetReadinessStateMap(states []TargetStagedReadinessState) map[uint64]TargetStagedReadinessState { + out := make(map[uint64]TargetStagedReadinessState, len(states)) + for _, state := range states { + cloned := cloneTargetStagedReadinessState(state) + out[cloned.JobID] = cloned + } + return out +} + func readMVCCSnapshotTree(r io.Reader) (*treemap.Map, error) { tree := treemap.NewWith(byteSliceComparator) for { diff --git a/store/mvcc_store_snapshot_test.go b/store/mvcc_store_snapshot_test.go index 3ebdbcac8..d7dfb93c4 100644 --- a/store/mvcc_store_snapshot_test.go +++ b/store/mvcc_store_snapshot_test.go @@ -38,6 +38,61 @@ func TestMVCCStore_SnapshotRestoreRoundTrip(t *testing.T) { require.Equal(t, []byte("v2"), v) } +func TestMVCCStore_SnapshotRestorePreservesTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + src := newTestMVCCStore(t) + + stale := TargetStagedReadinessState{ + JobID: 8, + RouteStart: []byte("old"), + RouteEnd: []byte("oldz"), + ExpectedCutoverVersion: 1, + MigrationJobID: 8, + MinWriteTSExclusive: 80, + Armed: true, + } + require.NoError(t, src.ApplyTargetStagedReadiness(ctx, stale)) + + snapState := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + require.NoError(t, src.ApplyTargetStagedReadiness(ctx, snapState)) + + snap, err := src.Snapshot() + require.NoError(t, err) + defer snap.Close() + raw := snapshotBytes(t, snap) + + dst := newTestMVCCStore(t) + require.NoError(t, dst.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 77, + RouteStart: []byte("dst-only"), + RouteEnd: []byte("dst-z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 77, + MinWriteTSExclusive: 700, + Armed: true, + })) + + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + states, err := dst.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{stale, snapState}, states) + + states[0].RouteStart[0] = 'x' + states, err = dst.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []byte("old"), states[0].RouteStart) +} + func TestMVCCStore_RestoreRejectsInvalidChecksum(t *testing.T) { t.Parallel() From b04d30fe3cfe3dab6bd7bfdadf07920d1a886681 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 03:40:50 +0900 Subject: [PATCH 023/162] store: keep promote replay ungated --- store/lsm_migration.go | 8 +++-- store/lsm_store_registration_gate_test.go | 44 +++++++++++++++++++++++ store/migration_promote.go | 4 ++- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index e0796cbf6..d0c66b822 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -247,7 +247,9 @@ func (s *pebbleStore) validatePebbleImportBatch(opts ImportVersionsOptions) (boo func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchMax uint64) error { batch := s.db.NewBatch() defer batch.Close() - if err := s.applyImportVersionsBatch(batch, opts.Versions); err != nil { + // Keep the existing import registration-gate behavior; promotion opts out + // at its call site because it replays committed Raft entries. + if err := s.applyImportVersionsBatch(batch, opts.Versions, true); err != nil { return err } if err := batch.Set(migrationAckKey(opts.JobID, opts.BracketID), encodeMigrationImportAck(migrationImportAck{ @@ -277,14 +279,14 @@ func (s *pebbleStore) stageMigrationClockMetadataIfNeeded(batch *pebble.Batch, j return s.stageMigrationClockMetadata(batch, jobID, batchMax) } -func (s *pebbleStore) applyImportVersionsBatch(batch *pebble.Batch, versions []MVCCVersion) error { +func (s *pebbleStore) applyImportVersionsBatch(batch *pebble.Batch, versions []MVCCVersion, gateRegistration bool) error { for _, version := range versions { k := encodeKey(version.Key, version.CommitTS) var encoded []byte if version.Tombstone { encoded = encodeValue(nil, true, 0, encStateCleartext) } else { - body, encState, err := s.encryptForKey(k, version.Value, version.ExpireAt, true) + body, encState, err := s.encryptForKey(k, version.Value, version.ExpireAt, gateRegistration) if err != nil { return err } diff --git a/store/lsm_store_registration_gate_test.go b/store/lsm_store_registration_gate_test.go index 4dbfeb6a5..11e6d14fc 100644 --- a/store/lsm_store_registration_gate_test.go +++ b/store/lsm_store_registration_gate_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "path/filepath" "testing" @@ -135,6 +136,49 @@ func TestRegistrationGate_FSMApplyPathNeverGated(t *testing.T) { mustGet(t, f.mvcc, []byte("raft"), 150, "applied") } +func TestRegistrationGate_PromoteVersionsNeverGated(t *testing.T) { + t.Parallel() + ctx := context.Background() + registered := true + f := newRegGateStore(t, ®istered) + stage := func(raw string) []byte { + return append([]byte("stage|"), []byte(raw)...) + } + targetKey := func(staged []byte) ([]byte, bool) { + return bytes.TrimPrefix(staged, []byte("stage|")), bytes.HasPrefix(staged, []byte("stage|")) + } + + if err := f.mvcc.PutAt(ctx, stage("promote"), []byte("value"), 100, 0); err != nil { + t.Fatalf("seed staged PutAt: %v", err) + } + registered = false + if err := f.mvcc.PutAt(ctx, []byte("direct"), []byte("blocked"), 110, 0); !errors.Is(err, ErrWriterNotRegistered) { + t.Fatalf("direct PutAt pre-registration: got %v, want ErrWriterNotRegistered", err) + } + promoter, ok := f.mvcc.(MigrationPromoter) + if !ok { + t.Fatalf("expected MigrationPromoter, got %T", f.mvcc) + } + + result, err := promoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 11, + StartKey: []byte("stage|"), + EndKey: PrefixScanEnd([]byte("stage|")), + MaxVersions: 10, + TargetKey: targetKey, + }) + if err != nil { + t.Fatalf("PromoteVersions pre-registration: %v", err) + } + if !result.Done || result.PromotedRows != 1 { + t.Fatalf("PromoteVersions result = %+v, want done with one promoted row", result) + } + mustGet(t, f.mvcc, []byte("promote"), 150, "value") + if _, err := f.mvcc.GetAt(ctx, stage("promote"), 150); !errors.Is(err, ErrKeyNotFound) { + t.Fatalf("staged version after promotion: got %v, want ErrKeyNotFound", err) + } +} + // TestRegistrationGate_NotEncryptingIsUngated confirms the gate is only // consulted when the write WOULD encrypt. When the cutover gate is off // (envelope inactive) the direct path writes cleartext and never diff --git a/store/migration_promote.go b/store/migration_promote.go index a0b8174a5..12cc79389 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -329,7 +329,9 @@ func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jo for _, version := range versions { targets = append(targets, version.target) } - if err := s.applyImportVersionsBatch(batch, targets); err != nil { + // Promotion is replayed from the Raft FSM, so it must not fail closed on + // this node's local writer-registration state. + if err := s.applyImportVersionsBatch(batch, targets, false); err != nil { return err } for _, version := range versions { From a2ee041227b34923434c2ada1f931c3c4593d128 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 04:19:48 +0900 Subject: [PATCH 024/162] Guard target readiness apply paths --- kv/fsm.go | 105 ++++++++++++++++-- kv/fsm_migration_fence_test.go | 82 ++++++++++++++ kv/route_history.go | 8 ++ kv/shard_store.go | 43 +++++++- kv/shard_store_test.go | 44 ++++++++ kv/shard_store_txn_lock_test.go | 128 ++++++++++++++++++++++ kv/sharded_coordinator.go | 21 ++++ kv/sharded_coordinator_del_prefix_test.go | 27 +++++ 8 files changed, 443 insertions(+), 15 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index a4bc98dfe..58fdccf32 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -9,6 +9,7 @@ import ( "log/slog" "os" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/encryption/fsmwire" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/internal/s3keys" @@ -121,6 +122,13 @@ type RouteSnapshot interface { // OwnerOf returns the Raft group ID that owned key at this // snapshot's version. (0, false) when no route covered key. OwnerOf(key []byte) (uint64, bool) + // RouteOf returns the route descriptor that covered key at this + // snapshot's version. Used by apply-time target-readiness checks to + // prove staged/cleared descriptor state before mutating the store. + RouteOf(key []byte) (distribution.Route, bool) + // IntersectingRoutes returns the route descriptors that intersect + // [start, end) at this snapshot's version. A nil end denotes +infinity. + IntersectingRoutes(start, end []byte) []distribution.Route // WriteFencedForKey reports whether key is currently inside a // WriteFenced route in this snapshot. WriteFencedForKey(key []byte) bool @@ -536,6 +544,9 @@ func (f *kvFSM) verifyRawMutationCanApply(ctx context.Context, mut *pb.Mutation, if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { return err } + if err := f.verifyTargetReadinessForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } if err := f.verifyRouteWriteFloorForKey(mut.Key, commitTS); err != nil { return err } @@ -562,6 +573,9 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { return err } + if err := f.verifyTargetReadinessForPrefix(ctx, prefix); err != nil { + return err + } if err := f.verifyRouteWriteFloorForPrefix(prefix, commitTS); err != nil { return err } @@ -626,6 +640,73 @@ func (f *kvFSM) verifyRouteWriteFloorForMutations(muts []*pb.Mutation, commitTS return nil } +func (f *kvFSM) verifyTargetReadinessForMutations(ctx context.Context, muts []*pb.Mutation) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyTargetReadinessForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyTargetReadinessForPrefix(ctx context.Context, prefix []byte) error { + start, end := routePrefixRange(prefix) + return f.verifyTargetReadinessForRouteRange(ctx, start, end) +} + +func (f *kvFSM) verifyTargetReadinessForRange(ctx context.Context, start []byte, end []byte) error { + routeStart, routeEnd := readinessRouteRange(start, end) + return f.verifyTargetReadinessForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) verifyTargetReadinessForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + if len(states) == 0 { + return nil + } + + var ( + snap RouteSnapshot + proof bool + ) + if f.routes != nil { + snap, proof = f.routes.Current() + } + for _, ready := range states { + if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + continue + } + if !proof || !routesSatisfyTargetReadiness(snap.IntersectingRoutes(routeStart, routeEnd), ready) { + return errors.WithStack(ErrRouteCutoverPending) + } + } + return nil +} + +func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState) bool { + matched := false + for _, route := range routes { + if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + continue + } + matched = true + if !routeSatisfiesTargetReadiness(route, ready) { + return false + } + } + return matched +} + func (f *kvFSM) verifyRouteWriteFloorForKey(key []byte, commitTS uint64) error { if f.routes == nil { return nil @@ -1026,16 +1107,10 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { } startTS := r.Ts - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, startTS) if err != nil { return err } - if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { - return err - } - if err := f.verifyRouteWriteFloorForMutations(uniq, startTS); err != nil { - return err - } if err := f.validateConflicts(ctx, uniq, startTS); err != nil { return errors.WithStack(err) } @@ -1102,7 +1177,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsNotFenced(muts, commitTS) + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, commitTS) if err != nil { return err } @@ -1118,7 +1193,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } -func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { +func (f *kvFSM) uniqueMutationsNotFenced(ctx context.Context, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { return nil, err @@ -1126,6 +1201,9 @@ func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation, commitTS uint64) ( if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { return nil, err } + if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { + return nil, err + } if err := f.verifyRouteWriteFloorForMutations(uniq, commitTS); err != nil { return nil, err } @@ -1171,7 +1249,7 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - uniq, err := f.uniqueMutationsAboveWriteFloor(muts, commitTS) + uniq, err := f.uniqueMutationsAboveWriteFloor(ctx, muts, commitTS) if err != nil { return err } @@ -1288,7 +1366,7 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u // shouldClearAbortKey (lock-missing ⇒ nothing to do) and for the // rollback-marker Put in appendRollbackRecord. - uniq, err := f.uniqueMutationsAboveWriteFloor(muts, abortTS) + uniq, err := f.uniqueMutationsAboveWriteFloor(ctx, muts, abortTS) if err != nil { return err } @@ -1308,11 +1386,14 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u return errors.WithStack(f.store.ApplyMutationsRaftAt(ctx, storeMuts, nil, startTS, abortTS, f.pendingApplyIdx)) } -func (f *kvFSM) uniqueMutationsAboveWriteFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { +func (f *kvFSM) uniqueMutationsAboveWriteFloor(ctx context.Context, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { return nil, err } + if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { + return nil, err + } if err := f.verifyRouteWriteFloorForMutations(uniq, commitTS); err != nil { return nil, err } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 00b02e31b..0f593df46 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -39,6 +39,26 @@ func newWriteFloorFSM(t *testing.T) *kvFSM { return newComposed1FSM(t, engine, 1) } +func newTargetReadinessFSM(t *testing.T, route distribution.RouteDescriptor) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{route}) + fsm := newComposed1FSM(t, engine, route.GroupID) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) + return fsm +} + func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() @@ -52,6 +72,46 @@ func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } +func TestFSMRejectsRawPointWriteWithoutTargetReadinessProof(t *testing.T) { + t.Parallel() + + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMAllowsRawPointWriteWithClearedTargetReadinessProof(t *testing.T) { + t.Parallel() + + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 101) + require.NoError(t, err) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { t.Parallel() @@ -68,6 +128,28 @@ func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMRejectsDelPrefixWithoutTargetReadinessProof(t *testing.T) { + t.Parallel() + + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("b"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("b")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() diff --git a/kv/route_history.go b/kv/route_history.go index bd6989513..94eec1076 100644 --- a/kv/route_history.go +++ b/kv/route_history.go @@ -61,6 +61,14 @@ func (s distributionRouteSnapshot) OwnerOf(key []byte) (uint64, bool) { return s.snap.OwnerOf(key) } +func (s distributionRouteSnapshot) RouteOf(key []byte) (distribution.Route, bool) { + return s.snap.RouteOf(key) +} + +func (s distributionRouteSnapshot) IntersectingRoutes(start, end []byte) []distribution.Route { + return s.snap.IntersectingRoutes(start, end) +} + func (s distributionRouteSnapshot) WriteFencedForKey(key []byte) bool { route, ok := s.snap.RouteOf(key) return ok && route.State == distribution.RouteStateWriteFenced diff --git a/kv/shard_store.go b/kv/shard_store.go index f2e1beba6..246fac205 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -92,6 +92,9 @@ func isLinearizableRaftLeader(ctx context.Context, engine raftengine.LeaderView) } func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return nil, err + } if !isTxnInternalKey(key) { if err := s.maybeResolveTxnLock(ctx, g, key, ts); err != nil { return nil, err @@ -132,6 +135,11 @@ func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetS } func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { + routeStart, routeEnd := readinessRouteRange(start, end) + return s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd) +} + +func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) error { if g == nil || g.Store == nil { return nil } @@ -143,7 +151,6 @@ func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *Shard if err != nil { return errors.WithStack(err) } - routeStart, routeEnd := readinessRouteRange(start, end) for _, ready := range states { if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { continue @@ -591,7 +598,7 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( if routeHasStagedVisibility(route) { return nil, true, nil } - return s.scanRouteAtLeaderPhysicalLimit(ctx, g, start, end, visibleLimit, physicalLimit, ts, reverse) + return s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) } // RawScanAt cannot enforce physicalLimit, so report truncation and let @@ -666,6 +673,7 @@ func (s *ShardStore) scanRouteLocal( func (s *ShardStore) scanRouteAtLeaderPhysicalLimit( ctx context.Context, g *ShardGroup, + route distribution.Route, start []byte, end []byte, visibleLimit int, @@ -682,7 +690,7 @@ func (s *ShardStore) scanRouteAtLeaderPhysicalLimit( if err != nil { return nil, limitReached, err } - resolved, err := s.resolveScanLocks(ctx, g, distribution.Route{}, kvs, lockKVs, ts) + resolved, err := s.resolveScanLocks(ctx, g, route, kvs, lockKVs, ts) return resolved, limitReached, err } @@ -1802,6 +1810,9 @@ func (s *ShardStore) resolveSingleShardGroup(mutations []*store.KVPairMutation) // DeletePrefixAt applies a prefix delete to every shard in the store. func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { + if err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue @@ -1815,6 +1826,9 @@ func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludeP // DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { + if err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue @@ -1841,6 +1855,9 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl // is the receiver only when an aggregate (admin / coordinator) path // is replaying a global FLUSHALL, which is not raft-applied. func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { + if err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue @@ -1862,6 +1879,26 @@ func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, ex return nil } +func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte, commitTS uint64) error { + if s == nil || s.engine == nil { + return nil + } + routeStart, routeEnd := routePrefixRange(prefix) + for _, route := range s.engine.GetIntersectingRoutes(routeStart, routeEnd) { + g, ok := s.groupForID(route.GroupID) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + if err := s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd); err != nil { + return err + } + if err := verifyRouteWriteFloor(route, commitTS); err != nil { + return err + } + } + return nil +} + func (s *ShardStore) LastCommitTS() uint64 { var max uint64 for _, g := range s.groups { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index c1fe77814..7088c01ec 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -248,6 +248,50 @@ func TestShardStoreApplyMutationsRaftAtEnforcesRouteFloor(t *testing.T) { require.ErrorIs(t, err, ErrRouteWriteBelowFloor) } +func TestShardStoreDeletePrefixAtChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAt(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtRaftEnforcesRouteFloorBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAtRaft(ctx, []byte("b"), nil, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { t.Parallel() diff --git a/kv/shard_store_txn_lock_test.go b/kv/shard_store_txn_lock_test.go index 8c50ff3a4..c395b3dc9 100644 --- a/kv/shard_store_txn_lock_test.go +++ b/kv/shard_store_txn_lock_test.go @@ -113,6 +113,36 @@ func TestShardStoreGetAt_ReturnsTxnLockedForPendingLock(t *testing.T) { require.True(t, errors.Is(err, ErrTxnLocked), "expected ErrTxnLocked, got %v", err) } +func TestShardStoreLeaderGetAtChecksReadinessBeforeResolvingPendingLock(t *testing.T) { + t.Parallel() + + ctx := context.Background() + route := distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + } + shardStore, group := newReadinessShardStore(t, route) + applyTargetReadiness(t, group) + + lockTS := uint64(1) + key := []byte("b") + require.NoError(t, group.Store.PutAt(ctx, txnLockKey(key), encodeTxnLock(txnLock{ + StartTS: lockTS, + PrimaryKey: key, + }), lockTS, 0)) + + resolvedRoute, _, ok := shardStore.routeAndGroupForKey(key) + require.True(t, ok) + _, err := shardStore.leaderGetAt(ctx, group, resolvedRoute, key, ^uint64(0)) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, lockErr := group.Store.GetAt(ctx, txnLockKey(key), ^uint64(0)) + require.NoError(t, lockErr) +} + func TestShardStoreGetAt_ReturnsTxnLockedForPendingCrossShardTxn(t *testing.T) { t.Parallel() @@ -441,6 +471,104 @@ func TestShardStoreScanAt_ResolvesCommittedSecondaryLockWithoutCommittedValue(t require.Equal(t, []byte("v2"), kvs[0].Value) } +func TestShardStorePhysicalLimitScanPreservesRouteProofDuringLockResolution(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + + st1 := store.NewMVCCStore() + fsm := NewKvFSMWithHLC(st1, NewHLC(), WithRouteHistory(WrapDistributionEngine(engine), 1)) + applyFSM, ok := fsm.(*kvFSM) + require.True(t, ok) + txn := &localApplyTransactional{fsm: applyFSM} + + groups := map[uint64]*ShardGroup{ + 1: {Store: st1, Txn: txn}, + } + shardStore := NewShardStore(engine, groups) + applyTargetReadiness(t, groups[1]) + + startTS := uint64(101) + commitTS := uint64(120) + primaryKey := []byte("b") + secondaryKey := []byte("c") + require.NoError(t, st1.PutAt(ctx, secondaryKey, []byte("old"), 1, 0)) + prepare := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: primaryKey, Value: []byte("vp")}, + {Op: pb.Op_PUT, Key: secondaryKey, Value: []byte("vs")}, + }, + } + _, err := groups[1].Txn.Commit(ctx, []*pb.Request{prepare}) + require.NoError(t, err) + commitPrimary := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: commitTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + _, err = groups[1].Txn.Commit(ctx, []*pb.Request{commitPrimary}) + require.NoError(t, err) + + route, ok := engine.GetRoute(secondaryKey) + require.True(t, ok) + kvs, _, err := shardStore.scanRouteAtLeaderPhysicalLimit(ctx, groups[1], route, []byte(""), nil, 10, 10, commitTS, false) + require.NoError(t, err) + got := map[string]string{} + for _, kvp := range kvs { + got[string(kvp.Key)] = string(kvp.Value) + } + require.Equal(t, "vs", got[string(secondaryKey)]) + + _, lockErr := st1.GetAt(ctx, txnLockKey(secondaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) +} + +type localApplyTransactional struct { + fsm *kvFSM +} + +func (t *localApplyTransactional) Commit(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + for _, req := range reqs { + if err := t.fsm.handleTxnRequest(ctx, req, txnRequestResolveTS(req)); err != nil { + return nil, err + } + } + return &TransactionResponse{}, nil +} + +func (t *localApplyTransactional) Abort(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + return t.Commit(ctx, reqs) +} + +func txnRequestResolveTS(req *pb.Request) uint64 { + meta, _, err := extractTxnMeta(req.GetMutations()) + if err == nil && meta.CommitTS != 0 { + return meta.CommitTS + } + return req.GetTs() +} + func TestShardStoreScanAt_FiltersTxnInternalKeysWithoutRaft(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 7ed2ccb7c..285036d3e 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1036,6 +1036,9 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT if err != nil { return nil, err } + if err := c.rejectDelPrefixesBelowWriteFloor(elems, ts); err != nil { + return nil, err + } requests := make([]*pb.Request, 0, len(elems)) for _, elem := range elems { requests = append(requests, &pb.Request{ @@ -1084,6 +1087,24 @@ func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) err return nil } +func (c *ShardedCoordinator) rejectDelPrefixesBelowWriteFloor(elems []*Elem[OP], commitTS uint64) error { + if c == nil || c.engine == nil { + return nil + } + for _, elem := range elems { + if elem == nil { + continue + } + start, end := routePrefixRange(elem.Key) + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if err := verifyRouteWriteFloor(route, commitTS); err != nil { + return errors.Wrapf(err, "prefix %q route range [%q,%q)", elem.Key, start, end) + } + } + } + return nil +} + // broadcastToAllGroups sends the same set of requests to every shard group in // parallel and returns the maximum commit index. func (c *ShardedCoordinator) broadcastToAllGroups(ctx context.Context, requests []*pb.Request) (*CoordinateResponse, error) { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 4b83dc131..5b712db58 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -173,6 +173,33 @@ func TestShardedCoordinatorRejectsDelPrefixIntersectingWriteFencedRoute(t *testi require.Empty(t, g2Txn.requests) } +func TestShardedCoordinatorRejectsDelPrefixBelowRouteFloorBeforeBroadcast(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive, MinWriteTSExclusive: ^uint64(0)}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + func TestShardedCoordinatorRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() From 643a5a6a5fc4698f516761a2d5a453ae490d52cc Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 04:20:43 +0900 Subject: [PATCH 025/162] store: fix migration export metadata handling --- store/lsm_migration.go | 112 +++++++++++++++----- store/lsm_store.go | 61 +++++++---- store/migration_versions.go | 171 ++++++++++++++++++++++++------- store/migration_versions_test.go | 113 ++++++++++++++++++++ store/mvcc_store.go | 6 +- store/snapshot_pebble.go | 3 + 6 files changed, 381 insertions(+), 85 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 1f6749e18..2be215212 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -11,7 +11,7 @@ import ( func (s *pebbleStore) ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { opts = normalizeExportVersionsOptions(opts) - pos, err := decodeExportCursor(opts.Cursor) + pos, err := decodeExportCursorForOptions(opts) if err != nil { return ExportVersionsResult{}, err } @@ -85,11 +85,8 @@ func pebbleExportSeekKey(opts ExportVersionsOptions, pos exportCursorPosition) ( if !pos.hasKey { return encodeKey(opts.StartKey, math.MaxUint64), nil } - if opts.StartKey != nil && bytes.Compare(pos.key, opts.StartKey) < 0 { - return nil, errors.WithStack(ErrInvalidExportCursor) - } - if opts.EndKey != nil && bytes.Compare(pos.key, opts.EndKey) >= 0 { - return nil, errors.WithStack(ErrInvalidExportCursor) + if err := validateExportCursorRange(opts, pos); err != nil { + return nil, err } return encodeKey(pos.key, pos.commitTS), nil } @@ -112,13 +109,8 @@ func (s *pebbleStore) exportPebbleIteratorPosition( if userKey == nil || pebbleExportCursorEqual(pos, userKey, commitTS) { return true, true, nil } - if opts.StartKey != nil && bytes.Compare(userKey, opts.StartKey) < 0 { - _ = s.skipToNextUserKey(iter, userKey) - return false, true, nil - } - if opts.EndKey != nil && bytes.Compare(userKey, opts.EndKey) >= 0 { - _ = s.skipToNextUserKey(iter, userKey) - return false, true, nil + if skipped, err := s.skipPebbleExportKeyOutsideRange(iter, opts, userKey); skipped || err != nil { + return false, true, err } if commitTS <= opts.MinCommitTSExclusive { _ = s.skipToNextUserKey(iter, userKey) @@ -128,6 +120,31 @@ func (s *pebbleStore) exportPebbleIteratorPosition( return true, done, err } +func (s *pebbleStore) skipPebbleExportKeyOutsideRange(iter *pebble.Iterator, opts ExportVersionsOptions, userKey []byte) (bool, error) { + if opts.StartKey != nil && bytes.Compare(userKey, opts.StartKey) < 0 { + _ = s.skipToNextUserKey(iter, userKey) + return true, nil + } + if opts.EndKey == nil || bytes.Compare(userKey, opts.EndKey) < 0 { + return false, nil + } + if pebbleExportCanStopAtEndKey(opts.StartKey, opts.EndKey, userKey) { + return true, errExportReachedEnd + } + _ = s.skipToNextUserKey(iter, userKey) + return true, nil +} + +func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { + for prefixLen := range userKey { + prefix := userKey[:prefixLen] + if (startKey == nil || bytes.Compare(prefix, startKey) >= 0) && bytes.Compare(prefix, endKey) < 0 { + return false + } + } + return true +} + func pebbleExportCursorEqual(pos exportCursorPosition, userKey []byte, commitTS uint64) bool { return pos.hasKey && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS } @@ -246,11 +263,11 @@ func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchM if err := s.applyImportVersionsBatch(batch, opts.Versions); err != nil { return err } - if err := batch.Set(migrationAckKey(opts.JobID, opts.BracketID), encodeMigrationImportAck(migrationImportAck{ + if err := s.stageMigrationImportAck(batch, opts.JobID, opts.BracketID, migrationImportAck{ batchSeq: opts.BatchSeq, cursor: opts.Cursor, - }), nil); err != nil { - return errors.WithStack(err) + }); err != nil { + return err } unlock, newLastTS, err := s.stageMigrationClockMetadataIfNeeded(batch, opts.JobID, batchMax) if err != nil { @@ -266,6 +283,18 @@ func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchM return nil } +func (s *pebbleStore) stageMigrationImportAck(batch *pebble.Batch, jobID, bracketID uint64, ack migrationImportAck) error { + acks, err := s.readMigrationImportAcks() + if err != nil { + return err + } + acks[migrationAckID{jobID: jobID, bracketID: bracketID}] = migrationImportAck{ + batchSeq: ack.batchSeq, + cursor: bytes.Clone(ack.cursor), + } + return errors.WithStack(batch.Set(migrationAckMetaKeyBytes, encodeMigrationImportAcks(acks), nil)) +} + func (s *pebbleStore) stageMigrationClockMetadataIfNeeded(batch *pebble.Batch, jobID, batchMax uint64) (func(), uint64, error) { if batchMax == 0 { return func() {}, 0, nil @@ -310,32 +339,67 @@ func (s *pebbleStore) stageMigrationClockMetadata(batch *pebble.Batch, jobID, ba return nil, 0, err } if batchMax > floor { - if err := setPebbleUint64InBatch(batch, migrationHLCFloorKey(jobID), batchMax); err != nil { + floors, err := s.readMigrationHLCFloors() + if err != nil { unlock() return nil, 0, err } + floors[jobID] = batchMax + if err := batch.Set(migrationHLCFloorMetaKeyBytes, encodeMigrationHLCFloors(floors), nil); err != nil { + unlock() + return nil, 0, errors.WithStack(err) + } } return unlock, newLastTS, nil } func (s *pebbleStore) readMigrationImportAck(jobID, bracketID uint64) (migrationImportAck, bool, error) { - val, closer, err := s.db.Get(migrationAckKey(jobID, bracketID)) + acks, err := s.readMigrationImportAcks() + if err != nil { + return migrationImportAck{}, false, err + } + ack, ok := acks[migrationAckID{jobID: jobID, bracketID: bracketID}] + return ack, ok, nil +} + +func (s *pebbleStore) readMigrationImportAcks() (map[migrationAckID]migrationImportAck, error) { + val, closer, err := s.db.Get(migrationAckMetaKeyBytes) if err != nil { if errors.Is(err, pebble.ErrNotFound) { - return migrationImportAck{}, false, nil + return make(map[migrationAckID]migrationImportAck), nil } - return migrationImportAck{}, false, errors.WithStack(err) + return nil, errors.WithStack(err) } defer func() { _ = closer.Close() }() - ack, ok := decodeMigrationImportAck(val) + acks, ok := decodeMigrationImportAcks(val) if !ok { - return migrationImportAck{}, false, errors.New("corrupt migration import ack") + return nil, errors.New("corrupt migration import ack metadata") } - return ack, true, nil + return acks, nil } func (s *pebbleStore) readMigrationHLCFloorLocked(jobID uint64) (uint64, error) { - return readPebbleUint64(s.db, migrationHLCFloorKey(jobID)) + floors, err := s.readMigrationHLCFloors() + if err != nil { + return 0, err + } + return floors[jobID], nil +} + +func (s *pebbleStore) readMigrationHLCFloors() (map[uint64]uint64, error) { + val, closer, err := s.db.Get(migrationHLCFloorMetaKeyBytes) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return make(map[uint64]uint64), nil + } + return nil, errors.WithStack(err) + } + defer func() { _ = closer.Close() }() + floors, ok := decodeMigrationHLCFloors(val) + if !ok { + return nil, errors.New("corrupt migration HLC floor metadata") + } + return floors, nil } func (s *pebbleStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64, error) { diff --git a/store/lsm_store.go b/store/lsm_store.go index 18913cddc..81c0dd1e9 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -994,6 +994,7 @@ func (s *pebbleStore) seekToVisibleVersion(iter *pebble.Iterator, userKey []byte } func (s *pebbleStore) skipToNextUserKey(iter *pebble.Iterator, userKey []byte) bool { + userKey = bytes.Clone(userKey) if !iter.SeekGE(encodeKey(userKey, 0)) { return false } @@ -2176,6 +2177,41 @@ func writeRestoreEntry(r io.Reader, batch *pebble.Batch, keyBuf []byte, kLen, vL return errors.WithStack(deferred.Finish()) } +func discardRestoreEntryValue(r io.Reader, vLen int) error { + _, err := io.CopyN(io.Discard, r, int64(vLen)) + return errors.WithStack(err) +} + +func restoreBatchLoopStep(r io.Reader, db *pebble.DB, batch **pebble.Batch, keyBuf *[]byte) (bool, error) { + kLen, vLen, eof, err := readRestoreEntry(r, keyBuf) + if err != nil { + return false, err + } + if eof { + return true, nil + } + if isMigrationMetadataKey((*keyBuf)[:kLen]) { + return false, discardRestoreEntryValue(r, vLen) + } + if err := flushSnapshotBatchIfNeeded(db, batch, kLen, vLen); err != nil { + return false, err + } + if err := writeRestoreEntry(r, *batch, *keyBuf, kLen, vLen); err != nil { + return false, err + } + if snapshotBatchShouldFlush(*batch) { + return false, flushSnapshotBatch(db, batch, pebble.NoSync) + } + return false, nil +} + +func flushSnapshotBatchIfNeeded(db *pebble.DB, batch **pebble.Batch, kLen, vLen int) error { + if !(*batch).Empty() && (*batch).Len()+kLen+vLen >= snapshotBatchByteLimit { + return flushSnapshotBatch(db, batch, pebble.NoSync) + } + return nil +} + // restoreBatchLoopInto reads raw Pebble key-value entries from r and writes // them into db using batched commits. It is used for both the direct and the // temp-dir atomic native Pebble restore paths. @@ -2184,33 +2220,14 @@ func restoreBatchLoopInto(r io.Reader, db *pebble.DB) error { var keyBuf []byte // reused across entries to reduce per-entry allocations for { - kLen, vLen, eof, err := readRestoreEntry(r, &keyBuf) - if err != nil { - _ = batch.Close() - return err - } - if eof { + done, err := restoreBatchLoopStep(r, db, &batch, &keyBuf) + if done { break } - - // Flush before adding when the batch is non-empty and the anticipated - // entry size would push the batch over the byte limit. - if !batch.Empty() && batch.Len()+kLen+vLen >= snapshotBatchByteLimit { - if err := flushSnapshotBatch(db, &batch, pebble.NoSync); err != nil { - return err - } - } - - if err := writeRestoreEntry(r, batch, keyBuf, kLen, vLen); err != nil { + if err != nil { _ = batch.Close() return err } - - if snapshotBatchShouldFlush(batch) { - if err := flushSnapshotBatch(db, &batch, pebble.NoSync); err != nil { - return err - } - } } return commitSnapshotBatch(batch, pebble.Sync) } diff --git a/store/migration_versions.go b/store/migration_versions.go index e9aa56546..d95ba6ab8 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "sort" "github.com/cockroachdb/errors" "github.com/emirpasic/gods/maps/treemap" @@ -13,14 +14,21 @@ const ( exportCursorTagEmitted byte = iota exportCursorTagScanned + migrationAckMetaKey = "_migack" + migrationHLCFloorMetaKey = "_mighlc" + migrationMetadataVersion = 1 + migrationAckPrefix = "!migstage|ack|" - migrationHLCFloorPrefix = "!migstage|hlc_floor|" migrationUint64Bytes = 8 - migrationAckKeyIDBytes = 2 * migrationUint64Bytes exportVersionSizeOverhead = 24 defaultSparseExportMaxScannedBytes = 1 << 20 ) +var ( + migrationAckMetaKeyBytes = []byte(migrationAckMetaKey) + migrationHLCFloorMetaKeyBytes = []byte(migrationHLCFloorMetaKey) +) + type exportCursorPosition struct { key []byte commitTS uint64 @@ -28,6 +36,11 @@ type exportCursorPosition struct { hasKey bool } +type migrationAckID struct { + jobID uint64 + bracketID uint64 +} + type migrationImportAck struct { batchSeq uint64 cursor []byte @@ -71,6 +84,30 @@ func decodeExportCursor(cursor []byte) (exportCursorPosition, error) { return exportCursorPosition{key: key, commitTS: commitTS, tag: tag, hasKey: true}, nil } +func decodeExportCursorForOptions(opts ExportVersionsOptions) (exportCursorPosition, error) { + pos, err := decodeExportCursor(opts.Cursor) + if err != nil { + return exportCursorPosition{}, err + } + if err := validateExportCursorRange(opts, pos); err != nil { + return exportCursorPosition{}, err + } + return pos, nil +} + +func validateExportCursorRange(opts ExportVersionsOptions, pos exportCursorPosition) error { + if !pos.hasKey { + return nil + } + if opts.StartKey != nil && bytes.Compare(pos.key, opts.StartKey) < 0 { + return errors.WithStack(ErrInvalidExportCursor) + } + if opts.EndKey != nil && bytes.Compare(pos.key, opts.EndKey) >= 0 { + return errors.WithStack(ErrInvalidExportCursor) + } + return nil +} + func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOptions { if opts.AcceptKey != nil && opts.MaxScannedBytes == 0 { opts.MaxScannedBytes = defaultSparseExportMaxScannedBytes @@ -78,49 +115,111 @@ func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOp return opts } -func migrationAckKey(jobID, bracketID uint64) []byte { - key := make([]byte, len(migrationAckPrefix)+migrationAckKeyIDBytes) - copy(key, migrationAckPrefix) - binary.BigEndian.PutUint64(key[len(migrationAckPrefix):], jobID) - binary.BigEndian.PutUint64(key[len(migrationAckPrefix)+migrationUint64Bytes:], bracketID) - return key +func isMigrationMetadataKey(rawKey []byte) bool { + return bytes.Equal(rawKey, migrationAckMetaKeyBytes) || + bytes.Equal(rawKey, migrationHLCFloorMetaKeyBytes) } -func migrationHLCFloorKey(jobID uint64) []byte { - key := make([]byte, len(migrationHLCFloorPrefix)+migrationUint64Bytes) - copy(key, migrationHLCFloorPrefix) - binary.BigEndian.PutUint64(key[len(migrationHLCFloorPrefix):], jobID) - return key +func encodeMigrationImportAcks(acks map[migrationAckID]migrationImportAck) []byte { + ids := make([]migrationAckID, 0, len(acks)) + for id := range acks { + ids = append(ids, id) + } + sort.Slice(ids, func(i, j int) bool { + if ids[i].jobID != ids[j].jobID { + return ids[i].jobID < ids[j].jobID + } + return ids[i].bracketID < ids[j].bracketID + }) + + buf := make([]byte, 0, 1+binary.MaxVarintLen64+len(ids)*(3*migrationUint64Bytes+binary.MaxVarintLen64)) + buf = append(buf, migrationMetadataVersion) + buf = binary.AppendUvarint(buf, lenAsUint64(len(ids))) + for _, id := range ids { + ack := acks[id] + buf = binary.BigEndian.AppendUint64(buf, id.jobID) + buf = binary.BigEndian.AppendUint64(buf, id.bracketID) + buf = binary.BigEndian.AppendUint64(buf, ack.batchSeq) + buf = binary.AppendUvarint(buf, lenAsUint64(len(ack.cursor))) + buf = append(buf, ack.cursor...) + } + return buf } -func isMigrationMetadataKey(rawKey []byte) bool { - return (len(rawKey) == len(migrationAckPrefix)+migrationAckKeyIDBytes && bytes.HasPrefix(rawKey, []byte(migrationAckPrefix))) || - (len(rawKey) == len(migrationHLCFloorPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationHLCFloorPrefix))) +func decodeMigrationImportAcks(data []byte) (map[migrationAckID]migrationImportAck, bool) { + if len(data) == 0 || data[0] != migrationMetadataVersion { + return nil, false + } + rest := data[1:] + count, n := binary.Uvarint(rest) + if n <= 0 { + return nil, false + } + rest = rest[n:] + acks := make(map[migrationAckID]migrationImportAck) + for i := uint64(0); i < count; i++ { + if len(rest) < 3*migrationUint64Bytes { + return nil, false + } + id := migrationAckID{ + jobID: binary.BigEndian.Uint64(rest[:migrationUint64Bytes]), + bracketID: binary.BigEndian.Uint64(rest[migrationUint64Bytes : 2*migrationUint64Bytes]), + } + ack := migrationImportAck{batchSeq: binary.BigEndian.Uint64(rest[2*migrationUint64Bytes : 3*migrationUint64Bytes])} + rest = rest[3*migrationUint64Bytes:] + cursorLen, n := binary.Uvarint(rest) + if n <= 0 { + return nil, false + } + rest = rest[n:] + if cursorLen > lenAsUint64(len(rest)) { + return nil, false + } + ack.cursor = bytes.Clone(rest[:cursorLen]) + rest = rest[cursorLen:] + acks[id] = ack + } + return acks, len(rest) == 0 } -func encodeMigrationImportAck(ack migrationImportAck) []byte { - buf := make([]byte, 0, migrationUint64Bytes+binary.MaxVarintLen64+len(ack.cursor)) - buf = binary.BigEndian.AppendUint64(buf, ack.batchSeq) - buf = binary.AppendUvarint(buf, lenAsUint64(len(ack.cursor))) - buf = append(buf, ack.cursor...) +func encodeMigrationHLCFloors(floors map[uint64]uint64) []byte { + jobIDs := make([]uint64, 0, len(floors)) + for jobID := range floors { + jobIDs = append(jobIDs, jobID) + } + sort.Slice(jobIDs, func(i, j int) bool { return jobIDs[i] < jobIDs[j] }) + + buf := make([]byte, 0, 1+binary.MaxVarintLen64+len(jobIDs)*2*migrationUint64Bytes) + buf = append(buf, migrationMetadataVersion) + buf = binary.AppendUvarint(buf, lenAsUint64(len(jobIDs))) + for _, jobID := range jobIDs { + buf = binary.BigEndian.AppendUint64(buf, jobID) + buf = binary.BigEndian.AppendUint64(buf, floors[jobID]) + } return buf } -func decodeMigrationImportAck(data []byte) (migrationImportAck, bool) { - if len(data) < migrationUint64Bytes { - return migrationImportAck{}, false +func decodeMigrationHLCFloors(data []byte) (map[uint64]uint64, bool) { + if len(data) == 0 || data[0] != migrationMetadataVersion { + return nil, false } - ack := migrationImportAck{batchSeq: binary.BigEndian.Uint64(data[:migrationUint64Bytes])} - cursorLen, n := binary.Uvarint(data[migrationUint64Bytes:]) + rest := data[1:] + count, n := binary.Uvarint(rest) if n <= 0 { - return migrationImportAck{}, false + return nil, false } - rest := data[migrationUint64Bytes+n:] - if cursorLen != lenAsUint64(len(rest)) { - return migrationImportAck{}, false + rest = rest[n:] + floors := make(map[uint64]uint64) + for i := uint64(0); i < count; i++ { + if len(rest) < 2*migrationUint64Bytes { + return nil, false + } + jobID := binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + floor := binary.BigEndian.Uint64(rest[migrationUint64Bytes : 2*migrationUint64Bytes]) + rest = rest[2*migrationUint64Bytes:] + floors[jobID] = floor } - ack.cursor = bytes.Clone(rest) - return ack, true + return floors, len(rest) == 0 } func validateImportVersion(version MVCCVersion) error { @@ -178,7 +277,7 @@ func validateNextImportBatch(existing migrationImportAck, hasExisting bool, batc func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptions) (ExportVersionsResult, error) { opts = normalizeExportVersionsOptions(opts) - pos, err := decodeExportCursor(opts.Cursor) + pos, err := decodeExportCursorForOptions(opts) if err != nil { return ExportVersionsResult{}, err } @@ -341,8 +440,8 @@ func (s *mvccStore) ImportVersions(_ context.Context, opts ImportVersionsOptions s.mtx.Lock() defer s.mtx.Unlock() - key := string(migrationAckKey(opts.JobID, opts.BracketID)) - existing, hasExisting := s.migrationAcks[key] + id := migrationAckID{jobID: opts.JobID, bracketID: opts.BracketID} + existing, hasExisting := s.migrationAcks[id] duplicate, err := validateNextImportBatch(existing, hasExisting, opts.BatchSeq) if err != nil { return ImportVersionsResult{}, err @@ -371,7 +470,7 @@ func (s *mvccStore) ImportVersions(_ context.Context, opts ImportVersionsOptions if batchMax > s.migrationHLCFloors[opts.JobID] { s.migrationHLCFloors[opts.JobID] = batchMax } - s.migrationAcks[key] = migrationImportAck{batchSeq: opts.BatchSeq, cursor: bytes.Clone(opts.Cursor)} + s.migrationAcks[id] = migrationImportAck{batchSeq: opts.BatchSeq, cursor: bytes.Clone(opts.Cursor)} return ImportVersionsResult{AckedCursor: bytes.Clone(opts.Cursor), MaxImportedTS: batchMax}, nil } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 900afedb4..3019375e4 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -3,6 +3,7 @@ package store import ( "bytes" "context" + "math" "os" "testing" @@ -199,6 +200,66 @@ func TestExportVersionsUsesUserKeyRangeBounds(t *testing.T) { }) } +func TestExportVersionsRejectsCursorOutsideRequestedRange(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("a10"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("a20"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("b30"), 30, 0)) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("b"), + EndKey: []byte("c"), + Cursor: encodeExportCursor([]byte("a"), 20, exportCursorTagEmitted), + MaxVersions: 10, + }) + require.ErrorIs(t, err, ErrInvalidExportCursor) + require.Empty(t, res.Versions) + }) +} + +func TestExportVersionsDoesNotTreatMigrationPrefixUserKeyAsMetadata(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + key := append([]byte(migrationAckPrefix), bytes.Repeat([]byte{0x7}, migrationUint64Bytes)...) + require.NoError(t, st.PutAt(ctx, key, []byte("value"), 10, 0)) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 10}) + require.NoError(t, err) + require.True(t, res.Done) + require.Equal(t, []MVCCVersion{{Key: key, CommitTS: 10, Value: []byte("value")}}, res.Versions) + }) +} + +func TestPebbleExportStopsAtEndKey(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-end-key-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + ps, ok := st.(*pebbleStore) + require.True(t, ok) + require.NoError(t, ps.PutAt(ctx, []byte("b"), []byte("later"), 10, 0)) + + iter, err := ps.db.NewIter(nil) + require.NoError(t, err) + defer func() { require.NoError(t, iter.Close()) }() + require.True(t, iter.SeekGE(encodeKey([]byte("b"), math.MaxUint64))) + + result := newExportVersionsResult(10) + advance, done, err := ps.exportPebbleIteratorPosition(ctx, iter, ExportVersionsOptions{ + StartKey: []byte("a"), + EndKey: []byte("b"), + }, exportCursorPosition{}, &result) + require.ErrorIs(t, err, errExportReachedEnd) + require.False(t, advance) + require.True(t, done) + require.Empty(t, result.Versions) + require.Zero(t, result.ScannedBytes) +} + func TestImportVersionsIdempotencyAndMetadata(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() @@ -302,3 +363,55 @@ func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { require.True(t, res.Duplicate) require.Equal(t, []byte("persisted"), res.AckedCursor) } + +func TestPebbleSnapshotExcludesMigrationMetadata(t *testing.T) { + ctx := context.Background() + srcDir, err := os.MkdirTemp("", "migration-snapshot-src-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(srcDir)) }) + src, err := NewPebbleStore(srcDir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, src.Close()) }) + + _, err = src.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("stale"), + Versions: []MVCCVersion{{Key: []byte("snapshotted"), CommitTS: 50, Value: []byte("v50")}}, + }) + require.NoError(t, err) + snap, err := src.Snapshot() + require.NoError(t, err) + raw := snapshotBytes(t, snap) + require.NoError(t, snap.Close()) + + dstDir, err := os.MkdirTemp("", "migration-snapshot-dst-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dstDir)) }) + dst, err := NewPebbleStore(dstDir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, dst.Close()) }) + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + + val, err := dst.GetAt(ctx, []byte("snapshotted"), 50) + require.NoError(t, err) + require.Equal(t, []byte("v50"), val) + floor, err := dst.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Zero(t, floor) + + res, err := dst.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("fresh"), + Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.False(t, res.Duplicate) + require.Equal(t, []byte("fresh"), res.AckedCursor) + val, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.NoError(t, err) + require.Equal(t, []byte("v60"), val) +} diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 46ded171e..8c8712eb9 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -65,7 +65,7 @@ type mvccStore struct { log *slog.Logger lastCommitTS uint64 minRetainedTS uint64 - migrationAcks map[string]migrationImportAck + migrationAcks map[migrationAckID]migrationImportAck migrationHLCFloors map[uint64]uint64 // writeConflicts mirrors the per-(kind, key_prefix) counter from // the pebble-backed store so the in-memory implementation shows up @@ -113,7 +113,7 @@ func NewMVCCStore(opts ...MVCCStoreOption) MVCCStore { log: slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ Level: slog.LevelWarn, })), - migrationAcks: make(map[string]migrationImportAck), + migrationAcks: make(map[migrationAckID]migrationImportAck), migrationHLCFloors: make(map[uint64]uint64), writeConflicts: newWriteConflictCounter(), } @@ -934,7 +934,7 @@ func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { s.tree = tree s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS - s.migrationAcks = make(map[string]migrationImportAck) + s.migrationAcks = make(map[migrationAckID]migrationImportAck) s.migrationHLCFloors = make(map[uint64]uint64) return nil } diff --git a/store/snapshot_pebble.go b/store/snapshot_pebble.go index 8bc34fc65..02f1844eb 100644 --- a/store/snapshot_pebble.go +++ b/store/snapshot_pebble.go @@ -79,6 +79,9 @@ func writePebbleSnapshotEntries(snap *pebble.Snapshot, w io.Writer) error { for iter.First(); iter.Valid(); iter.Next() { k := iter.Key() v := iter.Value() + if isMigrationMetadataKey(k) { + continue + } if err := binary.Write(w, binary.LittleEndian, uint64(len(k))); err != nil { _ = iter.Close() From 6ac85a1ecdc7ec247b14b8a6542cb11089f2fadc Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 05:03:15 +0900 Subject: [PATCH 026/162] Fix target readiness guard gaps --- adapter/redis_compat_commands_stream_test.go | 39 +++++++---- kv/fsm.go | 20 +++++- kv/fsm_migration_fence_test.go | 69 ++++++++++++++++++++ 3 files changed, 112 insertions(+), 16 deletions(-) diff --git a/adapter/redis_compat_commands_stream_test.go b/adapter/redis_compat_commands_stream_test.go index bc78c846f..cf666f497 100644 --- a/adapter/redis_compat_commands_stream_test.go +++ b/adapter/redis_compat_commands_stream_test.go @@ -278,7 +278,7 @@ func TestRedis_StreamXReadLatencyIsConstant(t *testing.T) { rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) defer func() { _ = rdb.Close() }() - ctx := context.Background() + ctx := t.Context() const ( total = 10_000 @@ -286,23 +286,36 @@ func TestRedis_StreamXReadLatencyIsConstant(t *testing.T) { ) lastID := "" for i := range total { - id, err := rdb.XAdd(ctx, &redis.XAddArgs{ - Stream: "stream-lat", - ID: "*", - Values: []string{"i", fmt.Sprint(i)}, - }).Result() + var id string + err := retryNotLeader(ctx, func() error { + var xerr error + id, xerr = rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: "stream-lat", + ID: "*", + Values: []string{"i", fmt.Sprint(i)}, + }).Result() + return xerr + }) require.NoError(t, err) lastID = id } measure := func() time.Duration { - start := time.Now() - streams, err := rdb.XRead(ctx, &redis.XReadArgs{ - Streams: []string{"stream-lat", lastID}, - Count: 10, - Block: 10 * time.Millisecond, - }).Result() - elapsed := time.Since(start) + var ( + streams []redis.XStream + elapsed time.Duration + ) + err := retryNotLeader(ctx, func() error { + start := time.Now() + var xerr error + streams, xerr = rdb.XRead(ctx, &redis.XReadArgs{ + Streams: []string{"stream-lat", lastID}, + Count: 10, + Block: 10 * time.Millisecond, + }).Result() + elapsed = time.Since(start) + return xerr + }) require.True(t, errors.Is(err, redis.Nil) || err == nil) require.Empty(t, streams) return elapsed diff --git a/kv/fsm.go b/kv/fsm.go index 58fdccf32..31f38b0b5 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -686,16 +686,19 @@ func (f *kvFSM) verifyTargetReadinessForRouteRange(ctx context.Context, routeSta if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { continue } - if !proof || !routesSatisfyTargetReadiness(snap.IntersectingRoutes(routeStart, routeEnd), ready) { + if !proof || !routesSatisfyTargetReadiness(snap.IntersectingRoutes(routeStart, routeEnd), ready, f.shardGroupID) { return errors.WithStack(ErrRouteCutoverPending) } } return nil } -func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState) bool { +func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState, groupID uint64) bool { matched := false for _, route := range routes { + if route.GroupID != groupID { + continue + } if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { continue } @@ -1366,7 +1369,7 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u // shouldClearAbortKey (lock-missing ⇒ nothing to do) and for the // rollback-marker Put in appendRollbackRecord. - uniq, err := f.uniqueMutationsAboveWriteFloor(ctx, muts, abortTS) + uniq, err := f.uniqueAbortCleanupMutations(ctx, muts) if err != nil { return err } @@ -1400,6 +1403,17 @@ func (f *kvFSM) uniqueMutationsAboveWriteFloor(ctx context.Context, muts []*pb.M return uniq, nil } +func (f *kvFSM) uniqueAbortCleanupMutations(ctx context.Context, muts []*pb.Mutation) ([]*pb.Mutation, error) { + uniq, err := uniqueMutations(muts) + if err != nil { + return nil, err + } + if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { + return nil, err + } + return uniq, nil +} + func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt uint64) ([]*store.KVPairMutation, error) { storeMuts := make([]*store.KVPairMutation, 0, len(muts)*txnPrepareStoreMutationFactor) for _, mut := range muts { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 0f593df46..d0d856c39 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -112,6 +112,39 @@ func TestFSMAllowsRawPointWriteWithClearedTargetReadinessProof(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMRejectsTargetReadinessProofFromAnotherGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + }) + fsm := newComposed1FSM(t, engine, 1) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 101) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { t.Parallel() @@ -251,3 +284,39 @@ func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { err := fsm.handleTxnRequest(ctx, abort, 11) require.False(t, errors.Is(err, ErrRouteWriteFenced), "ABORT must keep the narrow cleanup lane open") } + +func TestFSMAbortCleanupBypassesRetainedWriteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + startTS := uint64(10) + abortTS := uint64(11) + primaryKey := []byte("b") + require.NoError(t, fsm.store.PutAt(ctx, txnLockKey(primaryKey), encodeTxnLock(txnLock{ + StartTS: startTS, + PrimaryKey: primaryKey, + IsPrimaryKey: true, + }), startTS, 0)) + require.NoError(t, fsm.store.PutAt(ctx, txnIntentKey(primaryKey), encodeTxnIntent(txnIntent{ + StartTS: startTS, + Op: txnIntentOpPut, + Value: []byte("v"), + }), startTS, 0)) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: abortTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + require.NoError(t, fsm.handleTxnRequest(ctx, abort, abortTS)) + + _, lockErr := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) + _, intentErr := fsm.store.GetAt(ctx, txnIntentKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, intentErr, store.ErrKeyNotFound) +} From ce91b665196bcbe9a1817c2017b14dbdb052a0b4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 05:05:57 +0900 Subject: [PATCH 027/162] Guard manifest scan readiness routes --- kv/shard_store.go | 9 ++++++- kv/shard_store_test.go | 54 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 58 insertions(+), 5 deletions(-) diff --git a/kv/shard_store.go b/kv/shard_store.go index 246fac205..eaacf33a7 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -135,7 +135,7 @@ func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetS } func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { - routeStart, routeEnd := readinessRouteRange(start, end) + routeStart, routeEnd := readinessRouteRangeForScan(start, end) return s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd) } @@ -174,6 +174,13 @@ func readinessRouteRange(start []byte, end []byte) ([]byte, []byte) { return routeStart, routeEnd } +func readinessRouteRangeForScan(start []byte, end []byte) ([]byte, []byte) { + if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { + return routeStart, routeEnd + } + return readinessRouteRange(start, end) +} + func verifyRouteWriteFloor(route distribution.Route, commitTS uint64) error { if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { return nil diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 7088c01ec..baa5ef041 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -35,9 +35,7 @@ func newStagedVisibilityShardStore(t *testing.T) (*ShardStore, *ShardGroup) { func applyTargetReadiness(t *testing.T, group *ShardGroup) { t.Helper() - writer, ok := group.Store.(store.MigrationTargetReadinessWriter) - require.True(t, ok) - require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ JobID: 9, RouteStart: []byte("a"), RouteEnd: []byte("z"), @@ -45,7 +43,14 @@ func applyTargetReadiness(t *testing.T, group *ShardGroup) { MigrationJobID: 9, MinWriteTSExclusive: 100, Armed: true, - })) + }) +} + +func applyTargetReadinessState(t *testing.T, group *ShardGroup, state store.TargetStagedReadinessState) { + t.Helper() + writer, ok := group.Store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) } func newReadinessShardStore(t *testing.T, route distribution.RouteDescriptor) (*ShardStore, *ShardGroup) { @@ -227,6 +232,47 @@ func TestShardStorePhysicalLimitScanChecksTargetReadiness(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreS3ManifestScanChecksTargetReadinessInRouteSpace(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + end := prefixScanEnd(start) + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + key := s3keys.ObjectManifestKey("bucket-a", 1, "z/object-0") + require.NoError(t, group.Store.PutAt(ctx, key, []byte("manifest"), 120, 0)) + + _, err := st.ScanAt(ctx, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestShardStoreApplyMutationsRaftAtEnforcesRouteFloor(t *testing.T) { t.Parallel() From d33bf3589a6a1f84ad05fa57073c8fa69dee4ed2 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 05:26:57 +0900 Subject: [PATCH 028/162] Harden staged migration visibility --- adapter/internal.go | 14 +- adapter/internal_migration_test.go | 37 ++++ kv/fsm_migration_import.go | 2 +- kv/leader_routed_store.go | 8 + kv/shard_store.go | 226 +++++++++++++++++++++- kv/shard_store_test.go | 61 ++++++ kv/sharded_coordinator.go | 81 +++++++- kv/sharded_coordinator_txn_test.go | 69 +++++++ kv/tso_test.go | 15 ++ store/lsm_migration.go | 20 +- store/lsm_store_registration_gate_test.go | 38 ++++ store/migration_versions.go | 4 + store/store.go | 3 + 13 files changed, 558 insertions(+), 20 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index f72d116f2..a14fb9134 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -133,9 +133,19 @@ func (i *Internal) validateExportRangeVersionsRequest(req *pb.ExportRangeVersion if req.GetKeyFamily() == 0 { return errors.WithStack(status.Error(codes.InvalidArgument, "migration export key_family is required")) } + if exportRangeVersionsRequestFullyUnbounded(req) { + return errors.WithStack(status.Error(codes.InvalidArgument, "migration export requires a raw or route bound")) + } return nil } +func exportRangeVersionsRequestFullyUnbounded(req *pb.ExportRangeVersionsRequest) bool { + return len(req.GetRangeStart()) == 0 && + len(req.GetRangeEnd()) == 0 && + len(req.GetRouteStart()) == 0 && + len(req.GetRouteEnd()) == 0 +} + func (i *Internal) streamExportRangeVersions(req *pb.ExportRangeVersionsRequest, stream pb.Internal_ExportRangeVersionsServer) error { opts := exportRangeVersionsOptions(req) for { @@ -271,10 +281,10 @@ func decodedS3BucketRouteFilter(family uint32, routeStart, routeEnd []byte) func return false } routeKey := s3keys.RouteKey(bucket, 0, "") - if routeStart != nil && bytes.Compare(routeKey, routeStart) < 0 { + if len(routeStart) > 0 && bytes.Compare(routeKey, routeStart) < 0 { return false } - return routeEnd == nil || bytes.Compare(routeKey, routeEnd) < 0 + return len(routeEnd) == 0 || bytes.Compare(routeKey, routeEnd) < 0 } } diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index b43a8f4b9..57ca63494 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -124,6 +124,14 @@ func TestInternalExportRangeVersionsRejectsUnboundedExport(t *testing.T) { }, stream) require.Error(t, err) require.Equal(t, codes.InvalidArgument, status.Code(err)) + + stream = &captureExportRangeVersionsStream{ctx: context.Background()} + err = internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + KeyFamily: distribution.MigrationFamilyUser, + }, stream) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) } func TestInternalExportRangeVersionsUsesDecodedS3BucketRouteFilter(t *testing.T) { @@ -188,6 +196,35 @@ func TestInternalExportRangeVersionsUsesDecodedS3BucketRouteFilter(t *testing.T) } } +func TestInternalExportRangeVersionsDecodedS3EmptyRouteEndIsUnbounded(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + + inRouteKey := s3keys.BucketMetaKey("bucket-z") + outRouteKey := s3keys.BucketMetaKey("bucket-a") + require.NoError(t, st.PutAt(ctx, inRouteKey, []byte("meta-z"), 10, 0)) + require.NoError(t, st.PutAt(ctx, outRouteKey, []byte("skip"), 10, 0)) + + stream := &captureExportRangeVersionsStream{ctx: ctx} + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + RouteStart: s3keys.RouteKey("bucket-z", 0, ""), + RouteEnd: []byte{}, + KeyFamily: distribution.MigrationFamilyS3BucketMeta, + RangeStart: []byte(s3keys.BucketMetaPrefix), + RangeEnd: testPrefixScanEnd([]byte(s3keys.BucketMetaPrefix)), + MaxScannedBytes: 1 << 20, + }, stream) + require.NoError(t, err) + require.Len(t, stream.responses, 1) + require.Equal(t, []*pb.MVCCVersion{ + {Key: inRouteKey, CommitTs: 10, Value: []byte("meta-z"), KeyFamily: distribution.MigrationFamilyS3BucketMeta}, + }, stream.responses[0].GetVersions()) +} + func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() diff --git a/kv/fsm_migration_import.go b/kv/fsm_migration_import.go index 9de47d897..91dca3070 100644 --- a/kv/fsm_migration_import.go +++ b/kv/fsm_migration_import.go @@ -34,7 +34,7 @@ func (f *kvFSM) applyMigrationImport(ctx context.Context, data []byte) any { if err := proto.Unmarshal(data, req); err != nil { return errors.WithStack(err) } - result, err := f.store.ImportVersions(ctx, store.ImportVersionsOptions{ + result, err := f.store.ImportVersionsRaft(ctx, store.ImportVersionsOptions{ JobID: req.GetJobId(), BracketID: req.GetBracketId(), BatchSeq: req.GetBatchSeq(), diff --git a/kv/leader_routed_store.go b/kv/leader_routed_store.go index 3f6004242..45569789d 100644 --- a/kv/leader_routed_store.go +++ b/kv/leader_routed_store.go @@ -519,6 +519,14 @@ func (s *LeaderRoutedStore) ImportVersions(ctx context.Context, opts store.Impor return result, errors.WithStack(err) } +func (s *LeaderRoutedStore) ImportVersionsRaft(ctx context.Context, opts store.ImportVersionsOptions) (store.ImportVersionsResult, error) { + if s == nil || s.local == nil { + return store.ImportVersionsResult{}, errors.WithStack(store.ErrNotSupported) + } + result, err := s.local.ImportVersionsRaft(ctx, opts) + return result, errors.WithStack(err) +} + func (s *LeaderRoutedStore) MigrationHLCFloor(ctx context.Context, jobID uint64) (uint64, error) { if s == nil || s.local == nil { return 0, errors.WithStack(store.ErrNotSupported) diff --git a/kv/shard_store.go b/kv/shard_store.go index fcf47f15c..c69b47e3b 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -26,6 +26,7 @@ type ShardStore struct { } var ErrCrossShardMutationBatchNotSupported = errors.New("cross-shard mutation batches are not supported") +var ErrExplicitGroupStagedVisibilityUnresolved = errors.New("explicit group read cannot resolve staged visibility route") // NewShardStore creates a sharded MVCC store wrapper. func NewShardStore(engine *distribution.Engine, groups map[uint64]*ShardGroup) *ShardStore { @@ -61,12 +62,16 @@ func (s *ShardStore) GetGroupAt(ctx context.Context, groupID uint64, key []byte, if !ok || g.Store == nil { return nil, store.ErrKeyNotFound } + route, err := s.routeForExplicitGroupKey(groupID, key) + if err != nil { + return nil, err + } if engineForGroup(g) == nil { - return s.localGetAt(ctx, g, distribution.Route{}, key, ts) + return s.localGetAt(ctx, g, route, key, ts) } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.leaderGetAt(ctx, g, distribution.Route{}, key, ts) + return s.leaderGetAt(ctx, g, route, key, ts) } return s.proxyRawGet(ctx, g, key, ts, groupID) } @@ -112,6 +117,32 @@ func routeHasStagedVisibility(route distribution.Route) bool { return route.StagedVisibilityActive && route.MigrationJobID != 0 } +func (s *ShardStore) routeForExplicitGroupKey(groupID uint64, key []byte) (distribution.Route, error) { + fallback := distribution.Route{GroupID: groupID} + if s == nil || s.engine == nil { + return fallback, nil + } + if route, ok := s.engine.GetRoute(routeKey(key)); ok && route.GroupID == groupID { + return route, nil + } + if s.groupHasStagedVisibility(groupID) { + return distribution.Route{}, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d key=%q", groupID, key) + } + return fallback, nil +} + +func (s *ShardStore) groupHasStagedVisibility(groupID uint64) bool { + if s == nil || s.engine == nil { + return false + } + for _, route := range s.engine.Stats() { + if route.GroupID == groupID && routeHasStagedVisibility(route) { + return true + } + } + return false +} + func (s *ShardStore) getAtWithStagedVisibility(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { if err := ensureReadTSRetained(g.Store, ts); err != nil { return nil, err @@ -312,7 +343,29 @@ func (s *ShardStore) ScanGroupAt(ctx context.Context, groupID uint64, start []by if limit <= 0 { return []*store.KVPair{}, nil } - return s.scanRouteAtDirection(ctx, distribution.Route{GroupID: groupID}, start, end, limit, ts, false, true) + routes, clampToRoutes, err := s.routesForExplicitGroupScan(groupID, start, end) + if err != nil { + return nil, err + } + out := make([]*store.KVPair, 0) + for _, route := range routes { + scanStart := start + scanEnd := end + if clampToRoutes { + scanStart = clampScanStart(start, route.Start) + scanEnd = clampScanEnd(end, route.End) + } + kvs, err := s.scanRouteAtDirection(ctx, route, scanStart, scanEnd, limit, ts, false, true) + if err != nil { + return nil, err + } + out = append(out, kvs...) + if len(out) >= limit { + clear(out[limit:]) + return out[:limit], nil + } + } + return out, nil } func (s *ShardStore) ReverseScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { @@ -380,6 +433,27 @@ func routesContainStagedVisibility(routes []distribution.Route) bool { return false } +func (s *ShardStore) routesForExplicitGroupScan(groupID uint64, start []byte, end []byte) ([]distribution.Route, bool, error) { + fallback := []distribution.Route{{GroupID: groupID}} + if s == nil || s.engine == nil { + return fallback, false, nil + } + routes := s.engine.GetIntersectingRoutes(start, end) + matched := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID == groupID { + matched = append(matched, route) + } + } + if len(matched) > 0 { + return matched, true, nil + } + if s.groupHasStagedVisibility(groupID) { + return nil, false, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d range=[%q,%q)", groupID, start, end) + } + return fallback, false, nil +} + func (s *ShardStore) scanRoutesAt(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) for _, route := range routes { @@ -691,37 +765,127 @@ func (s *ShardStore) scanRouteWithStagedVisibility( if err := ensureReadTSRetained(g.Store, ts); err != nil { return nil, err } + out := make([]*store.KVPair, 0, limit) + scanStart := bytes.Clone(start) + scanEnd := bytes.Clone(end) + for len(out) < limit { + remaining := limit - len(out) + kvs, boundary, hasMore, err := s.scanRouteWithStagedVisibilityPage(ctx, g, route, scanStart, scanEnd, remaining, ts, reverse) + if err != nil { + return nil, err + } + out = append(out, kvs...) + if len(out) >= limit { + clear(out[limit:]) + return out[:limit], nil + } + if !hasMore { + return out, nil + } + if reverse { + scanEnd = boundary + } else { + scanStart = exclusiveScanStartAfter(boundary) + } + } + return out, nil +} + +func (s *ShardStore) scanRouteWithStagedVisibilityPage( + ctx context.Context, + g *ShardGroup, + route distribution.Route, + start []byte, + end []byte, + limit int, + ts uint64, + reverse bool, +) ([]*store.KVPair, []byte, bool, error) { stagedStart, stagedEnd := stagedVisibilityScanBounds(route.MigrationJobID, start, end) window := stagedVisibilityCandidateWindow(limit) for { liveKVs, err := scanVisibleCandidates(ctx, g.Store, start, end, window, ts, reverse) if err != nil { - return nil, err + return nil, nil, false, err } stagedKVs, err := scanVisibleCandidates(ctx, g.Store, stagedStart, stagedEnd, window, ts, reverse) if err != nil { - return nil, err + return nil, nil, false, err } versions, err := s.latestStagedVisibilityCandidates(ctx, g.Store, route, liveKVs, stagedKVs, ts) if err != nil { - return nil, err + return nil, nil, false, err } out := visibleLogicalKVs(versions, ts, reverse) + boundary, hasBoundary := stagedVisibilityCandidateBoundary(liveKVs, stagedKVs, reverse) + exhausted := len(liveKVs) < window && len(stagedKVs) < window if len(out) >= limit { clear(out[limit:]) - return out[:limit], nil + return out[:limit], boundary, !exhausted && hasBoundary, nil } - if len(liveKVs) < window && len(stagedKVs) < window { - return out, nil + if exhausted { + return out, nil, false, nil } nextWindow := nextStagedVisibilityCandidateWindow(window) if nextWindow == window { - return out, nil + return out, boundary, hasBoundary, nil } window = nextWindow } } +func stagedVisibilityCandidateBoundary(liveKVs []*store.KVPair, stagedKVs []*store.KVPair, reverse bool) ([]byte, bool) { + boundary := stagedVisibilityBoundary{reverse: reverse} + for _, kvp := range liveKVs { + if kvp == nil { + continue + } + boundary.visit(kvp.Key) + } + for _, kvp := range stagedKVs { + rawKey, stagedOK := stagedVisibilityRawCandidateKey(kvp) + if !stagedOK { + continue + } + boundary.visit(rawKey) + } + return boundary.key, boundary.ok +} + +type stagedVisibilityBoundary struct { + key []byte + ok bool + reverse bool +} + +func (b *stagedVisibilityBoundary) visit(key []byte) { + if !b.ok { + b.key = bytes.Clone(key) + b.ok = true + return + } + cmp := bytes.Compare(key, b.key) + if (!b.reverse && cmp > 0) || (b.reverse && cmp < 0) { + b.key = bytes.Clone(key) + } +} + +func stagedVisibilityRawCandidateKey(kvp *store.KVPair) ([]byte, bool) { + if kvp == nil { + return nil, false + } + _, rawKey, ok := distribution.MigrationStagedDataKeyParts(kvp.Key) + return rawKey, ok +} + +func exclusiveScanStartAfter(key []byte) []byte { + if key == nil { + return nil + } + out := bytes.Clone(key) + return append(out, 0) +} + func stagedVisibilityCandidateWindow(limit int) int { if limit <= 0 { return 0 @@ -1676,6 +1840,7 @@ func (s *ShardStore) ApplyMutations(ctx context.Context, mutations []*store.KVPa if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { return err } + readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) return errors.WithStack(group.Store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS)) } @@ -1689,6 +1854,7 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { return err } + readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) } @@ -1703,6 +1869,7 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { return err } + readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) } @@ -1732,6 +1899,41 @@ func (s *ShardStore) ensureMutationWriteTimestampFloors(mutations []*store.KVPai return nil } +func (s *ShardStore) readKeysWithStagedVisibilityAliases(group *ShardGroup, readKeys [][]byte) [][]byte { + if len(readKeys) == 0 || s == nil || s.engine == nil || group == nil { + return readKeys + } + var out [][]byte + for _, key := range readKeys { + alias, ok := s.stagedVisibilityReadKeyAlias(group, key) + if !ok { + continue + } + if out == nil { + out = append([][]byte(nil), readKeys...) + } + out = append(out, alias) + } + if out == nil { + return readKeys + } + return out +} + +func (s *ShardStore) stagedVisibilityReadKeyAlias(group *ShardGroup, key []byte) ([]byte, bool) { + if len(key) == 0 { + return nil, false + } + if _, _, ok := distribution.MigrationStagedDataKeyParts(key); ok { + return nil, false + } + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g != group || !routeHasStagedVisibility(route) { + return nil, false + } + return distribution.MigrationStagedDataKey(route.MigrationJobID, key), true +} + // resolveSingleShardGroup returns the shard group that owns every // mutation in the batch, or an error if the batch is cross-shard or // references an unknown group. A nil group with nil error means "empty @@ -1976,6 +2178,10 @@ func (s *ShardStore) ImportVersions(context.Context, store.ImportVersionsOptions return store.ImportVersionsResult{}, store.ErrNotSupported } +func (s *ShardStore) ImportVersionsRaft(context.Context, store.ImportVersionsOptions) (store.ImportVersionsResult, error) { + return store.ImportVersionsResult{}, store.ErrNotSupported +} + func (s *ShardStore) MigrationHLCFloor(context.Context, uint64) (uint64, error) { return 0, store.ErrNotSupported } diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index e3a3ea8d5..9ff8677a4 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -2,6 +2,7 @@ package kv import ( "context" + "fmt" "testing" "github.com/bootjp/elastickv/distribution" @@ -159,6 +160,66 @@ func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { require.Equal(t, uint64(40), ts) } +func TestShardStoreExplicitGroupReads_MergeStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live-b"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged-b"), 20, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("c")), []byte("staged-c"), 30, 0)) + + got, err := st.GetGroupAt(ctx, 1, []byte("b"), 25) + require.NoError(t, err) + require.Equal(t, []byte("staged-b"), got) + + kvs, err := st.ScanGroupAt(ctx, 1, []byte("a"), []byte("z"), 10, 35) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b"), Value: []byte("staged-b")}, + {Key: []byte("c"), Value: []byte("staged-c")}, + }, kvs) +} + +func TestShardStoreScanAt_ContinuesStagedVisibilityAfterCandidateWindow(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + limit := stagedVisibilityMaxCandidateWindow + 3 + for i := range limit { + key := []byte(fmt.Sprintf("k%05d", i)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, key), []byte(fmt.Sprintf("v%05d", i)), 10, 0)) + } + + kvs, err := st.ScanAt(ctx, []byte("a"), []byte("z"), limit, 20) + require.NoError(t, err) + require.Len(t, kvs, limit) + require.Equal(t, []byte("k00000"), kvs[0].Key) + require.Equal(t, []byte(fmt.Sprintf("k%05d", limit-1)), kvs[limit-1].Key) + + kvs, err = st.ReverseScanAt(ctx, []byte("a"), []byte("z"), limit, 20) + require.NoError(t, err) + require.Len(t, kvs, limit) + require.Equal(t, []byte(fmt.Sprintf("k%05d", limit-1)), kvs[0].Key) + require.Equal(t, []byte("k00000"), kvs[limit-1].Key) +} + +func TestShardStoreApplyMutations_ValidatesStagedReadKeys(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + readKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, readKey), []byte("staged"), 20, 0)) + + err := st.ApplyMutations(ctx, []*store.KVPairMutation{ + {Op: store.OpTypePut, Key: []byte("m"), Value: []byte("write")}, + }, [][]byte{readKey}, 10, 101) + require.ErrorIs(t, err, store.ErrWriteConflict) +} + func TestShardStorePhysicalLimitFailsClosedBeforeStagedVisibilityFallback(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index a071a7e86..9cb4b8eee 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1297,6 +1297,7 @@ func (c *ShardedCoordinator) dispatchSingleShardTxn(ctx context.Context, startTS if err != nil { return nil, err } + readKeys = c.readKeysWithStagedVisibilityAliasesForGroup(gid, readKeys) // ReadKeys are included in the Raft log entry so the FSM validates // read-write conflicts atomically under applyMu. prevCommitTS, when set, // carries the one-phase dedup probe key for a retry that reuses a failed @@ -1313,6 +1314,27 @@ func (c *ShardedCoordinator) dispatchSingleShardTxn(ctx context.Context, startTS return &CoordinateResponse{CommitIndex: resp.CommitIndex}, nil } +func (c *ShardedCoordinator) readKeysWithStagedVisibilityAliasesForGroup(gid uint64, readKeys [][]byte) [][]byte { + if len(readKeys) == 0 { + return readKeys + } + var out [][]byte + for _, key := range readKeys { + alias, ok := c.stagedVisibilityReadKeyAlias(gid, key) + if !ok { + continue + } + if out == nil { + out = append([][]byte(nil), readKeys...) + } + out = append(out, alias) + } + if out == nil { + return readKeys + } + return out +} + type preparedGroup struct { gid uint64 keys []*pb.Mutation @@ -1929,10 +1951,27 @@ func (c *ShardedCoordinator) groupReadKeysByShardID(readKeys [][]byte) (map[uint "preserve OCC read-set integrity", key) } grouped[gid] = append(grouped[gid], key) + if alias, ok := c.stagedVisibilityReadKeyAlias(gid, key); ok { + grouped[gid] = append(grouped[gid], alias) + } } return grouped, nil } +func (c *ShardedCoordinator) stagedVisibilityReadKeyAlias(gid uint64, key []byte) ([]byte, bool) { + if c == nil || c.engine == nil || len(key) == 0 { + return nil, false + } + if _, _, ok := distribution.MigrationStagedDataKeyParts(key); ok { + return nil, false + } + route, ok := c.engine.GetRoute(routeKey(key)) + if !ok || route.GroupID != gid || !routeHasStagedVisibility(route) { + return nil, false + } + return distribution.MigrationStagedDataKey(route.MigrationJobID, key), true +} + // validateReadOnlyShards checks read-write conflicts on shards that have // read keys but no mutations in this transaction. writeGIDs is the set of // shards that already received a PREPARE with their readKeys attached. @@ -1982,7 +2021,7 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return errors.WithStack(err) } for _, key := range keys { - ts, exists, err := g.Store.LatestCommitTS(ctx, key) + ts, exists, err := c.latestCommitTSForReadKeyOnShard(ctx, gid, g, key) if err != nil { return errors.WithStack(err) } @@ -1993,6 +2032,43 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return nil } +func (c *ShardedCoordinator) latestCommitTSForReadKeyOnShard(ctx context.Context, gid uint64, g *ShardGroup, key []byte) (uint64, bool, error) { + liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) + if err != nil { + return 0, false, errors.WithStack(err) + } + route, ok := c.stagedVisibilityRouteForReadKey(gid, key) + if !ok { + return liveTS, liveExists, nil + } + stagedTS, stagedExists, err := g.Store.LatestCommitTS(ctx, distribution.MigrationStagedDataKey(route.MigrationJobID, key)) + if err != nil { + return 0, false, errors.WithStack(err) + } + return maxStagedVisibilityLatestCommitTS(liveTS, liveExists, stagedTS, stagedExists), liveExists || stagedExists, nil +} + +func (c *ShardedCoordinator) stagedVisibilityRouteForReadKey(gid uint64, key []byte) (distribution.Route, bool) { + if c == nil || c.engine == nil { + return distribution.Route{}, false + } + if _, _, ok := distribution.MigrationStagedDataKeyParts(key); ok { + return distribution.Route{}, false + } + route, ok := c.engine.GetRoute(routeKey(key)) + return route, ok && route.GroupID == gid && routeHasStagedVisibility(route) +} + +func maxStagedVisibilityLatestCommitTS(liveTS uint64, liveExists bool, stagedTS uint64, stagedExists bool) uint64 { + if !liveExists { + return stagedTS + } + if !stagedExists || liveTS > stagedTS { + return liveTS + } + return stagedTS +} + var _ Coordinator = (*ShardedCoordinator)(nil) func validateOperationGroup(reqs *OperationGroup[OP]) error { @@ -2074,6 +2150,9 @@ func (c *ShardedCoordinator) stampRawRequestTimestamps(ctx context.Context, reqs return err } r.Ts = ts + if err := c.rejectWriteTimestampFloorMutations(r.Mutations, r.Ts); err != nil { + return err + } } return nil } diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index c07733d16..588404cfb 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -43,6 +43,75 @@ func (s *recordingTransactional) Abort(_ context.Context, _ []*pb.Request) (*Tra return &TransactionResponse{}, nil } +func TestShardedCoordinatorValidateReadKeysOnShard_UsesStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + }, + })) + st := store.NewMVCCStore() + readKey := []byte("k") + require.NoError(t, st.PutAt(ctx, distribution.MigrationStagedDataKey(9, readKey), []byte("staged"), 20, 0)) + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Engine: stubLeaderEngine{}, Store: st}, + }, 1, NewHLC(), nil) + + err := coord.validateReadKeysOnShard(ctx, 1, [][]byte{readKey}, 10) + require.ErrorIs(t, err, store.ErrWriteConflict) +} + +func TestShardedCoordinatorDispatchTxn_AddsStagedReadKeyAlias(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + }, + })) + txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: txn}, + }, 1, NewHLC(), nil) + + readKey := []byte("k") + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + IsTxn: true, + StartTS: 10, + CommitTS: 101, + Elems: []*Elem[OP]{{Op: Put, Key: []byte("m"), Value: []byte("write")}}, + ReadKeys: [][]byte{readKey}, + }) + require.NoError(t, err) + require.Len(t, txn.requests, 1) + require.Equal(t, [][]byte{ + readKey, + distribution.MigrationStagedDataKey(9, readKey), + }, txn.requests[0].ReadKeys) +} + func cloneTxnRequest(req *pb.Request) *pb.Request { if req == nil { return nil diff --git a/kv/tso_test.go b/kv/tso_test.go index 8daa8651d..fa256fb83 100644 --- a/kv/tso_test.go +++ b/kv/tso_test.go @@ -284,6 +284,21 @@ func TestShardedCoordinatorRawFollowerDefersTSOAllocationToLeaderPath(t *testing require.EqualValues(t, 0, txn.requests[0].Ts) } +func TestShardedCoordinatorRejectsTSORawPointWriteAfterStamping(t *testing.T) { + t.Parallel() + + txn := &recordingTransactional{} + coord := NewShardedCoordinator(newMigrationFloorEngine(t, testTSOInitialBase), map[uint64]*ShardGroup{ + 1: {Txn: txn}, + }, 1, NewHLC(), nil).WithTSOAllocator(&fakeTSOAllocator{nextBase: testTSOInitialBase, leader: true}) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: []byte("z"), Value: []byte("v")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + require.Empty(t, txn.requests, "coordinator must reject after TSO stamping before proposing") +} + func TestShardedCoordinatorUsesTSOAllocatorForRawTxnAndDelPrefix(t *testing.T) { t.Parallel() diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 3ae343afd..40c4efca8 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -192,6 +192,14 @@ func (s *pebbleStore) decodeExportedPebbleVersion(iter *pebble.Iterator, userKey } func (s *pebbleStore) ImportVersions(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) { + return s.importVersionsWithOpts(ctx, opts, s.directApplyWriteOpts(), true) +} + +func (s *pebbleStore) ImportVersionsRaft(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) { + return s.importVersionsWithOpts(ctx, opts, s.raftApplyWriteOpts(), false) +} + +func (s *pebbleStore) importVersionsWithOpts(ctx context.Context, opts ImportVersionsOptions, writeOpts *pebble.WriteOptions, gateRegistration bool) (ImportVersionsResult, error) { s.dbMu.RLock() defer s.dbMu.RUnlock() @@ -207,7 +215,7 @@ func (s *pebbleStore) ImportVersions(ctx context.Context, opts ImportVersionsOpt } batchMax := importBatchMaxTS(opts.Versions) - if err := s.commitPebbleImportBatch(opts, batchMax); err != nil { + if err := s.commitPebbleImportBatch(opts, batchMax, writeOpts, gateRegistration); err != nil { return ImportVersionsResult{}, errors.WithStack(err) } s.log.InfoContext(ctx, "import_versions", @@ -240,10 +248,10 @@ func (s *pebbleStore) validatePebbleImportBatch(opts ImportVersionsOptions) (boo return false, nil, nil } -func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchMax uint64) error { +func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchMax uint64, writeOpts *pebble.WriteOptions, gateRegistration bool) error { batch := s.db.NewBatch() defer batch.Close() - if err := s.applyImportVersionsBatch(batch, opts.Versions); err != nil { + if err := s.applyImportVersionsBatch(batch, opts.Versions, gateRegistration); err != nil { return err } if err := batch.Set(migrationAckKey(opts.JobID, opts.BracketID), encodeMigrationImportAck(migrationImportAck{ @@ -257,7 +265,7 @@ func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchM return err } defer unlock() - if err := batch.Commit(s.directApplyWriteOpts()); err != nil { + if err := batch.Commit(writeOpts); err != nil { return errors.WithStack(err) } if batchMax > 0 { @@ -273,14 +281,14 @@ func (s *pebbleStore) stageMigrationClockMetadataIfNeeded(batch *pebble.Batch, j return s.stageMigrationClockMetadata(batch, jobID, batchMax) } -func (s *pebbleStore) applyImportVersionsBatch(batch *pebble.Batch, versions []MVCCVersion) error { +func (s *pebbleStore) applyImportVersionsBatch(batch *pebble.Batch, versions []MVCCVersion, gateRegistration bool) error { for _, version := range versions { k := encodeKey(version.Key, version.CommitTS) var encoded []byte if version.Tombstone { encoded = encodeValue(nil, true, 0, encStateCleartext) } else { - body, encState, err := s.encryptForKey(k, version.Value, version.ExpireAt, true) + body, encState, err := s.encryptForKey(k, version.Value, version.ExpireAt, gateRegistration) if err != nil { return err } diff --git a/store/lsm_store_registration_gate_test.go b/store/lsm_store_registration_gate_test.go index 4dbfeb6a5..8039a9afb 100644 --- a/store/lsm_store_registration_gate_test.go +++ b/store/lsm_store_registration_gate_test.go @@ -96,6 +96,30 @@ func TestRegistrationGate_DirectPathFailsClosedBeforeRegistration(t *testing.T) mustGet(t, f.mvcc, []byte("a"), 250, "1") }) + t.Run("ImportVersions", func(t *testing.T) { + t.Parallel() + registered := false + f := newRegGateStore(t, ®istered) + opts := ImportVersionsOptions{ + JobID: 1, + BracketID: 1, + BatchSeq: 1, + Cursor: []byte("cursor"), + Versions: []MVCCVersion{ + {Key: []byte("import"), CommitTS: 100, Value: []byte("v")}, + }, + } + _, err := f.mvcc.ImportVersions(ctx, opts) + if !errors.Is(err, ErrWriterNotRegistered) { + t.Fatalf("ImportVersions pre-registration: got %v, want ErrWriterNotRegistered", err) + } + registered = true + if _, err := f.mvcc.ImportVersions(ctx, opts); err != nil { + t.Fatalf("ImportVersions post-registration: %v", err) + } + mustGet(t, f.mvcc, []byte("import"), 150, "v") + }) + t.Run("ExpireAt", func(t *testing.T) { t.Parallel() // ExpireAt re-encrypts the latest value, so seed a value first @@ -133,6 +157,20 @@ func TestRegistrationGate_FSMApplyPathNeverGated(t *testing.T) { t.Fatalf("ApplyMutationsRaft must not be gated, got: %v", err) } mustGet(t, f.mvcc, []byte("raft"), 150, "applied") + + _, err := f.mvcc.ImportVersionsRaft(ctx, ImportVersionsOptions{ + JobID: 2, + BracketID: 1, + BatchSeq: 1, + Cursor: []byte("cursor"), + Versions: []MVCCVersion{ + {Key: []byte("raft-import"), CommitTS: 200, Value: []byte("imported")}, + }, + }) + if err != nil { + t.Fatalf("ImportVersionsRaft must not be gated, got: %v", err) + } + mustGet(t, f.mvcc, []byte("raft-import"), 250, "imported") } // TestRegistrationGate_NotEncryptingIsUngated confirms the gate is only diff --git a/store/migration_versions.go b/store/migration_versions.go index 54b7dd19b..8211e060b 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -365,6 +365,10 @@ func (s *mvccStore) ImportVersions(_ context.Context, opts ImportVersionsOptions return ImportVersionsResult{AckedCursor: bytes.Clone(opts.Cursor), MaxImportedTS: batchMax}, nil } +func (s *mvccStore) ImportVersionsRaft(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) { + return s.ImportVersions(ctx, opts) +} + func (s *mvccStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64, error) { s.mtx.RLock() defer s.mtx.RUnlock() diff --git a/store/store.go b/store/store.go index 78cb59354..5e4e75b53 100644 --- a/store/store.go +++ b/store/store.go @@ -265,6 +265,9 @@ type MVCCStore interface { // ImportVersions applies a migration import batch idempotently by // (jobID, bracketID, batchSeq), preserving tombstones and expireAt. ImportVersions(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) + // ImportVersionsRaft is the raft-apply variant of ImportVersions. It + // preserves the same idempotency contract while using the FSM write path. + ImportVersionsRaft(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) // MigrationHLCFloor returns the full-HLC target-local migration floor // persisted by ImportVersions for jobID. MigrationHLCFloor(ctx context.Context, jobID uint64) (uint64, error) From f2bcd65bccee6f69e78ee7ff0673861db8d88172 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 05:36:59 +0900 Subject: [PATCH 029/162] Fix target readiness route proofs --- kv/fsm.go | 6 ++- kv/fsm_migration_fence_test.go | 40 +++++++++++++++ kv/shard_store.go | 14 ++++- kv/shard_store_test.go | 94 ++++++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 3 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 31f38b0b5..b3a8d6d42 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -1110,7 +1110,11 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { } startTS := r.Ts - uniq, err := f.uniqueMutationsNotFenced(ctx, muts, startTS) + floorTS := startTS + if meta.CommitTS != 0 { + floorTS = meta.CommitTS + } + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, floorTS) if err != nil { return err } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index d0d856c39..35cfee44d 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -256,6 +256,46 @@ func TestFSMRejectsOnePhaseTxnBelowRouteFloor(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } +func TestFSMPrepareUsesCommitTSForRouteFloorWhenPresent(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 50, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b"), LockTTLms: defaultTxnLockTTLms, CommitTS: 101}), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, + }, + }, 50) + require.NoError(t, err) +} + +func TestFSMPrepareWithoutCommitTSUsesStartTSForRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 100, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b"), LockTTLms: defaultTxnLockTTLms}), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, + }, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) +} + func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { t.Parallel() diff --git a/kv/shard_store.go b/kv/shard_store.go index eaacf33a7..06349ae36 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -152,7 +152,7 @@ func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g * return errors.WithStack(err) } for _, ready := range states { - if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + if !targetReadinessAppliesToRoute(route, routeStart, routeEnd, ready) { continue } if !routeSatisfiesTargetReadiness(route, ready) { @@ -162,6 +162,16 @@ func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g * return nil } +func targetReadinessAppliesToRoute(route distribution.Route, routeStart []byte, routeEnd []byte, ready store.TargetStagedReadinessState) bool { + if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + return false + } + if route.RouteID != 0 && !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + return false + } + return true +} + func readinessRouteRange(start []byte, end []byte) ([]byte, []byte) { routeStart := routeKey(start) if end == nil { @@ -2104,7 +2114,7 @@ func (s *ShardStore) explicitGroupRouteForRange(groupID uint64, start []byte, en if s == nil || s.engine == nil { return fallback } - routeStart, routeEnd := readinessRouteRange(start, end) + routeStart, routeEnd := readinessRouteRangeForScan(start, end) for _, route := range s.engine.GetIntersectingRoutes(routeStart, routeEnd) { if route.GroupID == groupID { return route diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index baa5ef041..5748cdfc0 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -273,6 +273,100 @@ func TestShardStoreS3ManifestScanChecksTargetReadinessInRouteSpace(t *testing.T) require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreTargetReadinessSkipsNonOverlappingGuardForScannedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "a/") + end := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + routeBoundary, _, ok := s3keys.ManifestScanRouteBounds( + s3keys.ObjectManifestScanStart("bucket-a", 1, "m/"), + end, + ) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: routeStart, + End: routeBoundary, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: routeBoundary, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeBoundary, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + kvs, err := st.ScanAt(ctx, start, end, 10, 130) + require.NoError(t, err) + require.Empty(t, kvs) +} + +func TestShardStoreExplicitGroupS3ManifestScanUsesRouteProofForReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + end := prefixScanEnd(start) + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + key := s3keys.ObjectManifestKey("bucket-a", 1, "z/object-0") + require.NoError(t, group.Store.PutAt(ctx, key, []byte("manifest"), 120, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, start, end, 10, 130) + require.NoError(t, err) + require.Len(t, kvs, 1) + require.Equal(t, key, kvs[0].Key) +} + func TestShardStoreApplyMutationsRaftAtEnforcesRouteFloor(t *testing.T) { t.Parallel() From f727f3079a06bd8cb8c772ba0e6331f0194aa230 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 06:11:36 +0900 Subject: [PATCH 030/162] Enable split migration job creation --- adapter/distribution_server.go | 200 +++++++++++++++++++++++++--- adapter/distribution_server_test.go | 168 +++++++++++++++++++++++ distribution/split_job_catalog.go | 1 + 3 files changed, 354 insertions(+), 15 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 78106c6bb..c1b1fa7f4 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -22,12 +22,13 @@ import ( // DistributionServer serves distribution related gRPC APIs. type DistributionServer struct { - mu sync.Mutex - engine *distribution.Engine - catalog *distribution.CatalogStore - coordinator kv.Coordinator - readTracker *kv.ActiveTimestampTracker - reloadRetry struct { + mu sync.Mutex + engine *distribution.Engine + catalog *distribution.CatalogStore + coordinator kv.Coordinator + readTracker *kv.ActiveTimestampTracker + migrationCapabilityGate SplitMigrationCapabilityGate + reloadRetry struct { attempts int interval time.Duration } @@ -37,6 +38,10 @@ type DistributionServer struct { // DistributionServerOption configures DistributionServer behavior. type DistributionServerOption func(*DistributionServer) +// SplitMigrationCapabilityGate reports whether this node can safely create +// migration-only side effects. A nil gate keeps StartSplitMigration fail-closed. +type SplitMigrationCapabilityGate func(context.Context) error + // WithDistributionCoordinator configures the coordinator used for Raft-backed // catalog mutations in SplitRange. func WithDistributionCoordinator(coordinator kv.Coordinator) DistributionServerOption { @@ -51,6 +56,12 @@ func WithDistributionActiveTimestampTracker(tracker *kv.ActiveTimestampTracker) } } +func WithSplitMigrationCapabilityGate(gate SplitMigrationCapabilityGate) DistributionServerOption { + return func(s *DistributionServer) { + s.migrationCapabilityGate = gate + } +} + // WithCatalogReloadRetryPolicy configures the retry policy used after split // commit when waiting for the local catalog snapshot to become visible. func WithCatalogReloadRetryPolicy(attempts int, interval time.Duration) DistributionServerOption { @@ -184,16 +195,97 @@ func (s *DistributionServer) GetIntersectingRoutes(ctx context.Context, req *pb. } func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.StartSplitMigrationRequest) (*pb.StartSplitMigrationResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.verifyStartSplitMigrationReady(ctx, req); err != nil { + return nil, err + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return nil, err + } + readPin := s.pinReadTS(snapshot.ReadTS) + defer readPin.Release() + + parent, err := s.startSplitMigrationParent(ctx, snapshot, req) + if err != nil { + return nil, err + } + job, err := s.newSplitMigrationJob(ctx, snapshot, parent, req) + if err != nil { + return nil, err + } + if err := s.createSplitJobViaCoordinator(ctx, snapshot.ReadTS, job); err != nil { + return nil, err + } + return &pb.StartSplitMigrationResponse{ + CatalogVersion: snapshot.Version, + JobId: job.JobID, + }, nil +} + +func (s *DistributionServer) verifyStartSplitMigrationReady(ctx context.Context, req *pb.StartSplitMigrationRequest) error { if req.GetTargetGroupId() == 0 { - return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) + return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) } if err := s.verifyCatalogLeader(ctx); err != nil { - return nil, err + return err } if s.catalog == nil { - return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + return grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + return s.verifySplitMigrationCapability(ctx) +} + +func (s *DistributionServer) startSplitMigrationParent( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + req *pb.StartSplitMigrationRequest, +) (distribution.RouteDescriptor, error) { + if err := validateExpectedCatalogVersion(snapshot.Version, req.GetExpectedCatalogVersion()); err != nil { + return distribution.RouteDescriptor{}, err + } + parent, found := findRouteByID(snapshot.Routes, req.GetRouteId()) + if !found { + return distribution.RouteDescriptor{}, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error()) + } + splitKey := distribution.CloneBytes(req.GetSplitKey()) + if err := validateSplitKey(parent, splitKey); err != nil { + return distribution.RouteDescriptor{}, err + } + if err := distribution.ValidateMigrationRouteRange(splitKey, parent.End); err != nil { + return distribution.RouteDescriptor{}, splitMigrationRangeStatusError(err) + } + if parent.GroupID == req.GetTargetGroupId() { + return distribution.RouteDescriptor{}, grpcStatusError(codes.InvalidArgument, "target group must differ from source route group") + } + if err := s.rejectLiveSplitJob(ctx, snapshot, parent); err != nil { + return distribution.RouteDescriptor{}, err + } + return parent, nil +} + +func (s *DistributionServer) newSplitMigrationJob( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + parent distribution.RouteDescriptor, + req *pb.StartSplitMigrationRequest, +) (distribution.SplitJob, error) { + jobID, err := s.catalog.NextSplitJobIDAt(ctx, snapshot.ReadTS) + if err != nil { + return distribution.SplitJob{}, splitJobCatalogStatusError(err) + } + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: jobID, + SourceRouteID: parent.RouteID, + SplitKey: distribution.CloneBytes(req.GetSplitKey()), + TargetGroupID: req.GetTargetGroupId(), + }, parent, time.Now().UnixMilli()) + if err != nil { + return distribution.SplitJob{}, splitJobCatalogStatusError(err) } - return nil, grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) + return job, nil } func (s *DistributionServer) GetSplitJob(ctx context.Context, req *pb.GetSplitJobRequest) (*pb.GetSplitJobResponse, error) { @@ -303,7 +395,7 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR return nil, err } - parent, _, found := findRouteByID(snapshot.Routes, req.GetRouteId()) + parent, found := findRouteByID(snapshot.Routes, req.GetRouteId()) if !found { return nil, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error()) } @@ -536,6 +628,51 @@ func (s *DistributionServer) updateSplitJobViaCoordinator( return nil } +func (s *DistributionServer) createSplitJobViaCoordinator(ctx context.Context, readTS uint64, job distribution.SplitJob) error { + if job.JobID == math.MaxUint64 { + return splitJobCatalogStatusError(distribution.ErrCatalogSplitJobIDOverflow) + } + encoded, err := distribution.EncodeSplitJob(job) + if err != nil { + return splitJobCatalogStatusError(err) + } + jobKey := distribution.CatalogSplitJobKey(job.JobID) + nextIDKey := distribution.CatalogNextSplitJobIDKey() + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + { + Op: kv.Put, + Key: jobKey, + Value: encoded, + }, + { + Op: kv.Put, + Key: nextIDKey, + Value: distribution.EncodeCatalogNextSplitJobID(job.JobID + 1), + }, + }, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{jobKey, nextIDKey}, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) verifySplitMigrationCapability(ctx context.Context) error { + if s.migrationCapabilityGate == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) + } + if err := s.migrationCapabilityGate(ctx); err != nil { + if status.Code(err) != codes.OK { + return err + } + return grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) + } + return nil +} + func validateExpectedCatalogVersion(currentVersion, expectedVersion uint64) error { if currentVersion != expectedVersion { return grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) @@ -583,6 +720,29 @@ func (s *DistributionServer) rejectLiveSplitJobOverlap(ctx context.Context, snap return nil } +func (s *DistributionServer) rejectLiveSplitJob(ctx context.Context, snapshot distribution.CatalogSnapshot, parent distribution.RouteDescriptor) error { + jobs, err := s.catalog.ListSplitJobsAt(ctx, snapshot.ReadTS) + if err != nil { + return grpcStatusErrorf(codes.Internal, "load split jobs: %v", err) + } + liveJobs := 0 + for _, job := range jobs { + if !splitJobIsLive(job) { + continue + } + liveJobs++ + for _, interval := range liveSplitJobIntervals(job, snapshot.Routes) { + if routeRangeIntersects(parent.Start, parent.End, interval.start, interval.end) { + return grpcStatusError(codes.Aborted, distribution.ErrSplitJobOverlap.Error()) + } + } + } + if liveJobs > 0 { + return grpcStatusError(codes.ResourceExhausted, distribution.ErrTooManyInFlightSplitJobs.Error()) + } + return nil +} + func splitJobIsLive(job distribution.SplitJob) bool { return job.Phase != distribution.SplitJobPhaseDone && job.Phase != distribution.SplitJobPhaseAbandoned } @@ -676,6 +836,16 @@ func splitJobCatalogStatusError(err error) error { } } +func splitMigrationRangeStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrMigrationReservedRange), + errors.Is(err, distribution.ErrMigrationInvalidRoute): + return grpcStatusError(codes.InvalidArgument, err.Error()) + default: + return grpcStatusErrorf(codes.Internal, "validate split migration range: %v", err) + } +} + func splitJobCoordinatorStatusError(err error) error { switch { case errors.Is(err, store.ErrWriteConflict): @@ -865,13 +1035,13 @@ func (s *DistributionServer) allocateChildRouteIDs(ctx context.Context, readTS u return leftID, rightID, nil } -func findRouteByID(routes []distribution.RouteDescriptor, routeID uint64) (distribution.RouteDescriptor, int, bool) { - for i, route := range routes { +func findRouteByID(routes []distribution.RouteDescriptor, routeID uint64) (distribution.RouteDescriptor, bool) { + for _, route := range routes { if route.RouteID == routeID { - return distribution.CloneRouteDescriptor(route), i, true + return distribution.CloneRouteDescriptor(route), true } } - return distribution.RouteDescriptor{}, -1, false + return distribution.RouteDescriptor{}, false } func toProtoRouteDescriptors(routes []distribution.RouteDescriptor) []*pb.RouteDescriptor { diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 109f21d62..8e51c4f99 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -156,6 +156,174 @@ func TestDistributionServerStartSplitMigration_FailsClosedUntilCapabilityGate(t require.Empty(t, jobs) } +func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + gateCalls := 0 + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { + gateCalls++ + return nil + }), + ) + + resp, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.NoError(t, err) + require.Equal(t, saved.Version, resp.CatalogVersion) + require.Equal(t, uint64(1), resp.JobId) + require.Equal(t, 1, gateCalls) + require.Equal(t, 1, coordinator.dispatchCalls) + + jobs, err := catalog.ListSplitJobs(ctx) + require.NoError(t, err) + require.Len(t, jobs, 1) + job := jobs[0] + require.Equal(t, uint64(1), job.JobID) + require.Equal(t, uint64(1), job.SourceRouteID) + require.Equal(t, []byte("g"), job.SplitKey) + require.Equal(t, uint64(2), job.TargetGroupID) + require.Equal(t, distribution.SplitJobPhasePlanned, job.Phase) + require.NotZero(t, job.StartedAtMs) + require.NotZero(t, job.UpdatedAtMs) + require.NotEmpty(t, job.BracketProgress) + next, err := catalog.NextSplitJobID(ctx) + require.NoError(t, err) + require.Equal(t, uint64(2), next) +} + +func TestDistributionServerStartSplitMigration_RejectsSecondLiveJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }, + }) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, distribution.SplitJob{ + JobID: 1, + SourceRouteID: 2, + SplitKey: []byte("t"), + TargetGroupID: 3, + Phase: distribution.SplitJobPhaseBackfill, + StartedAtMs: 1000, + UpdatedAtMs: 1000, + })) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 4, + }) + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.ErrorContains(t, err, distribution.ErrTooManyInFlightSplitJobs.Error()) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerStartSplitMigration_RejectsReservedMigrationRange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + splitKey []byte + end []byte + }{ + {name: "dist catalog", splitKey: []byte("!dist|"), end: nil}, + {name: "dist staged catalog", splitKey: []byte("!dist|migstage|"), end: nil}, + {name: "migration staged", splitKey: []byte("!migstage|ready|1"), end: nil}, + {name: "migration tracker", splitKey: []byte("!migwrite|"), end: nil}, + {name: "migration fence", splitKey: []byte("!migfence|"), end: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: tc.end, + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: tc.splitKey, + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.ErrorContains(t, err, distribution.ErrMigrationReservedRange.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) + }) + } +} + func TestDistributionServerSplitJobRPCs_ReadAndListCatalogJobs(t *testing.T) { t.Parallel() diff --git a/distribution/split_job_catalog.go b/distribution/split_job_catalog.go index 082f9aaf6..1f7fc241b 100644 --- a/distribution/split_job_catalog.go +++ b/distribution/split_job_catalog.go @@ -36,6 +36,7 @@ var ( ErrCatalogSplitJobConflict = errors.New("catalog split job conflict") ErrCatalogSplitJobTerminalRequired = errors.New("catalog split job terminal state is required") ErrSplitJobOverlap = errors.New("split job overlaps requested route") + ErrTooManyInFlightSplitJobs = errors.New("too many in-flight split jobs") ) // SplitJobPhase is the durable phase of a split migration job. From c72eace4cd5007b515cda3d7fb961b4893e55e48 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 06:18:35 +0900 Subject: [PATCH 031/162] Validate promote cursor before proposal --- adapter/internal.go | 14 ++++++++ adapter/internal_migration_test.go | 52 ++++++++++++++++++++++++++++++ store/migration_versions.go | 19 +++++++++++ 3 files changed, 85 insertions(+) diff --git a/adapter/internal.go b/adapter/internal.go index a909990f6..656b84380 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -216,6 +216,9 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { return nil, err } + if err := validatePromoteStagedVersionsRequest(req); err != nil { + return nil, errors.WithStack(err) + } result, err := i.proposeMigrationPromote(ctx, req) if err != nil { return nil, errors.WithStack(err) @@ -241,6 +244,17 @@ func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { return nil } +func validatePromoteStagedVersionsRequest(req *pb.PromoteStagedVersionsRequest) error { + prefix := distribution.MigrationStagedDataKeyPrefix(req.GetJobId()) + if err := store.ValidateExportCursorForRange(req.GetCursor(), prefix, store.PrefixScanEnd(prefix)); err != nil { + if errors.Is(err, store.ErrInvalidExportCursor) { + return errors.WithStack(status.Error(codes.InvalidArgument, store.ErrInvalidExportCursor.Error())) + } + return errors.WithStack(status.Errorf(codes.Internal, "validate promote cursor: %v", err)) + } + return nil +} + func defaultMigrationPromoteGate(context.Context) error { if migrationPromoteOpcodeEnabledFromEnv() { return nil diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 9760a61b0..c1cf5df04 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -2,6 +2,7 @@ package adapter import ( "context" + "encoding/binary" "testing" "github.com/bootjp/elastickv/distribution" @@ -71,6 +72,15 @@ func (s *captureExportRangeVersionsStream) Send(resp *pb.ExportRangeVersionsResp return nil } +func encodeTestExportCursor(key []byte, commitTS uint64, tag byte) []byte { + var out []byte + out = binary.AppendUvarint(out, uint64(len(key))) + out = append(out, key...) + out = binary.AppendUvarint(out, commitTS) + out = append(out, tag) + return out +} + func testPrefixScanEnd(prefix []byte) []byte { out := append([]byte(nil), prefix...) for i := len(out) - 1; i >= 0; i-- { @@ -291,6 +301,48 @@ func TestInternalPromoteStagedVersionsRejectsWhenOpcodeGateClosed(t *testing.T) require.Equal(t, uint64(0), proposer.calls) } +func TestInternalPromoteStagedVersionsRejectsInvalidCursorBeforePropose(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + cursor []byte + }{ + {name: "malformed cursor", cursor: []byte{0xff}}, + { + name: "cursor outside job staged prefix", + cursor: encodeTestExportCursor(distribution.MigrationStagedDataKey(8, []byte("k")), 30, 0), + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + st := store.NewMVCCStore() + clock := kv.NewHLC() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, clock), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), + ) + + resp, err := internal.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{ + JobId: 7, + Cursor: tc.cursor, + MaxVersions: 10, + }) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.ErrorContains(t, err, store.ErrInvalidExportCursor.Error()) + require.Equal(t, uint64(0), proposer.calls) + }) + } +} + func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() diff --git a/store/migration_versions.go b/store/migration_versions.go index cd4c535fb..93ce86819 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -70,6 +70,25 @@ func decodeExportCursor(cursor []byte) (exportCursorPosition, error) { return exportCursorPosition{key: key, commitTS: commitTS, tag: tag}, nil } +// ValidateExportCursorForRange verifies that an export cursor decodes and +// resumes inside the supplied key interval. +func ValidateExportCursorForRange(cursor, startKey, endKey []byte) error { + pos, err := decodeExportCursor(cursor) + if err != nil { + return err + } + if len(pos.key) == 0 { + return nil + } + if startKey != nil && bytes.Compare(pos.key, startKey) < 0 { + return errors.WithStack(ErrInvalidExportCursor) + } + if endKey != nil && bytes.Compare(pos.key, endKey) >= 0 { + return errors.WithStack(ErrInvalidExportCursor) + } + return nil +} + func migrationAckKey(jobID, bracketID uint64) []byte { key := make([]byte, len(migrationAckPrefix)+migrationAckKeyIDBytes) copy(key, migrationAckPrefix) From 53d38e31d0e8f9fa32a76a970218c0c32f7c44af Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 06:21:34 +0900 Subject: [PATCH 032/162] Harden split migration readiness checks --- adapter/distribution_server.go | 7 ++--- adapter/distribution_server_test.go | 46 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index c1b1fa7f4..389559b4e 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -431,7 +431,7 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR func (s *DistributionServer) pinReadTS(ts uint64) *kv.ActiveTimestampToken { if s == nil || s.readTracker == nil { - return nil + return &kv.ActiveTimestampToken{} } return s.readTracker.Pin(ts) } @@ -665,10 +665,7 @@ func (s *DistributionServer) verifySplitMigrationCapability(ctx context.Context) return grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) } if err := s.migrationCapabilityGate(ctx); err != nil { - if status.Code(err) != codes.OK { - return err - } - return grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) + return err } return nil } diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 8e51c4f99..ceb73fe36 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -74,6 +74,24 @@ func TestWithCatalogReloadRetryPolicy_OverridesDefaults(t *testing.T) { require.Equal(t, 5*time.Millisecond, s.reloadRetry.interval) } +func TestDistributionServerPinReadTSWithoutTrackerReturnsReleasableToken(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil) + token := s.pinReadTS(10) + require.NotNil(t, token) + require.NotPanics(t, func() { + token.Release() + }) + + var nilServer *DistributionServer + nilToken := nilServer.pinReadTS(10) + require.NotNil(t, nilToken) + require.NotPanics(t, func() { + nilToken.Release() + }) +} + func TestDistributionServerListRoutes_ReadsDurableCatalog(t *testing.T) { t.Parallel() @@ -156,6 +174,34 @@ func TestDistributionServerStartSplitMigration_FailsClosedUntilCapabilityGate(t require.Empty(t, jobs) } +func TestDistributionServerStartSplitMigrationReturnsCapabilityGateError(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { + return status.Error(codes.Unavailable, "split migration capability not ready") + }), + ) + + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: 1, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, "split migration capability not ready") + require.Zero(t, coordinator.dispatchCalls) +} + func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t *testing.T) { t.Parallel() From a624a2169ed8ab7d63a224274a022c86236cc8f9 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 06:32:03 +0900 Subject: [PATCH 033/162] Harden target readiness route proofs --- kv/fsm.go | 6 +- kv/fsm_migration_fence_test.go | 2 +- kv/shard_store.go | 57 +++++++++++++++---- kv/shard_store_test.go | 95 ++++++++++++++++++++++++++++++++ kv/shard_store_txn_lock_test.go | 2 +- store/migration_readiness.go | 8 +++ store/migration_versions_test.go | 36 ++++++++++++ 7 files changed, 190 insertions(+), 16 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index b3a8d6d42..85dd67590 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -686,14 +686,14 @@ func (f *kvFSM) verifyTargetReadinessForRouteRange(ctx context.Context, routeSta if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { continue } - if !proof || !routesSatisfyTargetReadiness(snap.IntersectingRoutes(routeStart, routeEnd), ready, f.shardGroupID) { + if !proof || !routesSatisfyTargetReadiness(snap.IntersectingRoutes(routeStart, routeEnd), ready, f.shardGroupID, snap.Version()) { return errors.WithStack(ErrRouteCutoverPending) } } return nil } -func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState, groupID uint64) bool { +func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState, groupID uint64, catalogVersion uint64) bool { matched := false for _, route := range routes { if route.GroupID != groupID { @@ -703,7 +703,7 @@ func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.Targe continue } matched = true - if !routeSatisfiesTargetReadiness(route, ready) { + if !routeSatisfiesTargetReadiness(route, ready, catalogVersion) { return false } } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 35cfee44d..9ebaab322 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -43,7 +43,7 @@ func newTargetReadinessFSM(t *testing.T, route distribution.RouteDescriptor) *kv t.Helper() engine := distribution.NewEngine() - applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{route}) + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{route}) fsm := newComposed1FSM(t, engine, route.GroupID) writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) require.True(t, ok) diff --git a/kv/shard_store.go b/kv/shard_store.go index 06349ae36..4e3fb96c4 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -121,7 +121,7 @@ func routeHasStagedVisibility(route distribution.Route) bool { return route.StagedVisibilityActive && route.MigrationJobID != 0 } -func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetStagedReadinessState) bool { +func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetStagedReadinessState, catalogVersion uint64) bool { if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { return false } @@ -131,7 +131,10 @@ func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetS if route.StagedVisibilityActive { return route.MigrationJobID == ready.MigrationJobID } - return route.MigrationJobID == 0 + if route.MigrationJobID != 0 { + return false + } + return ready.ExpectedCutoverVersion == 0 || catalogVersion >= ready.ExpectedCutoverVersion } func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { @@ -151,11 +154,15 @@ func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g * if err != nil { return errors.WithStack(err) } + var catalogVersion uint64 + if s != nil && s.engine != nil { + catalogVersion = s.engine.Version() + } for _, ready := range states { if !targetReadinessAppliesToRoute(route, routeStart, routeEnd, ready) { continue } - if !routeSatisfiesTargetReadiness(route, ready) { + if !routeSatisfiesTargetReadiness(route, ready, catalogVersion) { return errors.WithStack(ErrRouteCutoverPending) } } @@ -388,7 +395,11 @@ func (s *ShardStore) ScanGroupAt(ctx context.Context, groupID uint64, start []by if limit <= 0 { return []*store.KVPair{}, nil } - return s.scanRouteAtDirection(ctx, s.explicitGroupRouteForRange(groupID, start, end), start, end, limit, ts, false, true) + routes := s.explicitGroupRoutesForRange(groupID, start, end) + if err := s.verifyExplicitGroupRoutesForRange(ctx, groupID, routes, start, end); err != nil { + return nil, err + } + return s.scanRouteAtDirection(ctx, routes[0], start, end, limit, ts, false, true) } func (s *ShardStore) ReverseScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { @@ -1901,7 +1912,11 @@ func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte return nil } routeStart, routeEnd := routePrefixRange(prefix) - for _, route := range s.engine.GetIntersectingRoutes(routeStart, routeEnd) { + routes := s.engine.GetIntersectingRoutes(routeStart, routeEnd) + if len(routes) == 0 { + return errors.WithStack(ErrRouteCutoverPending) + } + for _, route := range routes { g, ok := s.groupForID(route.GroupID) if !ok || g == nil || g.Store == nil { return store.ErrNotSupported @@ -2109,18 +2124,38 @@ func (s *ShardStore) explicitGroupRouteForKey(groupID uint64, key []byte) distri return route } -func (s *ShardStore) explicitGroupRouteForRange(groupID uint64, start []byte, end []byte) distribution.Route { +func (s *ShardStore) explicitGroupRoutesForRange(groupID uint64, start []byte, end []byte) []distribution.Route { fallback := distribution.Route{GroupID: groupID} if s == nil || s.engine == nil { - return fallback + return []distribution.Route{fallback} + } + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + routes := s.engine.GetIntersectingRoutes(routeStart, routeEnd) + if len(routes) == 0 { + return []distribution.Route{fallback} + } + out := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID != groupID { + return []distribution.Route{fallback} + } + out = append(out, route) + } + return out +} + +func (s *ShardStore) verifyExplicitGroupRoutesForRange(ctx context.Context, groupID uint64, routes []distribution.Route, start []byte, end []byte) error { + g, ok := s.groupForID(groupID) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported } routeStart, routeEnd := readinessRouteRangeForScan(start, end) - for _, route := range s.engine.GetIntersectingRoutes(routeStart, routeEnd) { - if route.GroupID == groupID { - return route + for _, route := range routes { + if err := s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd); err != nil { + return err } } - return fallback + return nil } func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *ShardGroup, bool) { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 5748cdfc0..ffc34ff60 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -185,6 +185,31 @@ func TestShardStoreTargetReadinessAcceptsClearedDescriptorAndRetainsFloor(t *tes require.NoError(t, err) } +func TestShardStoreTargetReadinessRejectsClearedDescriptorBeforeCutoverVersion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 120, 0)) + + _, err := st.GetAt(ctx, []byte("k"), 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestShardStoreExplicitGroupReadUsesRouteProofForReadiness(t *testing.T) { t.Parallel() @@ -210,6 +235,49 @@ func TestShardStoreExplicitGroupReadUsesRouteProofForReadiness(t *testing.T) { require.Len(t, kvs, 2) } +func TestShardStoreScanGroupAtChecksEveryCoveredRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("left"), 120, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("x"), []byte("right"), 120, 0)) + + _, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestShardStorePhysicalLimitScanChecksTargetReadiness(t *testing.T) { t.Parallel() @@ -410,6 +478,33 @@ func TestShardStoreDeletePrefixAtChecksTargetReadinessBeforeDelete(t *testing.T) require.Equal(t, []byte("v"), got) } +func TestShardStoreDeletePrefixAtFailsClosedWithoutRouteProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAt(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + func TestShardStoreDeletePrefixAtRaftEnforcesRouteFloorBeforeDelete(t *testing.T) { t.Parallel() diff --git a/kv/shard_store_txn_lock_test.go b/kv/shard_store_txn_lock_test.go index c395b3dc9..aeb6a2e58 100644 --- a/kv/shard_store_txn_lock_test.go +++ b/kv/shard_store_txn_lock_test.go @@ -478,7 +478,7 @@ func TestShardStorePhysicalLimitScanPreservesRouteProofDuringLockResolution(t *t engine := distribution.NewEngine() require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, + Version: 2, Routes: []distribution.RouteDescriptor{{ RouteID: 1, Start: []byte("a"), diff --git a/store/migration_readiness.go b/store/migration_readiness.go index 7aa160ca3..e0c83a1df 100644 --- a/store/migration_readiness.go +++ b/store/migration_readiness.go @@ -79,6 +79,9 @@ func (s *pebbleStore) loadPebbleTargetReadinessStatesFromDB() ([]TargetStagedRea var out []TargetStagedReadinessState for valid := iter.First(); valid; valid = iter.Next() { + if !isTargetReadinessRecordKey(iter.Key()) { + continue + } state, ok := decodeTargetStagedReadinessState(iter.Value()) if !ok { return nil, errors.New("corrupt target staged readiness state") @@ -92,6 +95,11 @@ func (s *pebbleStore) loadPebbleTargetReadinessStatesFromDB() ([]TargetStagedRea return out, nil } +func isTargetReadinessRecordKey(rawKey []byte) bool { + return len(rawKey) == len(migrationReadyPrefix)+migrationUint64Bytes && + bytes.HasPrefix(rawKey, []byte(migrationReadyPrefix)) +} + func encodeTargetStagedReadinessState(state TargetStagedReadinessState) []byte { buf := make([]byte, 0, targetReadinessEncodedSize(state)) buf = append(buf, targetReadinessCodecVersion) diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 7332f318b..ee7ffaf42 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -358,6 +358,42 @@ func TestPebbleTargetStagedReadinessPersistsAcrossReopen(t *testing.T) { require.Equal(t, []TargetStagedReadinessState{state}, states) } +func TestPebbleTargetStagedReadinessIgnoresNonRecordPrefixKeys(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-ready-prefix-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + st, err := NewPebbleStore(dir) + require.NoError(t, err) + state := TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 22, + MigrationJobID: 11, + MinWriteTSExclusive: 333, + Armed: true, + } + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + pebbleStore, ok := st.(*pebbleStore) + require.True(t, ok) + junkKey := append([]byte(migrationReadyPrefix), []byte("side-record")...) + require.NoError(t, pebbleStore.db.Set(junkKey, []byte("not-readiness-state"), pebbleStore.directApplyWriteOpts())) + require.NoError(t, st.Close()) + + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + reader, ok := reopened.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) +} + func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-import-persist-*") From fa769ac5ccb1c888cd55d16134af78ce9cbc5a13 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 06:57:03 +0900 Subject: [PATCH 034/162] Harden split migration start guards --- adapter/distribution_server.go | 42 +++++++++++++++-- adapter/distribution_server_test.go | 56 +++++++++++++++++++++++ main.go | 11 +++++ main_encryption_rotate_on_startup_test.go | 1 + 4 files changed, 107 insertions(+), 3 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 389559b4e..ec1b9f7df 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -28,6 +28,7 @@ type DistributionServer struct { coordinator kv.Coordinator readTracker *kv.ActiveTimestampTracker migrationCapabilityGate SplitMigrationCapabilityGate + knownRaftGroups map[uint64]struct{} reloadRetry struct { attempts int interval time.Duration @@ -62,6 +63,21 @@ func WithSplitMigrationCapabilityGate(gate SplitMigrationCapabilityGate) Distrib } } +// WithDistributionKnownRaftGroups configures the Raft group IDs this node can +// route migration work to. +func WithDistributionKnownRaftGroups(groupIDs ...uint64) DistributionServerOption { + return func(s *DistributionServer) { + groups := make(map[uint64]struct{}, len(groupIDs)) + for _, groupID := range groupIDs { + if groupID == 0 { + continue + } + groups[groupID] = struct{}{} + } + s.knownRaftGroups = groups + } +} + // WithCatalogReloadRetryPolicy configures the retry policy used after split // commit when waiting for the local catalog snapshot to become visible. func WithCatalogReloadRetryPolicy(attempts int, interval time.Duration) DistributionServerOption { @@ -100,6 +116,8 @@ var ( errDistributionEngineNotConfigured = errors.New("distribution engine is not configured") errDistributionCatalogVersionNotFound = errors.New("route catalog version not found") errDistributionClusterNotReady = errors.New("cluster is not ready for split migration") + errDistributionRaftGroupsNotKnown = errors.New("distribution raft groups are not configured") + errDistributionUnknownTargetGroup = errors.New("unknown target group") ) // NewDistributionServer creates a new server. @@ -260,6 +278,9 @@ func (s *DistributionServer) startSplitMigrationParent( if parent.GroupID == req.GetTargetGroupId() { return distribution.RouteDescriptor{}, grpcStatusError(codes.InvalidArgument, "target group must differ from source route group") } + if err := s.verifyKnownTargetGroup(req.GetTargetGroupId()); err != nil { + return distribution.RouteDescriptor{}, err + } if err := s.rejectLiveSplitJob(ctx, snapshot, parent); err != nil { return distribution.RouteDescriptor{}, err } @@ -628,6 +649,16 @@ func (s *DistributionServer) updateSplitJobViaCoordinator( return nil } +func (s *DistributionServer) verifyKnownTargetGroup(groupID uint64) error { + if len(s.knownRaftGroups) == 0 { + return grpcStatusError(codes.FailedPrecondition, errDistributionRaftGroupsNotKnown.Error()) + } + if _, ok := s.knownRaftGroups[groupID]; !ok { + return grpcStatusError(codes.InvalidArgument, errDistributionUnknownTargetGroup.Error()) + } + return nil +} + func (s *DistributionServer) createSplitJobViaCoordinator(ctx context.Context, readTS uint64, job distribution.SplitJob) error { if job.JobID == math.MaxUint64 { return splitJobCatalogStatusError(distribution.ErrCatalogSplitJobIDOverflow) @@ -651,9 +682,14 @@ func (s *DistributionServer) createSplitJobViaCoordinator(ctx context.Context, r Value: distribution.EncodeCatalogNextSplitJobID(job.JobID + 1), }, }, - IsTxn: true, - StartTS: readTS, - ReadKeys: [][]byte{jobKey, nextIDKey}, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{ + jobKey, + nextIDKey, + distribution.CatalogVersionKey(), + distribution.CatalogRouteKey(job.SourceRouteID), + }, }); err != nil { return splitJobCoordinatorStatusError(err) } diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index ceb73fe36..5751377d8 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -224,6 +224,7 @@ func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t * distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), WithSplitMigrationCapabilityGate(func(context.Context) error { gateCalls++ return nil @@ -241,6 +242,12 @@ func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t * require.Equal(t, uint64(1), resp.JobId) require.Equal(t, 1, gateCalls) require.Equal(t, 1, coordinator.dispatchCalls) + require.ElementsMatch(t, []string{ + string(distribution.CatalogSplitJobKey(1)), + string(distribution.CatalogNextSplitJobIDKey()), + string(distribution.CatalogVersionKey()), + string(distribution.CatalogRouteKey(1)), + }, byteSliceStrings(coordinator.lastReadKeys)) jobs, err := catalog.ListSplitJobs(ctx) require.NoError(t, err) @@ -259,6 +266,43 @@ func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t * require.Equal(t, uint64(2), next) } +func TestDistributionServerStartSplitMigration_RejectsUnknownTargetGroup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 3, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.ErrorContains(t, err, errDistributionUnknownTargetGroup.Error()) + require.Zero(t, coordinator.dispatchCalls) +} + func TestDistributionServerStartSplitMigration_RejectsSecondLiveJob(t *testing.T) { t.Parallel() @@ -299,6 +343,7 @@ func TestDistributionServerStartSplitMigration_RejectsSecondLiveJob(t *testing.T distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2, 3, 4), WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), ) @@ -349,6 +394,7 @@ func TestDistributionServerStartSplitMigration_RejectsReservedMigrationRange(t * distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), ) @@ -1237,12 +1283,21 @@ func splitJobIDs(jobs []*pb.SplitJob) []uint64 { return ids } +func byteSliceStrings(in [][]byte) []string { + out := make([]string, 0, len(in)) + for _, item := range in { + out = append(out, string(item)) + } + return out +} + type distributionCoordinatorStub struct { store store.MVCCStore leader bool dispatchErr error nextTS uint64 lastStartTS uint64 + lastReadKeys [][]byte afterDispatch func(context.Context, store.MVCCStore, uint64) error asyncApplyDone chan error asyncApplyDelay time.Duration @@ -1266,6 +1321,7 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope } startTS, commitTS := s.nextTimestamps(reqs.StartTS) s.lastStartTS = startTS + s.lastReadKeys = cloneByteSlices(reqs.ReadKeys) mutations, err := coordinatorStubMutations(reqs.Elems) if err != nil { diff --git a/main.go b/main.go index c236598d8..b33c4cbcd 100644 --- a/main.go +++ b/main.go @@ -503,6 +503,7 @@ func run() error { distCatalog, adapter.WithDistributionCoordinator(coordinate), adapter.WithDistributionActiveTimestampTracker(readTracker), + adapter.WithDistributionKnownRaftGroups(shardGroupIDs(shardGroups)...), ) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) compactor := kv.NewFSMCompactor( @@ -655,6 +656,15 @@ func cloneRaftServers(in []raftengine.Server) []raftengine.Server { return append([]raftengine.Server(nil), in...) } +func shardGroupIDs(groups map[uint64]*kv.ShardGroup) []uint64 { + ids := make([]uint64, 0, len(groups)) + for id := range groups { + ids = append(ids, id) + } + slices.Sort(ids) + return ids +} + type runtimeConfig struct { groups []groupSpec defaultGroup uint64 @@ -1830,6 +1840,7 @@ func startupRotationGatedMethod(fullMethod string) bool { case pb.Internal_Forward_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, pb.RaftAdmin_AddLearner_FullMethodName, pb.RaftAdmin_PromoteLearner_FullMethodName, diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index 5e16348ad..d8a33706d 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -421,6 +421,7 @@ func TestStartupPublicKVGate_BlocksMutatorsUntilReady(t *testing.T) { pb.Internal_Forward_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, pb.RaftAdmin_AddLearner_FullMethodName, pb.RaftAdmin_PromoteLearner_FullMethodName, From 4b8e345aef5629f57d151647a25c9da73249bca4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 07:02:01 +0900 Subject: [PATCH 035/162] Harden staged visibility OCC guards --- kv/shard_store.go | 47 +++++++++++++++++++++------- kv/shard_store_test.go | 71 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 12 deletions(-) diff --git a/kv/shard_store.go b/kv/shard_store.go index c69b47e3b..ee7fa66a7 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -122,8 +122,13 @@ func (s *ShardStore) routeForExplicitGroupKey(groupID uint64, key []byte) (distr if s == nil || s.engine == nil { return fallback, nil } - if route, ok := s.engine.GetRoute(routeKey(key)); ok && route.GroupID == groupID { - return route, nil + if route, ok := s.engine.GetRoute(routeKey(key)); ok { + if route.GroupID == groupID { + return route, nil + } + if routeHasStagedVisibility(route) { + return distribution.Route{}, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d key=%q", groupID, key) + } } if s.groupHasStagedVisibility(groupID) { return distribution.Route{}, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d key=%q", groupID, key) @@ -443,6 +448,10 @@ func (s *ShardStore) routesForExplicitGroupScan(groupID uint64, start []byte, en for _, route := range routes { if route.GroupID == groupID { matched = append(matched, route) + continue + } + if routeHasStagedVisibility(route) { + return nil, false, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d range=[%q,%q)", groupID, start, end) } } if len(matched) > 0 { @@ -1841,6 +1850,7 @@ func (s *ShardStore) ApplyMutations(ctx context.Context, mutations []*store.KVPa return err } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) + readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS)) } @@ -1855,6 +1865,7 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. return err } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) + readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) } @@ -1870,6 +1881,7 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor return err } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) + readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) } @@ -1900,24 +1912,35 @@ func (s *ShardStore) ensureMutationWriteTimestampFloors(mutations []*store.KVPai } func (s *ShardStore) readKeysWithStagedVisibilityAliases(group *ShardGroup, readKeys [][]byte) [][]byte { - if len(readKeys) == 0 || s == nil || s.engine == nil || group == nil { + if len(readKeys) == 0 { return readKeys } - var out [][]byte for _, key := range readKeys { - alias, ok := s.stagedVisibilityReadKeyAlias(group, key) - if !ok { + readKeys = s.appendStagedVisibilityAlias(group, readKeys, key) + } + return readKeys +} + +func (s *ShardStore) readKeysWithStagedVisibilityMutationAliases(group *ShardGroup, readKeys [][]byte, mutations []*store.KVPairMutation) [][]byte { + for _, mut := range mutations { + if mut == nil { continue } - if out == nil { - out = append([][]byte(nil), readKeys...) - } - out = append(out, alias) + readKeys = s.appendStagedVisibilityAlias(group, readKeys, mut.Key) } - if out == nil { + return readKeys +} + +func (s *ShardStore) appendStagedVisibilityAlias(group *ShardGroup, readKeys [][]byte, key []byte) [][]byte { + if s == nil || s.engine == nil || group == nil { return readKeys } - return out + alias, ok := s.stagedVisibilityReadKeyAlias(group, key) + if !ok { + return readKeys + } + out := append([][]byte(nil), readKeys...) + return append(out, alias) } func (s *ShardStore) stagedVisibilityReadKeyAlias(group *ShardGroup, key []byte) ([]byte, bool) { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 9ff8677a4..2040402c3 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -182,6 +182,40 @@ func TestShardStoreExplicitGroupReads_MergeStagedVisibility(t *testing.T) { }, kvs) } +func TestShardStoreExplicitGroupReads_FailClosedWhenRouteMovedToStagedGroup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 2, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + }, + })) + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("b"), []byte("old-source"), 10, 0)) + require.NoError(t, groups[2].Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged-target"), 20, 0)) + + _, err := st.GetGroupAt(ctx, 1, []byte("b"), 25) + require.ErrorIs(t, err, ErrExplicitGroupStagedVisibilityUnresolved) + + _, err = st.ScanGroupAt(ctx, 1, []byte("a"), []byte("z"), 10, 25) + require.ErrorIs(t, err, ErrExplicitGroupStagedVisibilityUnresolved) +} + func TestShardStoreScanAt_ContinuesStagedVisibilityAfterCandidateWindow(t *testing.T) { t.Parallel() @@ -220,6 +254,43 @@ func TestShardStoreApplyMutations_ValidatesStagedReadKeys(t *testing.T) { require.ErrorIs(t, err, store.ErrWriteConflict) } +func TestShardStoreApplyMutations_ValidatesStagedWriteKeys(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + writeKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, writeKey), []byte("staged"), 20, 0)) + + apply := []struct { + name string + fn func(context.Context, []*store.KVPairMutation, [][]byte, uint64, uint64) error + }{ + { + name: "direct", + fn: st.ApplyMutations, + }, + { + name: "raft", + fn: st.ApplyMutationsRaft, + }, + { + name: "raft_at", + fn: func(ctx context.Context, muts []*store.KVPairMutation, readKeys [][]byte, startTS, commitTS uint64) error { + return st.ApplyMutationsRaftAt(ctx, muts, readKeys, startTS, commitTS, 1) + }, + }, + } + for _, tc := range apply { + t.Run(tc.name, func(t *testing.T) { + err := tc.fn(ctx, []*store.KVPairMutation{ + {Op: store.OpTypePut, Key: writeKey, Value: []byte("write")}, + }, nil, 10, 101) + require.ErrorIs(t, err, store.ErrWriteConflict) + }) + } +} + func TestShardStorePhysicalLimitFailsClosedBeforeStagedVisibilityFallback(t *testing.T) { t.Parallel() From 4447dd315213282d49714e9945f52aa2c0c08f3b Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 07:09:26 +0900 Subject: [PATCH 036/162] Reject non-active migration sources --- adapter/distribution_server.go | 4 +++ adapter/distribution_server_test.go | 52 +++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index ec1b9f7df..5098ab32e 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -118,6 +118,7 @@ var ( errDistributionClusterNotReady = errors.New("cluster is not ready for split migration") errDistributionRaftGroupsNotKnown = errors.New("distribution raft groups are not configured") errDistributionUnknownTargetGroup = errors.New("unknown target group") + errDistributionSourceRouteNotActive = errors.New("source route is not active") ) // NewDistributionServer creates a new server. @@ -268,6 +269,9 @@ func (s *DistributionServer) startSplitMigrationParent( if !found { return distribution.RouteDescriptor{}, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error()) } + if parent.State != distribution.RouteStateActive { + return distribution.RouteDescriptor{}, grpcStatusError(codes.FailedPrecondition, errDistributionSourceRouteNotActive.Error()) + } splitKey := distribution.CloneBytes(req.GetSplitKey()) if err := validateSplitKey(parent, splitKey); err != nil { return distribution.RouteDescriptor{}, err diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 5751377d8..ba8e0fd6f 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -303,6 +303,58 @@ func TestDistributionServerStartSplitMigration_RejectsUnknownTargetGroup(t *test require.Zero(t, coordinator.dispatchCalls) } +func TestDistributionServerStartSplitMigration_RejectsNonActiveSourceRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + state distribution.RouteState + }{ + {name: "write_fenced", state: distribution.RouteStateWriteFenced}, + {name: "migrating_source", state: distribution.RouteStateMigratingSource}, + {name: "migrating_target", state: distribution.RouteStateMigratingTarget}, + } { + t.Run(tc.name, func(t *testing.T) { + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: tc.state, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionSourceRouteNotActive.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) + }) + } +} + func TestDistributionServerStartSplitMigration_RejectsSecondLiveJob(t *testing.T) { t.Parallel() From 061275ab2ecbdcfd6baada23b5b20f3a0abe8bab Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 07:16:17 +0900 Subject: [PATCH 037/162] Bound migration export sparse scans --- store/lsm_migration.go | 35 +++++++++++++-- store/migration_versions.go | 20 +++++++-- store/migration_versions_test.go | 75 ++++++++++++++++++++++++++++++++ 3 files changed, 122 insertions(+), 8 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 2be215212..9558a12a0 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -113,8 +113,7 @@ func (s *pebbleStore) exportPebbleIteratorPosition( return false, true, err } if commitTS <= opts.MinCommitTSExclusive { - _ = s.skipToNextUserKey(iter, userKey) - return false, true, nil + return s.skipPebbleExportVersionBelowMinTS(iter, opts, userKey, commitTS, result) } done, err = s.exportPebbleVersion(iter, opts, userKey, commitTS, result) return true, done, err @@ -122,7 +121,7 @@ func (s *pebbleStore) exportPebbleIteratorPosition( func (s *pebbleStore) skipPebbleExportKeyOutsideRange(iter *pebble.Iterator, opts ExportVersionsOptions, userKey []byte) (bool, error) { if opts.StartKey != nil && bytes.Compare(userKey, opts.StartKey) < 0 { - _ = s.skipToNextUserKey(iter, userKey) + advancePebbleExportPastCurrentUserKey(iter, userKey) return true, nil } if opts.EndKey == nil || bytes.Compare(userKey, opts.EndKey) < 0 { @@ -131,10 +130,38 @@ func (s *pebbleStore) skipPebbleExportKeyOutsideRange(iter *pebble.Iterator, opt if pebbleExportCanStopAtEndKey(opts.StartKey, opts.EndKey, userKey) { return true, errExportReachedEnd } - _ = s.skipToNextUserKey(iter, userKey) + advancePebbleExportPastCurrentUserKey(iter, userKey) return true, nil } +func (s *pebbleStore) skipPebbleExportVersionBelowMinTS( + iter *pebble.Iterator, + opts ExportVersionsOptions, + userKey []byte, + commitTS uint64, + result *ExportVersionsResult, +) (advance bool, done bool, err error) { + rawValue := iter.Value() + result.ScannedBytes += versionExportSize(userKey, len(rawValue)) + result.NextCursor = encodeExportCursor(userKey, commitTS, exportCursorTagScanned) + if finishExportIfLimited(opts, result) { + result.Done = false + return false, false, nil + } + advancePebbleExportPastCurrentUserKey(iter, userKey) + return false, true, nil +} + +func advancePebbleExportPastCurrentUserKey(iter *pebble.Iterator, userKey []byte) { + userKey = bytes.Clone(userKey) + for iter.Next() { + currentUserKey, _ := decodeKeyView(iter.Key()) + if !bytes.Equal(currentUserKey, userKey) { + return + } + } +} + func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { for prefixLen := range userKey { prefix := userKey[:prefixLen] diff --git a/store/migration_versions.go b/store/migration_versions.go index d95ba6ab8..32db62ddf 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -109,12 +109,18 @@ func validateExportCursorRange(opts ExportVersionsOptions, pos exportCursorPosit } func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOptions { - if opts.AcceptKey != nil && opts.MaxScannedBytes == 0 { + if exportUsesSparseScanBudget(opts) && opts.MaxScannedBytes == 0 { opts.MaxScannedBytes = defaultSparseExportMaxScannedBytes } return opts } +func exportUsesSparseScanBudget(opts ExportVersionsOptions) bool { + return opts.AcceptKey != nil || + opts.MaxCommitTSInclusive != 0 || + opts.MinCommitTSExclusive != 0 +} + func isMigrationMetadataKey(rawKey []byte) bool { return bytes.Equal(rawKey, migrationAckMetaKeyBytes) || bytes.Equal(rawKey, migrationHLCFloorMetaKeyBytes) @@ -407,9 +413,6 @@ func exportMemoryVersion(opts ExportVersionsOptions, cursorCommitTS uint64, key if shouldSkipMemoryVersion(cursorCommitTS, version) { return true } - if version.TS <= opts.MinCommitTSExclusive { - return false - } tag := appendMemoryExportVersion(opts, key, version, result) return finishMemoryExportPosition(opts, key, version, tag, result) } @@ -426,6 +429,15 @@ func exportMemoryVersionsForKey( if err := ctx.Err(); err != nil { return false, errors.WithStack(err) } + if shouldSkipMemoryVersion(cursorCommitTS, versions[i]) { + continue + } + if versions[i].TS <= opts.MinCommitTSExclusive { + if !finishMemoryExportPosition(opts, key, versions[i], exportCursorTagScanned, result) { + return false, nil + } + return true, nil + } if !exportMemoryVersion(opts, cursorCommitTS, key, versions[i], result) { return !finishExportIfLimited(opts, result), nil } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 3019375e4..200e1cda5 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -174,6 +174,81 @@ func TestExportVersionsAppliesDefaultSparseScanBudget(t *testing.T) { }) } +func TestExportVersionsAppliesDefaultScanBudgetForTimestampFilter(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + value := bytes.Repeat([]byte("x"), defaultSparseExportMaxScannedBytes) + require.NoError(t, st.PutAt(ctx, []byte("drop-new"), value, 30, 0)) + require.NoError(t, st.PutAt(ctx, []byte("keep-old"), []byte("v"), 10, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + MaxCommitTSInclusive: 20, + MaxVersions: 10, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.GreaterOrEqual(t, first.ScannedBytes, uint64(defaultSparseExportMaxScannedBytes)) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: first.NextCursor, + MaxCommitTSInclusive: 20, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("keep-old"), CommitTS: 10, Value: []byte("v")}}, second.Versions) + }) +} + +func TestExportVersionsMinTSPruneDoesNotSkipPrefixedKeys(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + prefixed := []byte{'a', 0xff, 0x00} + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("old"), 5, 0)) + require.NoError(t, st.PutAt(ctx, prefixed, []byte("new"), 20, 0)) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("a"), + EndKey: []byte("b"), + MinCommitTSExclusive: 10, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, res.Done) + require.Equal(t, []MVCCVersion{{Key: prefixed, CommitTS: 20, Value: []byte("new")}}, res.Versions) + }) +} + +func TestExportVersionsMinTSSkipHonorsScanBudget(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + value := bytes.Repeat([]byte("x"), defaultSparseExportMaxScannedBytes) + require.NoError(t, st.PutAt(ctx, []byte("old"), value, 5, 0)) + require.NoError(t, st.PutAt(ctx, []byte("tail"), []byte("v"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + MinCommitTSExclusive: 10, + MaxVersions: 10, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.GreaterOrEqual(t, first.ScannedBytes, uint64(defaultSparseExportMaxScannedBytes)) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: first.NextCursor, + MinCommitTSExclusive: 10, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("tail"), CommitTS: 20, Value: []byte("v")}}, second.Versions) + }) +} + func TestExportVersionsUsesUserKeyRangeBounds(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() From b5723b5694a08d24b5d5af78a4b908db0fc65e59 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 07:31:55 +0900 Subject: [PATCH 038/162] Fix target readiness staged delete paths --- kv/fsm.go | 26 ++++ kv/fsm_migration_fence_test.go | 27 ++++ kv/shard_store.go | 149 +++++++++++++++++++--- kv/shard_store_test.go | 69 ++++++++++ kv/sharded_coordinator.go | 19 +++ kv/sharded_coordinator_del_prefix_test.go | 37 ++++++ 6 files changed, 307 insertions(+), 20 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 85dd67590..4954209f5 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -579,6 +579,13 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin if err := f.verifyRouteWriteFloorForPrefix(prefix, commitTS); err != nil { return err } + routes := f.stagedVisibilityRoutesForPrefix(prefix) + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return f.store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, f.pendingApplyIdx) + } + if err := deleteStagedVisibilityPrefixes(routes, prefix, txnCommonPrefix, deleteStagedPrefix); err != nil { + return err + } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -586,6 +593,25 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin return nil } +func (f *kvFSM) stagedVisibilityRoutesForPrefix(prefix []byte) []distribution.Route { + if f.routes == nil { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + start, end := routePrefixRange(prefix) + routes := snap.IntersectingRoutes(start, end) + out := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID == f.shardGroupID && routeHasStagedVisibility(route) { + out = append(out, route) + } + } + return out +} + func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation) error { for _, mut := range muts { if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 9ebaab322..09ab1163a 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -183,6 +183,33 @@ func TestFSMRejectsDelPrefixWithoutTargetReadinessProof(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMDelPrefixTombstonesStagedVisibilityRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, fsm.store.PutAt(ctx, stagedKey, []byte("staged"), 110, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: rawKey}}, + }, 120) + require.NoError(t, err) + + _, err = fsm.store.GetAt(ctx, stagedKey, 130) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() diff --git a/kv/shard_store.go b/kv/shard_store.go index 4e3fb96c4..181f3286a 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -304,7 +304,7 @@ func (s *ShardStore) ExistsAt(ctx context.Context, key []byte, ts uint64) (bool, // pending.length fast-path during churn. Mirrors LeaderRoutedStore's fix // for codex P1 #796. func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitTS uint64) (bool, error) { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return false, nil } @@ -312,11 +312,7 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT // without raft; preserve the existing local-only fallback there. engine := engineForGroup(g) if engine == nil { - exists, err := g.Store.CommittedVersionAt(ctx, key, commitTS) - if err != nil { - return false, errors.WithStack(err) - } - return exists, nil + return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) } if !isLinearizableRaftLeader(ctx, engine) && !tryEngineLinearizableFence(ctx, engine) { // Not the linearizable leader for this group AND the ReadIndex @@ -326,7 +322,19 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT // serialization. return false, nil } - exists, err := g.Store.CommittedVersionAt(ctx, key, commitTS) + return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) +} + +func committedVersionAtForRoute(ctx context.Context, st store.MVCCStore, route distribution.Route, key []byte, commitTS uint64) (bool, error) { + exists, err := st.CommittedVersionAt(ctx, key, commitTS) + if err != nil { + return false, errors.WithStack(err) + } + if exists || !routeHasStagedVisibility(route) { + return exists, nil + } + stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, key) + exists, err = st.CommittedVersionAt(ctx, stagedKey, commitTS) if err != nil { return false, errors.WithStack(err) } @@ -399,7 +407,55 @@ func (s *ShardStore) ScanGroupAt(ctx context.Context, groupID uint64, start []by if err := s.verifyExplicitGroupRoutesForRange(ctx, groupID, routes, start, end); err != nil { return nil, err } - return s.scanRouteAtDirection(ctx, routes[0], start, end, limit, ts, false, true) + return s.scanExplicitGroupRoutesAt(ctx, routes, start, end, limit, ts) +} + +func (s *ShardStore) scanExplicitGroupRoutesAt(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + if len(routes) == 1 { + return s.scanRouteAtDirection(ctx, routes[0], start, end, limit, ts, false, true) + } + out := make([]*store.KVPair, 0, limit) + rawRouteBounds := rawScanBoundsMatchRouteKeyspace(start, end) + for _, route := range routes { + scanStart, scanEnd := start, end + if rawRouteBounds { + scanStart = clampScanStart(start, route.Start) + scanEnd = clampScanEnd(end, route.End) + } + kvs, err := s.scanRouteAtDirection(ctx, route, scanStart, scanEnd, limit-len(out), ts, false, true) + if err != nil { + return nil, err + } + for _, kvp := range kvs { + if !kvBelongsToRoute(kvp, route) { + continue + } + out = append(out, kvp) + if len(out) == limit { + return out, nil + } + } + } + return out, nil +} + +func rawScanBoundsMatchRouteKeyspace(start []byte, end []byte) bool { + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + if !bytes.Equal(routeStart, start) { + return false + } + if end == nil { + return routeEnd == nil + } + return bytes.Equal(routeEnd, end) +} + +func kvBelongsToRoute(kvp *store.KVPair, route distribution.Route) bool { + if kvp == nil || route.RouteID == 0 { + return true + } + key := routeKey(kvp.Key) + return routeRangeIntersects(key, nextScanCursor(key), route.Start, route.End) } func (s *ShardStore) ReverseScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { @@ -1838,13 +1894,20 @@ func (s *ShardStore) resolveSingleShardGroup(mutations []*store.KVPairMutation) // DeletePrefixAt applies a prefix delete to every shard in the store. func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { - if err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS); err != nil { + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { return err } - for _, g := range s.groups { + for groupID, g := range s.groups { if g == nil || g.Store == nil { continue } + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return g.Store.DeletePrefixAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS) + } + if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { + return err + } if err := g.Store.DeletePrefixAt(ctx, prefix, excludePrefix, commitTS); err != nil { return errors.WithStack(err) } @@ -1854,13 +1917,20 @@ func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludeP // DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { - if err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS); err != nil { + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { return err } - for _, g := range s.groups { + for groupID, g := range s.groups { if g == nil || g.Store == nil { continue } + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return g.Store.DeletePrefixAtRaft(ctx, stagedPrefix, stagedExcludePrefix, commitTS) + } + if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { + return err + } if err := g.Store.DeletePrefixAtRaft(ctx, prefix, excludePrefix, commitTS); err != nil { return errors.WithStack(err) } @@ -1883,13 +1953,20 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl // is the receiver only when an aggregate (admin / coordinator) path // is replaying a global FLUSHALL, which is not raft-applied. func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { - if err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS); err != nil { + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { return err } - for _, g := range s.groups { + for groupID, g := range s.groups { if g == nil || g.Store == nil { continue } + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return g.Store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, appliedIndex) + } + if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { + return err + } // Pass appliedIndex through to every group. In the // single-group call-path (the production raft-apply case) // this is correct: appliedIndex IS that group's raft entry @@ -1907,25 +1984,57 @@ func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, ex return nil } -func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte, commitTS uint64) error { +func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte, commitTS uint64) ([]distribution.Route, error) { if s == nil || s.engine == nil { - return nil + return nil, nil } routeStart, routeEnd := routePrefixRange(prefix) routes := s.engine.GetIntersectingRoutes(routeStart, routeEnd) if len(routes) == 0 { - return errors.WithStack(ErrRouteCutoverPending) + return nil, errors.WithStack(ErrRouteCutoverPending) } for _, route := range routes { g, ok := s.groupForID(route.GroupID) if !ok || g == nil || g.Store == nil { - return store.ErrNotSupported + return nil, store.ErrNotSupported } if err := s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd); err != nil { - return err + return nil, err } if err := verifyRouteWriteFloor(route, commitTS); err != nil { - return err + return nil, err + } + } + return routes, nil +} + +func routesForGroupID(routes []distribution.Route, groupID uint64) []distribution.Route { + out := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID == groupID { + out = append(out, route) + } + } + return out +} + +func deleteStagedVisibilityPrefixes(routes []distribution.Route, prefix []byte, excludePrefix []byte, deletePrefix func([]byte, []byte) error) error { + seen := make(map[uint64]struct{}) + for _, route := range routes { + if !routeHasStagedVisibility(route) { + continue + } + if _, ok := seen[route.MigrationJobID]; ok { + continue + } + seen[route.MigrationJobID] = struct{}{} + stagedPrefix := distribution.MigrationStagedDataKey(route.MigrationJobID, prefix) + var stagedExcludePrefix []byte + if len(excludePrefix) > 0 { + stagedExcludePrefix = distribution.MigrationStagedDataKey(route.MigrationJobID, excludePrefix) + } + if err := deletePrefix(stagedPrefix, stagedExcludePrefix); err != nil { + return errors.WithStack(err) } } return nil diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index ffc34ff60..72f23b6ad 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -88,6 +88,19 @@ func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) } +func TestShardStoreCommittedVersionAtChecksStagedVisibilityKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, rawKey, 77) + require.NoError(t, err) + require.True(t, landed) +} + func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T) { t.Parallel() @@ -278,6 +291,45 @@ func TestShardStoreScanGroupAtChecksEveryCoveredRoute(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreScanGroupAtSplitsCoveredRoutesForStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 42, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 10, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("left-live"), 110, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(10, []byte("x")), []byte("right-staged"), 120, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b"), Value: []byte("left-live")}, + {Key: []byte("x"), Value: []byte("right-staged")}, + }, kvs) +} + func TestShardStorePhysicalLimitScanChecksTargetReadiness(t *testing.T) { t.Parallel() @@ -527,6 +579,23 @@ func TestShardStoreDeletePrefixAtRaftEnforcesRouteFloorBeforeDelete(t *testing.T require.Equal(t, []byte("v"), got) } +func TestShardStoreDeletePrefixAtTombstonesStagedVisibilityRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, st.DeletePrefixAt(ctx, rawKey, nil, 130)) + + _, err := st.GetAt(ctx, rawKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = group.Store.GetAt(ctx, stagedKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 285036d3e..a31e02e6d 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1039,6 +1039,9 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT if err := c.rejectDelPrefixesBelowWriteFloor(elems, ts); err != nil { return nil, err } + if err := c.rejectDelPrefixesWithoutTargetReadinessProof(ctx, elems, ts); err != nil { + return nil, err + } requests := make([]*pb.Request, 0, len(elems)) for _, elem := range elems { requests = append(requests, &pb.Request{ @@ -1087,6 +1090,22 @@ func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) err return nil } +func (c *ShardedCoordinator) rejectDelPrefixesWithoutTargetReadinessProof(ctx context.Context, elems []*Elem[OP], commitTS uint64) error { + shards, ok := c.store.(*ShardStore) + if !ok || shards == nil { + return nil + } + for _, elem := range elems { + if elem == nil { + continue + } + if _, err := shards.verifyPrefixDeleteRoutes(ctx, elem.Key, commitTS); err != nil { + return errors.Wrapf(err, "prefix %q", elem.Key) + } + } + return nil +} + func (c *ShardedCoordinator) rejectDelPrefixesBelowWriteFloor(elems []*Elem[OP], commitTS uint64) error { if c == nil || c.engine == nil { return nil diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 5b712db58..e2767eeb0 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -200,6 +200,43 @@ func TestShardedCoordinatorRejectsDelPrefixBelowRouteFloorBeforeBroadcast(t *tes require.Empty(t, g2Txn.requests) } +func TestShardedCoordinatorRejectsDelPrefixWithoutTargetReadinessProofBeforeBroadcast(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + g1 := &ShardGroup{Txn: g1Txn, Store: store.NewMVCCStore()} + g2 := &ShardGroup{Txn: g2Txn, Store: store.NewMVCCStore()} + applyTargetReadinessState(t, g2, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + groups := map[uint64]*ShardGroup{1: g1, 2: g2} + shardStore := NewShardStore(engine, groups) + coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), shardStore) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteCutoverPending) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + func TestShardedCoordinatorRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() From ca503cff888dc1c5cdd979932cbbed0b537a5c7e Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 07:35:11 +0900 Subject: [PATCH 039/162] Fix staged migration read safety --- kv/fsm.go | 82 ++-------------------- kv/fsm_migration_fence_test.go | 45 +++---------- kv/shard_store.go | 94 ++++++++++++++++++-------- kv/shard_store_test.go | 105 +++++++++++++++++++++++++++++ kv/sharded_coordinator.go | 57 ++++++++++++++++ kv/sharded_coordinator_txn_test.go | 83 +++++++++++++++++++++++ 6 files changed, 329 insertions(+), 137 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 1693d5eaa..9e82f62b5 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -501,7 +501,7 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui return f.handleDelPrefix(ctx, prefix, commitTS) } - if err := f.validateRawMutationsForApply(ctx, r.Mutations, commitTS); err != nil { + if err := f.validateRawMutationsForApply(ctx, r.Mutations); err != nil { return err } @@ -518,16 +518,16 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui return nil } -func (f *kvFSM) validateRawMutationsForApply(ctx context.Context, muts []*pb.Mutation, commitTS uint64) error { +func (f *kvFSM) validateRawMutationsForApply(ctx context.Context, muts []*pb.Mutation) error { for _, mut := range muts { - if err := f.validateRawMutationForApply(ctx, mut, commitTS); err != nil { + if err := f.validateRawMutationForApply(ctx, mut); err != nil { return err } } return nil } -func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutation, commitTS uint64) error { +func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutation) error { if mut == nil || len(mut.Key) == 0 { return errors.WithStack(ErrInvalidRequest) } @@ -538,9 +538,6 @@ func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutatio if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { return err } - if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { - return err - } if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { return err } @@ -564,9 +561,6 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { return err } - if err := f.verifyRouteWriteTimestampFloorForPrefix(prefix, commitTS); err != nil { - return err - } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -574,51 +568,6 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin return nil } -func (f *kvFSM) verifyRouteWriteTimestampFloorForKey(key []byte, commitTS uint64) error { - if f.routes == nil || commitTS == 0 { - return nil - } - snap, ok := f.routes.Current() - if !ok { - return nil - } - rkey := routeKey(key) - route, ok := snap.RouteOf(rkey) - if !ok || route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { - return nil - } - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, rkey, commitTS, route.MinWriteTSExclusive) -} - -func (f *kvFSM) verifyRouteWriteTimestampFloorForPrefix(prefix []byte, commitTS uint64) error { - if f.routes == nil || commitTS == 0 { - return nil - } - snap, ok := f.routes.Current() - if !ok { - return nil - } - start, end := routePrefixRange(prefix) - for _, route := range snap.IntersectingRoutes(start, end) { - if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", prefix, start, end, commitTS, route.MinWriteTSExclusive) - } - } - return nil -} - -func (f *kvFSM) verifyRouteWriteTimestampFloorForMutations(muts []*pb.Mutation, commitTS uint64) error { - for _, mut := range muts { - if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { - continue - } - if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { - return err - } - } - return nil -} - func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation) error { for _, mut := range muts { if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { @@ -1036,9 +985,6 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { return err } - if err := f.verifyRouteWriteTimestampFloorForMutations(uniq, meta.CommitTS); err != nil { - return err - } if err := f.validateConflicts(ctx, uniq, startTS); err != nil { return errors.WithStack(err) } @@ -1105,7 +1051,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsNotFencedAboveWriteFloor(muts, commitTS) + uniq, err := f.uniqueMutationsNotFenced(muts) if err != nil { return err } @@ -1132,25 +1078,11 @@ func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation) ([]*pb.Mutation, e return uniq, nil } -func (f *kvFSM) uniqueMutationsNotFencedAboveWriteFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { - uniq, err := f.uniqueMutationsNotFenced(muts) - if err != nil { - return nil, err - } - if err := f.verifyRouteWriteTimestampFloorForMutations(uniq, commitTS); err != nil { - return nil, err - } - return uniq, nil -} - -func (f *kvFSM) uniqueMutationsAboveWriteFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { +func uniqueTxnMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { return nil, err } - if err := f.verifyRouteWriteTimestampFloorForMutations(uniq, commitTS); err != nil { - return nil, err - } return uniq, nil } @@ -1193,7 +1125,7 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - uniq, err := f.uniqueMutationsAboveWriteFloor(muts, commitTS) + uniq, err := uniqueTxnMutations(muts) if err != nil { return err } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 045a9d04d..70be77390 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -123,29 +123,21 @@ func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { require.False(t, errors.Is(err, ErrRouteWriteFenced), "ABORT must keep the narrow cleanup lane open") } -func TestFSMRejectsRawPointWriteAtMigrationTimestampFloor(t *testing.T) { +func TestFSMAllowsRawPointWriteAtMigrationTimestampFloorDuringReplay(t *testing.T) { t.Parallel() ctx := context.Background() fsm := newWriteFloorFSM(t) err := fsm.handleRawRequest(ctx, &pb.Request{ - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("low")}}, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("replayed")}}, }, 100) - require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) - - _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.ErrorIs(t, getErr, store.ErrKeyNotFound) - - err = fsm.handleRawRequest(ctx, &pb.Request{ - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("ok")}}, - }, 101) require.NoError(t, err) got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) require.NoError(t, getErr) - require.Equal(t, []byte("ok"), got) + require.Equal(t, []byte("replayed"), got) } -func TestFSMRejectsDelPrefixAtMigrationTimestampFloor(t *testing.T) { +func TestFSMAllowsDelPrefixAtMigrationTimestampFloorDuringReplay(t *testing.T) { t.Parallel() ctx := context.Background() @@ -155,14 +147,13 @@ func TestFSMRejectsDelPrefixAtMigrationTimestampFloor(t *testing.T) { err := fsm.handleRawRequest(ctx, &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, }, 100) - require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + require.NoError(t, err) - got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) } -func TestFSMRejectsOnePhaseTxnAtMigrationTimestampFloor(t *testing.T) { +func TestFSMAllowsOnePhaseTxnAtMigrationTimestampFloorDuringReplay(t *testing.T) { t.Parallel() ctx := context.Background() @@ -177,21 +168,13 @@ func TestFSMRejectsOnePhaseTxnAtMigrationTimestampFloor(t *testing.T) { }, } err := fsm.handleTxnRequest(ctx, req, 100) - require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) - - _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.ErrorIs(t, getErr, store.ErrKeyNotFound) - - req.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 101}) - req.Mutations[1].Value = []byte("ok") - err = fsm.handleTxnRequest(ctx, req, 101) require.NoError(t, err) got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) require.NoError(t, getErr) - require.Equal(t, []byte("ok"), got) + require.Equal(t, []byte("low"), got) } -func TestFSMRejectsCommitButNotPrepareAtMigrationTimestampFloor(t *testing.T) { +func TestFSMAllowsCommitAtMigrationTimestampFloorDuringReplay(t *testing.T) { t.Parallel() ctx := context.Background() @@ -217,14 +200,6 @@ func TestFSMRejectsCommitButNotPrepareAtMigrationTimestampFloor(t *testing.T) { }, } err := fsm.handleTxnRequest(ctx, commit, 100) - require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) - - _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.ErrorIs(t, getErr, store.ErrKeyNotFound) - - commit.Mutations[0].Value = EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 101}) - commit.Mutations[1].Value = []byte("ignored") - err = fsm.handleTxnRequest(ctx, commit, 101) require.NoError(t, err) got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) require.NoError(t, getErr) diff --git a/kv/shard_store.go b/kv/shard_store.go index ee7fa66a7..7e681d53e 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -130,24 +130,9 @@ func (s *ShardStore) routeForExplicitGroupKey(groupID uint64, key []byte) (distr return distribution.Route{}, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d key=%q", groupID, key) } } - if s.groupHasStagedVisibility(groupID) { - return distribution.Route{}, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d key=%q", groupID, key) - } return fallback, nil } -func (s *ShardStore) groupHasStagedVisibility(groupID uint64) bool { - if s == nil || s.engine == nil { - return false - } - for _, route := range s.engine.Stats() { - if route.GroupID == groupID && routeHasStagedVisibility(route) { - return true - } - } - return false -} - func (s *ShardStore) getAtWithStagedVisibility(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { if err := ensureReadTSRetained(g.Store, ts); err != nil { return nil, err @@ -443,7 +428,8 @@ func (s *ShardStore) routesForExplicitGroupScan(groupID uint64, start []byte, en if s == nil || s.engine == nil { return fallback, false, nil } - routes := s.engine.GetIntersectingRoutes(start, end) + routeStart, routeEnd, routeMapped := explicitGroupScanRouteBounds(start, end) + routes := s.engine.GetIntersectingRoutes(routeStart, routeEnd) matched := make([]distribution.Route, 0, len(routes)) for _, route := range routes { if route.GroupID == groupID { @@ -455,14 +441,30 @@ func (s *ShardStore) routesForExplicitGroupScan(groupID uint64, start []byte, en } } if len(matched) > 0 { - return matched, true, nil - } - if s.groupHasStagedVisibility(groupID) { - return nil, false, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d range=[%q,%q)", groupID, start, end) + return matched, !routeMapped, nil } return fallback, false, nil } +func explicitGroupScanRouteBounds(start []byte, end []byte) ([]byte, []byte, bool) { + routeStart := routeKey(start) + if len(start) == 0 { + routeStart = []byte("") + } + routeEnd := end + routeMapped := !bytes.Equal(routeStart, start) + if end != nil { + normalizedEnd := routeKey(end) + if !bytes.Equal(normalizedEnd, end) { + routeMapped = true + } + } + if routeMapped && len(routeStart) != 0 { + routeEnd = prefixScanEnd(routeStart) + } + return routeStart, routeEnd, routeMapped +} + func (s *ShardStore) scanRoutesAt(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) for _, route := range routes { @@ -826,8 +828,10 @@ func (s *ShardStore) scanRouteWithStagedVisibilityPage( return nil, nil, false, err } out := visibleLogicalKVs(versions, ts, reverse) - boundary, hasBoundary := stagedVisibilityCandidateBoundary(liveKVs, stagedKVs, reverse) - exhausted := len(liveKVs) < window && len(stagedKVs) < window + liveExhausted := len(liveKVs) < window + stagedExhausted := len(stagedKVs) < window + boundary, hasBoundary := stagedVisibilityCandidateBoundary(liveKVs, stagedKVs, liveExhausted, stagedExhausted, reverse) + exhausted := liveExhausted && stagedExhausted if len(out) >= limit { clear(out[limit:]) return out[:limit], boundary, !exhausted && hasBoundary, nil @@ -843,22 +847,58 @@ func (s *ShardStore) scanRouteWithStagedVisibilityPage( } } -func stagedVisibilityCandidateBoundary(liveKVs []*store.KVPair, stagedKVs []*store.KVPair, reverse bool) ([]byte, bool) { - boundary := stagedVisibilityBoundary{reverse: reverse} +func stagedVisibilityCandidateBoundary(liveKVs []*store.KVPair, stagedKVs []*store.KVPair, liveExhausted bool, stagedExhausted bool, reverse bool) ([]byte, bool) { + liveBoundary := stagedVisibilityBoundary{reverse: reverse} for _, kvp := range liveKVs { if kvp == nil { continue } - boundary.visit(kvp.Key) + liveBoundary.visit(kvp.Key) } + stagedBoundary := stagedVisibilityBoundary{reverse: reverse} for _, kvp := range stagedKVs { rawKey, stagedOK := stagedVisibilityRawCandidateKey(kvp) if !stagedOK { continue } - boundary.visit(rawKey) + stagedBoundary.visit(rawKey) + } + return mergeStagedVisibilityBoundaries(liveBoundary, stagedBoundary, liveExhausted, stagedExhausted, reverse) +} + +func mergeStagedVisibilityBoundaries(live stagedVisibilityBoundary, staged stagedVisibilityBoundary, liveExhausted bool, stagedExhausted bool, reverse bool) ([]byte, bool) { + if !live.ok { + return staged.key, staged.ok + } + if !staged.ok { + return live.key, live.ok + } + if !liveExhausted && !stagedExhausted { + return nearerStagedVisibilityBoundary(live.key, staged.key, reverse), true + } + if liveExhausted && stagedExhausted { + return fartherStagedVisibilityBoundary(live.key, staged.key, reverse), true + } + if liveExhausted { + return staged.key, true + } + return live.key, true +} + +func nearerStagedVisibilityBoundary(a []byte, b []byte, reverse bool) []byte { + cmp := bytes.Compare(a, b) + if (!reverse && cmp <= 0) || (reverse && cmp >= 0) { + return a + } + return b +} + +func fartherStagedVisibilityBoundary(a []byte, b []byte, reverse bool) []byte { + cmp := bytes.Compare(a, b) + if (!reverse && cmp >= 0) || (reverse && cmp <= 0) { + return a } - return boundary.key, boundary.ok + return b } type stagedVisibilityBoundary struct { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 2040402c3..cfa7bc3e3 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -216,6 +216,74 @@ func TestShardStoreExplicitGroupReads_FailClosedWhenRouteMovedToStagedGroup(t *t require.ErrorIs(t, err, ErrExplicitGroupStagedVisibilityUnresolved) } +func TestShardStoreExplicitGroupScan_NormalizesRouteMappedBoundsForStagedRoutes(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: sqsGlobalRouteKey, + End: prefixScanEnd(sqsGlobalRouteKey), + GroupID: 2, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + }, + })) + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + start := []byte("!sqs|msg|vis|p|") + end := prefixScanEnd(start) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("!sqs|msg|vis|p|orders|1"), []byte("old-source"), 10, 0)) + + _, err := st.ScanGroupAt(ctx, 1, start, end, 10, 25) + require.ErrorIs(t, err, ErrExplicitGroupStagedVisibilityUnresolved) +} + +func TestShardStoreExplicitGroupRead_IgnoresUnrelatedStagedRoutes(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + }, + })) + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("b"), []byte("explicit-group"), 10, 0)) + + got, err := st.GetGroupAt(ctx, 1, []byte("b"), 25) + require.NoError(t, err) + require.Equal(t, []byte("explicit-group"), got) + + kvs, err := st.ScanGroupAt(ctx, 1, []byte("b"), []byte("c"), 10, 25) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: []byte("b"), Value: []byte("explicit-group")}}, kvs) +} + func TestShardStoreScanAt_ContinuesStagedVisibilityAfterCandidateWindow(t *testing.T) { t.Parallel() @@ -240,6 +308,23 @@ func TestShardStoreScanAt_ContinuesStagedVisibilityAfterCandidateWindow(t *testi require.Equal(t, []byte("k00000"), kvs[limit-1].Key) } +func TestStagedVisibilityCandidateBoundary_UsesSafeFrontier(t *testing.T) { + t.Parallel() + + live := []*store.KVPair{{Key: []byte("a")}, {Key: []byte("c")}} + staged := []*store.KVPair{ + {Key: distribution.MigrationStagedDataKey(9, []byte("b"))}, + {Key: distribution.MigrationStagedDataKey(9, []byte("z"))}, + } + boundary, ok := stagedVisibilityCandidateBoundary(live, staged, false, false, false) + require.True(t, ok) + require.Equal(t, []byte("c"), boundary) + + boundary, ok = stagedVisibilityCandidateBoundary(live, staged, false, false, true) + require.True(t, ok) + require.Equal(t, []byte("b"), boundary) +} + func TestShardStoreApplyMutations_ValidatesStagedReadKeys(t *testing.T) { t.Parallel() @@ -402,6 +487,26 @@ func TestShardStoreScanGroupAt_UsesExplicitGroup(t *testing.T) { require.Equal(t, []byte("msg-2"), kvs[0].Value) } +func TestShardStoreScanGroupAt_DoesNotClampRouteMappedRawBounds(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), nil, 42) + groups := map[uint64]*ShardGroup{ + 42: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + + start := []byte("!sqs|msg|vis|p|") + key := []byte("!sqs|msg|vis|p|orders|partition-2") + require.NoError(t, groups[42].Store.PutAt(ctx, key, []byte("msg-2"), 7, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, start, prefixScanEnd(start), 10, 7) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: key, Value: []byte("msg-2")}}, kvs) +} + func TestShardStoreGetGroupAt_UsesExplicitGroup(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 9cb4b8eee..44805ce8d 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1223,6 +1223,10 @@ func (c *ShardedCoordinator) dispatchMultiShardTxn(ctx context.Context, startTS, if err != nil { return nil, err } + groupedReadKeys = c.groupedReadKeysWithStagedVisibilityMutationAliases(groupedReadKeys, grouped) + if groupedReadKeyCount(groupedReadKeys) > maxReadKeys { + return nil, errors.WithStack(ErrInvalidRequest) + } prepared, err := c.prewriteTxn(ctx, startTS, commitTS, primaryKey, grouped, gids, groupedReadKeys, observedRouteVersion) if err != nil { return nil, err @@ -1298,6 +1302,10 @@ func (c *ShardedCoordinator) dispatchSingleShardTxn(ctx context.Context, startTS return nil, err } readKeys = c.readKeysWithStagedVisibilityAliasesForGroup(gid, readKeys) + readKeys = c.readKeysWithStagedVisibilityMutationAliasesForGroup(gid, readKeys, elems) + if len(readKeys) > maxReadKeys { + return nil, errors.WithStack(ErrInvalidRequest) + } // ReadKeys are included in the Raft log entry so the FSM validates // read-write conflicts atomically under applyMu. prevCommitTS, when set, // carries the one-phase dedup probe key for a retry that reuses a failed @@ -1335,6 +1343,27 @@ func (c *ShardedCoordinator) readKeysWithStagedVisibilityAliasesForGroup(gid uin return out } +func (c *ShardedCoordinator) readKeysWithStagedVisibilityMutationAliasesForGroup(gid uint64, readKeys [][]byte, elems []*Elem[OP]) [][]byte { + var out [][]byte + for _, elem := range elems { + if elem == nil { + continue + } + alias, ok := c.stagedVisibilityReadKeyAlias(gid, elem.Key) + if !ok { + continue + } + if out == nil { + out = append([][]byte(nil), readKeys...) + } + out = append(out, alias) + } + if out == nil { + return readKeys + } + return out +} + type preparedGroup struct { gid uint64 keys []*pb.Mutation @@ -1958,6 +1987,34 @@ func (c *ShardedCoordinator) groupReadKeysByShardID(readKeys [][]byte) (map[uint return grouped, nil } +func (c *ShardedCoordinator) groupedReadKeysWithStagedVisibilityMutationAliases(groupedReadKeys map[uint64][][]byte, groupedMutations map[uint64][]*pb.Mutation) map[uint64][][]byte { + out := groupedReadKeys + for gid, muts := range groupedMutations { + for _, mut := range muts { + if mut == nil { + continue + } + alias, ok := c.stagedVisibilityReadKeyAlias(gid, mut.Key) + if !ok { + continue + } + if out == nil { + out = make(map[uint64][][]byte) + } + out[gid] = append(out[gid], alias) + } + } + return out +} + +func groupedReadKeyCount(grouped map[uint64][][]byte) int { + var count int + for _, keys := range grouped { + count += len(keys) + } + return count +} + func (c *ShardedCoordinator) stagedVisibilityReadKeyAlias(gid uint64, key []byte) ([]byte, bool) { if c == nil || c.engine == nil || len(key) == 0 { return nil, false diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 588404cfb..e40318e00 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -109,9 +109,92 @@ func TestShardedCoordinatorDispatchTxn_AddsStagedReadKeyAlias(t *testing.T) { require.Equal(t, [][]byte{ readKey, distribution.MigrationStagedDataKey(9, readKey), + distribution.MigrationStagedDataKey(9, []byte("m")), }, txn.requests[0].ReadKeys) } +func TestShardedCoordinatorDispatchTxn_AddsStagedWriteKeyAlias(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + }, + })) + txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: txn}, + }, 1, NewHLC(), nil) + + writeKey := []byte("k") + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + IsTxn: true, + StartTS: 10, + CommitTS: 101, + Elems: []*Elem[OP]{{Op: Put, Key: writeKey, Value: []byte("write")}}, + }) + require.NoError(t, err) + require.Len(t, txn.requests, 1) + require.Equal(t, [][]byte{ + distribution.MigrationStagedDataKey(9, writeKey), + }, txn.requests[0].ReadKeys) +} + +func TestShardedCoordinatorPrewrite_AddsStagedWriteKeyAlias(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + writeKey := []byte("b") + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + IsTxn: true, + StartTS: 10, + CommitTS: 101, + Elems: []*Elem[OP]{ + {Op: Put, Key: writeKey, Value: []byte("write-b")}, + {Op: Put, Key: []byte("x"), Value: []byte("write-x")}, + }, + }) + require.NoError(t, err) + require.NotEmpty(t, g1Txn.requests) + require.NotEmpty(t, g2Txn.requests) + require.Equal(t, [][]byte{ + distribution.MigrationStagedDataKey(9, writeKey), + }, g1Txn.requests[0].ReadKeys) + require.Empty(t, g2Txn.requests[0].ReadKeys) +} + func cloneTxnRequest(req *pb.Request) *pb.Request { if req == nil { return nil From f22c4ff9edd3d778a18d285432f7261f42394ba4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 07:44:28 +0900 Subject: [PATCH 040/162] Stabilize urgent compactor pagination test --- adapter/redis_delta_compactor_test.go | 29 ++++++++++----------------- 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index c3f6f1518..358241350 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -411,27 +411,20 @@ func TestDeltaCompactor_UrgentCompactionPagination(t *testing.T) { require.NoError(t, st.PutAt(ctx, dKey, delta, i+1, 0)) } - // Queue and process the urgent compaction. - c.TriggerUrgentCompaction("hash", userKey) - - runCtx, cancel := context.WithCancel(ctx) - defer cancel() - go func() { _ = c.Run(runCtx) }() + // Exercise the urgent pagination loop directly. Running through c.Run would + // also start an initial background SyncOnce; the local test coordinator applies + // elems one-by-one instead of atomically, so a test read can observe the meta + // update before all delete elems have been applied under the race detector. + c.compactUrgentKey(ctx, urgentCompactionRequest{typeName: "hash", userKey: userKey}) - // Wait until the base meta holds the accumulated total. - // The pagination loop should take two passes: first 1025, then 9. - require.Eventually(t, func() bool { - readTS := st.LastCommitTS() - raw, err := st.GetAt(ctx, store.HashMetaKey(userKey), readTS) - if err != nil { - return false - } - got, err := store.UnmarshalHashMeta(raw) - return err == nil && got.Len == int64(totalDeltasU64) - }, 5*time.Second, 20*time.Millisecond, "all %d delta keys should be compacted into base meta", totalDeltasU64) + readTS := st.LastCommitTS() + raw, err := st.GetAt(ctx, store.HashMetaKey(userKey), readTS) + require.NoError(t, err) + got, err := store.UnmarshalHashMeta(raw) + require.NoError(t, err) + require.Equal(t, int64(totalDeltasU64), got.Len, "all %d delta keys should be compacted into base meta", totalDeltasU64) // No delta keys should remain after pagination compaction. - readTS := st.LastCommitTS() prefix := store.HashMetaDeltaScanPrefix(userKey) end := store.PrefixScanEnd(prefix) remaining, err := st.ScanAt(ctx, prefix, end, int(totalDeltasU64)+1, readTS) From 31ed5074c4da099c71a490769f5d1160e774fb71 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 08:01:12 +0900 Subject: [PATCH 041/162] Fix migration export metadata handling --- store/lsm_migration.go | 17 ++++++-- store/lsm_store.go | 11 +---- store/migration_versions.go | 8 +++- store/migration_versions_test.go | 73 ++++++++++++++++++++++++++++---- store/snapshot_pebble.go | 3 -- 5 files changed, 86 insertions(+), 26 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 9558a12a0..31d1e9a77 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -106,7 +106,14 @@ func (s *pebbleStore) exportPebbleIteratorPosition( return true, true, nil } userKey, commitTS := decodeKeyView(rawKey) - if userKey == nil || pebbleExportCursorEqual(pos, userKey, commitTS) { + if userKey == nil { + return true, true, nil + } + if pebbleExportCursorPrunedKey(pos, userKey, commitTS) { + advancePebbleExportPastCurrentUserKey(iter, userKey) + return false, true, nil + } + if pebbleExportCursorEqual(pos, userKey, commitTS) { return true, true, nil } if skipped, err := s.skipPebbleExportKeyOutsideRange(iter, opts, userKey); skipped || err != nil { @@ -143,7 +150,7 @@ func (s *pebbleStore) skipPebbleExportVersionBelowMinTS( ) (advance bool, done bool, err error) { rawValue := iter.Value() result.ScannedBytes += versionExportSize(userKey, len(rawValue)) - result.NextCursor = encodeExportCursor(userKey, commitTS, exportCursorTagScanned) + result.NextCursor = encodeExportCursor(userKey, commitTS, exportCursorTagPrunedKey) if finishExportIfLimited(opts, result) { result.Done = false return false, false, nil @@ -163,7 +170,7 @@ func advancePebbleExportPastCurrentUserKey(iter *pebble.Iterator, userKey []byte } func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { - for prefixLen := range userKey { + for prefixLen := 1; prefixLen <= len(userKey); prefixLen++ { prefix := userKey[:prefixLen] if (startKey == nil || bytes.Compare(prefix, startKey) >= 0) && bytes.Compare(prefix, endKey) < 0 { return false @@ -176,6 +183,10 @@ func pebbleExportCursorEqual(pos exportCursorPosition, userKey []byte, commitTS return pos.hasKey && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS } +func pebbleExportCursorPrunedKey(pos exportCursorPosition, userKey []byte, commitTS uint64) bool { + return pos.hasKey && pos.tag == exportCursorTagPrunedKey && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS +} + func (s *pebbleStore) exportPebbleVersion( iter *pebble.Iterator, opts ExportVersionsOptions, diff --git a/store/lsm_store.go b/store/lsm_store.go index 81c0dd1e9..d7f6e5f85 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -552,7 +552,8 @@ func isPebbleMetaKey(rawKey []byte) bool { bytes.Equal(rawKey, metaMinRetainedTSBytes) || bytes.Equal(rawKey, metaPendingMinRetainedTSBytes) || bytes.Equal(rawKey, metaAppliedIndexBytes) || - isMigrationMetadataKey(rawKey) + isMigrationMetadataKey(rawKey) || + bytes.HasPrefix(rawKey, encryption.WriterRegistryPrefix) } func (s *pebbleStore) findMaxCommitTS() (uint64, error) { @@ -2177,11 +2178,6 @@ func writeRestoreEntry(r io.Reader, batch *pebble.Batch, keyBuf []byte, kLen, vL return errors.WithStack(deferred.Finish()) } -func discardRestoreEntryValue(r io.Reader, vLen int) error { - _, err := io.CopyN(io.Discard, r, int64(vLen)) - return errors.WithStack(err) -} - func restoreBatchLoopStep(r io.Reader, db *pebble.DB, batch **pebble.Batch, keyBuf *[]byte) (bool, error) { kLen, vLen, eof, err := readRestoreEntry(r, keyBuf) if err != nil { @@ -2190,9 +2186,6 @@ func restoreBatchLoopStep(r io.Reader, db *pebble.DB, batch **pebble.Batch, keyB if eof { return true, nil } - if isMigrationMetadataKey((*keyBuf)[:kLen]) { - return false, discardRestoreEntryValue(r, vLen) - } if err := flushSnapshotBatchIfNeeded(db, batch, kLen, vLen); err != nil { return false, err } diff --git a/store/migration_versions.go b/store/migration_versions.go index 32db62ddf..043dcba8f 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -13,6 +13,7 @@ import ( const ( exportCursorTagEmitted byte = iota exportCursorTagScanned + exportCursorTagPrunedKey migrationAckMetaKey = "_migack" migrationHLCFloorMetaKey = "_mighlc" @@ -78,7 +79,7 @@ func decodeExportCursor(cursor []byte) (exportCursorPosition, error) { return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) } tag := rest[0] - if tag != exportCursorTagEmitted && tag != exportCursorTagScanned { + if tag != exportCursorTagEmitted && tag != exportCursorTagScanned && tag != exportCursorTagPrunedKey { return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) } return exportCursorPosition{key: key, commitTS: commitTS, tag: tag, hasKey: true}, nil @@ -362,6 +363,9 @@ func exportMemoryIteratorKey( result *ExportVersionsResult, ) (bool, error) { versions, _ := value.([]VersionedValue) + if pos.hasKey && pos.tag == exportCursorTagPrunedKey && bytes.Equal(key, pos.key) { + return true, nil + } cursorCommitTS := uint64(0) if pos.hasKey && bytes.Equal(key, pos.key) { cursorCommitTS = pos.commitTS @@ -433,7 +437,7 @@ func exportMemoryVersionsForKey( continue } if versions[i].TS <= opts.MinCommitTSExclusive { - if !finishMemoryExportPosition(opts, key, versions[i], exportCursorTagScanned, result) { + if !finishMemoryExportPosition(opts, key, versions[i], exportCursorTagPrunedKey, result) { return false, nil } return true, nil diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 200e1cda5..d8863cfbe 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -7,6 +7,7 @@ import ( "os" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/stretchr/testify/require" ) @@ -249,6 +250,35 @@ func TestExportVersionsMinTSSkipHonorsScanBudget(t *testing.T) { }) } +func TestExportVersionsMinTSPruneCursorSkipsWholeKey(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + value := bytes.Repeat([]byte("x"), defaultSparseExportMaxScannedBytes) + require.NoError(t, st.PutAt(ctx, []byte("old"), value, 1, 0)) + require.NoError(t, st.PutAt(ctx, []byte("old"), value, 2, 0)) + require.NoError(t, st.PutAt(ctx, []byte("old"), value, 3, 0)) + require.NoError(t, st.PutAt(ctx, []byte("tail"), []byte("v20"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + MinCommitTSExclusive: 10, + MaxVersions: 10, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: first.NextCursor, + MinCommitTSExclusive: 10, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("tail"), CommitTS: 20, Value: []byte("v20")}}, second.Versions) + }) +} + func TestExportVersionsUsesUserKeyRangeBounds(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() @@ -306,6 +336,33 @@ func TestExportVersionsDoesNotTreatMigrationPrefixUserKeyAsMetadata(t *testing.T }) } +func TestPebbleExportSkipsWriterRegistryRows(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-writer-registry-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + registry, err := WriterRegistryFor(st) + require.NoError(t, err) + require.NoError(t, registry.SetRegistryRow( + encryption.RegistryKey(1, 2), + encryption.EncodeRegistryValue(encryption.RegistryValue{ + FullNodeID: 2, + FirstSeenLocalEpoch: 1, + LastSeenLocalEpoch: 1, + }), + )) + require.NoError(t, st.PutAt(ctx, []byte("user"), []byte("value"), 10, 0)) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 10}) + require.NoError(t, err) + require.True(t, res.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("user"), CommitTS: 10, Value: []byte("value")}}, res.Versions) +} + func TestPebbleExportStopsAtEndKey(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-end-key-*") @@ -325,8 +382,7 @@ func TestPebbleExportStopsAtEndKey(t *testing.T) { result := newExportVersionsResult(10) advance, done, err := ps.exportPebbleIteratorPosition(ctx, iter, ExportVersionsOptions{ - StartKey: []byte("a"), - EndKey: []byte("b"), + EndKey: []byte("b"), }, exportCursorPosition{}, &result) require.ErrorIs(t, err, errExportReachedEnd) require.False(t, advance) @@ -439,7 +495,7 @@ func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { require.Equal(t, []byte("persisted"), res.AckedCursor) } -func TestPebbleSnapshotExcludesMigrationMetadata(t *testing.T) { +func TestPebbleSnapshotPreservesMigrationMetadata(t *testing.T) { ctx := context.Background() srcDir, err := os.MkdirTemp("", "migration-snapshot-src-*") require.NoError(t, err) @@ -474,7 +530,7 @@ func TestPebbleSnapshotExcludesMigrationMetadata(t *testing.T) { require.Equal(t, []byte("v50"), val) floor, err := dst.MigrationHLCFloor(ctx, 7) require.NoError(t, err) - require.Zero(t, floor) + require.Equal(t, uint64(50), floor) res, err := dst.ImportVersions(ctx, ImportVersionsOptions{ JobID: 7, @@ -484,9 +540,8 @@ func TestPebbleSnapshotExcludesMigrationMetadata(t *testing.T) { Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, }) require.NoError(t, err) - require.False(t, res.Duplicate) - require.Equal(t, []byte("fresh"), res.AckedCursor) - val, err = dst.GetAt(ctx, []byte("fresh"), 60) - require.NoError(t, err) - require.Equal(t, []byte("v60"), val) + require.True(t, res.Duplicate) + require.Equal(t, []byte("stale"), res.AckedCursor) + _, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.ErrorIs(t, err, ErrKeyNotFound) } diff --git a/store/snapshot_pebble.go b/store/snapshot_pebble.go index 02f1844eb..8bc34fc65 100644 --- a/store/snapshot_pebble.go +++ b/store/snapshot_pebble.go @@ -79,9 +79,6 @@ func writePebbleSnapshotEntries(snap *pebble.Snapshot, w io.Writer) error { for iter.First(); iter.Valid(); iter.Next() { k := iter.Key() v := iter.Value() - if isMigrationMetadataKey(k) { - continue - } if err := binary.Write(w, binary.LittleEndian, uint64(len(k))); err != nil { _ = iter.Close() From 2065650a0b6b1444389cd3d428bf71b3d0072088 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 08:29:17 +0900 Subject: [PATCH 042/162] Fix migration route bracket filtering --- distribution/migrator.go | 52 ++++++++++++++- distribution/migrator_export_plan_test.go | 80 +++++++++++++++++++++++ internal/s3keys/keys.go | 18 +++++ internal/s3keys/keys_test.go | 22 +++++++ store/list_helpers.go | 13 ++-- store/list_helpers_test.go | 28 ++++++++ 6 files changed, 208 insertions(+), 5 deletions(-) create mode 100644 store/list_helpers_test.go diff --git a/distribution/migrator.go b/distribution/migrator.go index c795f70f6..561f0a5a6 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -226,6 +226,26 @@ func (b MigrationBracket) ContainsRawKey(rawKey []byte) bool { return !hasAnyPrefix(rawKey, b.ExcludePrefixes) } +// ContainsRoutedKey applies both the bracket's raw family interval and its +// route ownership predicate. S3 bucket-level auxiliary rows do not encode an +// object route key, so they are matched by bucket route-prefix intersection. +func (b MigrationBracket) ContainsRoutedKey(rawKey, routeStart, routeEnd []byte, routeKey func([]byte) []byte) bool { + if !b.ContainsRawKey(rawKey) { + return false + } + routeEnd = normalizeMigrationRouteEnd(routeEnd) + if b.RequiresDecodedS3 { + return b.containsDecodedS3Route(rawKey, routeStart, routeEnd) + } + if !b.RequiresRouteKeyCheck { + return true + } + if routeKey == nil { + return false + } + return routeKeyInRange(routeKey(rawKey), routeStart, routeEnd) +} + func (b MigrationBracket) containsFamilyShape(rawKey []byte) bool { switch b.Family { case MigrationFamilyListMetaDelta: @@ -237,6 +257,36 @@ func (b MigrationBracket) containsFamilyShape(rawKey []byte) bool { } } +func (b MigrationBracket) containsDecodedS3Route(rawKey, routeStart, routeEnd []byte) bool { + bucket, ok := b.decodedS3Bucket(rawKey) + if !ok { + return false + } + bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + return rangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) +} + +func (b MigrationBracket) decodedS3Bucket(rawKey []byte) (string, bool) { + switch b.Family { + case MigrationFamilyS3BucketMeta: + return s3keys.ParseBucketMetaKey(rawKey) + case MigrationFamilyS3BucketGeneration: + return s3keys.ParseBucketGenerationKey(rawKey) + default: + return "", false + } +} + +func routeKeyInRange(routeKey, routeStart, routeEnd []byte) bool { + if len(routeKey) == 0 { + return false + } + if bytes.Compare(routeKey, routeStart) < 0 { + return false + } + return len(routeEnd) == 0 || bytes.Compare(routeKey, routeEnd) < 0 +} + // InitializeSplitJobPlan validates the source route and seeds the job's // bracket progress for the moving right child [SplitKey, source.End). func InitializeSplitJobPlan(job SplitJob, source RouteDescriptor, nowMs int64) (SplitJob, error) { @@ -244,7 +294,7 @@ func InitializeSplitJobPlan(job SplitJob, source RouteDescriptor, nowMs int64) ( return SplitJob{}, err } routeStart := CloneBytes(job.SplitKey) - routeEnd := CloneBytes(source.End) + routeEnd := CloneBytes(normalizeMigrationRouteEnd(source.End)) brackets, err := PlanExportBrackets(routeStart, routeEnd) if err != nil { return SplitJob{}, err diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index 6888aa3bf..630c2dbc2 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -145,6 +145,86 @@ func TestPlanMigrationBracketsNormalizesEmptyRouteEnd(t *testing.T) { require.True(t, user.ContainsRawKey([]byte("z"))) } +func TestSplitJobPlanNormalizesEmptySourceRouteEnd(t *testing.T) { + t.Parallel() + + source := RouteDescriptor{ + RouteID: 9, + Start: []byte("a"), + End: []byte{}, + GroupID: 3, + State: RouteStateActive, + } + job := SplitJob{ + JobID: 1, + SourceRouteID: source.RouteID, + SplitKey: []byte("m"), + TargetGroupID: source.GroupID, + Phase: SplitJobPhasePlanned, + } + + planned, err := InitializeSplitJobPlan(job, source, 1000) + require.NoError(t, err) + for _, progress := range planned.BracketProgress { + if progress.Family != MigrationFamilyUser { + continue + } + require.False(t, progress.Done) + return + } + require.Fail(t, "missing user bracket progress") +} + +func TestMigrationBracketContainsRoutedKeyForS3BucketAuxiliaryState(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("m"), []byte("z")) + require.NoError(t, err) + byFamily := bracketsByFamily(brackets) + + routeStart := s3keys.RouteKey("bucket-b", 7, "a") + routeEnd := s3keys.RouteKey("bucket-b", 7, "z") + for _, tc := range []struct { + name string + family uint32 + key []byte + want bool + }{ + {name: "meta same bucket", family: MigrationFamilyS3BucketMeta, key: s3keys.BucketMetaKey("bucket-b"), want: true}, + {name: "generation same bucket", family: MigrationFamilyS3BucketGeneration, key: s3keys.BucketGenerationKey("bucket-b"), want: true}, + {name: "meta different bucket", family: MigrationFamilyS3BucketMeta, key: s3keys.BucketMetaKey("bucket-c"), want: false}, + {name: "generation different bucket", family: MigrationFamilyS3BucketGeneration, key: s3keys.BucketGenerationKey("bucket-c"), want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := byFamily[tc.family].ContainsRoutedKey(tc.key, routeStart, routeEnd, s3keys.ExtractRouteKey) + require.Equal(t, tc.want, got) + }) + } +} + +func TestMigrationBracketContainsRoutedKeyUsesObjectRoutes(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("m"), []byte("z")) + require.NoError(t, err) + manifest := bracketsByFamily(brackets)[MigrationFamilyS3ObjectManifest] + + key := s3keys.ObjectManifestKey("bucket-b", 7, "m") + require.True(t, manifest.ContainsRoutedKey( + key, + s3keys.RouteKey("bucket-b", 7, "a"), + s3keys.RouteKey("bucket-b", 7, "z"), + s3keys.ExtractRouteKey, + )) + require.False(t, manifest.ContainsRoutedKey( + key, + s3keys.RouteKey("bucket-c", 1, "a"), + nil, + s3keys.ExtractRouteKey, + )) +} + func TestMigrationKnownInternalPrefixesAreConcreteOnly(t *testing.T) { t.Parallel() diff --git a/internal/s3keys/keys.go b/internal/s3keys/keys.go index 682105219..c90324219 100644 --- a/internal/s3keys/keys.go +++ b/internal/s3keys/keys.go @@ -80,6 +80,17 @@ func ParseBucketMetaKey(key []byte) (string, bool) { return string(segment), true } +func ParseBucketGenerationKey(key []byte) (string, bool) { + if !bytes.HasPrefix(key, bucketGenerationPrefixBytes) { + return "", false + } + segment, next, ok := decodeSegment(key, len(bucketGenerationPrefixBytes)) + if !ok || next != len(key) { + return "", false + } + return string(segment), true +} + func ObjectManifestKey(bucket string, generation uint64, object string) []byte { return buildObjectKey(objectManifestPrefixBytes, bucket, generation, object, "", 0, 0) } @@ -227,6 +238,13 @@ func RoutePrefixForBucket(bucket string, generation uint64) []byte { return bucketScopedPrefix(routePrefixBytes, bucket, generation) } +func RoutePrefixForBucketAnyGeneration(bucket string) []byte { + out := make([]byte, 0, len(RoutePrefix)+len(bucket)+segmentEscapeOverhead) + out = append(out, routePrefixBytes...) + out = append(out, EncodeSegment([]byte(bucket))...) + return out +} + func bucketScopedPrefix(prefix []byte, bucket string, generation uint64) []byte { out := make([]byte, 0, len(prefix)+len(bucket)+u64Bytes+segmentEscapeOverhead) out = append(out, prefix...) diff --git a/internal/s3keys/keys_test.go b/internal/s3keys/keys_test.go index e5e5fca5a..2079fb76c 100644 --- a/internal/s3keys/keys_test.go +++ b/internal/s3keys/keys_test.go @@ -18,6 +18,17 @@ func TestBucketMetaKey_RoundTripsZeroByteSegments(t *testing.T) { require.Equal(t, bucket, parsed) } +func TestBucketGenerationKey_RoundTripsZeroByteSegments(t *testing.T) { + t.Parallel() + + bucket := string([]byte{'b', 'u', 0x00, 'c', 'k', 'e', 't'}) + key := BucketGenerationKey(bucket) + + parsed, ok := ParseBucketGenerationKey(key) + require.True(t, ok) + require.Equal(t, bucket, parsed) +} + func TestObjectManifestKey_RoundTripsZeroByteSegments(t *testing.T) { t.Parallel() @@ -52,6 +63,17 @@ func TestExtractRouteKey_ObjectScopedKeys(t *testing.T) { } } +func TestRoutePrefixForBucketAnyGeneration(t *testing.T) { + t.Parallel() + + bucket := string([]byte{'b', 0x00, 'k'}) + prefix := RoutePrefixForBucketAnyGeneration(bucket) + + require.True(t, bytes.HasPrefix(RouteKey(bucket, 1, "a"), prefix)) + require.True(t, bytes.HasPrefix(RouteKey(bucket, 2, "b"), prefix)) + require.False(t, bytes.HasPrefix(RouteKey("other", 1, "a"), prefix)) +} + func TestManifestScanRouteBounds(t *testing.T) { t.Parallel() diff --git a/store/list_helpers.go b/store/list_helpers.go index 051233724..252d681cf 100644 --- a/store/list_helpers.go +++ b/store/list_helpers.go @@ -121,7 +121,7 @@ func ListClaimScanPrefix(userKey []byte) []byte { // IsListMetaDeltaKey reports whether the key is a list metadata delta key. func IsListMetaDeltaKey(key []byte) bool { - return bytes.HasPrefix(key, []byte(ListMetaDeltaPrefix)) + return ExtractListUserKeyFromDelta(key) != nil } // IsListClaimKey reports whether the key is a list claim key. @@ -131,15 +131,20 @@ func IsListClaimKey(key []byte) bool { // ExtractListUserKeyFromDelta extracts the logical user key from a list delta key. func ExtractListUserKeyFromDelta(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(ListMetaDeltaPrefix)) + if !bytes.HasPrefix(key, []byte(ListMetaDeltaPrefix)) { + return nil + } + trimmed := key[len(ListMetaDeltaPrefix):] if len(trimmed) < wideColKeyLenSize+deltaKeyTSSize+deltaKeySeqSize { return nil } ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen+uint32(deltaKeyTSSize+deltaKeySeqSize) { //nolint:gosec // constants fit in uint32 + wantLen := uint64(wideColKeyLenSize) + uint64(ukLen) + uint64(deltaKeyTSSize+deltaKeySeqSize) + if uint64(len(trimmed)) != wantLen { return nil } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + userEnd := wideColKeyLenSize + int(ukLen) //nolint:gosec // exact length check above bounds ukLen to len(trimmed) + return trimmed[wideColKeyLenSize:userEnd] } // ExtractListUserKeyFromClaim extracts the logical user key from a list claim key. diff --git a/store/list_helpers_test.go b/store/list_helpers_test.go new file mode 100644 index 000000000..eca80eec1 --- /dev/null +++ b/store/list_helpers_test.go @@ -0,0 +1,28 @@ +package store + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExtractListUserKeyFromDeltaRequiresExactDeltaShape(t *testing.T) { + t.Parallel() + + userKey := []byte("d|list") + deltaKey := ListMetaDeltaKey(userKey, 42, 7) + require.True(t, IsListMetaDeltaKey(deltaKey)) + require.Equal(t, userKey, ExtractListUserKeyFromDelta(deltaKey)) + + baseMetaWithDeltaLookingUserKey := ListMetaKey(userKey) + require.False(t, IsListMetaDeltaKey(baseMetaWithDeltaLookingUserKey)) + require.Nil(t, ExtractListUserKeyFromDelta(baseMetaWithDeltaLookingUserKey)) + + trailingGarbage := append([]byte{}, deltaKey...) + trailingGarbage = append(trailingGarbage, 0) + require.False(t, IsListMetaDeltaKey(trailingGarbage)) + require.Nil(t, ExtractListUserKeyFromDelta(trailingGarbage)) + + require.False(t, IsListMetaDeltaKey([]byte("not-a-delta-key"))) + require.Nil(t, ExtractListUserKeyFromDelta([]byte("not-a-delta-key"))) +} From 119dda295c940fe8cec6d619d40942dc76f9b767 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 08:49:12 +0900 Subject: [PATCH 043/162] Guard staged readiness probes --- kv/fsm.go | 2 +- kv/fsm_migration_fence_test.go | 61 ++++++++++++++++++++++++++++++++++ kv/shard_store.go | 5 ++- kv/shard_store_test.go | 37 +++++++++++++++++++++ 4 files changed, 103 insertions(+), 2 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 4954209f5..9a0c10714 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -581,7 +581,7 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin } routes := f.stagedVisibilityRoutesForPrefix(prefix) deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { - return f.store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, f.pendingApplyIdx) + return f.store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, 0) } if err := deleteStagedVisibilityPrefixes(routes, prefix, txnCommonPrefix, deleteStagedPrefix); err != nil { return err diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 09ab1163a..e42514f43 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -11,6 +11,35 @@ import ( "github.com/stretchr/testify/require" ) +type deletePrefixIndexRecordingStore struct { + store.MVCCStore + + deletePrefixIndexes []uint64 + deletePrefixPrefixes [][]byte +} + +func (s *deletePrefixIndexRecordingStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { + s.deletePrefixIndexes = append(s.deletePrefixIndexes, appliedIndex) + s.deletePrefixPrefixes = append(s.deletePrefixPrefixes, append([]byte(nil), prefix...)) + return s.MVCCStore.DeletePrefixAtRaftAt(ctx, prefix, excludePrefix, commitTS, appliedIndex) +} + +func (s *deletePrefixIndexRecordingStore) MigrationTargetReadinessStates(ctx context.Context) ([]store.TargetStagedReadinessState, error) { + reader, ok := s.MVCCStore.(store.MigrationTargetReadinessReader) + if !ok { + return nil, nil + } + return reader.MigrationTargetReadinessStates(ctx) +} + +func (s *deletePrefixIndexRecordingStore) ApplyTargetStagedReadiness(ctx context.Context, state store.TargetStagedReadinessState) error { + writer, ok := s.MVCCStore.(store.MigrationTargetReadinessWriter) + if !ok { + return store.ErrNotSupported + } + return writer.ApplyTargetStagedReadiness(ctx, state) +} + func newWriteFencedFSM(t *testing.T) *kvFSM { t.Helper() @@ -210,6 +239,38 @@ func TestFSMDelPrefixTombstonesStagedVisibilityRows(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) } +func TestFSMDelPrefixAdvancesApplyIndexOnlyOnLiveDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + recording := &deletePrefixIndexRecordingStore{MVCCStore: fsm.store} + fsm.store = recording + fsm.SetApplyIndex(55) + + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, fsm.store.PutAt(ctx, stagedKey, []byte("staged"), 110, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: rawKey}}, + }, 120) + require.NoError(t, err) + + require.Equal(t, []uint64{0, 55}, recording.deletePrefixIndexes) + require.Equal(t, stagedKey, recording.deletePrefixPrefixes[0]) + require.Equal(t, rawKey, recording.deletePrefixPrefixes[1]) +} + func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() diff --git a/kv/shard_store.go b/kv/shard_store.go index 181f3286a..c27183717 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -308,6 +308,9 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT if !ok || g.Store == nil { return false, nil } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return false, err + } // engineForGroup may be nil in test fixtures that wire ShardStore // without raft; preserve the existing local-only fallback there. engine := engineForGroup(g) @@ -1962,7 +1965,7 @@ func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, ex continue } deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { - return g.Store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, appliedIndex) + return g.Store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, 0) } if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { return err diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 72f23b6ad..c41b7c655 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -101,6 +101,25 @@ func TestShardStoreCommittedVersionAtChecksStagedVisibilityKey(t *testing.T) { require.True(t, landed) } +func TestShardStoreCommittedVersionAtChecksTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, []byte("k"), 77) + require.False(t, landed) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T) { t.Parallel() @@ -596,6 +615,24 @@ func TestShardStoreDeletePrefixAtTombstonesStagedVisibilityRows(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) } +func TestShardStoreDeletePrefixAtRaftAtAdvancesApplyIndexOnlyOnLiveDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + recording := &deletePrefixIndexRecordingStore{MVCCStore: group.Store} + group.Store = recording + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, st.DeletePrefixAtRaftAt(ctx, rawKey, nil, 130, 88)) + + require.Equal(t, []uint64{0, 88}, recording.deletePrefixIndexes) + require.Equal(t, stagedKey, recording.deletePrefixPrefixes[0]) + require.Equal(t, rawKey, recording.deletePrefixPrefixes[1]) +} + func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { t.Parallel() From 2b501297445704217a48cc86b7315fe2a59d885b Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 09:07:17 +0900 Subject: [PATCH 044/162] Fix staged migration guard gaps --- adapter/internal.go | 35 +++++++++++++++++++++++++++++++++++ adapter/internal_test.go | 34 ++++++++++++++++++++++++++++++++++ kv/fsm.go | 30 ++++++++++++++++++++++++++++++ kv/fsm_onephase_dedup_test.go | 32 ++++++++++++++++++++++++++++++++ kv/shard_store.go | 27 +++++++++++++++++++++++++++ kv/shard_store_test.go | 22 ++++++++++++++++++++++ main.go | 5 +++++ main_bootstrap_e2e_test.go | 11 +++++++++-- 8 files changed, 194 insertions(+), 2 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index a14fb9134..515faf9aa 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -35,6 +35,12 @@ func WithInternalMigrationProposer(proposer raftengine.Proposer) InternalOption } } +func WithInternalRouteEngine(engine *distribution.Engine) InternalOption { + return func(i *Internal) { + i.routeEngine = engine + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ leader: leader, @@ -56,6 +62,7 @@ type Internal struct { relay *RedisPubSubRelay store store.MVCCStore migrationProposer raftengine.Proposer + routeEngine *distribution.Engine pb.UnimplementedInternalServer } @@ -408,10 +415,38 @@ func (i *Internal) stampRawTimestamps(ctx context.Context, reqs []*pb.Request) e return err } r.Ts = ts + if err := i.rejectWriteTimestampFloorMutations(r.Mutations, r.Ts); err != nil { + return err + } + } + return nil +} + +func (i *Internal) rejectWriteTimestampFloorMutations(muts []*pb.Mutation, commitTS uint64) error { + if i == nil || i.routeEngine == nil || commitTS == 0 { + return nil + } + routes := i.routeEngine.Stats() + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 { + continue + } + for _, route := range routes { + if routeWriteTimestampFloorApplies(route, mut.Key, commitTS) { + return errors.Wrapf(kv.ErrRouteWriteTimestampTooLow, "key %q commit_ts=%d floor=%d", mut.Key, commitTS, route.MinWriteTSExclusive) + } + } } return nil } +func routeWriteTimestampFloorApplies(route distribution.Route, key []byte, commitTS uint64) bool { + if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return false + } + return kv.RouteKeyFilter(route.Start, route.End)(key) +} + func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) error { startTS := forwardedTxnStartTS(reqs) if startTS == 0 { diff --git a/adapter/internal_test.go b/adapter/internal_test.go index 49802d20a..a028aa65d 100644 --- a/adapter/internal_test.go +++ b/adapter/internal_test.go @@ -4,11 +4,18 @@ import ( "context" "testing" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" ) +type fixedInternalTimestampAllocator uint64 + +func (a fixedInternalTimestampAllocator) Next(context.Context) (uint64, error) { + return uint64(a), nil +} + func TestStampTxnTimestamps_RejectsMaxStartTS(t *testing.T) { t.Parallel() @@ -170,3 +177,30 @@ func TestStampTxnTimestamps_UsesSingleTxnStartTS(t *testing.T) { require.NoError(t, err) require.Greater(t, meta.CommitTS, uint64(9)) } + +func TestStampRawTimestampsRejectsRouteWriteFloor(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + i := &Internal{ + tsAllocator: fixedInternalTimestampAllocator(100), + routeEngine: engine, + } + reqs := []*pb.Request{{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}}, + }} + + err := i.stampRawTimestamps(context.Background(), reqs) + require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) +} diff --git a/kv/fsm.go b/kv/fsm.go index 9e82f62b5..9e703a926 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -1102,9 +1102,39 @@ func (f *kvFSM) dedupProbeOnePhase(ctx context.Context, meta TxnMeta) (bool, err if err != nil { return false, errors.WithStack(err) } + if landed { + return true, nil + } + route, ok := f.currentStagedVisibilityRouteForKey(meta.PrimaryKey) + if !ok { + return false, nil + } + stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, meta.PrimaryKey) + landed, err = f.store.CommittedVersionAt(ctx, stagedKey, meta.PrevCommitTS) + if err != nil { + return false, errors.WithStack(err) + } return landed, nil } +func (f *kvFSM) currentStagedVisibilityRouteForKey(key []byte) (distribution.Route, bool) { + if f == nil || f.routes == nil || len(key) == 0 { + return distribution.Route{}, false + } + if _, _, ok := distribution.MigrationStagedDataKeyParts(key); ok { + return distribution.Route{}, false + } + snap, ok := f.routes.Current() + if !ok { + return distribution.Route{}, false + } + route, ok := snap.RouteOf(routeKey(key)) + if !ok || route.GroupID != f.shardGroupID || !routeHasStagedVisibility(route) { + return distribution.Route{}, false + } + return route, true +} + func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { meta, muts, err := extractTxnMeta(r.Mutations) if err != nil { diff --git a/kv/fsm_onephase_dedup_test.go b/kv/fsm_onephase_dedup_test.go index aaf027c1d..8b3f50f28 100644 --- a/kv/fsm_onephase_dedup_test.go +++ b/kv/fsm_onephase_dedup_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/bootjp/elastickv/distribution" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -67,6 +68,37 @@ func TestOnePhaseDedup_NoOpsWhenPriorAttemptLanded(t *testing.T) { require.Equal(t, uint64(20), latest, "newest version must remain attempt 1's at 20") } +func TestOnePhaseDedup_NoOpsWhenPriorAttemptLandedAsStagedVersion(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }}) + fsmIface := NewKvFSMWithHLC(st, NewHLC(), WithRouteHistory(WrapDistributionEngine(engine), 1)) + fsm, ok := fsmIface.(*kvFSM) + require.True(t, ok) + + key := []byte("list-item") + require.NoError(t, st.PutAt(ctx, distribution.MigrationStagedDataKey(9, key), []byte("v"), 20, 0)) + + require.NoError(t, applyFSMRequest(t, fsm, onePhaseReq(30, 40, 20, key, []byte("v")))) + + at40, err := st.CommittedVersionAt(ctx, key, 40) + require.NoError(t, err) + require.False(t, at40, "retry must not write a live version when the prior attempt is staged") + stagedAt20, err := st.CommittedVersionAt(ctx, distribution.MigrationStagedDataKey(9, key), 20) + require.NoError(t, err) + require.True(t, stagedAt20) +} + // TestOnePhaseDedup_AppliesWhenPriorAttemptDidNotLand covers the truncated / // never-applied case: prev_commit_ts is set but no version exists at exactly // that timestamp (attempt 1's entry lost the log race). The probe misses and diff --git a/kv/shard_store.go b/kv/shard_store.go index 7e681d53e..217f630ad 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -832,6 +832,7 @@ func (s *ShardStore) scanRouteWithStagedVisibilityPage( stagedExhausted := len(stagedKVs) < window boundary, hasBoundary := stagedVisibilityCandidateBoundary(liveKVs, stagedKVs, liveExhausted, stagedExhausted, reverse) exhausted := liveExhausted && stagedExhausted + out = stagedVisibilityKVsWithinPageBoundary(out, boundary, hasBoundary, exhausted, reverse) if len(out) >= limit { clear(out[limit:]) return out[:limit], boundary, !exhausted && hasBoundary, nil @@ -847,6 +848,32 @@ func (s *ShardStore) scanRouteWithStagedVisibilityPage( } } +func stagedVisibilityKVsWithinPageBoundary(kvs []*store.KVPair, boundary []byte, hasBoundary bool, exhausted bool, reverse bool) []*store.KVPair { + if exhausted || !hasBoundary { + return kvs + } + return stagedVisibilityKVsWithinBoundary(kvs, boundary, reverse) +} + +func stagedVisibilityKVsWithinBoundary(kvs []*store.KVPair, boundary []byte, reverse bool) []*store.KVPair { + if len(boundary) == 0 { + return kvs + } + n := 0 + for _, kvp := range kvs { + if kvp == nil { + continue + } + cmp := bytes.Compare(kvp.Key, boundary) + if (!reverse && cmp <= 0) || (reverse && cmp >= 0) { + kvs[n] = kvp + n++ + } + } + clear(kvs[n:]) + return kvs[:n] +} + func stagedVisibilityCandidateBoundary(liveKVs []*store.KVPair, stagedKVs []*store.KVPair, liveExhausted bool, stagedExhausted bool, reverse bool) ([]byte, bool) { liveBoundary := stagedVisibilityBoundary{reverse: reverse} for _, kvp := range liveKVs { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index cfa7bc3e3..cc58a4af6 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -308,6 +308,28 @@ func TestShardStoreScanAt_ContinuesStagedVisibilityAfterCandidateWindow(t *testi require.Equal(t, []byte("k00000"), kvs[limit-1].Key) } +func TestShardStoreScanAtRestrictsStagedVisibilityToSafeFrontier(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + limit := stagedVisibilityMaxCandidateWindow + 1 + for i := range limit { + key := []byte(fmt.Sprintf("k%05d", i)) + require.NoError(t, group.Store.PutAt(ctx, key, []byte(fmt.Sprintf("live%05d", i)), 10, 0)) + } + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("x-staged")), []byte("staged"), 10, 0)) + + kvs, err := st.ScanAt(ctx, []byte("a"), []byte("z"), limit, 20) + require.NoError(t, err) + require.Len(t, kvs, limit) + require.Equal(t, []byte("k00000"), kvs[0].Key) + require.Equal(t, []byte(fmt.Sprintf("k%05d", limit-1)), kvs[limit-1].Key) + for _, kvp := range kvs { + require.NotEqual(t, []byte("x-staged"), kvp.Key) + } +} + func TestStagedVisibilityCandidateBoundary_UsesSafeFrontier(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index c236598d8..c77369312 100644 --- a/main.go +++ b/main.go @@ -1491,6 +1491,7 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, shardStore: in.shardStore, coordinate: adapterCoordinate, distServer: in.distServer, + routeEngine: in.cfg.engine, adminServer: adminServer, adminGRPCOpts: adminGRPCOpts, redisAddress: *redisAddr, @@ -2053,6 +2054,7 @@ func startRaftServers( shardStore *kv.ShardStore, coordinate kv.Coordinator, distServer *adapter.DistributionServer, + routeEngine *distribution.Engine, relay *adapter.RedisPubSubRelay, proposalObserverForGroup func(uint64) kv.ProposalObserver, adminServer *adapter.AdminServer, @@ -2097,6 +2099,7 @@ func startRaftServers( internalTimestampOptions(coordinate), adapter.WithInternalStore(rt.store), adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), + adapter.WithInternalRouteEngine(routeEngine), )..., )) pb.RegisterDistributionServer(gs, distServer) @@ -2439,6 +2442,7 @@ type runtimeServerRunner struct { shardStore *kv.ShardStore coordinate kv.Coordinator distServer *adapter.DistributionServer + routeEngine *distribution.Engine adminServer *adapter.AdminServer adminGRPCOpts adminGRPCInterceptors redisAddress string @@ -2552,6 +2556,7 @@ func (r *runtimeServerRunner) startRaftTransport() error { r.shardStore, r.coordinate, r.distServer, + r.routeEngine, r.pubsubRelay, func(groupID uint64) kv.ProposalObserver { return r.metricsRegistry.RaftProposalObserver(groupID) diff --git a/main_bootstrap_e2e_test.go b/main_bootstrap_e2e_test.go index 81ca49dce..f4483c137 100644 --- a/main_bootstrap_e2e_test.go +++ b/main_bootstrap_e2e_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/distribution" internalraftadmin "github.com/bootjp/elastickv/internal/raftadmin" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" @@ -577,6 +578,7 @@ func startBootstrapE2ENode( shardStore, coordinate, distServer, + cfg.engine, cfg.leaderRedis, listeners, ) @@ -660,6 +662,7 @@ func startBootstrapE2EMultiGroupNode( shardStore, coordinate, distServer, + cfg.engine, cfg.leaderRedis, listeners, ) @@ -688,6 +691,7 @@ func startRuntimeServersWithBoundListeners( shardStore *kv.ShardStore, coordinate kv.Coordinator, distServer *adapter.DistributionServer, + routeEngine *distribution.Engine, leaderRedis map[string]string, listeners bootstrapE2EListeners, ) error { @@ -701,7 +705,7 @@ func startRuntimeServersWithBoundListeners( if err := startBoundRedisServer(ctx, eg, listeners.redis, shardStore, coordinate, leaderRedis, redisAddr, relay); err != nil { return waitErrgroupAfterStartupFailure(cancel, eg, err) } - if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, relay, listeners.grpc); err != nil { + if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, routeEngine, relay, listeners.grpc); err != nil { return waitErrgroupAfterStartupFailure(cancel, eg, err) } if err := startBoundDynamoDBServer(ctx, eg, listeners.dynamo, shardStore, coordinate); err != nil { @@ -718,6 +722,7 @@ func startRuntimeServersWithBoundMultiGroupListeners( shardStore *kv.ShardStore, coordinate kv.Coordinator, distServer *adapter.DistributionServer, + routeEngine *distribution.Engine, leaderRedis map[string]string, listeners bootstrapE2EMultiGroupListeners, ) error { @@ -730,7 +735,7 @@ func startRuntimeServersWithBoundMultiGroupListeners( return waitErrgroupAfterStartupFailure(cancel, eg, err) } for _, rt := range runtimes { - if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, relay, listeners.grpc[rt.spec.id]); err != nil { + if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, routeEngine, relay, listeners.grpc[rt.spec.id]); err != nil { return waitErrgroupAfterStartupFailure(cancel, eg, err) } } @@ -747,6 +752,7 @@ func startBoundGRPCServer( shardStore *kv.ShardStore, coordinate kv.Coordinator, distServer *adapter.DistributionServer, + routeEngine *distribution.Engine, relay *adapter.RedisPubSubRelay, listener net.Listener, ) error { @@ -769,6 +775,7 @@ func startBoundGRPCServer( relay, adapter.WithInternalStore(rt.store), adapter.WithInternalMigrationProposer(rt.engine), + adapter.WithInternalRouteEngine(routeEngine), )) pb.RegisterDistributionServer(gs, distServer) rt.registerGRPC(gs) From 9f0b7f1e03aa2742d6be96e40c1716c025c9bbe2 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 09:17:15 +0900 Subject: [PATCH 045/162] store: skip export writer registry rows --- store/lsm_migration.go | 7 ++++++- store/migration_versions_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 31d1e9a77..aa3f0ebca 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -5,6 +5,7 @@ import ( "context" "math" + "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/v2" ) @@ -102,7 +103,7 @@ func (s *pebbleStore) exportPebbleIteratorPosition( return false, false, errors.WithStack(err) } rawKey := iter.Key() - if isPebbleMetaKey(rawKey) { + if isPebbleExportMetadataKey(rawKey) { return true, true, nil } userKey, commitTS := decodeKeyView(rawKey) @@ -126,6 +127,10 @@ func (s *pebbleStore) exportPebbleIteratorPosition( return true, done, err } +func isPebbleExportMetadataKey(rawKey []byte) bool { + return isPebbleMetaKey(rawKey) || bytes.HasPrefix(rawKey, encryption.WriterRegistryPrefix) +} + func (s *pebbleStore) skipPebbleExportKeyOutsideRange(iter *pebble.Iterator, opts ExportVersionsOptions, userKey []byte) (bool, error) { if opts.StartKey != nil && bytes.Compare(userKey, opts.StartKey) < 0 { advancePebbleExportPastCurrentUserKey(iter, userKey) diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index d8863cfbe..3d8e7da37 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -363,6 +363,35 @@ func TestPebbleExportSkipsWriterRegistryRows(t *testing.T) { require.Equal(t, []MVCCVersion{{Key: []byte("user"), CommitTS: 10, Value: []byte("value")}}, res.Versions) } +func TestPebbleExportSkipsWriterRegistryRowsWithoutUserVersions(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-writer-registry-only-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + registry, err := WriterRegistryFor(st) + require.NoError(t, err) + require.NoError(t, registry.SetRegistryRow( + encryption.RegistryKey(1, 2), + encryption.EncodeRegistryValue(encryption.RegistryValue{ + FullNodeID: 2, + FirstSeenLocalEpoch: 1, + LastSeenLocalEpoch: 1, + }), + )) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 10}) + require.NoError(t, err) + require.True(t, res.Done) + require.Empty(t, res.NextCursor) + require.Empty(t, res.Versions) + require.Zero(t, res.ScannedBytes) + require.Zero(t, res.AcceptedRows) +} + func TestPebbleExportStopsAtEndKey(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-end-key-*") From e6f88bd725d288296529be840e1f162ccda0b4ff Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 09:19:51 +0900 Subject: [PATCH 046/162] Guard target readiness retry paths --- kv/fsm.go | 43 ++++++++++++--- kv/fsm_migration_fence_test.go | 95 ++++++++++++++++++++++++++++++++++ kv/shard_store.go | 3 ++ kv/shard_store_test.go | 66 +++++++++++++++++++++++ 4 files changed, 201 insertions(+), 6 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 9a0c10714..8a7727464 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -678,6 +678,30 @@ func (f *kvFSM) verifyTargetReadinessForMutations(ctx context.Context, muts []*p return nil } +func (f *kvFSM) verifyTargetReadinessForReadKeys(ctx context.Context, keys [][]byte) error { + for _, key := range keys { + if isTxnInternalKey(key) { + continue + } + if err := f.verifyTargetReadinessForRange(ctx, key, nextScanCursor(key)); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyTargetReadinessForTxnFootprint(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, primaryKey []byte) error { + if len(primaryKey) != 0 && !isTxnInternalKey(primaryKey) { + if err := f.verifyTargetReadinessForRange(ctx, primaryKey, nextScanCursor(primaryKey)); err != nil { + return err + } + } + if err := f.verifyTargetReadinessForReadKeys(ctx, readKeys); err != nil { + return err + } + return f.verifyTargetReadinessForMutations(ctx, muts) +} + func (f *kvFSM) verifyTargetReadinessForPrefix(ctx context.Context, prefix []byte) error { start, end := routePrefixRange(prefix) return f.verifyTargetReadinessForRouteRange(ctx, start, end) @@ -1140,6 +1164,9 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if meta.CommitTS != 0 { floorTS = meta.CommitTS } + if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, r.ReadKeys, meta.PrimaryKey); err != nil { + return err + } uniq, err := f.uniqueMutationsNotFenced(ctx, muts, floorTS) if err != nil { return err @@ -1202,7 +1229,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com // applying this log entry. The retention-window > max-retry-latency // invariant prevents the rare case where a real never-landed retry // arrives with PrevCommitTS below pebble's compacted floor. - dedup, err := f.dedupProbeOnePhase(ctx, meta) + dedup, err := f.dedupProbeOnePhase(ctx, meta, muts, r.ReadKeys) if err != nil { return err } @@ -1243,15 +1270,19 @@ func (f *kvFSM) uniqueMutationsNotFenced(ctx context.Context, muts []*pb.Mutatio return uniq, nil } -// dedupProbeOnePhase decides whether handleOnePhaseTxnRequest should no-op -// because the entry is a retry whose prior attempt already landed. Extracted -// to keep handleOnePhaseTxnRequest under the cyclop budget; the determinism -// rationale lives at the call site. +// dedupProbeOnePhase first fail-closes the target readiness footprint, then +// decides whether handleOnePhaseTxnRequest should no-op because the entry is a +// retry whose prior attempt already landed. Extracted to keep +// handleOnePhaseTxnRequest under the cyclop budget; the determinism rationale +// lives at the call site. // // Returns (true, nil) → the entry must no-op (prior attempt landed). // Returns (false, nil) → fall through to normal apply. // Returns (false, err) → propagate err; apply must not proceed. -func (f *kvFSM) dedupProbeOnePhase(ctx context.Context, meta TxnMeta) (bool, error) { +func (f *kvFSM) dedupProbeOnePhase(ctx context.Context, meta TxnMeta, muts []*pb.Mutation, readKeys [][]byte) (bool, error) { + if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, readKeys, meta.PrimaryKey); err != nil { + return false, err + } if meta.PrevCommitTS == 0 { return false, nil } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index e42514f43..8fb30e168 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -88,6 +88,35 @@ func newTargetReadinessFSM(t *testing.T, route distribution.RouteDescriptor) *kv return fsm } +func applyTargetReadinessToFSM(t *testing.T, fsm *kvFSM, state store.TargetStagedReadinessState) { + t.Helper() + + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) +} + +func newReadinessReadKeyFSM(t *testing.T) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + return fsm +} + func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() @@ -344,6 +373,72 @@ func TestFSMRejectsOnePhaseTxnBelowRouteFloor(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } +func TestFSMOnePhaseDedupChecksTargetReadinessBeforeNoOp(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + require.NoError(t, fsm.store.PutAt(ctx, []byte("b"), []byte("landed"), 20, 0)) + + err := fsm.handleTxnRequest(ctx, onePhaseReq(30, 40, 20, []byte("b"), []byte("retry")), 40) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + landedAt40, probeErr := fsm.store.CommittedVersionAt(ctx, []byte("b"), 40) + require.NoError(t, probeErr) + require.False(t, landedAt40) +} + +func TestFSMOnePhaseTxnChecksReadKeysForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newReadinessReadKeyFSM(t) + req := onePhaseReq(10, 20, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 20) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMPrepareTxnChecksReadKeysForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newReadinessReadKeyFSM(t) + req := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + ReadKeys: [][]byte{[]byte("n")}, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, + }, + } + + err := fsm.handleTxnRequest(ctx, req, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("b")), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + func TestFSMPrepareUsesCommitTSForRouteFloorWhenPresent(t *testing.T) { t.Parallel() diff --git a/kv/shard_store.go b/kv/shard_store.go index c27183717..716acedc5 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -325,6 +325,9 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT // serialization. return false, nil } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return false, err + } return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) } diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index c41b7c655..b3ff784a1 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -2,9 +2,11 @@ package kv import ( "context" + "errors" "testing" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/internal/s3keys" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -64,6 +66,49 @@ func newReadinessShardStore(t *testing.T, route distribution.RouteDescriptor) (* return NewShardStore(engine, map[uint64]*ShardGroup{route.GroupID: group}), group } +type readinessFenceEngine struct { + onLinearizableRead func() +} + +func (e *readinessFenceEngine) State() raftengine.State { + return raftengine.StateFollower +} + +func (e *readinessFenceEngine) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "leader"} +} + +func (e *readinessFenceEngine) VerifyLeader(context.Context) error { + return nil +} + +func (e *readinessFenceEngine) LinearizableRead(context.Context) (uint64, error) { + if e.onLinearizableRead != nil { + e.onLinearizableRead() + } + return 1, nil +} + +func (e *readinessFenceEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, errors.New("unexpected propose") +} + +func (e *readinessFenceEngine) ProposeAdmin(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, errors.New("unexpected propose admin") +} + +func (e *readinessFenceEngine) Status() raftengine.Status { + return raftengine.Status{State: e.State()} +} + +func (e *readinessFenceEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return raftengine.Configuration{}, nil +} + +func (e *readinessFenceEngine) Close() error { + return nil +} + func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { t.Parallel() @@ -120,6 +165,27 @@ func TestShardStoreCommittedVersionAtChecksTargetReadiness(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreCommittedVersionAtRechecksTargetReadinessAfterFence(t *testing.T) { + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + group.Engine = &readinessFenceEngine{ + onLinearizableRead: func() { + applyTargetReadiness(t, group) + }, + } + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, []byte("k"), 77) + require.False(t, landed) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T) { t.Parallel() From 79b34092b76a6c0ea4749b9a12dbcc04036df59b Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 10:02:40 +0900 Subject: [PATCH 047/162] Tighten target readiness proof --- kv/fsm.go | 11 +- kv/fsm_migration_fence_test.go | 42 ++++++++ kv/shard_store.go | 180 +++++++++++++++++++++++++------ kv/shard_store_test.go | 89 +++++++++++++++ store/lsm_migration.go | 3 + store/migration_versions.go | 14 +++ store/migration_versions_test.go | 28 +++++ store/store.go | 14 +-- 8 files changed, 332 insertions(+), 49 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 8a7727464..8c88d21d1 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -1464,15 +1464,8 @@ func (f *kvFSM) uniqueMutationsAboveWriteFloor(ctx context.Context, muts []*pb.M return uniq, nil } -func (f *kvFSM) uniqueAbortCleanupMutations(ctx context.Context, muts []*pb.Mutation) ([]*pb.Mutation, error) { - uniq, err := uniqueMutations(muts) - if err != nil { - return nil, err - } - if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { - return nil, err - } - return uniq, nil +func (f *kvFSM) uniqueAbortCleanupMutations(_ context.Context, muts []*pb.Mutation) ([]*pb.Mutation, error) { + return uniqueMutations(muts) } func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt uint64) ([]*store.KVPairMutation, error) { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 8fb30e168..82c2d5410 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -543,3 +543,45 @@ func TestFSMAbortCleanupBypassesRetainedWriteFloor(t *testing.T) { _, intentErr := fsm.store.GetAt(ctx, txnIntentKey(primaryKey), ^uint64(0)) require.ErrorIs(t, intentErr, store.ErrKeyNotFound) } + +func TestFSMAbortCleanupBypassesTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + startTS := uint64(10) + abortTS := uint64(11) + primaryKey := []byte("b") + require.NoError(t, fsm.store.PutAt(ctx, txnLockKey(primaryKey), encodeTxnLock(txnLock{ + StartTS: startTS, + PrimaryKey: primaryKey, + IsPrimaryKey: true, + }), startTS, 0)) + require.NoError(t, fsm.store.PutAt(ctx, txnIntentKey(primaryKey), encodeTxnIntent(txnIntent{ + StartTS: startTS, + Op: txnIntentOpPut, + Value: []byte("v"), + }), startTS, 0)) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: abortTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + require.NoError(t, fsm.handleTxnRequest(ctx, abort, abortTS)) + + _, lockErr := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) + _, intentErr := fsm.store.GetAt(ctx, txnIntentKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, intentErr, store.ErrKeyNotFound) +} diff --git a/kv/shard_store.go b/kv/shard_store.go index 716acedc5..cb7fef43c 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -92,7 +92,9 @@ func isLinearizableRaftLeader(ctx context.Context, engine raftengine.LeaderView) } func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return nil, err } if !isTxnInternalKey(key) { @@ -104,7 +106,9 @@ func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distr } func (s *ShardStore) localGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return nil, err } if routeHasStagedVisibility(route) { @@ -137,36 +141,101 @@ func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetS return ready.ExpectedCutoverVersion == 0 || catalogVersion >= ready.ExpectedCutoverVersion } -func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { +func (s *ShardStore) targetReadyRouteForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) (distribution.Route, error) { routeStart, routeEnd := readinessRouteRangeForScan(start, end) - return s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd) + return s.targetReadyRouteForRouteRange(ctx, g, route, routeStart, routeEnd) } -func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) error { +func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { + _, err := s.targetReadyRouteForRange(ctx, g, route, start, end) + return err +} + +func (s *ShardStore) targetReadyRouteForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) (distribution.Route, error) { if g == nil || g.Store == nil { - return nil + return route, nil } reader, ok := g.Store.(store.MigrationTargetReadinessReader) if !ok { - return nil + return route, nil } states, err := reader.MigrationTargetReadinessStates(ctx) if err != nil { - return errors.WithStack(err) + return route, errors.WithStack(err) } - var catalogVersion uint64 - if s != nil && s.engine != nil { - catalogVersion = s.engine.Version() + applicable := targetReadinessApplicableStates(states, route, routeStart, routeEnd) + if len(applicable) == 0 { + return route, nil + } + + proofRoutes, catalogVersion, ok := s.readinessProofRoutes(route, routeStart, routeEnd) + if !ok { + return route, errors.WithStack(ErrRouteCutoverPending) + } + if !readinessProofSatisfiesStates(applicable, proofRoutes, route.GroupID, catalogVersion) { + return route, errors.WithStack(ErrRouteCutoverPending) + } + if len(proofRoutes) == 1 { + return proofRoutes[0], nil + } + return route, nil +} + +func targetReadinessApplicableStates( + states []store.TargetStagedReadinessState, + route distribution.Route, + routeStart []byte, + routeEnd []byte, +) []store.TargetStagedReadinessState { + applicable := make([]store.TargetStagedReadinessState, 0, len(states)) + for _, ready := range states { + if targetReadinessAppliesToRoute(route, routeStart, routeEnd, ready) { + applicable = append(applicable, ready) + } } + return applicable +} + +func readinessProofSatisfiesStates( + states []store.TargetStagedReadinessState, + routes []distribution.Route, + groupID uint64, + catalogVersion uint64, +) bool { for _, ready := range states { - if !targetReadinessAppliesToRoute(route, routeStart, routeEnd, ready) { + if !routesSatisfyTargetReadiness(routes, ready, groupID, catalogVersion) { + return false + } + } + return true +} + +func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) error { + _, err := s.targetReadyRouteForRouteRange(ctx, g, route, routeStart, routeEnd) + return err +} + +func (s *ShardStore) readinessProofRoutes(route distribution.Route, routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if s == nil || s.engine == nil { + return []distribution.Route{route}, 0, true + } + snap, ok := s.engine.Current() + if !ok { + return nil, 0, false + } + routes := snap.IntersectingRoutes(routeStart, routeEnd) + proof := routes[:0] + for _, candidate := range routes { + if candidate.GroupID != route.GroupID { continue } - if !routeSatisfiesTargetReadiness(route, ready, catalogVersion) { - return errors.WithStack(ErrRouteCutoverPending) + if (route.Start != nil || route.End != nil || route.RouteID != 0) && + !routeRangeIntersects(candidate.Start, candidate.End, route.Start, route.End) { + continue } + proof = append(proof, candidate) } - return nil + return proof, snap.Version(), len(proof) > 0 } func targetReadinessAppliesToRoute(route distribution.Route, routeStart []byte, routeEnd []byte, ready store.TargetStagedReadinessState) bool { @@ -240,6 +309,7 @@ func latestMVCCVersionAt(ctx context.Context, st store.MVCCStore, key []byte, ts StartKey: key, EndKey: nextScanCursor(key), MaxCommitTSInclusive: ts, + ReadTS: ts, MaxVersions: 1, MaxScannedBytes: 0, MinCommitTSExclusive: 0, @@ -308,7 +378,9 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT if !ok || g.Store == nil { return false, nil } - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return false, err } // engineForGroup may be nil in test fixtures that wire ShardStore @@ -325,7 +397,8 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT // serialization. return false, nil } - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return false, err } return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) @@ -669,7 +742,9 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( if !ok || g == nil || g.Store == nil { return nil, false, nil } - if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { return nil, false, err } @@ -685,10 +760,7 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - if routeHasStagedVisibility(route) { - return nil, true, nil - } - return s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) + return s.scanReadyLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) } // RawScanAt cannot enforce physicalLimit, so report truncation and let @@ -696,6 +768,27 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( return nil, true, nil } +func (s *ShardStore) scanReadyLeaderPhysicalLimit( + ctx context.Context, + g *ShardGroup, + route distribution.Route, + start []byte, + end []byte, + visibleLimit int, + physicalLimit int, + ts uint64, + reverse bool, +) ([]*store.KVPair, bool, error) { + route, err := s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } + if routeHasStagedVisibility(route) { + return nil, true, nil + } + return s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) +} + func scanLocalPhysicalLimit( ctx context.Context, st store.MVCCStore, @@ -746,7 +839,9 @@ func (s *ShardStore) scanRouteLocal( ts uint64, reverse bool, ) ([]*store.KVPair, error) { - if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { return nil, err } if routeHasStagedVisibility(route) { @@ -771,6 +866,11 @@ func (s *ShardStore) scanRouteAtLeaderPhysicalLimit( ts uint64, reverse bool, ) ([]*store.KVPair, bool, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } kvs, limitReached, err := scanLocalPhysicalLimit(ctx, g.Store, start, end, visibleLimit, physicalLimit, ts, reverse) if err != nil { return nil, limitReached, errors.WithStack(err) @@ -794,13 +894,12 @@ func (s *ShardStore) scanRouteAtLeader( ts uint64, reverse bool, ) ([]*store.KVPair, error) { - if err := s.verifyTargetReadinessForRange(ctx, g, route, start, end); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { return nil, err } - var ( - kvs []*store.KVPair - err error - ) + var kvs []*store.KVPair switch { case routeHasStagedVisibility(route): kvs, err = s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) @@ -894,6 +993,7 @@ func collectLatestLogicalVersions( StartKey: scanStart, EndKey: scanEnd, MaxCommitTSInclusive: ts, + ReadTS: ts, Cursor: cursor, MaxVersions: stagedVisibilityExportPageSize, }) @@ -1088,7 +1188,9 @@ func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commit if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return err } if err := verifyRouteWriteFloor(route, commitTS); err != nil { @@ -1102,7 +1204,9 @@ func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return err } if err := verifyRouteWriteFloor(route, commitTS); err != nil { @@ -1116,7 +1220,9 @@ func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return err } if err := verifyRouteWriteFloor(route, commitTS); err != nil { @@ -1130,7 +1236,9 @@ func (s *ShardStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return err } if err := verifyRouteWriteFloor(route, commitTS); err != nil { @@ -1170,7 +1278,9 @@ func (s *ShardStore) LatestCommitTS(ctx context.Context, key []byte) (uint64, bo } func (s *ShardStore) localLatestCommitTS(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte) (uint64, bool, error) { - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return 0, false, err } liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) @@ -1841,7 +1951,9 @@ func (s *ShardStore) verifyMutationWriteRoute(ctx context.Context, key []byte, c if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } - if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { return err } return verifyRouteWriteFloor(route, commitTS) diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index b3ff784a1..9f343cc32 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -67,10 +67,14 @@ func newReadinessShardStore(t *testing.T, route distribution.RouteDescriptor) (* } type readinessFenceEngine struct { + state raftengine.State onLinearizableRead func() } func (e *readinessFenceEngine) State() raftengine.State { + if e.state != "" { + return e.state + } return raftengine.StateFollower } @@ -308,6 +312,49 @@ func TestShardStoreTargetReadinessRejectsClearedDescriptorBeforeCutoverVersion(t require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreTargetReadinessUsesSingleCatalogSnapshotProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + staleRoute, ok := engine.GetRoute([]byte("b")) + require.True(t, ok) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live-old"), 80, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged-new"), 120, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + got, err := shards.localGetAt(ctx, group, staleRoute, []byte("b"), 130) + require.NoError(t, err) + require.Equal(t, []byte("staged-new"), got) +} + func TestShardStoreExplicitGroupReadUsesRouteProofForReadiness(t *testing.T) { t.Parallel() @@ -437,6 +484,31 @@ func TestShardStorePhysicalLimitScanChecksTargetReadiness(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStorePhysicalLimitScanRechecksTargetReadinessAfterFence(t *testing.T) { + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + group.Engine = &readinessFenceEngine{ + state: raftengine.StateLeader, + onLinearizableRead: func() { + applyTargetReadiness(t, group) + }, + } + + userKey := []byte("b") + start := store.ListItemKey(userKey, 0) + end := store.ListItemKey(userKey, 2) + require.NoError(t, group.Store.PutAt(ctx, start, []byte("v"), 120, 0)) + + _, _, err := st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestShardStoreS3ManifestScanChecksTargetReadinessInRouteSpace(t *testing.T) { t.Parallel() @@ -744,6 +816,23 @@ func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { require.Equal(t, uint64(40), ts) } +func TestShardStoreStagedVisibilityPreservesCompactionErrors(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + retention, ok := group.Store.(store.RetentionController) + require.True(t, ok) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged"), 10, 0)) + retention.SetMinRetainedTS(20) + + _, err := st.GetAt(ctx, []byte("b"), 15) + require.ErrorIs(t, err, store.ErrReadTSCompacted) + + _, err = st.ScanAt(ctx, []byte("a"), []byte("z"), 10, 15) + require.ErrorIs(t, err, store.ErrReadTSCompacted) +} + func TestShardStoreScanAt_IncludesListKeysAcrossShards(t *testing.T) { t.Parallel() diff --git a/store/lsm_migration.go b/store/lsm_migration.go index e0796cbf6..f60f5ae02 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -24,6 +24,9 @@ func (s *pebbleStore) exportVersionsLocked(ctx context.Context, opts ExportVersi if opts.MaxVersions <= 0 { return ExportVersionsResult{Done: true}, nil } + if readTSCompacted(opts.ReadTS, s.effectiveMinRetainedTS()) { + return ExportVersionsResult{}, ErrReadTSCompacted + } iter, err := s.db.NewIter(pebbleExportIterOptions(opts)) if err != nil { diff --git a/store/migration_versions.go b/store/migration_versions.go index b1e275ecb..4c5f4edb8 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -196,7 +196,14 @@ func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptio s.mtx.RLock() defer s.mtx.RUnlock() + if err := s.checkExportReadTSLocked(opts); err != nil { + return ExportVersionsResult{}, err + } + + return s.exportVersionsLocked(ctx, opts, pos) +} +func (s *mvccStore) exportVersionsLocked(ctx context.Context, opts ExportVersionsOptions, pos exportCursorPosition) (ExportVersionsResult, error) { result := newExportVersionsResult(opts.MaxVersions) it := s.tree.Iterator() if !s.seekMemoryExportStart(&it, opts.StartKey, pos.key) { @@ -227,6 +234,13 @@ func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptio return result, nil } +func (s *mvccStore) checkExportReadTSLocked(opts ExportVersionsOptions) error { + if readTSCompacted(opts.ReadTS, s.minRetainedTS) { + return ErrReadTSCompacted + } + return nil +} + var errExportReachedEnd = errors.New("export reached end") var errExportChunkFull = errors.New("export chunk full") diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index ee7ffaf42..2fb9ee72c 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -53,6 +53,34 @@ func TestExportVersionsPreservesRawVersionMetadata(t *testing.T) { }) } +func TestExportVersionsReadTSPreservesCompactionErrors(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v10"), 10, 0)) + retention, ok := st.(RetentionController) + require.True(t, ok) + retention.SetMinRetainedTS(20) + + _, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 15, + ReadTS: 15, + MaxVersions: 10, + }) + require.ErrorIs(t, err, ErrReadTSCompacted) + + result, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 15, + MaxVersions: 10, + }) + require.NoError(t, err) + require.Len(t, result.Versions, 1) + }) +} + func TestExportVersionsCursorResumesWithinHotKey(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() diff --git a/store/store.go b/store/store.go index ae53203f9..b0dea1e79 100644 --- a/store/store.go +++ b/store/store.go @@ -82,12 +82,14 @@ type ExportVersionsOptions struct { EndKey []byte MinCommitTSExclusive uint64 MaxCommitTSInclusive uint64 - Cursor []byte - MaxVersions int - MaxBytes uint64 - MaxScannedBytes uint64 - KeyFamily uint32 - AcceptKey func([]byte) bool + // ReadTS asks export to enforce the same retention watermark as GetAt/ScanAt. + ReadTS uint64 + Cursor []byte + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + KeyFamily uint32 + AcceptKey func([]byte) bool } // ExportVersionsResult is one resumable chunk of raw MVCC versions. From ce0c2e1b4ed48e7c3f8a2708a580bd1dd6dea1ed Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 10:21:55 +0900 Subject: [PATCH 048/162] Wire split migration capability gate --- main.go | 5 +++++ main_catalog_test.go | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/main.go b/main.go index b33c4cbcd..6cfd9abd6 100644 --- a/main.go +++ b/main.go @@ -504,6 +504,7 @@ func run() error { adapter.WithDistributionCoordinator(coordinate), adapter.WithDistributionActiveTimestampTracker(readTracker), adapter.WithDistributionKnownRaftGroups(shardGroupIDs(shardGroups)...), + adapter.WithSplitMigrationCapabilityGate(splitMigrationCapabilityGate), ) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) compactor := kv.NewFSMCompactor( @@ -665,6 +666,10 @@ func shardGroupIDs(groups map[uint64]*kv.ShardGroup) []uint64 { return ids } +func splitMigrationCapabilityGate(context.Context) error { + return nil +} + type runtimeConfig struct { groups []groupSpec defaultGroup uint64 diff --git a/main_catalog_test.go b/main_catalog_test.go index 193a98219..843b2dea8 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -68,3 +68,9 @@ func TestSetupDistributionCatalog_UsesResolvedCatalogGroup(t *testing.T) { require.NoError(t, err) require.NotNil(t, catalog) } + +func TestSplitMigrationCapabilityGateIsReady(t *testing.T) { + t.Parallel() + + require.NoError(t, splitMigrationCapabilityGate(context.Background())) +} From f884ed2fff8f74896e7eb1cc4d42bc10587168b3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 10:49:51 +0900 Subject: [PATCH 049/162] Replicate target readiness guards --- adapter/internal.go | 40 ++++++ adapter/internal_migration_test.go | 40 ++++++ kv/fsm.go | 6 + kv/fsm_migration_readiness.go | 59 ++++++++ kv/fsm_migration_readiness_test.go | 44 ++++++ kv/shard_store.go | 47 ++++-- kv/shard_store_test.go | 86 +++++++++++ main.go | 1 + main_encryption_rotate_on_startup_test.go | 1 + proto/internal.pb.go | 167 ++++++++++++++++++++-- proto/internal.proto | 13 ++ proto/internal_grpc.pb.go | 48 ++++++- 12 files changed, 522 insertions(+), 30 deletions(-) create mode 100644 kv/fsm_migration_readiness.go create mode 100644 kv/fsm_migration_readiness_test.go diff --git a/adapter/internal.go b/adapter/internal.go index efacdab71..3773ec185 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -207,6 +207,28 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta }, nil } +func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) { + if req == nil { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness request is nil")) + } + if req.GetJobId() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness job_id is required")) + } + if req.GetMigrationJobId() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness migration_job_id is required")) + } + if i.migrationProposer == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "target staged readiness proposer is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + if err := i.proposeTargetStagedReadiness(ctx, req); err != nil { + return nil, errors.WithStack(err) + } + return &pb.TargetStagedReadinessResponse{}, nil +} + func (i *Internal) verifyInternalLeader(ctx context.Context) error { if i.leader.State() != raftengine.StateLeader { return errors.WithStack(ErrNotLeader) @@ -265,6 +287,24 @@ func (i *Internal) proposeMigrationPromote(ctx context.Context, req *pb.PromoteS } } +func (i *Internal) proposeTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) error { + cmd, err := kv.MarshalTargetStagedReadinessCommand(req) + if err != nil { + return errors.WithStack(err) + } + resp, err := i.proposeMigrationCommand(ctx, cmd, "target staged readiness") + if err != nil { + return errors.WithStack(err) + } + if resp == nil { + return nil + } + if err, ok := resp.(error); ok { + return errors.WithStack(err) + } + return errors.WithStack(errors.Newf("unexpected target readiness apply response type %T", resp)) +} + func (i *Internal) proposeMigrationCommand(ctx context.Context, cmd []byte, label string) (any, error) { result, err := i.migrationProposer.Propose(ctx, cmd) if err != nil { diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 54227d418..5a641f7c2 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -187,3 +187,43 @@ func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) require.GreaterOrEqual(t, clock.Current(), uint64(30)) } + +func TestInternalApplyTargetStagedReadinessProposesThroughRaft(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + ) + + _, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), proposer.calls) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} diff --git a/kv/fsm.go b/kv/fsm.go index 8c88d21d1..e4f1650fd 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -380,6 +380,8 @@ func (f *kvFSM) applyReservedOpcode(ctx context.Context, data []byte) (any, bool return f.applyMigrationImport(ctx, data[1:]), true case data[0] == raftEncodeMigrationPromote: return f.applyMigrationPromote(ctx, data[1:]), true + case data[0] == raftEncodeTargetReadiness: + return f.applyTargetStagedReadiness(ctx, data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true default: @@ -419,6 +421,10 @@ const ( // data promotion chunk. Every target voter atomically copies staged MVCC // versions into the live keyspace and removes the promoted staged rows. raftEncodeMigrationPromote byte = 0x0b + // raftEncodeTargetReadiness carries the target-local staged-readiness + // guard. It must be replicated through the target Raft group before the + // migration controller can treat a target as fail-closed for cutover. + raftEncodeTargetReadiness byte = 0x0c ) func decodeRaftRequests(data []byte) ([]*pb.Request, error) { diff --git a/kv/fsm_migration_readiness.go b/kv/fsm_migration_readiness.go new file mode 100644 index 000000000..91ea21cd8 --- /dev/null +++ b/kv/fsm_migration_readiness.go @@ -0,0 +1,59 @@ +package kv + +import ( + "bytes" + "context" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// MarshalTargetStagedReadinessCommand encodes a target-group readiness guard +// as a Raft FSM command. The target Internal RPC handler uses this instead of +// mutating its local store directly so an acknowledged guard has been applied +// by the target group's voters. +func MarshalTargetStagedReadinessCommand(req *pb.TargetStagedReadinessRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled target readiness request too large") + } + return prependByte(raftEncodeTargetReadiness, b), nil +} + +func (f *kvFSM) applyTargetStagedReadiness(ctx context.Context, data []byte) any { + req := &pb.TargetStagedReadinessRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + writer, ok := f.store.(store.MigrationTargetReadinessWriter) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + if err := writer.ApplyTargetStagedReadiness(ctx, targetStagedReadinessStateFromProto(req)); err != nil { + return errors.WithStack(err) + } + return nil +} + +func targetStagedReadinessStateFromProto(req *pb.TargetStagedReadinessRequest) store.TargetStagedReadinessState { + if req == nil { + return store.TargetStagedReadinessState{} + } + return store.TargetStagedReadinessState{ + JobID: req.GetJobId(), + RouteStart: bytes.Clone(req.GetRouteStart()), + RouteEnd: bytes.Clone(req.GetRouteEnd()), + ExpectedCutoverVersion: req.GetExpectedCutoverVersion(), + MigrationJobID: req.GetMigrationJobId(), + MinWriteTSExclusive: req.GetMinWriteTsExclusive(), + Armed: req.GetArmed(), + } +} diff --git a/kv/fsm_migration_readiness_test.go b/kv/fsm_migration_readiness_test.go new file mode 100644 index 000000000..8d5939b84 --- /dev/null +++ b/kv/fsm_migration_readiness_test.go @@ -0,0 +1,44 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestApplyTargetStagedReadinessCommandPersistsGuard(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + fsm := &kvFSM{store: st} + + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Nil(t, fsm.Apply(cmd)) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} diff --git a/kv/shard_store.go b/kv/shard_store.go index cb7fef43c..100407e4f 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -152,33 +152,41 @@ func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *Shard } func (s *ShardStore) targetReadyRouteForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) (distribution.Route, error) { + routes, err := s.targetReadyRoutesForRouteRange(ctx, g, route, routeStart, routeEnd) + if err != nil { + return route, err + } + if len(routes) == 1 { + return routes[0], nil + } + return route, nil +} + +func (s *ShardStore) targetReadyRoutesForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) ([]distribution.Route, error) { if g == nil || g.Store == nil { - return route, nil + return []distribution.Route{route}, nil } reader, ok := g.Store.(store.MigrationTargetReadinessReader) if !ok { - return route, nil + return []distribution.Route{route}, nil } states, err := reader.MigrationTargetReadinessStates(ctx) if err != nil { - return route, errors.WithStack(err) + return nil, errors.WithStack(err) } applicable := targetReadinessApplicableStates(states, route, routeStart, routeEnd) if len(applicable) == 0 { - return route, nil + return []distribution.Route{route}, nil } proofRoutes, catalogVersion, ok := s.readinessProofRoutes(route, routeStart, routeEnd) if !ok { - return route, errors.WithStack(ErrRouteCutoverPending) + return nil, errors.WithStack(ErrRouteCutoverPending) } if !readinessProofSatisfiesStates(applicable, proofRoutes, route.GroupID, catalogVersion) { - return route, errors.WithStack(ErrRouteCutoverPending) - } - if len(proofRoutes) == 1 { - return proofRoutes[0], nil + return nil, errors.WithStack(ErrRouteCutoverPending) } - return route, nil + return proofRoutes, nil } func targetReadinessApplicableStates( @@ -2111,19 +2119,32 @@ func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte if len(routes) == 0 { return nil, errors.WithStack(ErrRouteCutoverPending) } + verified := make([]distribution.Route, 0, len(routes)) for _, route := range routes { g, ok := s.groupForID(route.GroupID) if !ok || g == nil || g.Store == nil { return nil, store.ErrNotSupported } - if err := s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd); err != nil { + proofRoutes, err := s.verifyPrefixDeleteRoute(ctx, g, route, routeStart, routeEnd, commitTS) + if err != nil { return nil, err } - if err := verifyRouteWriteFloor(route, commitTS); err != nil { + verified = append(verified, proofRoutes...) + } + return verified, nil +} + +func (s *ShardStore) verifyPrefixDeleteRoute(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte, commitTS uint64) ([]distribution.Route, error) { + proofRoutes, err := s.targetReadyRoutesForRouteRange(ctx, g, route, routeStart, routeEnd) + if err != nil { + return nil, err + } + for _, proofRoute := range proofRoutes { + if err := verifyRouteWriteFloor(proofRoute, commitTS); err != nil { return nil, err } } - return routes, nil + return proofRoutes, nil } func routesForGroupID(routes []distribution.Route, groupID uint64) []distribution.Route { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 9f343cc32..ac3449cc5 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -355,6 +355,92 @@ func TestShardStoreTargetReadinessUsesSingleCatalogSnapshotProof(t *testing.T) { require.Equal(t, []byte("staged-new"), got) } +func TestShardStoreDeletePrefixAtUsesReadinessProofRouteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live"), 80, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + err := shards.DeletePrefixAt(ctx, []byte("b"), nil, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("live"), got) +} + +func TestShardStoreDeletePrefixAtUsesReadinessProofRouteForStagedCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + stagedKey := distribution.MigrationStagedDataKey(9, []byte("b")) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + require.NoError(t, shards.DeletePrefixAt(ctx, []byte("b"), nil, 130)) + + _, err := group.Store.GetAt(ctx, stagedKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = shards.GetAt(ctx, []byte("b"), 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + func TestShardStoreExplicitGroupReadUsesRouteProofForReadiness(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index c236598d8..75fc290d9 100644 --- a/main.go +++ b/main.go @@ -1828,6 +1828,7 @@ func (g *startupPublicKVGate) blocked() bool { func startupRotationGatedMethod(fullMethod string) bool { switch fullMethod { case pb.Internal_Forward_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index 5e16348ad..750dc673c 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -419,6 +419,7 @@ func TestStartupPublicKVGate_BlocksMutatorsUntilReady(t *testing.T) { pb.RawKV_RawGet_FullMethodName, pb.TransactionalKV_Get_FullMethodName, pb.Internal_Forward_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 91bcff36b..bfdd7db15 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -1075,6 +1075,134 @@ func (x *PromoteStagedVersionsResponse) GetMaxPromotedTs() uint64 { return 0 } +type TargetStagedReadinessRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + RouteStart []byte `protobuf:"bytes,2,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,3,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExpectedCutoverVersion uint64 `protobuf:"varint,4,opt,name=expected_cutover_version,json=expectedCutoverVersion,proto3" json:"expected_cutover_version,omitempty"` + MigrationJobId uint64 `protobuf:"varint,5,opt,name=migration_job_id,json=migrationJobId,proto3" json:"migration_job_id,omitempty"` + MinWriteTsExclusive uint64 `protobuf:"varint,6,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` + Armed bool `protobuf:"varint,7,opt,name=armed,proto3" json:"armed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TargetStagedReadinessRequest) Reset() { + *x = TargetStagedReadinessRequest{} + mi := &file_internal_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TargetStagedReadinessRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TargetStagedReadinessRequest) ProtoMessage() {} + +func (x *TargetStagedReadinessRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TargetStagedReadinessRequest.ProtoReflect.Descriptor instead. +func (*TargetStagedReadinessRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{14} +} + +func (x *TargetStagedReadinessRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *TargetStagedReadinessRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *TargetStagedReadinessRequest) GetExpectedCutoverVersion() uint64 { + if x != nil { + return x.ExpectedCutoverVersion + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetMigrationJobId() uint64 { + if x != nil { + return x.MigrationJobId + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetMinWriteTsExclusive() uint64 { + if x != nil { + return x.MinWriteTsExclusive + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetArmed() bool { + if x != nil { + return x.Armed + } + return false +} + +type TargetStagedReadinessResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TargetStagedReadinessResponse) Reset() { + *x = TargetStagedReadinessResponse{} + mi := &file_internal_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TargetStagedReadinessResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TargetStagedReadinessResponse) ProtoMessage() {} + +func (x *TargetStagedReadinessResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TargetStagedReadinessResponse.ProtoReflect.Descriptor instead. +func (*TargetStagedReadinessResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{15} +} + var File_internal_proto protoreflect.FileDescriptor const file_internal_proto_rawDesc = "" + @@ -1155,7 +1283,17 @@ const file_internal_proto_rawDesc = "" + "nextCursor\x12\x12\n" + "\x04done\x18\x02 \x01(\bR\x04done\x12#\n" + "\rpromoted_rows\x18\x03 \x01(\x04R\fpromotedRows\x12&\n" + - "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs*&\n" + + "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs\"\xa2\x02\n" + + "\x1cTargetStagedReadinessRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x1f\n" + + "\vroute_start\x18\x02 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x03 \x01(\fR\brouteEnd\x128\n" + + "\x18expected_cutover_version\x18\x04 \x01(\x04R\x16expectedCutoverVersion\x12(\n" + + "\x10migration_job_id\x18\x05 \x01(\x04R\x0emigrationJobId\x123\n" + + "\x16min_write_ts_exclusive\x18\x06 \x01(\x04R\x13minWriteTsExclusive\x12\x14\n" + + "\x05armed\x18\a \x01(\bR\x05armed\"\x1f\n" + + "\x1dTargetStagedReadinessResponse*&\n" + "\x02Op\x12\a\n" + "\x03PUT\x10\x00\x12\a\n" + "\x03DEL\x10\x01\x12\x0e\n" + @@ -1166,13 +1304,14 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\xfd\x02\n" + + "\x05ABORT\x10\x032\xdc\x03\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00\x12T\n" + "\x13ExportRangeVersions\x12\x1b.ExportRangeVersionsRequest\x1a\x1c.ExportRangeVersionsResponse\"\x000\x01\x12R\n" + "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00\x12X\n" + - "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00\x12]\n" + + "\x1aApplyTargetStagedReadiness\x12\x1d.TargetStagedReadinessRequest\x1a\x1e.TargetStagedReadinessResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_internal_proto_rawDescOnce sync.Once @@ -1187,7 +1326,7 @@ func file_internal_proto_rawDescGZIP() []byte { } var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_internal_proto_goTypes = []any{ (Op)(0), // 0: Op (Phase)(0), // 1: Phase @@ -1205,6 +1344,8 @@ var file_internal_proto_goTypes = []any{ (*ImportRangeVersionsResponse)(nil), // 13: ImportRangeVersionsResponse (*PromoteStagedVersionsRequest)(nil), // 14: PromoteStagedVersionsRequest (*PromoteStagedVersionsResponse)(nil), // 15: PromoteStagedVersionsResponse + (*TargetStagedReadinessRequest)(nil), // 16: TargetStagedReadinessRequest + (*TargetStagedReadinessResponse)(nil), // 17: TargetStagedReadinessResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op @@ -1219,13 +1360,15 @@ var file_internal_proto_depIdxs = []int32{ 9, // 9: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest 12, // 10: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest 14, // 11: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest - 6, // 12: Internal.Forward:output_type -> ForwardResponse - 8, // 13: Internal.RelayPublish:output_type -> RelayPublishResponse - 10, // 14: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse - 13, // 15: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse - 15, // 16: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse - 12, // [12:17] is the sub-list for method output_type - 7, // [7:12] is the sub-list for method input_type + 16, // 12: Internal.ApplyTargetStagedReadiness:input_type -> TargetStagedReadinessRequest + 6, // 13: Internal.Forward:output_type -> ForwardResponse + 8, // 14: Internal.RelayPublish:output_type -> RelayPublishResponse + 10, // 15: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse + 13, // 16: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse + 15, // 17: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse + 17, // 18: Internal.ApplyTargetStagedReadiness:output_type -> TargetStagedReadinessResponse + 13, // [13:19] is the sub-list for method output_type + 7, // [7:13] is the sub-list for method input_type 7, // [7:7] is the sub-list for extension type_name 7, // [7:7] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name @@ -1242,7 +1385,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), NumEnums: 2, - NumMessages: 14, + NumMessages: 16, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index b45bf8fb5..1b168812a 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -10,6 +10,7 @@ service Internal { rpc ExportRangeVersions(ExportRangeVersionsRequest) returns (stream ExportRangeVersionsResponse) {} rpc ImportRangeVersions(ImportRangeVersionsRequest) returns (ImportRangeVersionsResponse) {} rpc PromoteStagedVersions(PromoteStagedVersionsRequest) returns (PromoteStagedVersionsResponse) {} + rpc ApplyTargetStagedReadiness(TargetStagedReadinessRequest) returns (TargetStagedReadinessResponse) {} } // internal.proto is node to node communication message in raft replication. @@ -144,3 +145,15 @@ message PromoteStagedVersionsResponse { uint64 promoted_rows = 3; uint64 max_promoted_ts = 4; } + +message TargetStagedReadinessRequest { + uint64 job_id = 1; + bytes route_start = 2; + bytes route_end = 3; + uint64 expected_cutover_version = 4; + uint64 migration_job_id = 5; + uint64 min_write_ts_exclusive = 6; + bool armed = 7; +} + +message TargetStagedReadinessResponse {} diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index ba679ed80..6af68894d 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -19,11 +19,12 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Internal_Forward_FullMethodName = "/Internal/Forward" - Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" - Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" - Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" - Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" + Internal_Forward_FullMethodName = "/Internal/Forward" + Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" + Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" + Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" + Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" + Internal_ApplyTargetStagedReadiness_FullMethodName = "/Internal/ApplyTargetStagedReadiness" ) // InternalClient is the client API for Internal service. @@ -36,6 +37,7 @@ type InternalClient interface { ExportRangeVersions(ctx context.Context, in *ExportRangeVersionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ExportRangeVersionsResponse], error) ImportRangeVersions(ctx context.Context, in *ImportRangeVersionsRequest, opts ...grpc.CallOption) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(ctx context.Context, in *PromoteStagedVersionsRequest, opts ...grpc.CallOption) (*PromoteStagedVersionsResponse, error) + ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) } type internalClient struct { @@ -105,6 +107,16 @@ func (c *internalClient) PromoteStagedVersions(ctx context.Context, in *PromoteS return out, nil } +func (c *internalClient) ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(TargetStagedReadinessResponse) + err := c.cc.Invoke(ctx, Internal_ApplyTargetStagedReadiness_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // InternalServer is the server API for Internal service. // All implementations must embed UnimplementedInternalServer // for forward compatibility. @@ -115,6 +127,7 @@ type InternalServer interface { ExportRangeVersions(*ExportRangeVersionsRequest, grpc.ServerStreamingServer[ExportRangeVersionsResponse]) error ImportRangeVersions(context.Context, *ImportRangeVersionsRequest) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) + ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) mustEmbedUnimplementedInternalServer() } @@ -140,6 +153,9 @@ func (UnimplementedInternalServer) ImportRangeVersions(context.Context, *ImportR func (UnimplementedInternalServer) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) { return nil, status.Error(codes.Unimplemented, "method PromoteStagedVersions not implemented") } +func (UnimplementedInternalServer) ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ApplyTargetStagedReadiness not implemented") +} func (UnimplementedInternalServer) mustEmbedUnimplementedInternalServer() {} func (UnimplementedInternalServer) testEmbeddedByValue() {} @@ -244,6 +260,24 @@ func _Internal_PromoteStagedVersions_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _Internal_ApplyTargetStagedReadiness_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TargetStagedReadinessRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ApplyTargetStagedReadiness(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ApplyTargetStagedReadiness_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ApplyTargetStagedReadiness(ctx, req.(*TargetStagedReadinessRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Internal_ServiceDesc is the grpc.ServiceDesc for Internal service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -267,6 +301,10 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "PromoteStagedVersions", Handler: _Internal_PromoteStagedVersions_Handler, }, + { + MethodName: "ApplyTargetStagedReadiness", + Handler: _Internal_ApplyTargetStagedReadiness_Handler, + }, }, Streams: []grpc.StreamDesc{ { From 3cb3aad44078caf0a86233faf4dbac742fdc915d Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 10:52:15 +0900 Subject: [PATCH 050/162] store: advance promotion commit watermark --- store/migration_promote.go | 31 ++++++++++++++--- store/migration_versions_test.go | 58 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 5 deletions(-) diff --git a/store/migration_promote.go b/store/migration_promote.go index 12cc79389..3d2d9de57 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -219,7 +219,7 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO } if opts.JobID != 0 && state.Done { result := PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows} - return s.finishPebblePromotion(nil, opts.JobID, nil, result, opts.AppliedIndex) + return s.finishPebblePromotion(nil, opts.JobID, nil, result, opts.AppliedIndex, 0) } opts.Cursor = cursor exported, toPromote, promoted, err := s.planPebblePromotionLocked(ctx, opts) @@ -227,7 +227,7 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO return PromoteVersionsResult{}, err } result, stateToWrite := finishPromotionResult(opts, state, exported, promoted) - return s.finishPebblePromotion(toPromote, opts.JobID, stateToWrite, result, opts.AppliedIndex) + return s.finishPebblePromotion(toPromote, opts.JobID, stateToWrite, result, opts.AppliedIndex, promoted.MaxPromotedTS) } func (s *pebbleStore) finishPebblePromotion( @@ -236,11 +236,12 @@ func (s *pebbleStore) finishPebblePromotion( stateToWrite *PromotionState, result PromoteVersionsResult, appliedIndex uint64, + maxPromotedTS uint64, ) (PromoteVersionsResult, error) { - if len(toPromote) == 0 && stateToWrite == nil && appliedIndex == 0 { + if len(toPromote) == 0 && stateToWrite == nil && appliedIndex == 0 && maxPromotedTS == 0 { return result, nil } - if err := s.commitPebblePromoteVersions(toPromote, jobID, stateToWrite, appliedIndex); err != nil { + if err := s.commitPebblePromoteVersions(toPromote, jobID, stateToWrite, appliedIndex, maxPromotedTS); err != nil { return PromoteVersionsResult{}, err } return result, nil @@ -322,7 +323,7 @@ func (s *pebbleStore) readPebblePromotionState(jobID uint64) (PromotionState, bo return state, true, nil } -func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jobID uint64, state *PromotionState, appliedIndex uint64) error { +func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jobID uint64, state *PromotionState, appliedIndex, maxPromotedTS uint64) error { batch := s.db.NewBatch() defer batch.Close() targets := make([]MVCCVersion, 0, len(versions)) @@ -349,6 +350,26 @@ func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jo return err } } + return s.commitPebblePromotionBatch(batch, maxPromotedTS) +} + +func (s *pebbleStore) commitPebblePromotionBatch(batch *pebble.Batch, maxPromotedTS uint64) error { + if maxPromotedTS > 0 { + s.mtx.Lock() + defer s.mtx.Unlock() + newLastTS := s.lastCommitTS + if maxPromotedTS > newLastTS { + newLastTS = maxPromotedTS + } + if err := setPebbleUint64InBatch(batch, metaLastCommitTSBytes, newLastTS); err != nil { + return err + } + if err := batch.Commit(s.directApplyWriteOpts()); err != nil { + return errors.WithStack(err) + } + s.updateLastCommitTS(newLastTS) + return nil + } if err := batch.Commit(s.directApplyWriteOpts()); err != nil { return errors.WithStack(err) } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 91548bb00..d11fefaed 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/cockroachdb/pebble/v2" "github.com/stretchr/testify/require" ) @@ -279,6 +280,63 @@ func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { }) } +func TestPebblePromoteVersionsAdvancesLastCommitTS(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + st, err := NewPebbleStore(dir) + require.NoError(t, err) + closed := false + t.Cleanup(func() { + if !closed { + require.NoError(t, st.Close()) + } + }) + ps, ok := st.(*pebbleStore) + require.True(t, ok) + + stage := func(raw string) []byte { + return append([]byte("stage|"), []byte(raw)...) + } + targetKey := func(staged []byte) ([]byte, bool) { + return bytes.TrimPrefix(staged, []byte("stage|")), bytes.HasPrefix(staged, []byte("stage|")) + } + prefix := []byte("stage|") + + const promotedTS uint64 = 100 + require.NoError(t, ps.db.Set(encodeKey(stage("k"), promotedTS), encodeValue([]byte("v100"), false, 0, encStateCleartext), pebble.NoSync)) + require.Zero(t, ps.LastCommitTS()) + + result, err := ps.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 101, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, result.Done) + require.Equal(t, uint64(1), result.PromotedRows) + require.Equal(t, promotedTS, result.MaxPromotedTS) + require.Equal(t, promotedTS, ps.LastCommitTS()) + + metaTS, err := readPebbleUint64(ps.db, metaLastCommitTSBytes) + require.NoError(t, err) + require.Equal(t, promotedTS, metaTS) + val, err := ps.GetAt(ctx, []byte("k"), ps.LastCommitTS()) + require.NoError(t, err) + require.Equal(t, []byte("v100"), val) + + require.NoError(t, st.Close()) + closed = true + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + defer func() { require.NoError(t, reopened.Close()) }() + require.Equal(t, promotedTS, reopened.LastCommitTS()) + val, err = reopened.GetAt(ctx, []byte("k"), reopened.LastCommitTS()) + require.NoError(t, err) + require.Equal(t, []byte("v100"), val) +} + func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-import-persist-*") From c5dae8f86afa49ce3b5ea34a3b95fb0bc964db19 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 10:53:25 +0900 Subject: [PATCH 051/162] Fix bounded store export edge cases --- store/lsm_migration.go | 18 +++--- store/lsm_store.go | 7 ++- store/migration_versions.go | 3 + store/migration_versions_test.go | 104 ++++++++++++++++++++++++++++++- 4 files changed, 120 insertions(+), 12 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index aa3f0ebca..dbf1e257e 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -5,7 +5,6 @@ import ( "context" "math" - "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/v2" ) @@ -128,7 +127,7 @@ func (s *pebbleStore) exportPebbleIteratorPosition( } func isPebbleExportMetadataKey(rawKey []byte) bool { - return isPebbleMetaKey(rawKey) || bytes.HasPrefix(rawKey, encryption.WriterRegistryPrefix) + return isPebbleMetaKey(rawKey) } func (s *pebbleStore) skipPebbleExportKeyOutsideRange(iter *pebble.Iterator, opts ExportVersionsOptions, userKey []byte) (bool, error) { @@ -175,9 +174,12 @@ func advancePebbleExportPastCurrentUserKey(iter *pebble.Iterator, userKey []byte } func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { + if len(startKey) == 0 { + return false + } for prefixLen := 1; prefixLen <= len(userKey); prefixLen++ { prefix := userKey[:prefixLen] - if (startKey == nil || bytes.Compare(prefix, startKey) >= 0) && bytes.Compare(prefix, endKey) < 0 { + if bytes.Compare(prefix, startKey) >= 0 && bytes.Compare(prefix, endKey) < 0 { return false } } @@ -232,14 +234,12 @@ func (s *pebbleStore) decodeExportedPebbleVersion(iter *pebble.Iterator, userKey if err != nil { return MVCCVersion{}, errors.WithStack(err) } - var value []byte + value, err := s.decryptForKey(iter.Key(), sv, sv.Value) + if err != nil { + return MVCCVersion{}, err + } if sv.Tombstone { value = nil - } else { - value, err = s.decryptForKey(iter.Key(), sv, sv.Value) - if err != nil { - return MVCCVersion{}, err - } } return MVCCVersion{ Key: bytes.Clone(userKey), diff --git a/store/lsm_store.go b/store/lsm_store.go index d7f6e5f85..da40ee41e 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -553,7 +553,12 @@ func isPebbleMetaKey(rawKey []byte) bool { bytes.Equal(rawKey, metaPendingMinRetainedTSBytes) || bytes.Equal(rawKey, metaAppliedIndexBytes) || isMigrationMetadataKey(rawKey) || - bytes.HasPrefix(rawKey, encryption.WriterRegistryPrefix) + isPebbleWriterRegistryKey(rawKey) +} + +func isPebbleWriterRegistryKey(rawKey []byte) bool { + _, _, err := encryption.DecodeRegistryKey(rawKey) + return err == nil } func (s *pebbleStore) findMaxCommitTS() (uint64, error) { diff --git a/store/migration_versions.go b/store/migration_versions.go index 043dcba8f..492a0c13f 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -110,6 +110,9 @@ func validateExportCursorRange(opts ExportVersionsOptions, pos exportCursorPosit } func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOptions { + if opts.EndKey != nil && len(opts.EndKey) == 0 { + opts.EndKey = nil + } if exportUsesSparseScanBudget(opts) && opts.MaxScannedBytes == 0 { opts.MaxScannedBytes = defaultSparseExportMaxScannedBytes } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 3d8e7da37..41e027b4a 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -305,6 +305,63 @@ func TestExportVersionsUsesUserKeyRangeBounds(t *testing.T) { }) } +func TestExportVersionsEmptyEndKeyIsUnbounded(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("m"), []byte("m-value"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("z"), []byte("z-value"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("m"), + EndKey: []byte{}, + MaxVersions: 1, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("m"), CommitTS: 10, Value: []byte("m-value")}}, first.Versions) + require.NotEmpty(t, first.NextCursor) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("m"), + EndKey: []byte{}, + Cursor: first.NextCursor, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Empty(t, second.NextCursor) + require.Equal(t, []MVCCVersion{{Key: []byte("z"), CommitTS: 20, Value: []byte("z-value")}}, second.Versions) + }) +} + +func TestPebbleExportDoesNotStopBeforeTrailingEmptyKey(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-trailing-empty-key-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("a-value"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("b-value"), 20, 0)) + require.NoError(t, st.PutAt(ctx, nil, []byte("empty-value"), 30, 0)) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("aa"), + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, res.Done) + require.Len(t, res.Versions, 2) + require.Equal(t, []byte("a"), res.Versions[0].Key) + require.Equal(t, uint64(10), res.Versions[0].CommitTS) + require.Equal(t, []byte("a-value"), res.Versions[0].Value) + require.Empty(t, res.Versions[1].Key) + require.Equal(t, uint64(30), res.Versions[1].CommitTS) + require.Equal(t, []byte("empty-value"), res.Versions[1].Value) +} + func TestExportVersionsRejectsCursorOutsideRequestedRange(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() @@ -392,7 +449,49 @@ func TestPebbleExportSkipsWriterRegistryRowsWithoutUserVersions(t *testing.T) { require.Zero(t, res.AcceptedRows) } -func TestPebbleExportStopsAtEndKey(t *testing.T) { +func TestPebbleExportDoesNotDropWriterRegistryPrefixUserKey(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-writer-registry-user-key-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + registryKey := encryption.RegistryKey(1, 2) + registry, err := WriterRegistryFor(st) + require.NoError(t, err) + require.NoError(t, registry.SetRegistryRow( + registryKey, + encryption.EncodeRegistryValue(encryption.RegistryValue{ + FullNodeID: 2, + FirstSeenLocalEpoch: 1, + LastSeenLocalEpoch: 1, + }), + )) + require.NoError(t, st.PutAt(ctx, registryKey, []byte("user-value"), 10, 0)) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 10}) + require.NoError(t, err) + require.True(t, res.Done) + require.Equal(t, []MVCCVersion{{Key: registryKey, CommitTS: 10, Value: []byte("user-value")}}, res.Versions) +} + +func TestPebbleExportAuthenticatesEncryptedTombstoneHeader(t *testing.T) { + ctx := context.Background() + f := newEncryptedStoreFixture(t, 81) + require.NoError(t, f.mvcc.PutAt(ctx, []byte("tampered-export"), []byte("payload"), 100, 0)) + f.tamperPebbleValue(t, []byte("tampered-export"), 100, func(raw []byte) []byte { + raw[0] |= tombstoneMask + return raw + }) + + res, err := f.mvcc.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 10}) + require.ErrorIs(t, err, ErrEncryptedReadIntegrity) + require.Empty(t, res.Versions) +} + +func TestPebbleExportStopsAtEndKeyWhenNoLaterInRangeKeyCanTrail(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-end-key-*") require.NoError(t, err) @@ -411,7 +510,8 @@ func TestPebbleExportStopsAtEndKey(t *testing.T) { result := newExportVersionsResult(10) advance, done, err := ps.exportPebbleIteratorPosition(ctx, iter, ExportVersionsOptions{ - EndKey: []byte("b"), + StartKey: []byte("a"), + EndKey: []byte("b"), }, exportCursorPosition{}, &result) require.ErrorIs(t, err, errExportReachedEnd) require.False(t, advance) From 49adab3dd41acf0042af2724ac2f5f8dc47291f6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 10:55:58 +0900 Subject: [PATCH 052/162] Gate split migration on peer capability --- adapter/distribution_server.go | 8 + adapter/distribution_server_test.go | 10 + main.go | 86 ++++++++- main_catalog_test.go | 76 +++++++- proto/distribution.pb.go | 279 +++++++++++++++++++--------- proto/distribution.proto | 8 + proto/distribution_grpc.pb.go | 60 ++++-- 7 files changed, 421 insertions(+), 106 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 5098ab32e..42360803a 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -99,6 +99,7 @@ const ( splitJobListCursorTerminalOff = 1 splitJobListCursorJobIDOff = splitJobListCursorTerminalOff + 8 splitJobListCursorEncodedBytes = splitJobListCursorJobIDOff + 8 + splitMigrationCapabilityV2 = "split_migration_v2" ) var ( @@ -244,6 +245,13 @@ func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.St }, nil } +func (s *DistributionServer) GetSplitMigrationCapability(context.Context, *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + return &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{splitMigrationCapabilityV2}, + }, nil +} + func (s *DistributionServer) verifyStartSplitMigrationReady(ctx context.Context, req *pb.StartSplitMigrationRequest) error { if req.GetTargetGroupId() == 0 { return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index ba8e0fd6f..21a34d96b 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -149,6 +149,16 @@ func TestDistributionServerListRoutes_RequiresCatalog(t *testing.T) { require.ErrorContains(t, err, errDistributionCatalogNotConfigured.Error()) } +func TestDistributionServerGetSplitMigrationCapabilityReportsReady(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.True(t, resp.GetMigrationCapable()) + require.Contains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + func TestDistributionServerStartSplitMigration_FailsClosedUntilCapabilityGate(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 6cfd9abd6..93fed90ce 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,8 @@ const ( etcdMaxSizePerMsg = 1 << 20 etcdMaxInflightMsg = 1024 defaultTSOBatchSize = 256 + + splitMigrationCapabilityProbeTimeout = 2 * time.Second ) func newRaftFactory(engineType raftEngineType, coldStartObs raftengine.ColdStartObserver) (raftengine.Factory, error) { @@ -498,13 +500,18 @@ func run() error { startKeyVizFlusher(runCtx, eg, sampler) startKeyVizLeaderTermPublisher(runCtx, eg, sampler, runtimes) startMemoryWatchdog(runCtx, eg, cancel) + splitMigrationGate := newSplitMigrationCapabilityGate( + splitMigrationCapabilityPeers(bootstrapCfg, cfg.defaultGroup), + splitMigrationCapabilityProbeTimeout, + nil, + ) distServer := adapter.NewDistributionServer( cfg.engine, distCatalog, adapter.WithDistributionCoordinator(coordinate), adapter.WithDistributionActiveTimestampTracker(readTracker), adapter.WithDistributionKnownRaftGroups(shardGroupIDs(shardGroups)...), - adapter.WithSplitMigrationCapabilityGate(splitMigrationCapabilityGate), + adapter.WithSplitMigrationCapabilityGate(splitMigrationGate), ) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) compactor := kv.NewFSMCompactor( @@ -666,7 +673,82 @@ func shardGroupIDs(groups map[uint64]*kv.ShardGroup) []uint64 { return ids } -func splitMigrationCapabilityGate(context.Context) error { +type splitMigrationCapabilityPeer struct { + ID string + Address string +} + +type splitMigrationCapabilityProbe func(context.Context, string) error + +func splitMigrationCapabilityPeers(cfg raftBootstrapConfig, defaultGroup uint64) []splitMigrationCapabilityPeer { + servers := cfg.adminSeed(defaultGroup) + if len(servers) == 0 { + return nil + } + peers := make([]splitMigrationCapabilityPeer, 0, len(servers)) + seen := make(map[string]struct{}, len(servers)) + for _, server := range servers { + key := server.ID + "\x00" + server.Address + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + id := server.ID + if id == "" { + id = server.Address + } + peers = append(peers, splitMigrationCapabilityPeer{ + ID: id, + Address: server.Address, + }) + } + return peers +} + +func newSplitMigrationCapabilityGate(peers []splitMigrationCapabilityPeer, timeout time.Duration, probe splitMigrationCapabilityProbe) adapter.SplitMigrationCapabilityGate { + if probe == nil { + probe = probeSplitMigrationCapabilityPeer + } + return func(ctx context.Context) error { + if len(peers) == 0 { + return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") + } + for _, peer := range peers { + peerCtx := ctx + cancel := func() {} + if timeout > 0 { + var cancelCtx context.CancelFunc + peerCtx, cancelCtx = context.WithTimeout(ctx, timeout) + cancel = cancelCtx + } + err := probe(peerCtx, peer.Address) + cancel() + if err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration capability peer %s is not ready: %v", peer.ID, err) + } + } + return nil + } +} + +func probeSplitMigrationCapabilityPeer(ctx context.Context, address string) error { + if strings.TrimSpace(address) == "" { + return errors.New("empty split migration capability peer address") + } + conn, err := grpc.NewClient(address, internalutil.GRPCDialOptions()...) + if err != nil { + return errors.Wrapf(err, "dial split migration capability peer %s", address) + } + defer func() { + _ = conn.Close() + }() + resp, err := pb.NewDistributionClient(conn).GetSplitMigrationCapability(ctx, &pb.GetSplitMigrationCapabilityRequest{}) + if err != nil { + return errors.Wrapf(err, "probe split migration capability peer %s", address) + } + if resp == nil || !resp.GetMigrationCapable() { + return errors.New("peer does not advertise split migration capability") + } return nil } diff --git a/main_catalog_test.go b/main_catalog_test.go index 843b2dea8..d2e63118b 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -3,10 +3,14 @@ package main import ( "context" "testing" + "time" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestDistributionCatalogGroupID_UsesCatalogKeyRoute(t *testing.T) { @@ -69,8 +73,76 @@ func TestSetupDistributionCatalog_UsesResolvedCatalogGroup(t *testing.T) { require.NotNil(t, catalog) } -func TestSplitMigrationCapabilityGateIsReady(t *testing.T) { +func TestSplitMigrationCapabilityGateChecksAllPeers(t *testing.T) { t.Parallel() - require.NoError(t, splitMigrationCapabilityGate(context.Background())) + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + } + var probed []string + gate := newSplitMigrationCapabilityGate(peers, time.Second, func(_ context.Context, address string) error { + probed = append(probed, address) + return nil + }) + + require.NoError(t, gate(context.Background())) + require.Equal(t, []string{"10.0.0.11:50051", "10.0.0.12:50051"}, probed) +} + +func TestSplitMigrationCapabilityGateFailsClosedWithoutPeers(t *testing.T) { + t.Parallel() + + gate := newSplitMigrationCapabilityGate(nil, time.Second, func(context.Context, string) error { + t.Fatal("probe must not run without peers") + return nil + }) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "peers are not configured") +} + +func TestSplitMigrationCapabilityGateFailsClosedWhenPeerMissingCapability(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + } + gate := newSplitMigrationCapabilityGate(peers, time.Second, func(_ context.Context, address string) error { + if address == "10.0.0.12:50051" { + return status.Error(codes.Unimplemented, "method not found") + } + return nil + }) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "n2") + require.ErrorContains(t, err, "method not found") +} + +func TestSplitMigrationCapabilityPeersUseDefaultGroupAdminSeed(t *testing.T) { + t.Parallel() + + cfg := raftBootstrapConfig{ + groupServers: map[uint64][]raftengine.Server{ + 1: { + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "voter", ID: "n2", Address: "10.0.0.12:50051"}, + }, + 2: { + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50052"}, + {Suffrage: "voter", ID: "n2", Address: "10.0.0.12:50052"}, + }, + }, + } + + require.Equal(t, []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50052"}, + {ID: "n2", Address: "10.0.0.12:50052"}, + }, splitMigrationCapabilityPeers(cfg, 2)) } diff --git a/proto/distribution.pb.go b/proto/distribution.pb.go index 21d15180c..a4a586aa9 100644 --- a/proto/distribution.pb.go +++ b/proto/distribution.pb.go @@ -1280,6 +1280,94 @@ func (x *StartSplitMigrationResponse) GetJobId() uint64 { return 0 } +type GetSplitMigrationCapabilityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSplitMigrationCapabilityRequest) Reset() { + *x = GetSplitMigrationCapabilityRequest{} + mi := &file_distribution_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSplitMigrationCapabilityRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSplitMigrationCapabilityRequest) ProtoMessage() {} + +func (x *GetSplitMigrationCapabilityRequest) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSplitMigrationCapabilityRequest.ProtoReflect.Descriptor instead. +func (*GetSplitMigrationCapabilityRequest) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{13} +} + +type GetSplitMigrationCapabilityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MigrationCapable bool `protobuf:"varint,1,opt,name=migration_capable,json=migrationCapable,proto3" json:"migration_capable,omitempty"` + Capabilities []string `protobuf:"bytes,2,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSplitMigrationCapabilityResponse) Reset() { + *x = GetSplitMigrationCapabilityResponse{} + mi := &file_distribution_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSplitMigrationCapabilityResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSplitMigrationCapabilityResponse) ProtoMessage() {} + +func (x *GetSplitMigrationCapabilityResponse) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSplitMigrationCapabilityResponse.ProtoReflect.Descriptor instead. +func (*GetSplitMigrationCapabilityResponse) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{14} +} + +func (x *GetSplitMigrationCapabilityResponse) GetMigrationCapable() bool { + if x != nil { + return x.MigrationCapable + } + return false +} + +func (x *GetSplitMigrationCapabilityResponse) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + type GetRouteOwnershipRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -1290,7 +1378,7 @@ type GetRouteOwnershipRequest struct { func (x *GetRouteOwnershipRequest) Reset() { *x = GetRouteOwnershipRequest{} - mi := &file_distribution_proto_msgTypes[13] + mi := &file_distribution_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1302,7 +1390,7 @@ func (x *GetRouteOwnershipRequest) String() string { func (*GetRouteOwnershipRequest) ProtoMessage() {} func (x *GetRouteOwnershipRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[13] + mi := &file_distribution_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1315,7 +1403,7 @@ func (x *GetRouteOwnershipRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteOwnershipRequest.ProtoReflect.Descriptor instead. func (*GetRouteOwnershipRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{13} + return file_distribution_proto_rawDescGZIP(), []int{15} } func (x *GetRouteOwnershipRequest) GetKey() []byte { @@ -1343,7 +1431,7 @@ type GetRouteOwnershipResponse struct { func (x *GetRouteOwnershipResponse) Reset() { *x = GetRouteOwnershipResponse{} - mi := &file_distribution_proto_msgTypes[14] + mi := &file_distribution_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1355,7 +1443,7 @@ func (x *GetRouteOwnershipResponse) String() string { func (*GetRouteOwnershipResponse) ProtoMessage() {} func (x *GetRouteOwnershipResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[14] + mi := &file_distribution_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1368,7 +1456,7 @@ func (x *GetRouteOwnershipResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteOwnershipResponse.ProtoReflect.Descriptor instead. func (*GetRouteOwnershipResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{14} + return file_distribution_proto_rawDescGZIP(), []int{16} } func (x *GetRouteOwnershipResponse) GetRoute() *RouteDescriptor { @@ -1403,7 +1491,7 @@ type GetIntersectingRoutesRequest struct { func (x *GetIntersectingRoutesRequest) Reset() { *x = GetIntersectingRoutesRequest{} - mi := &file_distribution_proto_msgTypes[15] + mi := &file_distribution_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1415,7 +1503,7 @@ func (x *GetIntersectingRoutesRequest) String() string { func (*GetIntersectingRoutesRequest) ProtoMessage() {} func (x *GetIntersectingRoutesRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[15] + mi := &file_distribution_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1428,7 +1516,7 @@ func (x *GetIntersectingRoutesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetIntersectingRoutesRequest.ProtoReflect.Descriptor instead. func (*GetIntersectingRoutesRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{15} + return file_distribution_proto_rawDescGZIP(), []int{17} } func (x *GetIntersectingRoutesRequest) GetStart() []byte { @@ -1462,7 +1550,7 @@ type GetIntersectingRoutesResponse struct { func (x *GetIntersectingRoutesResponse) Reset() { *x = GetIntersectingRoutesResponse{} - mi := &file_distribution_proto_msgTypes[16] + mi := &file_distribution_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1474,7 +1562,7 @@ func (x *GetIntersectingRoutesResponse) String() string { func (*GetIntersectingRoutesResponse) ProtoMessage() {} func (x *GetIntersectingRoutesResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[16] + mi := &file_distribution_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1487,7 +1575,7 @@ func (x *GetIntersectingRoutesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetIntersectingRoutesResponse.ProtoReflect.Descriptor instead. func (*GetIntersectingRoutesResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{16} + return file_distribution_proto_rawDescGZIP(), []int{18} } func (x *GetIntersectingRoutesResponse) GetRoutes() []*RouteDescriptor { @@ -1513,7 +1601,7 @@ type GetSplitJobRequest struct { func (x *GetSplitJobRequest) Reset() { *x = GetSplitJobRequest{} - mi := &file_distribution_proto_msgTypes[17] + mi := &file_distribution_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1525,7 +1613,7 @@ func (x *GetSplitJobRequest) String() string { func (*GetSplitJobRequest) ProtoMessage() {} func (x *GetSplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[17] + mi := &file_distribution_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1538,7 +1626,7 @@ func (x *GetSplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSplitJobRequest.ProtoReflect.Descriptor instead. func (*GetSplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{17} + return file_distribution_proto_rawDescGZIP(), []int{19} } func (x *GetSplitJobRequest) GetJobId() uint64 { @@ -1557,7 +1645,7 @@ type GetSplitJobResponse struct { func (x *GetSplitJobResponse) Reset() { *x = GetSplitJobResponse{} - mi := &file_distribution_proto_msgTypes[18] + mi := &file_distribution_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1569,7 +1657,7 @@ func (x *GetSplitJobResponse) String() string { func (*GetSplitJobResponse) ProtoMessage() {} func (x *GetSplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[18] + mi := &file_distribution_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1582,7 +1670,7 @@ func (x *GetSplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSplitJobResponse.ProtoReflect.Descriptor instead. func (*GetSplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{18} + return file_distribution_proto_rawDescGZIP(), []int{20} } func (x *GetSplitJobResponse) GetJob() *SplitJob { @@ -1603,7 +1691,7 @@ type ListSplitJobsRequest struct { func (x *ListSplitJobsRequest) Reset() { *x = ListSplitJobsRequest{} - mi := &file_distribution_proto_msgTypes[19] + mi := &file_distribution_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1615,7 +1703,7 @@ func (x *ListSplitJobsRequest) String() string { func (*ListSplitJobsRequest) ProtoMessage() {} func (x *ListSplitJobsRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[19] + mi := &file_distribution_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1628,7 +1716,7 @@ func (x *ListSplitJobsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSplitJobsRequest.ProtoReflect.Descriptor instead. func (*ListSplitJobsRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{19} + return file_distribution_proto_rawDescGZIP(), []int{21} } func (x *ListSplitJobsRequest) GetSinceTerminalAtMs() uint64 { @@ -1662,7 +1750,7 @@ type ListSplitJobsResponse struct { func (x *ListSplitJobsResponse) Reset() { *x = ListSplitJobsResponse{} - mi := &file_distribution_proto_msgTypes[20] + mi := &file_distribution_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1674,7 +1762,7 @@ func (x *ListSplitJobsResponse) String() string { func (*ListSplitJobsResponse) ProtoMessage() {} func (x *ListSplitJobsResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[20] + mi := &file_distribution_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1687,7 +1775,7 @@ func (x *ListSplitJobsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSplitJobsResponse.ProtoReflect.Descriptor instead. func (*ListSplitJobsResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{20} + return file_distribution_proto_rawDescGZIP(), []int{22} } func (x *ListSplitJobsResponse) GetJobs() []*SplitJob { @@ -1713,7 +1801,7 @@ type AbandonSplitJobRequest struct { func (x *AbandonSplitJobRequest) Reset() { *x = AbandonSplitJobRequest{} - mi := &file_distribution_proto_msgTypes[21] + mi := &file_distribution_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1725,7 +1813,7 @@ func (x *AbandonSplitJobRequest) String() string { func (*AbandonSplitJobRequest) ProtoMessage() {} func (x *AbandonSplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[21] + mi := &file_distribution_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1738,7 +1826,7 @@ func (x *AbandonSplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AbandonSplitJobRequest.ProtoReflect.Descriptor instead. func (*AbandonSplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{21} + return file_distribution_proto_rawDescGZIP(), []int{23} } func (x *AbandonSplitJobRequest) GetJobId() uint64 { @@ -1756,7 +1844,7 @@ type AbandonSplitJobResponse struct { func (x *AbandonSplitJobResponse) Reset() { *x = AbandonSplitJobResponse{} - mi := &file_distribution_proto_msgTypes[22] + mi := &file_distribution_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1768,7 +1856,7 @@ func (x *AbandonSplitJobResponse) String() string { func (*AbandonSplitJobResponse) ProtoMessage() {} func (x *AbandonSplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[22] + mi := &file_distribution_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1781,7 +1869,7 @@ func (x *AbandonSplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AbandonSplitJobResponse.ProtoReflect.Descriptor instead. func (*AbandonSplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{22} + return file_distribution_proto_rawDescGZIP(), []int{24} } type RetrySplitJobRequest struct { @@ -1793,7 +1881,7 @@ type RetrySplitJobRequest struct { func (x *RetrySplitJobRequest) Reset() { *x = RetrySplitJobRequest{} - mi := &file_distribution_proto_msgTypes[23] + mi := &file_distribution_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1805,7 +1893,7 @@ func (x *RetrySplitJobRequest) String() string { func (*RetrySplitJobRequest) ProtoMessage() {} func (x *RetrySplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[23] + mi := &file_distribution_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1818,7 +1906,7 @@ func (x *RetrySplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RetrySplitJobRequest.ProtoReflect.Descriptor instead. func (*RetrySplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{23} + return file_distribution_proto_rawDescGZIP(), []int{25} } func (x *RetrySplitJobRequest) GetJobId() uint64 { @@ -1836,7 +1924,7 @@ type RetrySplitJobResponse struct { func (x *RetrySplitJobResponse) Reset() { *x = RetrySplitJobResponse{} - mi := &file_distribution_proto_msgTypes[24] + mi := &file_distribution_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1848,7 +1936,7 @@ func (x *RetrySplitJobResponse) String() string { func (*RetrySplitJobResponse) ProtoMessage() {} func (x *RetrySplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[24] + mi := &file_distribution_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1861,7 +1949,7 @@ func (x *RetrySplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RetrySplitJobResponse.ProtoReflect.Descriptor instead. func (*RetrySplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{24} + return file_distribution_proto_rawDescGZIP(), []int{26} } var File_distribution_proto protoreflect.FileDescriptor @@ -1960,7 +2048,11 @@ const file_distribution_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"]\n" + "\x1bStartSplitMigrationResponse\x12'\n" + "\x0fcatalog_version\x18\x01 \x01(\x04R\x0ecatalogVersion\x12\x15\n" + - "\x06job_id\x18\x02 \x01(\x04R\x05jobId\"U\n" + + "\x06job_id\x18\x02 \x01(\x04R\x05jobId\"$\n" + + "\"GetSplitMigrationCapabilityRequest\"v\n" + + "#GetSplitMigrationCapabilityResponse\x12+\n" + + "\x11migration_capable\x18\x01 \x01(\bR\x10migrationCapable\x12\"\n" + + "\fcapabilities\x18\x02 \x03(\tR\fcapabilities\"U\n" + "\x18GetRouteOwnershipRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\fR\x03key\x12'\n" + "\x0fcatalog_version\x18\x02 \x01(\x04R\x0ecatalogVersion\"\x82\x01\n" + @@ -2021,7 +2113,7 @@ const file_distribution_proto_rawDesc = "" + "\x13SplitJobExportPhase\x12\x1f\n" + "\x1bSPLIT_JOB_EXPORT_PHASE_NONE\x10\x00\x12#\n" + "\x1fSPLIT_JOB_EXPORT_PHASE_BACKFILL\x10\x01\x12%\n" + - "!SPLIT_JOB_EXPORT_PHASE_DELTA_COPY\x10\x022\xf6\x05\n" + + "!SPLIT_JOB_EXPORT_PHASE_DELTA_COPY\x10\x022\xe2\x06\n" + "\fDistribution\x121\n" + "\bGetRoute\x12\x10.GetRouteRequest\x1a\x11.GetRouteResponse\"\x00\x12=\n" + "\fGetTimestamp\x12\x14.GetTimestampRequest\x1a\x15.GetTimestampResponse\"\x00\x127\n" + @@ -2029,7 +2121,8 @@ const file_distribution_proto_rawDesc = "" + "ListRoutes\x12\x12.ListRoutesRequest\x1a\x13.ListRoutesResponse\"\x00\x127\n" + "\n" + "SplitRange\x12\x12.SplitRangeRequest\x1a\x13.SplitRangeResponse\"\x00\x12R\n" + - "\x13StartSplitMigration\x12\x1b.StartSplitMigrationRequest\x1a\x1c.StartSplitMigrationResponse\"\x00\x12L\n" + + "\x13StartSplitMigration\x12\x1b.StartSplitMigrationRequest\x1a\x1c.StartSplitMigrationResponse\"\x00\x12j\n" + + "\x1bGetSplitMigrationCapability\x12#.GetSplitMigrationCapabilityRequest\x1a$.GetSplitMigrationCapabilityResponse\"\x00\x12L\n" + "\x11GetRouteOwnership\x12\x19.GetRouteOwnershipRequest\x1a\x1a.GetRouteOwnershipResponse\"\x00\x12X\n" + "\x15GetIntersectingRoutes\x12\x1d.GetIntersectingRoutesRequest\x1a\x1e.GetIntersectingRoutesResponse\"\x00\x12:\n" + "\vGetSplitJob\x12\x13.GetSplitJobRequest\x1a\x14.GetSplitJobResponse\"\x00\x12@\n" + @@ -2050,38 +2143,40 @@ func file_distribution_proto_rawDescGZIP() []byte { } var file_distribution_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_distribution_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_distribution_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_distribution_proto_goTypes = []any{ - (RouteState)(0), // 0: RouteState - (SplitJobPhase)(0), // 1: SplitJobPhase - (SplitJobBarrierState)(0), // 2: SplitJobBarrierState - (SplitJobExportPhase)(0), // 3: SplitJobExportPhase - (*GetRouteRequest)(nil), // 4: GetRouteRequest - (*GetRouteResponse)(nil), // 5: GetRouteResponse - (*GetTimestampRequest)(nil), // 6: GetTimestampRequest - (*GetTimestampResponse)(nil), // 7: GetTimestampResponse - (*RouteDescriptor)(nil), // 8: RouteDescriptor - (*SplitJobBracketProgress)(nil), // 9: SplitJobBracketProgress - (*SplitJob)(nil), // 10: SplitJob - (*ListRoutesRequest)(nil), // 11: ListRoutesRequest - (*ListRoutesResponse)(nil), // 12: ListRoutesResponse - (*SplitRangeRequest)(nil), // 13: SplitRangeRequest - (*SplitRangeResponse)(nil), // 14: SplitRangeResponse - (*StartSplitMigrationRequest)(nil), // 15: StartSplitMigrationRequest - (*StartSplitMigrationResponse)(nil), // 16: StartSplitMigrationResponse - (*GetRouteOwnershipRequest)(nil), // 17: GetRouteOwnershipRequest - (*GetRouteOwnershipResponse)(nil), // 18: GetRouteOwnershipResponse - (*GetIntersectingRoutesRequest)(nil), // 19: GetIntersectingRoutesRequest - (*GetIntersectingRoutesResponse)(nil), // 20: GetIntersectingRoutesResponse - (*GetSplitJobRequest)(nil), // 21: GetSplitJobRequest - (*GetSplitJobResponse)(nil), // 22: GetSplitJobResponse - (*ListSplitJobsRequest)(nil), // 23: ListSplitJobsRequest - (*ListSplitJobsResponse)(nil), // 24: ListSplitJobsResponse - (*AbandonSplitJobRequest)(nil), // 25: AbandonSplitJobRequest - (*AbandonSplitJobResponse)(nil), // 26: AbandonSplitJobResponse - (*RetrySplitJobRequest)(nil), // 27: RetrySplitJobRequest - (*RetrySplitJobResponse)(nil), // 28: RetrySplitJobResponse - nil, // 29: StartSplitMigrationRequest.OptionsEntry + (RouteState)(0), // 0: RouteState + (SplitJobPhase)(0), // 1: SplitJobPhase + (SplitJobBarrierState)(0), // 2: SplitJobBarrierState + (SplitJobExportPhase)(0), // 3: SplitJobExportPhase + (*GetRouteRequest)(nil), // 4: GetRouteRequest + (*GetRouteResponse)(nil), // 5: GetRouteResponse + (*GetTimestampRequest)(nil), // 6: GetTimestampRequest + (*GetTimestampResponse)(nil), // 7: GetTimestampResponse + (*RouteDescriptor)(nil), // 8: RouteDescriptor + (*SplitJobBracketProgress)(nil), // 9: SplitJobBracketProgress + (*SplitJob)(nil), // 10: SplitJob + (*ListRoutesRequest)(nil), // 11: ListRoutesRequest + (*ListRoutesResponse)(nil), // 12: ListRoutesResponse + (*SplitRangeRequest)(nil), // 13: SplitRangeRequest + (*SplitRangeResponse)(nil), // 14: SplitRangeResponse + (*StartSplitMigrationRequest)(nil), // 15: StartSplitMigrationRequest + (*StartSplitMigrationResponse)(nil), // 16: StartSplitMigrationResponse + (*GetSplitMigrationCapabilityRequest)(nil), // 17: GetSplitMigrationCapabilityRequest + (*GetSplitMigrationCapabilityResponse)(nil), // 18: GetSplitMigrationCapabilityResponse + (*GetRouteOwnershipRequest)(nil), // 19: GetRouteOwnershipRequest + (*GetRouteOwnershipResponse)(nil), // 20: GetRouteOwnershipResponse + (*GetIntersectingRoutesRequest)(nil), // 21: GetIntersectingRoutesRequest + (*GetIntersectingRoutesResponse)(nil), // 22: GetIntersectingRoutesResponse + (*GetSplitJobRequest)(nil), // 23: GetSplitJobRequest + (*GetSplitJobResponse)(nil), // 24: GetSplitJobResponse + (*ListSplitJobsRequest)(nil), // 25: ListSplitJobsRequest + (*ListSplitJobsResponse)(nil), // 26: ListSplitJobsResponse + (*AbandonSplitJobRequest)(nil), // 27: AbandonSplitJobRequest + (*AbandonSplitJobResponse)(nil), // 28: AbandonSplitJobResponse + (*RetrySplitJobRequest)(nil), // 29: RetrySplitJobRequest + (*RetrySplitJobResponse)(nil), // 30: RetrySplitJobResponse + nil, // 31: StartSplitMigrationRequest.OptionsEntry } var file_distribution_proto_depIdxs = []int32{ 0, // 0: RouteDescriptor.state:type_name -> RouteState @@ -2095,7 +2190,7 @@ var file_distribution_proto_depIdxs = []int32{ 8, // 8: ListRoutesResponse.routes:type_name -> RouteDescriptor 8, // 9: SplitRangeResponse.left:type_name -> RouteDescriptor 8, // 10: SplitRangeResponse.right:type_name -> RouteDescriptor - 29, // 11: StartSplitMigrationRequest.options:type_name -> StartSplitMigrationRequest.OptionsEntry + 31, // 11: StartSplitMigrationRequest.options:type_name -> StartSplitMigrationRequest.OptionsEntry 8, // 12: GetRouteOwnershipResponse.route:type_name -> RouteDescriptor 8, // 13: GetIntersectingRoutesResponse.routes:type_name -> RouteDescriptor 10, // 14: GetSplitJobResponse.job:type_name -> SplitJob @@ -2105,25 +2200,27 @@ var file_distribution_proto_depIdxs = []int32{ 11, // 18: Distribution.ListRoutes:input_type -> ListRoutesRequest 13, // 19: Distribution.SplitRange:input_type -> SplitRangeRequest 15, // 20: Distribution.StartSplitMigration:input_type -> StartSplitMigrationRequest - 17, // 21: Distribution.GetRouteOwnership:input_type -> GetRouteOwnershipRequest - 19, // 22: Distribution.GetIntersectingRoutes:input_type -> GetIntersectingRoutesRequest - 21, // 23: Distribution.GetSplitJob:input_type -> GetSplitJobRequest - 23, // 24: Distribution.ListSplitJobs:input_type -> ListSplitJobsRequest - 25, // 25: Distribution.AbandonSplitJob:input_type -> AbandonSplitJobRequest - 27, // 26: Distribution.RetrySplitJob:input_type -> RetrySplitJobRequest - 5, // 27: Distribution.GetRoute:output_type -> GetRouteResponse - 7, // 28: Distribution.GetTimestamp:output_type -> GetTimestampResponse - 12, // 29: Distribution.ListRoutes:output_type -> ListRoutesResponse - 14, // 30: Distribution.SplitRange:output_type -> SplitRangeResponse - 16, // 31: Distribution.StartSplitMigration:output_type -> StartSplitMigrationResponse - 18, // 32: Distribution.GetRouteOwnership:output_type -> GetRouteOwnershipResponse - 20, // 33: Distribution.GetIntersectingRoutes:output_type -> GetIntersectingRoutesResponse - 22, // 34: Distribution.GetSplitJob:output_type -> GetSplitJobResponse - 24, // 35: Distribution.ListSplitJobs:output_type -> ListSplitJobsResponse - 26, // 36: Distribution.AbandonSplitJob:output_type -> AbandonSplitJobResponse - 28, // 37: Distribution.RetrySplitJob:output_type -> RetrySplitJobResponse - 27, // [27:38] is the sub-list for method output_type - 16, // [16:27] is the sub-list for method input_type + 17, // 21: Distribution.GetSplitMigrationCapability:input_type -> GetSplitMigrationCapabilityRequest + 19, // 22: Distribution.GetRouteOwnership:input_type -> GetRouteOwnershipRequest + 21, // 23: Distribution.GetIntersectingRoutes:input_type -> GetIntersectingRoutesRequest + 23, // 24: Distribution.GetSplitJob:input_type -> GetSplitJobRequest + 25, // 25: Distribution.ListSplitJobs:input_type -> ListSplitJobsRequest + 27, // 26: Distribution.AbandonSplitJob:input_type -> AbandonSplitJobRequest + 29, // 27: Distribution.RetrySplitJob:input_type -> RetrySplitJobRequest + 5, // 28: Distribution.GetRoute:output_type -> GetRouteResponse + 7, // 29: Distribution.GetTimestamp:output_type -> GetTimestampResponse + 12, // 30: Distribution.ListRoutes:output_type -> ListRoutesResponse + 14, // 31: Distribution.SplitRange:output_type -> SplitRangeResponse + 16, // 32: Distribution.StartSplitMigration:output_type -> StartSplitMigrationResponse + 18, // 33: Distribution.GetSplitMigrationCapability:output_type -> GetSplitMigrationCapabilityResponse + 20, // 34: Distribution.GetRouteOwnership:output_type -> GetRouteOwnershipResponse + 22, // 35: Distribution.GetIntersectingRoutes:output_type -> GetIntersectingRoutesResponse + 24, // 36: Distribution.GetSplitJob:output_type -> GetSplitJobResponse + 26, // 37: Distribution.ListSplitJobs:output_type -> ListSplitJobsResponse + 28, // 38: Distribution.AbandonSplitJob:output_type -> AbandonSplitJobResponse + 30, // 39: Distribution.RetrySplitJob:output_type -> RetrySplitJobResponse + 28, // [28:40] is the sub-list for method output_type + 16, // [16:28] is the sub-list for method input_type 16, // [16:16] is the sub-list for extension type_name 16, // [16:16] is the sub-list for extension extendee 0, // [0:16] is the sub-list for field type_name @@ -2140,7 +2237,7 @@ func file_distribution_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_distribution_proto_rawDesc), len(file_distribution_proto_rawDesc)), NumEnums: 4, - NumMessages: 26, + NumMessages: 28, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/distribution.proto b/proto/distribution.proto index eb9746f18..4f65163b0 100644 --- a/proto/distribution.proto +++ b/proto/distribution.proto @@ -8,6 +8,7 @@ service Distribution { rpc ListRoutes (ListRoutesRequest) returns (ListRoutesResponse) {} rpc SplitRange (SplitRangeRequest) returns (SplitRangeResponse) {} rpc StartSplitMigration (StartSplitMigrationRequest) returns (StartSplitMigrationResponse) {} + rpc GetSplitMigrationCapability (GetSplitMigrationCapabilityRequest) returns (GetSplitMigrationCapabilityResponse) {} rpc GetRouteOwnership (GetRouteOwnershipRequest) returns (GetRouteOwnershipResponse) {} rpc GetIntersectingRoutes (GetIntersectingRoutesRequest) returns (GetIntersectingRoutesResponse) {} rpc GetSplitJob (GetSplitJobRequest) returns (GetSplitJobResponse) {} @@ -160,6 +161,13 @@ message StartSplitMigrationResponse { uint64 job_id = 2; } +message GetSplitMigrationCapabilityRequest {} + +message GetSplitMigrationCapabilityResponse { + bool migration_capable = 1; + repeated string capabilities = 2; +} + message GetRouteOwnershipRequest { bytes key = 1; uint64 catalog_version = 2; diff --git a/proto/distribution_grpc.pb.go b/proto/distribution_grpc.pb.go index 66ebf8511..ec02eabee 100644 --- a/proto/distribution_grpc.pb.go +++ b/proto/distribution_grpc.pb.go @@ -19,17 +19,18 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Distribution_GetRoute_FullMethodName = "/Distribution/GetRoute" - Distribution_GetTimestamp_FullMethodName = "/Distribution/GetTimestamp" - Distribution_ListRoutes_FullMethodName = "/Distribution/ListRoutes" - Distribution_SplitRange_FullMethodName = "/Distribution/SplitRange" - Distribution_StartSplitMigration_FullMethodName = "/Distribution/StartSplitMigration" - Distribution_GetRouteOwnership_FullMethodName = "/Distribution/GetRouteOwnership" - Distribution_GetIntersectingRoutes_FullMethodName = "/Distribution/GetIntersectingRoutes" - Distribution_GetSplitJob_FullMethodName = "/Distribution/GetSplitJob" - Distribution_ListSplitJobs_FullMethodName = "/Distribution/ListSplitJobs" - Distribution_AbandonSplitJob_FullMethodName = "/Distribution/AbandonSplitJob" - Distribution_RetrySplitJob_FullMethodName = "/Distribution/RetrySplitJob" + Distribution_GetRoute_FullMethodName = "/Distribution/GetRoute" + Distribution_GetTimestamp_FullMethodName = "/Distribution/GetTimestamp" + Distribution_ListRoutes_FullMethodName = "/Distribution/ListRoutes" + Distribution_SplitRange_FullMethodName = "/Distribution/SplitRange" + Distribution_StartSplitMigration_FullMethodName = "/Distribution/StartSplitMigration" + Distribution_GetSplitMigrationCapability_FullMethodName = "/Distribution/GetSplitMigrationCapability" + Distribution_GetRouteOwnership_FullMethodName = "/Distribution/GetRouteOwnership" + Distribution_GetIntersectingRoutes_FullMethodName = "/Distribution/GetIntersectingRoutes" + Distribution_GetSplitJob_FullMethodName = "/Distribution/GetSplitJob" + Distribution_ListSplitJobs_FullMethodName = "/Distribution/ListSplitJobs" + Distribution_AbandonSplitJob_FullMethodName = "/Distribution/AbandonSplitJob" + Distribution_RetrySplitJob_FullMethodName = "/Distribution/RetrySplitJob" ) // DistributionClient is the client API for Distribution service. @@ -41,6 +42,7 @@ type DistributionClient interface { ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc.CallOption) (*ListRoutesResponse, error) SplitRange(ctx context.Context, in *SplitRangeRequest, opts ...grpc.CallOption) (*SplitRangeResponse, error) StartSplitMigration(ctx context.Context, in *StartSplitMigrationRequest, opts ...grpc.CallOption) (*StartSplitMigrationResponse, error) + GetSplitMigrationCapability(ctx context.Context, in *GetSplitMigrationCapabilityRequest, opts ...grpc.CallOption) (*GetSplitMigrationCapabilityResponse, error) GetRouteOwnership(ctx context.Context, in *GetRouteOwnershipRequest, opts ...grpc.CallOption) (*GetRouteOwnershipResponse, error) GetIntersectingRoutes(ctx context.Context, in *GetIntersectingRoutesRequest, opts ...grpc.CallOption) (*GetIntersectingRoutesResponse, error) GetSplitJob(ctx context.Context, in *GetSplitJobRequest, opts ...grpc.CallOption) (*GetSplitJobResponse, error) @@ -107,6 +109,16 @@ func (c *distributionClient) StartSplitMigration(ctx context.Context, in *StartS return out, nil } +func (c *distributionClient) GetSplitMigrationCapability(ctx context.Context, in *GetSplitMigrationCapabilityRequest, opts ...grpc.CallOption) (*GetSplitMigrationCapabilityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetSplitMigrationCapabilityResponse) + err := c.cc.Invoke(ctx, Distribution_GetSplitMigrationCapability_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *distributionClient) GetRouteOwnership(ctx context.Context, in *GetRouteOwnershipRequest, opts ...grpc.CallOption) (*GetRouteOwnershipResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetRouteOwnershipResponse) @@ -176,6 +188,7 @@ type DistributionServer interface { ListRoutes(context.Context, *ListRoutesRequest) (*ListRoutesResponse, error) SplitRange(context.Context, *SplitRangeRequest) (*SplitRangeResponse, error) StartSplitMigration(context.Context, *StartSplitMigrationRequest) (*StartSplitMigrationResponse, error) + GetSplitMigrationCapability(context.Context, *GetSplitMigrationCapabilityRequest) (*GetSplitMigrationCapabilityResponse, error) GetRouteOwnership(context.Context, *GetRouteOwnershipRequest) (*GetRouteOwnershipResponse, error) GetIntersectingRoutes(context.Context, *GetIntersectingRoutesRequest) (*GetIntersectingRoutesResponse, error) GetSplitJob(context.Context, *GetSplitJobRequest) (*GetSplitJobResponse, error) @@ -207,6 +220,9 @@ func (UnimplementedDistributionServer) SplitRange(context.Context, *SplitRangeRe func (UnimplementedDistributionServer) StartSplitMigration(context.Context, *StartSplitMigrationRequest) (*StartSplitMigrationResponse, error) { return nil, status.Error(codes.Unimplemented, "method StartSplitMigration not implemented") } +func (UnimplementedDistributionServer) GetSplitMigrationCapability(context.Context, *GetSplitMigrationCapabilityRequest) (*GetSplitMigrationCapabilityResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetSplitMigrationCapability not implemented") +} func (UnimplementedDistributionServer) GetRouteOwnership(context.Context, *GetRouteOwnershipRequest) (*GetRouteOwnershipResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetRouteOwnership not implemented") } @@ -336,6 +352,24 @@ func _Distribution_StartSplitMigration_Handler(srv interface{}, ctx context.Cont return interceptor(ctx, in, info, handler) } +func _Distribution_GetSplitMigrationCapability_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetSplitMigrationCapabilityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DistributionServer).GetSplitMigrationCapability(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Distribution_GetSplitMigrationCapability_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DistributionServer).GetSplitMigrationCapability(ctx, req.(*GetSplitMigrationCapabilityRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Distribution_GetRouteOwnership_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetRouteOwnershipRequest) if err := dec(in); err != nil { @@ -471,6 +505,10 @@ var Distribution_ServiceDesc = grpc.ServiceDesc{ MethodName: "StartSplitMigration", Handler: _Distribution_StartSplitMigration_Handler, }, + { + MethodName: "GetSplitMigrationCapability", + Handler: _Distribution_GetSplitMigrationCapability_Handler, + }, { MethodName: "GetRouteOwnership", Handler: _Distribution_GetRouteOwnership_Handler, From c459255d8159bad7cea625469563ea23a02fe3e6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 11:23:31 +0900 Subject: [PATCH 053/162] Fix migration route edge cases --- distribution/migrator.go | 2 +- distribution/migrator_export_plan_test.go | 23 +++++++ kv/migrator_filter.go | 31 ++++++++- kv/shard_key_test.go | 48 ++++++++++++++ store/hash_helpers.go | 31 +-------- store/list_helpers.go | 35 +++++----- store/set_helpers.go | 31 +-------- store/wide_column_helpers_test.go | 79 +++++++++++++++++++++++ store/zset_helpers.go | 41 ++---------- 9 files changed, 208 insertions(+), 113 deletions(-) create mode 100644 store/wide_column_helpers_test.go diff --git a/distribution/migrator.go b/distribution/migrator.go index 561f0a5a6..3b33a5318 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -278,7 +278,7 @@ func (b MigrationBracket) decodedS3Bucket(rawKey []byte) (string, bool) { } func routeKeyInRange(routeKey, routeStart, routeEnd []byte) bool { - if len(routeKey) == 0 { + if routeKey == nil { return false } if bytes.Compare(routeKey, routeStart) < 0 { diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index 630c2dbc2..ffe5d80c6 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -225,6 +225,29 @@ func TestMigrationBracketContainsRoutedKeyUsesObjectRoutes(t *testing.T) { )) } +func TestMigrationBracketContainsRoutedKeyAcceptsEmptyLogicalRouteKey(t *testing.T) { + t.Parallel() + + routeEnd := []byte{0x01} + brackets, err := PlanMigrationBrackets(nil, routeEnd) + require.NoError(t, err) + hash := bracketsByFamily(brackets)[MigrationFamilyHash] + rawKey := store.HashMetaKey(nil) + + require.True(t, hash.ContainsRoutedKey( + rawKey, + nil, + routeEnd, + store.ExtractHashUserKeyFromMeta, + )) + require.False(t, hash.ContainsRoutedKey( + rawKey, + nil, + routeEnd, + func([]byte) []byte { return nil }, + )) +} + func TestMigrationKnownInternalPrefixesAreConcreteOnly(t *testing.T) { t.Parallel() diff --git a/kv/migrator_filter.go b/kv/migrator_filter.go index d1b4ddc19..19cbb23da 100644 --- a/kv/migrator_filter.go +++ b/kv/migrator_filter.go @@ -1,6 +1,10 @@ package kv -import "bytes" +import ( + "bytes" + + "github.com/bootjp/elastickv/internal/s3keys" +) // RouteKeyFilter returns the migration export predicate for raw MVCC keys. // rangeEnd nil or empty means +infinity, matching the route descriptor wire @@ -9,6 +13,9 @@ func RouteKeyFilter(rangeStart, rangeEnd []byte) func([]byte) bool { start := bytes.Clone(rangeStart) end := bytes.Clone(rangeEnd) return func(rawKey []byte) bool { + if s3BucketAuxiliaryRouteInRange(rawKey, start, end) { + return true + } rkey := routeKey(rawKey) if bytes.Compare(rkey, start) < 0 { return false @@ -19,3 +26,25 @@ func RouteKeyFilter(rangeStart, rangeEnd []byte) func([]byte) bool { return true } } + +func s3BucketAuxiliaryRouteInRange(rawKey, routeStart, routeEnd []byte) bool { + bucket, ok := s3keys.ParseBucketMetaKey(rawKey) + if !ok { + bucket, ok = s3keys.ParseBucketGenerationKey(rawKey) + } + if !ok { + return false + } + bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + return migrationRouteRangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) +} + +func migrationRouteRangesIntersect(aStart, aEnd, bStart, bEnd []byte) bool { + if len(aEnd) > 0 && bytes.Compare(aEnd, bStart) <= 0 { + return false + } + if len(bEnd) > 0 && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true +} diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index e7d6c55d7..373ca144c 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -2,6 +2,7 @@ package kv import ( "encoding/base64" + "encoding/binary" "testing" "github.com/bootjp/elastickv/internal/s3keys" @@ -123,6 +124,38 @@ func TestRouteKey_NormalizesCollectionMigrationFamilies(t *testing.T) { } } +func TestRouteKey_MalformedWideColumnKeysFallBackToRaw(t *testing.T) { + t.Parallel() + + for _, raw := range [][]byte{ + malformedWideColumnKey(store.ListClaimPrefix, 8), + malformedWideColumnKey(store.HashMetaPrefix, 0), + malformedWideColumnKey(store.HashFieldPrefix, 0), + malformedWideColumnKey(store.HashMetaDeltaPrefix, 12), + malformedWideColumnKey(store.SetMetaPrefix, 0), + malformedWideColumnKey(store.SetMemberPrefix, 0), + malformedWideColumnKey(store.SetMetaDeltaPrefix, 12), + malformedWideColumnKey(store.ZSetMetaPrefix, 0), + malformedWideColumnKey(store.ZSetMemberPrefix, 0), + malformedWideColumnKey(store.ZSetScorePrefix, 8), + malformedWideColumnKey(store.ZSetMetaDeltaPrefix, 12), + } { + require.NotPanics(t, func() { + require.Equal(t, raw, routeKey(raw), "malformed key %q must not decode to a logical route", raw) + }) + } +} + +func malformedWideColumnKey(prefix string, suffixLen int) []byte { + key := make([]byte, 0, len(prefix)+4+suffixLen) + key = append(key, prefix...) + var lenPrefix [4]byte + binary.BigEndian.PutUint32(lenPrefix[:], ^uint32(0)) + key = append(key, lenPrefix[:]...) + key = append(key, make([]byte, suffixLen)...) + return key +} + func TestRouteKey_NormalizesTxnSuccessMarkerByLockedKey(t *testing.T) { t.Parallel() @@ -194,3 +227,18 @@ func TestRouteKeyFilterTreatsNilAndEmptyEndAsInfinity(t *testing.T) { }) } } + +func TestRouteKeyFilterIncludesS3BucketAuxiliaryKeys(t *testing.T) { + t.Parallel() + + filter := RouteKeyFilter( + s3keys.RouteKey("bucket-b", 7, "a"), + s3keys.RouteKey("bucket-b", 7, "z"), + ) + + require.True(t, filter(s3keys.BucketMetaKey("bucket-b"))) + require.True(t, filter(s3keys.BucketGenerationKey("bucket-b"))) + require.True(t, filter(s3keys.ObjectManifestKey("bucket-b", 7, "m"))) + require.False(t, filter(s3keys.BucketMetaKey("bucket-c"))) + require.False(t, filter(s3keys.BucketGenerationKey("bucket-c"))) +} diff --git a/store/hash_helpers.go b/store/hash_helpers.go index 76d5f53ec..56c3268d5 100644 --- a/store/hash_helpers.go +++ b/store/hash_helpers.go @@ -150,40 +150,15 @@ func IsHashMetaDeltaKey(key []byte) bool { // ExtractHashUserKeyFromMeta extracts the logical user key from a hash meta key. func ExtractHashUserKeyFromMeta(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(HashMetaPrefix)) - if len(trimmed) < wideColKeyLenSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen { //nolint:gosec // wideColKeyLenSize fits in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(HashMetaPrefix), 0, true) } // ExtractHashUserKeyFromField extracts the logical user key from a hash field key. func ExtractHashUserKeyFromField(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(HashFieldPrefix)) - if len(trimmed) < wideColKeyLenSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen { //nolint:gosec // wideColKeyLenSize fits in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(HashFieldPrefix), 0, false) } // ExtractHashUserKeyFromDelta extracts the logical user key from a hash delta key. func ExtractHashUserKeyFromDelta(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(HashMetaDeltaPrefix)) - minLen := wideColKeyLenSize + deltaKeyTSSize + deltaKeySeqSize - if len(trimmed) < minLen { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen+uint32(deltaKeyTSSize+deltaKeySeqSize) { //nolint:gosec // constants fit in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(HashMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) } diff --git a/store/list_helpers.go b/store/list_helpers.go index 252d681cf..e01458fc7 100644 --- a/store/list_helpers.go +++ b/store/list_helpers.go @@ -131,33 +131,32 @@ func IsListClaimKey(key []byte) bool { // ExtractListUserKeyFromDelta extracts the logical user key from a list delta key. func ExtractListUserKeyFromDelta(key []byte) []byte { - if !bytes.HasPrefix(key, []byte(ListMetaDeltaPrefix)) { - return nil - } - trimmed := key[len(ListMetaDeltaPrefix):] - if len(trimmed) < wideColKeyLenSize+deltaKeyTSSize+deltaKeySeqSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - wantLen := uint64(wideColKeyLenSize) + uint64(ukLen) + uint64(deltaKeyTSSize+deltaKeySeqSize) - if uint64(len(trimmed)) != wantLen { - return nil - } - userEnd := wideColKeyLenSize + int(ukLen) //nolint:gosec // exact length check above bounds ukLen to len(trimmed) - return trimmed[wideColKeyLenSize:userEnd] + return extractWideColumnUserKey(key, []byte(ListMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) } // ExtractListUserKeyFromClaim extracts the logical user key from a list claim key. func ExtractListUserKeyFromClaim(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(ListClaimPrefix)) - if len(trimmed) < wideColKeyLenSize+sortableInt64Bytes { + return extractWideColumnUserKey(key, []byte(ListClaimPrefix), sortableInt64Bytes, true) +} + +func extractWideColumnUserKey(key, prefix []byte, suffixLen uint64, exactLen bool) []byte { + if !bytes.HasPrefix(key, prefix) { + return nil + } + trimmed := key[len(prefix):] + if uint64(len(trimmed)) < uint64(wideColKeyLenSize)+suffixLen { return nil } ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen+uint32(sortableInt64Bytes) { //nolint:gosec // constants fit in uint32 + userEnd := uint64(wideColKeyLenSize) + uint64(ukLen) + minLen := userEnd + suffixLen + if minLen > uint64(len(trimmed)) { + return nil + } + if exactLen && minLen != uint64(len(trimmed)) { return nil } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return trimmed[wideColKeyLenSize:int(userEnd)] //nolint:gosec // userEnd is bounded by len(trimmed) above. } // PrefixScanEnd returns the exclusive end key for a prefix scan. diff --git a/store/set_helpers.go b/store/set_helpers.go index 44da629b4..ff8822f52 100644 --- a/store/set_helpers.go +++ b/store/set_helpers.go @@ -150,40 +150,15 @@ func IsSetMetaDeltaKey(key []byte) bool { // ExtractSetUserKeyFromMeta extracts the logical user key from a set meta key. func ExtractSetUserKeyFromMeta(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(SetMetaPrefix)) - if len(trimmed) < wideColKeyLenSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen { //nolint:gosec // wideColKeyLenSize fits in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(SetMetaPrefix), 0, true) } // ExtractSetUserKeyFromMember extracts the logical user key from a set member key. func ExtractSetUserKeyFromMember(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(SetMemberPrefix)) - if len(trimmed) < wideColKeyLenSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen { //nolint:gosec // wideColKeyLenSize fits in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(SetMemberPrefix), 0, false) } // ExtractSetUserKeyFromDelta extracts the logical user key from a set delta key. func ExtractSetUserKeyFromDelta(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(SetMetaDeltaPrefix)) - minLen := wideColKeyLenSize + deltaKeyTSSize + deltaKeySeqSize - if len(trimmed) < minLen { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen+uint32(deltaKeyTSSize+deltaKeySeqSize) { //nolint:gosec // constants fit in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(SetMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) } diff --git a/store/wide_column_helpers_test.go b/store/wide_column_helpers_test.go new file mode 100644 index 000000000..f49822573 --- /dev/null +++ b/store/wide_column_helpers_test.go @@ -0,0 +1,79 @@ +package store + +import ( + "encoding/binary" + "testing" +) + +func TestWideColumnExtractorsRejectOverflowingUserKeyLength(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + prefix string + suffixLen int + extract func([]byte) []byte + }{ + {name: "list claim", prefix: ListClaimPrefix, suffixLen: sortableInt64Bytes, extract: ExtractListUserKeyFromClaim}, + {name: "hash meta", prefix: HashMetaPrefix, extract: ExtractHashUserKeyFromMeta}, + {name: "hash field", prefix: HashFieldPrefix, extract: ExtractHashUserKeyFromField}, + {name: "hash delta", prefix: HashMetaDeltaPrefix, suffixLen: deltaKeyTSSize + deltaKeySeqSize, extract: ExtractHashUserKeyFromDelta}, + {name: "set meta", prefix: SetMetaPrefix, extract: ExtractSetUserKeyFromMeta}, + {name: "set member", prefix: SetMemberPrefix, extract: ExtractSetUserKeyFromMember}, + {name: "set delta", prefix: SetMetaDeltaPrefix, suffixLen: deltaKeyTSSize + deltaKeySeqSize, extract: ExtractSetUserKeyFromDelta}, + {name: "zset meta", prefix: ZSetMetaPrefix, extract: ExtractZSetUserKeyFromMeta}, + {name: "zset member", prefix: ZSetMemberPrefix, extract: ExtractZSetUserKeyFromMember}, + {name: "zset score", prefix: ZSetScorePrefix, suffixLen: zsetMetaSizeBytes, extract: ExtractZSetUserKeyFromScore}, + {name: "zset delta", prefix: ZSetMetaDeltaPrefix, suffixLen: deltaKeyTSSize + deltaKeySeqSize, extract: ExtractZSetUserKeyFromDelta}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := tc.extract(malformedWideColumnStorageKey(tc.prefix, tc.suffixLen)); got != nil { + t.Fatalf("overflowing user-key length: want nil, got %q", got) + } + }) + } +} + +func TestWideColumnExtractorsRoundTripEmptyUserKey(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + key []byte + extract func([]byte) []byte + }{ + {name: "list claim", key: ListClaimKey(nil, 1), extract: ExtractListUserKeyFromClaim}, + {name: "hash meta", key: HashMetaKey(nil), extract: ExtractHashUserKeyFromMeta}, + {name: "hash field", key: HashFieldKey(nil, []byte("field")), extract: ExtractHashUserKeyFromField}, + {name: "hash delta", key: HashMetaDeltaKey(nil, 2, 3), extract: ExtractHashUserKeyFromDelta}, + {name: "set meta", key: SetMetaKey(nil), extract: ExtractSetUserKeyFromMeta}, + {name: "set member", key: SetMemberKey(nil, []byte("member")), extract: ExtractSetUserKeyFromMember}, + {name: "set delta", key: SetMetaDeltaKey(nil, 2, 3), extract: ExtractSetUserKeyFromDelta}, + {name: "zset meta", key: ZSetMetaKey(nil), extract: ExtractZSetUserKeyFromMeta}, + {name: "zset member", key: ZSetMemberKey(nil, []byte("member")), extract: ExtractZSetUserKeyFromMember}, + {name: "zset score", key: ZSetScoreKey(nil, 1.5, []byte("member")), extract: ExtractZSetUserKeyFromScore}, + {name: "zset delta", key: ZSetMetaDeltaKey(nil, 2, 3), extract: ExtractZSetUserKeyFromDelta}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := tc.extract(tc.key) + if got == nil { + t.Fatal("empty user-key extraction returned nil") + } + if len(got) != 0 { + t.Fatalf("empty user-key extraction: want empty, got %q", got) + } + }) + } +} + +func malformedWideColumnStorageKey(prefix string, suffixLen int) []byte { + key := make([]byte, 0, len(prefix)+wideColKeyLenSize+suffixLen) + key = append(key, prefix...) + var lenPrefix [wideColKeyLenSize]byte + binary.BigEndian.PutUint32(lenPrefix[:], ^uint32(0)) + key = append(key, lenPrefix[:]...) + key = append(key, make([]byte, suffixLen)...) + return key +} diff --git a/store/zset_helpers.go b/store/zset_helpers.go index 0f4b42ea7..8bed4b8a0 100644 --- a/store/zset_helpers.go +++ b/store/zset_helpers.go @@ -264,53 +264,20 @@ func IsZSetMetaDeltaKey(key []byte) bool { // ExtractZSetUserKeyFromDelta extracts the logical user key from a zset delta key. func ExtractZSetUserKeyFromDelta(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(ZSetMetaDeltaPrefix)) - minLen := wideColKeyLenSize + deltaKeyTSSize + deltaKeySeqSize - if len(trimmed) < minLen { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen+uint32(deltaKeyTSSize+deltaKeySeqSize) { //nolint:gosec // constants fit in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(ZSetMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) } // ExtractZSetUserKeyFromMeta extracts the logical user key from a zset meta key. func ExtractZSetUserKeyFromMeta(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(ZSetMetaPrefix)) - if len(trimmed) < wideColKeyLenSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen { //nolint:gosec // wideColKeyLenSize fits in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(ZSetMetaPrefix), 0, true) } // ExtractZSetUserKeyFromMember extracts the logical user key from a zset member key. func ExtractZSetUserKeyFromMember(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(ZSetMemberPrefix)) - if len(trimmed) < wideColKeyLenSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen { //nolint:gosec // wideColKeyLenSize fits in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(ZSetMemberPrefix), 0, false) } // ExtractZSetUserKeyFromScore extracts the logical user key from a zset score index key. func ExtractZSetUserKeyFromScore(key []byte) []byte { - trimmed := bytes.TrimPrefix(key, []byte(ZSetScorePrefix)) - if len(trimmed) < wideColKeyLenSize { - return nil - } - ukLen := binary.BigEndian.Uint32(trimmed[:wideColKeyLenSize]) - if uint32(len(trimmed)) < uint32(wideColKeyLenSize)+ukLen { //nolint:gosec // wideColKeyLenSize fits in uint32 - return nil - } - return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] + return extractWideColumnUserKey(key, []byte(ZSetScorePrefix), zsetMetaSizeBytes, false) } From c64ff5fef248e4c6d94f69b3c1a00965dfac7001 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 11:38:13 +0900 Subject: [PATCH 054/162] migration: harden split capability gate --- adapter/distribution_server.go | 29 +++++++++-- adapter/distribution_server_test.go | 75 ++++++++++++++++++++++++++++- main.go | 36 ++++++++++++-- main_catalog_test.go | 56 ++++++++++++++------- 4 files changed, 169 insertions(+), 27 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 42360803a..40e76ba03 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -28,6 +28,7 @@ type DistributionServer struct { coordinator kv.Coordinator readTracker *kv.ActiveTimestampTracker migrationCapabilityGate SplitMigrationCapabilityGate + splitJobRunnerReady bool knownRaftGroups map[uint64]struct{} reloadRetry struct { attempts int @@ -63,6 +64,14 @@ func WithSplitMigrationCapabilityGate(gate SplitMigrationCapabilityGate) Distrib } } +// WithSplitJobRunnerReady opens the split migration capability probe once this +// node can advance planned split jobs. +func WithSplitJobRunnerReady() DistributionServerOption { + return func(s *DistributionServer) { + s.splitJobRunnerReady = true + } +} + // WithDistributionKnownRaftGroups configures the Raft group IDs this node can // route migration work to. func WithDistributionKnownRaftGroups(groupIDs ...uint64) DistributionServerOption { @@ -215,10 +224,14 @@ func (s *DistributionServer) GetIntersectingRoutes(ctx context.Context, req *pb. } func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.StartSplitMigrationRequest) (*pb.StartSplitMigrationResponse, error) { + if err := s.verifyStartSplitMigrationPreflight(ctx, req); err != nil { + return nil, err + } + s.mu.Lock() defer s.mu.Unlock() - if err := s.verifyStartSplitMigrationReady(ctx, req); err != nil { + if err := s.verifyStartSplitMigrationCatalogReady(ctx); err != nil { return nil, err } snapshot, err := s.loadCatalogSnapshot(ctx) @@ -246,23 +259,33 @@ func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.St } func (s *DistributionServer) GetSplitMigrationCapability(context.Context, *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + if !s.splitJobRunnerReady { + return &pb.GetSplitMigrationCapabilityResponse{}, nil + } return &pb.GetSplitMigrationCapabilityResponse{ MigrationCapable: true, Capabilities: []string{splitMigrationCapabilityV2}, }, nil } -func (s *DistributionServer) verifyStartSplitMigrationReady(ctx context.Context, req *pb.StartSplitMigrationRequest) error { +func (s *DistributionServer) verifyStartSplitMigrationPreflight(ctx context.Context, req *pb.StartSplitMigrationRequest) error { if req.GetTargetGroupId() == 0 { return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) } + if err := s.verifyStartSplitMigrationCatalogReady(ctx); err != nil { + return err + } + return s.verifySplitMigrationCapability(ctx) +} + +func (s *DistributionServer) verifyStartSplitMigrationCatalogReady(ctx context.Context) error { if err := s.verifyCatalogLeader(ctx); err != nil { return err } if s.catalog == nil { return grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) } - return s.verifySplitMigrationCapability(ctx) + return nil } func (s *DistributionServer) startSplitMigrationParent( diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 21a34d96b..5190498e6 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -149,12 +149,22 @@ func TestDistributionServerListRoutes_RequiresCatalog(t *testing.T) { require.ErrorContains(t, err, errDistributionCatalogNotConfigured.Error()) } -func TestDistributionServerGetSplitMigrationCapabilityReportsReady(t *testing.T) { +func TestDistributionServerGetSplitMigrationCapabilityReportsNotReadyUntilRunnerReady(t *testing.T) { t.Parallel() s := NewDistributionServer(distribution.NewEngine(), nil) resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) require.NoError(t, err) + require.False(t, resp.GetMigrationCapable()) + require.NotContains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + +func TestDistributionServerGetSplitMigrationCapabilityReportsReadyWhenRunnerReady(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil, WithSplitJobRunnerReady()) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) require.True(t, resp.GetMigrationCapable()) require.Contains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) } @@ -212,6 +222,69 @@ func TestDistributionServerStartSplitMigrationReturnsCapabilityGateError(t *test require.Zero(t, coordinator.dispatchCalls) } +func TestDistributionServerStartSplitMigrationCapabilityGateRunsOutsideCatalogLock(t *testing.T) { + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + gateEntered := make(chan struct{}) + releaseGate := make(chan struct{}) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { + close(gateEntered) + <-releaseGate + return status.Error(codes.Unavailable, "split migration capability not ready") + }), + ) + + startDone := make(chan error, 1) + go func() { + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + startDone <- err + }() + <-gateEntered + + splitDone := make(chan error, 1) + go func() { + _, err := s.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + }) + splitDone <- err + }() + select { + case err := <-splitDone: + require.NoError(t, err) + case <-time.After(500 * time.Millisecond): + t.Fatal("SplitRange blocked behind StartSplitMigration capability gate") + } + + close(releaseGate) + err = <-startDone + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) +} + func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 93fed90ce..628ced4e9 100644 --- a/main.go +++ b/main.go @@ -500,8 +500,9 @@ func run() error { startKeyVizFlusher(runCtx, eg, sampler) startKeyVizLeaderTermPublisher(runCtx, eg, sampler, runtimes) startMemoryWatchdog(runCtx, eg, cancel) + defaultRuntime := findDefaultGroupRuntime(runtimes, cfg.defaultGroup) splitMigrationGate := newSplitMigrationCapabilityGate( - splitMigrationCapabilityPeers(bootstrapCfg, cfg.defaultGroup), + splitMigrationCapabilityPeerSourceForRuntime(defaultRuntime), splitMigrationCapabilityProbeTimeout, nil, ) @@ -529,7 +530,6 @@ func run() error { // group), in which case raftadmin.Server skips the pre-step. encryptionConfChangeInterceptor := newEncryptionPreRegister( coordinate, shardGroups[cfg.defaultGroup], encWiring.cache, *encryptionSidecarPath, etcdraftengine.DeriveNodeID) - defaultRuntime := findDefaultGroupRuntime(runtimes, cfg.defaultGroup) rotateOnStartupDeregister, waitRotateOnStartup := installEncryptionRotateOnStartup( runCtx, *encryptionRotateOnStartup, @@ -678,16 +678,35 @@ type splitMigrationCapabilityPeer struct { Address string } +type splitMigrationCapabilityPeerSource func(context.Context) ([]splitMigrationCapabilityPeer, error) + type splitMigrationCapabilityProbe func(context.Context, string) error -func splitMigrationCapabilityPeers(cfg raftBootstrapConfig, defaultGroup uint64) []splitMigrationCapabilityPeer { - servers := cfg.adminSeed(defaultGroup) +func splitMigrationCapabilityPeerSourceForRuntime(rt *raftGroupRuntime) splitMigrationCapabilityPeerSource { + return func(ctx context.Context) ([]splitMigrationCapabilityPeer, error) { + engine := rt.snapshotEngine() + if engine == nil { + return nil, errors.New("default raft group engine is not configured") + } + cfg, err := engine.Configuration(ctx) + if err != nil { + return nil, errors.Wrap(err, "default raft group configuration") + } + return splitMigrationCapabilityPeersFromConfiguration(cfg), nil + } +} + +func splitMigrationCapabilityPeersFromConfiguration(cfg raftengine.Configuration) []splitMigrationCapabilityPeer { + servers := cfg.Servers if len(servers) == 0 { return nil } peers := make([]splitMigrationCapabilityPeer, 0, len(servers)) seen := make(map[string]struct{}, len(servers)) for _, server := range servers { + if server.Suffrage == etcdraftengine.SuffrageLearner { + continue + } key := server.ID + "\x00" + server.Address if _, ok := seen[key]; ok { continue @@ -705,11 +724,18 @@ func splitMigrationCapabilityPeers(cfg raftBootstrapConfig, defaultGroup uint64) return peers } -func newSplitMigrationCapabilityGate(peers []splitMigrationCapabilityPeer, timeout time.Duration, probe splitMigrationCapabilityProbe) adapter.SplitMigrationCapabilityGate { +func newSplitMigrationCapabilityGate(source splitMigrationCapabilityPeerSource, timeout time.Duration, probe splitMigrationCapabilityProbe) adapter.SplitMigrationCapabilityGate { if probe == nil { probe = probeSplitMigrationCapabilityPeer } return func(ctx context.Context) error { + if source == nil { + return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") + } + peers, err := source(ctx) + if err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration capability peers are not available: %v", err) + } if len(peers) == 0 { return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") } diff --git a/main_catalog_test.go b/main_catalog_test.go index d2e63118b..968a0fefb 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "testing" "time" @@ -81,7 +82,7 @@ func TestSplitMigrationCapabilityGateChecksAllPeers(t *testing.T) { {ID: "n2", Address: "10.0.0.12:50051"}, } var probed []string - gate := newSplitMigrationCapabilityGate(peers, time.Second, func(_ context.Context, address string) error { + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(_ context.Context, address string) error { probed = append(probed, address) return nil }) @@ -111,7 +112,7 @@ func TestSplitMigrationCapabilityGateFailsClosedWhenPeerMissingCapability(t *tes {ID: "n1", Address: "10.0.0.11:50051"}, {ID: "n2", Address: "10.0.0.12:50051"}, } - gate := newSplitMigrationCapabilityGate(peers, time.Second, func(_ context.Context, address string) error { + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(_ context.Context, address string) error { if address == "10.0.0.12:50051" { return status.Error(codes.Unimplemented, "method not found") } @@ -125,24 +126,43 @@ func TestSplitMigrationCapabilityGateFailsClosedWhenPeerMissingCapability(t *tes require.ErrorContains(t, err, "method not found") } -func TestSplitMigrationCapabilityPeersUseDefaultGroupAdminSeed(t *testing.T) { +func TestSplitMigrationCapabilityGateFailsClosedWhenPeerSourceErrors(t *testing.T) { t.Parallel() - cfg := raftBootstrapConfig{ - groupServers: map[uint64][]raftengine.Server{ - 1: { - {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, - {Suffrage: "voter", ID: "n2", Address: "10.0.0.12:50051"}, - }, - 2: { - {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50052"}, - {Suffrage: "voter", ID: "n2", Address: "10.0.0.12:50052"}, - }, - }, - } + errPeerSourceUnavailable := errors.New("configuration unavailable") + gate := newSplitMigrationCapabilityGate(func(context.Context) ([]splitMigrationCapabilityPeer, error) { + return nil, errPeerSourceUnavailable + }, time.Second, func(context.Context, string) error { + t.Fatal("probe must not run when peer source fails") + return nil + }) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "peers are not available") +} + +func TestSplitMigrationCapabilityPeersFromConfigurationUsesCurrentVoters(t *testing.T) { + t.Parallel() + + cfg := raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "learner", ID: "n2", Address: "10.0.0.12:50051"}, + {Suffrage: "", ID: "n3", Address: "10.0.0.13:50051"}, + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + }} require.Equal(t, []splitMigrationCapabilityPeer{ - {ID: "n1", Address: "10.0.0.11:50052"}, - {ID: "n2", Address: "10.0.0.12:50052"}, - }, splitMigrationCapabilityPeers(cfg, 2)) + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n3", Address: "10.0.0.13:50051"}, + }, splitMigrationCapabilityPeersFromConfiguration(cfg)) +} + +func staticSplitMigrationCapabilityPeerSource(peers []splitMigrationCapabilityPeer) splitMigrationCapabilityPeerSource { + return func(context.Context) ([]splitMigrationCapabilityPeer, error) { + out := make([]splitMigrationCapabilityPeer, len(peers)) + copy(out, peers) + return out, nil + } } From 90fa6544b08d45dc6c963ddb584caf104047ae3c Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 11:45:49 +0900 Subject: [PATCH 055/162] store: honor raft promotion sync mode --- store/lsm_store_sync_mode_test.go | 60 +++++++++++++++++++++++++++++++ store/migration_promote.go | 51 +++++++++++++++++++++----- 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/store/lsm_store_sync_mode_test.go b/store/lsm_store_sync_mode_test.go index e5e065bfd..29349b997 100644 --- a/store/lsm_store_sync_mode_test.go +++ b/store/lsm_store_sync_mode_test.go @@ -1,6 +1,7 @@ package store import ( + "bytes" "context" "testing" @@ -116,6 +117,65 @@ func TestDirectApplyWriteOpts_AlwaysSync(t *testing.T) { }) } +func TestPromoteVersionsWriteOptsFollowApplyContext(t *testing.T) { + t.Run("raft-applied promotion observes nosync", func(t *testing.T) { + ps := newPebbleStoreWithFSMApplyWriteOptsForTest(t, t.TempDir(), pebble.NoSync, fsmSyncModeNoSync) + defer ps.Close() + + require.Same(t, pebble.NoSync, ps.promotionWriteOpts(123), + "promotion with an applied index must use raft apply write options") + }) + + t.Run("direct promotion stays sync", func(t *testing.T) { + ps := newPebbleStoreWithFSMApplyWriteOptsForTest(t, t.TempDir(), pebble.NoSync, fsmSyncModeNoSync) + defer ps.Close() + + require.Same(t, pebble.Sync, ps.promotionWriteOpts(0), + "promotion without an applied index has no raft durability backstop") + }) +} + +func TestPromoteVersionsRaftNoSyncFunctionalEquivalence(t *testing.T) { + dir := t.TempDir() + ps := newPebbleStoreWithFSMApplyWriteOptsForTest(t, dir, pebble.NoSync, fsmSyncModeNoSync) + defer ps.Close() + + ctx := context.Background() + stage := func(raw string) []byte { + return append([]byte("stage|"), []byte(raw)...) + } + targetKey := func(staged []byte) ([]byte, bool) { + return bytes.TrimPrefix(staged, []byte("stage|")), bytes.HasPrefix(staged, []byte("stage|")) + } + prefix := []byte("stage|") + + require.NoError(t, ps.PutAt(ctx, stage("k"), []byte("v10"), 10, 0)) + + const entryIdx uint64 = 88 + result, err := ps.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 12, + AppliedIndex: entryIdx, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, result.Done) + require.Equal(t, uint64(1), result.PromotedRows) + + val, err := ps.GetAt(ctx, []byte("k"), 10) + require.NoError(t, err) + require.Equal(t, []byte("v10"), val) + _, err = ps.GetAt(ctx, stage("k"), 10) + require.ErrorIs(t, err, ErrKeyNotFound) + + got, present, err := ps.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, entryIdx, got) +} + // TestDirectApplyMutations_NoSyncConfigured_StillWritesDurably is the // functional twin of TestDirectApplyWriteOpts_AlwaysSync: it exercises // the public ApplyMutations and DeletePrefixAt entry points with a diff --git a/store/migration_promote.go b/store/migration_promote.go index 3d2d9de57..b0e5b7fa5 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -217,9 +217,10 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO if err != nil { return PromoteVersionsResult{}, err } + writeOpts := s.promotionWriteOpts(opts.AppliedIndex) if opts.JobID != 0 && state.Done { result := PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows} - return s.finishPebblePromotion(nil, opts.JobID, nil, result, opts.AppliedIndex, 0) + return s.finishPebblePromotion(nil, opts.JobID, nil, result, opts.AppliedIndex, 0, writeOpts) } opts.Cursor = cursor exported, toPromote, promoted, err := s.planPebblePromotionLocked(ctx, opts) @@ -227,7 +228,22 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO return PromoteVersionsResult{}, err } result, stateToWrite := finishPromotionResult(opts, state, exported, promoted) - return s.finishPebblePromotion(toPromote, opts.JobID, stateToWrite, result, opts.AppliedIndex, promoted.MaxPromotedTS) + return s.finishPebblePromotion( + toPromote, + opts.JobID, + stateToWrite, + result, + opts.AppliedIndex, + promoted.MaxPromotedTS, + writeOpts, + ) +} + +func (s *pebbleStore) promotionWriteOpts(appliedIndex uint64) *pebble.WriteOptions { + if appliedIndex > 0 { + return s.raftApplyWriteOpts() + } + return s.directApplyWriteOpts() } func (s *pebbleStore) finishPebblePromotion( @@ -237,11 +253,19 @@ func (s *pebbleStore) finishPebblePromotion( result PromoteVersionsResult, appliedIndex uint64, maxPromotedTS uint64, + writeOpts *pebble.WriteOptions, ) (PromoteVersionsResult, error) { if len(toPromote) == 0 && stateToWrite == nil && appliedIndex == 0 && maxPromotedTS == 0 { return result, nil } - if err := s.commitPebblePromoteVersions(toPromote, jobID, stateToWrite, appliedIndex, maxPromotedTS); err != nil { + if err := s.commitPebblePromoteVersions( + toPromote, + jobID, + stateToWrite, + appliedIndex, + maxPromotedTS, + writeOpts, + ); err != nil { return PromoteVersionsResult{}, err } return result, nil @@ -323,7 +347,14 @@ func (s *pebbleStore) readPebblePromotionState(jobID uint64) (PromotionState, bo return state, true, nil } -func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jobID uint64, state *PromotionState, appliedIndex, maxPromotedTS uint64) error { +func (s *pebbleStore) commitPebblePromoteVersions( + versions []promotedVersion, + jobID uint64, + state *PromotionState, + appliedIndex uint64, + maxPromotedTS uint64, + writeOpts *pebble.WriteOptions, +) error { batch := s.db.NewBatch() defer batch.Close() targets := make([]MVCCVersion, 0, len(versions)) @@ -350,10 +381,14 @@ func (s *pebbleStore) commitPebblePromoteVersions(versions []promotedVersion, jo return err } } - return s.commitPebblePromotionBatch(batch, maxPromotedTS) + return s.commitPebblePromotionBatch(batch, maxPromotedTS, writeOpts) } -func (s *pebbleStore) commitPebblePromotionBatch(batch *pebble.Batch, maxPromotedTS uint64) error { +func (s *pebbleStore) commitPebblePromotionBatch( + batch *pebble.Batch, + maxPromotedTS uint64, + writeOpts *pebble.WriteOptions, +) error { if maxPromotedTS > 0 { s.mtx.Lock() defer s.mtx.Unlock() @@ -364,13 +399,13 @@ func (s *pebbleStore) commitPebblePromotionBatch(batch *pebble.Batch, maxPromote if err := setPebbleUint64InBatch(batch, metaLastCommitTSBytes, newLastTS); err != nil { return err } - if err := batch.Commit(s.directApplyWriteOpts()); err != nil { + if err := batch.Commit(writeOpts); err != nil { return errors.WithStack(err) } s.updateLastCommitTS(newLastTS) return nil } - if err := batch.Commit(s.directApplyWriteOpts()); err != nil { + if err := batch.Commit(writeOpts); err != nil { return errors.WithStack(err) } return nil From a775fdaaa41f8073a77a725e17ddaababa01a320 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 11:53:08 +0900 Subject: [PATCH 056/162] Trim Lua negative cache bound test --- adapter/redis_lua_context.go | 26 ++++++++------ adapter/redis_lua_negative_type_cache_test.go | 35 ++++++------------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 4a57051b5..447423ee4 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -478,20 +478,26 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) { if err != nil { return redisTypeNone, err } - if typ == redisTypeNone && len(c.negativeType) < maxNegativeTypeCacheEntries { - // Pin the absence result for the rest of this Eval so repeated - // BullMQ-style polling of a missing key (e.g. a "delayed" zset) - // does not re-run the ~8-seek rawKeyTypeAt probe on every - // redis.call. - // - // Bounded to keep adversarial scripts from growing the map - // unboundedly; once full, subsequent misses correctly fall - // through to the server probe without caching. - c.negativeType[string(key)] = true + if typ == redisTypeNone { + c.rememberNegativeType(key) } return typ, nil } +func (c *luaScriptContext) rememberNegativeType(key []byte) { + if len(c.negativeType) >= maxNegativeTypeCacheEntries { + return + } + // Pin the absence result for the rest of this Eval so repeated + // BullMQ-style polling of a missing key (e.g. a "delayed" zset) does + // not re-run the ~8-seek rawKeyTypeAt probe on every redis.call. + // + // Bounded to keep adversarial scripts from growing the map + // unboundedly; once full, subsequent misses correctly fall through to + // the server probe without caching. + c.negativeType[string(key)] = true +} + func (c *luaScriptContext) ensureKeyNotExpired(key []byte) error { ttl, err := c.loadTTL(key) if err != nil { diff --git a/adapter/redis_lua_negative_type_cache_test.go b/adapter/redis_lua_negative_type_cache_test.go index 6253f7e30..f44aec35a 100644 --- a/adapter/redis_lua_negative_type_cache_test.go +++ b/adapter/redis_lua_negative_type_cache_test.go @@ -149,44 +149,31 @@ func TestLuaNegativeTypeCache_SingleProbePerKey(t *testing.T) { // itself stays bounded. func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) { t.Parallel() - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) - ctx := context.Background() - sc, err := newLuaScriptContext(ctx, nodes[0].redisServer) - require.NoError(t, err) - defer sc.Close() + sc := &luaScriptContext{negativeType: map[string]bool{}} // Probe cap+overflow unique missing keys. Each probe is a miss // (redisTypeNone); only the first `cap` should be memoized. const overflow = 50 total := maxNegativeTypeCacheEntries + overflow for i := 0; i < total; i++ { - typ, kerr := sc.keyType([]byte(fmt.Sprintf("lua:neg:cap:%d", i))) - require.NoError(t, kerr) - require.Equal(t, redisTypeNone, typ) + sc.rememberNegativeType([]byte(fmt.Sprintf("lua:neg:cap:%d", i))) } require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), "negativeType map must be capped at maxNegativeTypeCacheEntries") - // Each unique key above required exactly one probe on first access. - require.Equal(t, total, sc.keyTypeProbeCount, - "each unique key must have triggered exactly one server probe") - - // Re-probing one of the first `cap` keys must hit the cache - // (no additional server probe). Re-probing an overflow key must - // miss the cache and issue another server probe. + // One of the first `cap` keys must be cached. An overflow key must + // remain uncached so keyType falls back to a server probe in real Eval + // execution while the map size stays bounded. cachedKey := []byte("lua:neg:cap:0") - _, kerr := sc.keyType(cachedKey) - require.NoError(t, kerr) - require.Equal(t, total, sc.keyTypeProbeCount, - "a key inserted before the cap must remain cached") + typ, ok := sc.cachedType(cachedKey) + require.True(t, ok, "a key inserted before the cap must remain cached") + require.Equal(t, redisTypeNone, typ) overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", maxNegativeTypeCacheEntries+1)) - _, kerr = sc.keyType(overflowKey) - require.NoError(t, kerr) - require.Equal(t, total+1, sc.keyTypeProbeCount, - "a key probed after the cap was reached must fall back to the server probe") + _, ok = sc.cachedType(overflowKey) + require.False(t, ok, "a key probed after the cap must fall back to the server probe") + sc.rememberNegativeType(overflowKey) require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), "fallback probe must NOT grow the bounded cache") } From 2b2cbd249c75b6e2c14d7277c1f22111300fd356 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 11:53:58 +0900 Subject: [PATCH 057/162] store: harden Pebble migration export --- store/lsm_migration.go | 31 +++++++-- store/lsm_store.go | 108 ++++++++++++++++++++++--------- store/migration_versions.go | 12 +++- store/migration_versions_test.go | 96 +++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 37 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index dbf1e257e..109b44e22 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -109,14 +109,14 @@ func (s *pebbleStore) exportPebbleIteratorPosition( if userKey == nil { return true, true, nil } - if pebbleExportCursorPrunedKey(pos, userKey, commitTS) { + if pebbleExportCursorWholeKeySkipped(pos, userKey, commitTS) { advancePebbleExportPastCurrentUserKey(iter, userKey) return false, true, nil } if pebbleExportCursorEqual(pos, userKey, commitTS) { return true, true, nil } - if skipped, err := s.skipPebbleExportKeyOutsideRange(iter, opts, userKey); skipped || err != nil { + if skipped, err := s.skipPebbleExportKeyOutsideRange(iter, opts, userKey, commitTS, result); skipped || err != nil { return false, true, err } if commitTS <= opts.MinCommitTSExclusive { @@ -130,7 +130,13 @@ func isPebbleExportMetadataKey(rawKey []byte) bool { return isPebbleMetaKey(rawKey) } -func (s *pebbleStore) skipPebbleExportKeyOutsideRange(iter *pebble.Iterator, opts ExportVersionsOptions, userKey []byte) (bool, error) { +func (s *pebbleStore) skipPebbleExportKeyOutsideRange( + iter *pebble.Iterator, + opts ExportVersionsOptions, + userKey []byte, + commitTS uint64, + result *ExportVersionsResult, +) (bool, error) { if opts.StartKey != nil && bytes.Compare(userKey, opts.StartKey) < 0 { advancePebbleExportPastCurrentUserKey(iter, userKey) return true, nil @@ -141,6 +147,13 @@ func (s *pebbleStore) skipPebbleExportKeyOutsideRange(iter *pebble.Iterator, opt if pebbleExportCanStopAtEndKey(opts.StartKey, opts.EndKey, userKey) { return true, errExportReachedEnd } + rawValue := iter.Value() + result.ScannedBytes += versionExportSize(userKey, len(rawValue)) + result.NextCursor = encodeExportCursor(userKey, commitTS, exportCursorTagSkippedKey) + if finishExportIfLimited(opts, result) { + result.Done = false + return true, errExportChunkFull + } advancePebbleExportPastCurrentUserKey(iter, userKey) return true, nil } @@ -190,8 +203,11 @@ func pebbleExportCursorEqual(pos exportCursorPosition, userKey []byte, commitTS return pos.hasKey && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS } -func pebbleExportCursorPrunedKey(pos exportCursorPosition, userKey []byte, commitTS uint64) bool { - return pos.hasKey && pos.tag == exportCursorTagPrunedKey && bytes.Equal(userKey, pos.key) && commitTS == pos.commitTS +func pebbleExportCursorWholeKeySkipped(pos exportCursorPosition, userKey []byte, commitTS uint64) bool { + return pos.hasKey && + (pos.tag == exportCursorTagPrunedKey || pos.tag == exportCursorTagSkippedKey) && + bytes.Equal(userKey, pos.key) && + commitTS == pos.commitTS } func (s *pebbleStore) exportPebbleVersion( @@ -347,7 +363,10 @@ func (s *pebbleStore) stageMigrationClockMetadataIfNeeded(batch *pebble.Batch, j func (s *pebbleStore) applyImportVersionsBatch(batch *pebble.Batch, versions []MVCCVersion) error { for _, version := range versions { - k := encodeKey(version.Key, version.CommitTS) + k, err := encodePebbleUserVersionKey(version.Key, version.CommitTS) + if err != nil { + return err + } var encoded []byte if version.Tombstone { encoded = encodeValue(nil, true, 0, encStateCleartext) diff --git a/store/lsm_store.go b/store/lsm_store.go index da40ee41e..ba787b94b 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -561,6 +561,16 @@ func isPebbleWriterRegistryKey(rawKey []byte) bool { return err == nil } +var errMVCCMetadataKeyCollision = errors.New("store: mvcc encoded key collides with reserved pebble metadata key") + +func encodePebbleUserVersionKey(key []byte, commitTS uint64) ([]byte, error) { + encoded := encodeKey(key, commitTS) + if isPebbleMetaKey(encoded) { + return nil, errors.WithStack(errMVCCMetadataKeyCollision) + } + return encoded, nil +} + func (s *pebbleStore) findMaxCommitTS() (uint64, error) { return readPebbleUint64(s.db, metaLastCommitTSBytes) } @@ -882,26 +892,32 @@ func (s *pebbleStore) getAt(_ context.Context, key []byte, ts uint64) ([]byte, e // values, in which case the visibility checks below are operating // on authenticated bytes. func (s *pebbleStore) readVisibleVersion(iter *pebble.Iterator, key []byte, ts uint64) ([]byte, error) { - k := iter.Key() - userKey, _ := decodeKeyView(k) - if !bytes.Equal(userKey, key) { - return nil, ErrKeyNotFound - } - sv, err := decodeValue(iter.Value()) - if err != nil { - return nil, errors.WithStack(err) - } - plain, err := s.decryptForKey(k, sv, sv.Value) - if err != nil { - return nil, err - } - if sv.Tombstone { - return nil, ErrKeyNotFound - } - if sv.ExpireAt != 0 && sv.ExpireAt <= ts { - return nil, ErrKeyNotFound + for ; iter.Valid(); iter.Next() { + k := iter.Key() + if isPebbleMetaKey(k) { + continue + } + userKey, _ := decodeKeyView(k) + if !bytes.Equal(userKey, key) { + return nil, ErrKeyNotFound + } + sv, err := decodeValue(iter.Value()) + if err != nil { + return nil, errors.WithStack(err) + } + plain, err := s.decryptForKey(k, sv, sv.Value) + if err != nil { + return nil, err + } + if sv.Tombstone { + return nil, ErrKeyNotFound + } + if sv.ExpireAt != 0 && sv.ExpireAt <= ts { + return nil, ErrKeyNotFound + } + return plain, nil } - return plain, nil + return nil, ErrKeyNotFound } func (s *pebbleStore) GetAt(ctx context.Context, key []byte, ts uint64) ([]byte, error) { @@ -948,7 +964,11 @@ func (s *pebbleStore) ExistsAt(ctx context.Context, key []byte, ts uint64) (bool func (s *pebbleStore) CommittedVersionAt(_ context.Context, key []byte, commitTS uint64) (bool, error) { s.dbMu.RLock() defer s.dbMu.RUnlock() - _, closer, err := s.db.Get(encodeKey(key, commitTS)) + encoded := encodeKey(key, commitTS) + if isPebbleMetaKey(encoded) { + return false, nil + } + _, closer, err := s.db.Get(encoded) if err != nil { if errors.Is(err, pebble.ErrNotFound) { return false, nil @@ -1337,11 +1357,14 @@ func (s *pebbleStore) PutAt(ctx context.Context, key []byte, value []byte, commi if err := validateValueSize(value); err != nil { return err } + k, err := encodePebbleUserVersionKey(key, commitTS) + if err != nil { + return err + } s.dbMu.RLock() defer s.dbMu.RUnlock() commitTS = s.alignCommitTS(commitTS) - k := encodeKey(key, commitTS) // gateRegistration=true: PutAt is a direct (non-raft) write path. body, encState, err := s.encryptForKey(k, value, expireAt, true) if err != nil { @@ -1357,11 +1380,14 @@ func (s *pebbleStore) PutAt(ctx context.Context, key []byte, value []byte, commi } func (s *pebbleStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) error { + k, err := encodePebbleUserVersionKey(key, commitTS) + if err != nil { + return err + } s.dbMu.RLock() defer s.dbMu.RUnlock() commitTS = s.alignCommitTS(commitTS) - k := encodeKey(key, commitTS) v := encodeValue(nil, true, 0, encStateCleartext) if err := s.db.Set(k, v, pebble.NoSync); err != nil { @@ -1376,6 +1402,10 @@ func (s *pebbleStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte } func (s *pebbleStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, commitTS uint64) error { + k, err := encodePebbleUserVersionKey(key, commitTS) + if err != nil { + return err + } s.dbMu.RLock() defer s.dbMu.RUnlock() @@ -1386,8 +1416,7 @@ func (s *pebbleStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, return err } - commitTS = s.alignCommitTS(commitTS) - k := encodeKey(key, commitTS) + s.alignCommitTS(commitTS) // gateRegistration=true: ExpireAt is a direct (non-raft) write path // that calls encryptForKey directly (it does not delegate to PutAt). body, encState, err := s.encryptForKey(k, val, expireAt, true) @@ -1414,12 +1443,16 @@ func (s *pebbleStore) latestCommitTS(_ context.Context, key []byte) (uint64, boo } defer iter.Close() - if iter.First() { + for ok := iter.First(); ok; ok = iter.Next() { k := iter.Key() + if isPebbleMetaKey(k) { + continue + } userKey, version := decodeKeyView(k) if bytes.Equal(userKey, key) { return version, true, nil } + return 0, false, nil } return 0, false, nil } @@ -1483,7 +1516,10 @@ func (s *pebbleStore) WriteConflictCount() uint64 { func (s *pebbleStore) applyMutationsBatch(b *pebble.Batch, mutations []*KVPairMutation, commitTS uint64, gateRegistration bool) error { for _, mut := range mutations { - k := encodeKey(mut.Key, commitTS) + k, err := encodePebbleUserVersionKey(mut.Key, commitTS) + if err != nil { + return err + } var v []byte switch mut.Op { @@ -1751,8 +1787,8 @@ func (s *pebbleStore) scanDeletePrefix(iter *pebble.Iterator, batch *pebble.Batc return err } if needsTombstone { - if err := batch.Set(encodeKey(userKey, commitTS), tombstoneVal, nil); err != nil { - return errors.WithStack(err) + if err := setDeletePrefixTombstone(batch, userKey, commitTS, tombstoneVal); err != nil { + return err } } if !s.skipToNextUserKey(iter, userKey) { @@ -1762,6 +1798,14 @@ func (s *pebbleStore) scanDeletePrefix(iter *pebble.Iterator, batch *pebble.Batc return nil } +func setDeletePrefixTombstone(batch *pebble.Batch, userKey []byte, commitTS uint64, tombstoneVal []byte) error { + k, err := encodePebbleUserVersionKey(userKey, commitTS) + if err != nil { + return err + } + return errors.WithStack(batch.Set(k, tombstoneVal, nil)) +} + type deletePrefixAction int const ( @@ -2158,8 +2202,12 @@ func flushSnapshotBatch(db *pebble.DB, batch **pebble.Batch, opts *pebble.WriteO } func setEncodedVersionInBatch(batch *pebble.Batch, key []byte, version VersionedValue) error { - deferred := batch.SetDeferred(encodedKeyLen(key), encodedValueLen(len(version.Value))) - fillEncodedKey(deferred.Key, key, version.TS) + encodedKey, err := encodePebbleUserVersionKey(key, version.TS) + if err != nil { + return err + } + deferred := batch.SetDeferred(len(encodedKey), encodedValueLen(len(version.Value))) + copy(deferred.Key, encodedKey) // MVCC snapshot format v2 does not carry encryption_state — Stage 8 of // the encryption rollout (per docs/design/2026_04_29_proposed...) bumps // the format to v3 to round-trip encrypted entries through this path. diff --git a/store/migration_versions.go b/store/migration_versions.go index 492a0c13f..e5c047a64 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -14,6 +14,7 @@ const ( exportCursorTagEmitted byte = iota exportCursorTagScanned exportCursorTagPrunedKey + exportCursorTagSkippedKey migrationAckMetaKey = "_migack" migrationHLCFloorMetaKey = "_mighlc" @@ -79,7 +80,10 @@ func decodeExportCursor(cursor []byte) (exportCursorPosition, error) { return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) } tag := rest[0] - if tag != exportCursorTagEmitted && tag != exportCursorTagScanned && tag != exportCursorTagPrunedKey { + if tag != exportCursorTagEmitted && + tag != exportCursorTagScanned && + tag != exportCursorTagPrunedKey && + tag != exportCursorTagSkippedKey { return exportCursorPosition{}, errors.WithStack(ErrInvalidExportCursor) } return exportCursorPosition{key: key, commitTS: commitTS, tag: tag, hasKey: true}, nil @@ -103,6 +107,12 @@ func validateExportCursorRange(opts ExportVersionsOptions, pos exportCursorPosit if opts.StartKey != nil && bytes.Compare(pos.key, opts.StartKey) < 0 { return errors.WithStack(ErrInvalidExportCursor) } + if pos.tag == exportCursorTagSkippedKey { + if opts.EndKey == nil || bytes.Compare(pos.key, opts.EndKey) < 0 { + return errors.WithStack(ErrInvalidExportCursor) + } + return nil + } if opts.EndKey != nil && bytes.Compare(pos.key, opts.EndKey) >= 0 { return errors.WithStack(ErrInvalidExportCursor) } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 41e027b4a..7e76c5987 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -3,6 +3,7 @@ package store import ( "bytes" "context" + "encoding/binary" "math" "os" "testing" @@ -477,6 +478,68 @@ func TestPebbleExportDoesNotDropWriterRegistryPrefixUserKey(t *testing.T) { require.Equal(t, []MVCCVersion{{Key: registryKey, CommitTS: 10, Value: []byte("user-value")}}, res.Versions) } +func TestPebbleRejectsMVCCKeyThatEncodesAsWriterRegistryRow(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-writer-registry-collision-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + rawRegistryKey := encryption.RegistryKey(1, 2) + userKey := bytes.Clone(rawRegistryKey[:len(rawRegistryKey)-timestampSize]) + commitTS := ^binary.BigEndian.Uint64(rawRegistryKey[len(rawRegistryKey)-timestampSize:]) + require.True(t, isPebbleWriterRegistryKey(encodeKey(userKey, commitTS))) + + require.ErrorIs(t, st.PutAt(ctx, userKey, []byte("value"), commitTS, 0), errMVCCMetadataKeyCollision) + _, err = st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 1, + BracketID: 1, + BatchSeq: 1, + Versions: []MVCCVersion{{ + Key: userKey, + CommitTS: commitTS, + Value: []byte("value"), + }}, + }) + require.ErrorIs(t, err, errMVCCMetadataKeyCollision) +} + +func TestPebbleWriterRegistryRowIsNotVisibleAsMVCCCollision(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-writer-registry-read-collision-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + registryKey := encryption.RegistryKey(1, 2) + registry, err := WriterRegistryFor(st) + require.NoError(t, err) + require.NoError(t, registry.SetRegistryRow( + registryKey, + encryption.EncodeRegistryValue(encryption.RegistryValue{ + FullNodeID: 2, + FirstSeenLocalEpoch: 1, + LastSeenLocalEpoch: 1, + }), + )) + + userKey := bytes.Clone(registryKey[:len(registryKey)-timestampSize]) + commitTS := ^binary.BigEndian.Uint64(registryKey[len(registryKey)-timestampSize:]) + _, err = st.GetAt(ctx, userKey, commitTS) + require.ErrorIs(t, err, ErrKeyNotFound) + ok, err := st.CommittedVersionAt(ctx, userKey, commitTS) + require.NoError(t, err) + require.False(t, ok) + latest, ok, err := st.LatestCommitTS(ctx, userKey) + require.NoError(t, err) + require.False(t, ok) + require.Zero(t, latest) +} + func TestPebbleExportAuthenticatesEncryptedTombstoneHeader(t *testing.T) { ctx := context.Background() f := newEncryptedStoreFixture(t, 81) @@ -520,6 +583,39 @@ func TestPebbleExportStopsAtEndKeyWhenNoLaterInRangeKeyCanTrail(t *testing.T) { require.Zero(t, result.ScannedBytes) } +func TestPebbleExportOutOfRangeEndSkipReturnsResumableCursor(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-out-of-range-end-skip-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("later"), 10, 0)) + require.NoError(t, st.PutAt(ctx, nil, []byte("empty"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("a"), + MaxVersions: 10, + MaxScannedBytes: 1, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.NotEmpty(t, first.NextCursor) + require.Greater(t, first.ScannedBytes, uint64(0)) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("a"), + Cursor: first.NextCursor, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte{}, CommitTS: 20, Value: []byte("empty")}}, second.Versions) +} + func TestImportVersionsIdempotencyAndMetadata(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() From a9c9afef3b080c6979440b1b703faea673eebd9b Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 12:00:25 +0900 Subject: [PATCH 058/162] Trim Lua negative cache bound test --- adapter/redis_lua_context.go | 26 ++++++++------ adapter/redis_lua_negative_type_cache_test.go | 35 ++++++------------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 4a57051b5..447423ee4 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -478,20 +478,26 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) { if err != nil { return redisTypeNone, err } - if typ == redisTypeNone && len(c.negativeType) < maxNegativeTypeCacheEntries { - // Pin the absence result for the rest of this Eval so repeated - // BullMQ-style polling of a missing key (e.g. a "delayed" zset) - // does not re-run the ~8-seek rawKeyTypeAt probe on every - // redis.call. - // - // Bounded to keep adversarial scripts from growing the map - // unboundedly; once full, subsequent misses correctly fall - // through to the server probe without caching. - c.negativeType[string(key)] = true + if typ == redisTypeNone { + c.rememberNegativeType(key) } return typ, nil } +func (c *luaScriptContext) rememberNegativeType(key []byte) { + if len(c.negativeType) >= maxNegativeTypeCacheEntries { + return + } + // Pin the absence result for the rest of this Eval so repeated + // BullMQ-style polling of a missing key (e.g. a "delayed" zset) does + // not re-run the ~8-seek rawKeyTypeAt probe on every redis.call. + // + // Bounded to keep adversarial scripts from growing the map + // unboundedly; once full, subsequent misses correctly fall through to + // the server probe without caching. + c.negativeType[string(key)] = true +} + func (c *luaScriptContext) ensureKeyNotExpired(key []byte) error { ttl, err := c.loadTTL(key) if err != nil { diff --git a/adapter/redis_lua_negative_type_cache_test.go b/adapter/redis_lua_negative_type_cache_test.go index 6253f7e30..f44aec35a 100644 --- a/adapter/redis_lua_negative_type_cache_test.go +++ b/adapter/redis_lua_negative_type_cache_test.go @@ -149,44 +149,31 @@ func TestLuaNegativeTypeCache_SingleProbePerKey(t *testing.T) { // itself stays bounded. func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) { t.Parallel() - nodes, _, _ := createNode(t, 3) - defer shutdown(nodes) - ctx := context.Background() - sc, err := newLuaScriptContext(ctx, nodes[0].redisServer) - require.NoError(t, err) - defer sc.Close() + sc := &luaScriptContext{negativeType: map[string]bool{}} // Probe cap+overflow unique missing keys. Each probe is a miss // (redisTypeNone); only the first `cap` should be memoized. const overflow = 50 total := maxNegativeTypeCacheEntries + overflow for i := 0; i < total; i++ { - typ, kerr := sc.keyType([]byte(fmt.Sprintf("lua:neg:cap:%d", i))) - require.NoError(t, kerr) - require.Equal(t, redisTypeNone, typ) + sc.rememberNegativeType([]byte(fmt.Sprintf("lua:neg:cap:%d", i))) } require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), "negativeType map must be capped at maxNegativeTypeCacheEntries") - // Each unique key above required exactly one probe on first access. - require.Equal(t, total, sc.keyTypeProbeCount, - "each unique key must have triggered exactly one server probe") - - // Re-probing one of the first `cap` keys must hit the cache - // (no additional server probe). Re-probing an overflow key must - // miss the cache and issue another server probe. + // One of the first `cap` keys must be cached. An overflow key must + // remain uncached so keyType falls back to a server probe in real Eval + // execution while the map size stays bounded. cachedKey := []byte("lua:neg:cap:0") - _, kerr := sc.keyType(cachedKey) - require.NoError(t, kerr) - require.Equal(t, total, sc.keyTypeProbeCount, - "a key inserted before the cap must remain cached") + typ, ok := sc.cachedType(cachedKey) + require.True(t, ok, "a key inserted before the cap must remain cached") + require.Equal(t, redisTypeNone, typ) overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", maxNegativeTypeCacheEntries+1)) - _, kerr = sc.keyType(overflowKey) - require.NoError(t, kerr) - require.Equal(t, total+1, sc.keyTypeProbeCount, - "a key probed after the cap was reached must fall back to the server probe") + _, ok = sc.cachedType(overflowKey) + require.False(t, ok, "a key probed after the cap must fall back to the server probe") + sc.rememberNegativeType(overflowKey) require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), "fallback probe must NOT grow the bounded cache") } From 67f3e9de64d82099b2975b4bc815e34119cc5096 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 12:06:14 +0900 Subject: [PATCH 059/162] store: separate list delta key prefix --- distribution/migrator_export_plan_test.go | 18 +++++++++++- internal/backup/redis_list.go | 20 ++++++------- internal/backup/redis_list_test.go | 9 +++--- kv/shard_key_test.go | 27 ++++++++++++++++++ store/list_helpers.go | 4 +-- store/list_helpers_test.go | 34 +++++++++++++++++++++++ 6 files changed, 92 insertions(+), 20 deletions(-) diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index ffe5d80c6..ca04ec701 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -2,6 +2,7 @@ package distribution import ( "bytes" + "encoding/binary" "testing" "github.com/bootjp/elastickv/internal/s3keys" @@ -101,7 +102,7 @@ func TestPlanMigrationBracketsDisjointPrefixContainment(t *testing.T) { require.True(t, byFamily[MigrationFamilyListMetaDelta].ContainsRawKey(listDelta)) require.False(t, byFamily[MigrationFamilyListMeta].ContainsRawKey(listDelta)) - listMetaWithDeltaLookingUserKey := store.ListMetaKey([]byte("d|list")) + listMetaWithDeltaLookingUserKey := store.ListMetaKey(deltaLookingListMetaUserKey([]byte("list"), 2, 0)) require.True(t, byFamily[MigrationFamilyListMeta].ContainsRawKey(listMetaWithDeltaLookingUserKey)) require.False(t, byFamily[MigrationFamilyListMetaDelta].ContainsRawKey(listMetaWithDeltaLookingUserKey)) @@ -356,3 +357,18 @@ func bracketsByFamily(brackets []MigrationBracket) map[uint32]MigrationBracket { } return out } + +func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { + key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) + key = append(key, "d|"...) + var lenPrefix [4]byte + binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. + key = append(key, lenPrefix[:]...) + key = append(key, fakeUserKey...) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + key = append(key, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(key, seq[:]...) +} diff --git a/internal/backup/redis_list.go b/internal/backup/redis_list.go index bcc1bc40c..8b9b579a0 100644 --- a/internal/backup/redis_list.go +++ b/internal/backup/redis_list.go @@ -33,7 +33,7 @@ import ( // item record for the popped // seq. The encoder therefore // skips claim keys entirely. -// - !lst|meta|d|... -> meta delta. The hash encoder +// - !lst|delta|... -> meta delta. The hash encoder // skips its analogous deltas // and treats !hs|fld| as the // source of truth; the list @@ -45,7 +45,7 @@ import ( const ( ListMetaPrefix = "!lst|meta|" ListItemPrefix = "!lst|itm|" - ListMetaDeltaPrefix = "!lst|meta|d|" + ListMetaDeltaPrefix = "!lst|delta|" ListClaimPrefix = "!lst|claim|" // listMetaBinarySize mirrors store/list_helpers.go (24 bytes: @@ -85,11 +85,8 @@ type redisListState struct { // register the user key so a later !redis|ttl| record routes // back to this list state. // -// !lst|meta|d|... delta keys share the !lst|meta| string -// prefix, so a snapshot dispatcher that routes by "starts with -// ListMetaPrefix" lands delta records here too. The hash encoder -// solved the analogous problem (Codex P1 round 14 PR #725) by silently -// skipping the delta family; we mirror that policy because !lst|itm| +// List deltas normally dispatch through HandleListMetaDelta, but keep +// the guard here so direct callers also skip the delta family. !lst|itm| // records are the source of truth for the restored list contents and // the delta arithmetic does not need to be replayed at backup time. func (r *RedisDB) HandleListMeta(key, value []byte) error { @@ -140,7 +137,7 @@ func (r *RedisDB) HandleListItem(key, value []byte) error { // therefore reflect the post-POP state without any claim replay. func (r *RedisDB) HandleListClaim(_, _ []byte) error { return nil } -// HandleListMetaDelta accepts and discards one !lst|meta|d|... record. +// HandleListMetaDelta accepts and discards one !lst|delta|... record. // See HandleListMeta's docstring for the rationale; !lst|itm| is the // source of truth at backup time. func (r *RedisDB) HandleListMetaDelta(_, _ []byte) error { return nil } @@ -162,10 +159,9 @@ func (r *RedisDB) listState(userKey []byte) *redisListState { // parseListMetaKey strips !lst|meta| from a meta key and returns // (userKey, true). The list meta key shape is `prefix + userKey` with // no length prefix (mirror of store.ListMetaKey), so the trimmed -// remainder is the userKey verbatim. Delta keys (!lst|meta|d|...) -// share the meta string prefix and must be rejected here so a -// misrouted delta surfaces a parse failure rather than silent state -// corruption — analogous to parseHashMetaKey's delta guard. +// remainder is the userKey verbatim. Delta keys are rejected here so +// a misrouted delta surfaces a parse failure rather than silent state +// corruption. func parseListMetaKey(key []byte) ([]byte, bool) { if bytes.HasPrefix(key, []byte(ListMetaDeltaPrefix)) { return nil, false diff --git a/internal/backup/redis_list_test.go b/internal/backup/redis_list_test.go index baa691c2d..a41dc3ac2 100644 --- a/internal/backup/redis_list_test.go +++ b/internal/backup/redis_list_test.go @@ -40,7 +40,7 @@ func listItemKey(userKey string, seq int64) []byte { } // listMetaDeltaKey mirrors store.ListMetaDeltaKey: -// !lst|meta|d|. +// !lst|delta|. // The shape is irrelevant to the encoder (it skips deltas), but we // build a well-formed key here so the dispatcher integration test // would exercise the same byte sequence the live store emits. @@ -275,10 +275,9 @@ func TestRedisDB_ListBinaryItemUsesBase64Envelope(t *testing.T) { } // TestRedisDB_ListHandleListMetaSkipsDeltaKey pins that the -// !lst|meta|d|... family is silently skipped by HandleListMeta. Without +// !lst|delta|... family is silently skipped by HandleListMeta. Without // this, parsing the delta's userKeyLen prefix as the start of a -// userKey would corrupt the lists map. Mirrors the hash delta-key -// guard (Codex P1 round 14 PR #725). +// userKey would corrupt the lists map. Mirrors the hash delta-key guard. func TestRedisDB_ListHandleListMetaSkipsDeltaKey(t *testing.T) { t.Parallel() db, _ := newRedisDB(t) @@ -353,7 +352,7 @@ func TestRedisDB_ListRejectsMalformedMetaValueLength(t *testing.T) { // firing the declared-vs-observed length mismatch warning (because // metaSeen=false means we have no "declared" baseline to compare // against). Mirrors the items-as-source-of-truth contract that -// makes the !lst|meta|d| delta family safe to skip. +// makes the !lst|delta| delta family safe to skip. func TestRedisDB_ListItemsWithoutMetaStillEmitsFile(t *testing.T) { t.Parallel() db, root := newRedisDB(t) diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 373ca144c..93050bfd1 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -124,6 +124,18 @@ func TestRouteKey_NormalizesCollectionMigrationFamilies(t *testing.T) { } } +func TestRouteKey_ListMetaKeyThatLooksLikeDeltaRoutesByRealListKey(t *testing.T) { + t.Parallel() + + fakeUserKey := []byte("fake:user") + userKey := deltaLookingListMetaUserKey(fakeUserKey, 42, 7) + baseMeta := store.ListMetaKey(userKey) + deltaKey := store.ListMetaDeltaKey(userKey, 10, 1) + + require.Equal(t, userKey, routeKey(baseMeta), "base list metadata must not decode as a delta for %q", fakeUserKey) + require.Equal(t, userKey, routeKey(deltaKey), "real list deltas must still route by the logical list key") +} + func TestRouteKey_MalformedWideColumnKeysFallBackToRaw(t *testing.T) { t.Parallel() @@ -156,6 +168,21 @@ func malformedWideColumnKey(prefix string, suffixLen int) []byte { return key } +func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { + key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) + key = append(key, "d|"...) + var lenPrefix [4]byte + binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. + key = append(key, lenPrefix[:]...) + key = append(key, fakeUserKey...) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + key = append(key, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(key, seq[:]...) +} + func TestRouteKey_NormalizesTxnSuccessMarkerByLockedKey(t *testing.T) { t.Parallel() diff --git a/store/list_helpers.go b/store/list_helpers.go index e01458fc7..896ae230a 100644 --- a/store/list_helpers.go +++ b/store/list_helpers.go @@ -11,8 +11,8 @@ import ( // Delta/Claim key constants. const ( // ListMetaDeltaPrefix is the prefix for all list metadata delta keys. - // Layout: !lst|meta|d| - ListMetaDeltaPrefix = "!lst|meta|d|" + // Layout: !lst|delta| + ListMetaDeltaPrefix = "!lst|delta|" // ListClaimPrefix is the prefix for list claim keys used by POP operations. // Layout: !lst|claim| diff --git a/store/list_helpers_test.go b/store/list_helpers_test.go index eca80eec1..efd8c8559 100644 --- a/store/list_helpers_test.go +++ b/store/list_helpers_test.go @@ -1,6 +1,7 @@ package store import ( + "encoding/binary" "testing" "github.com/stretchr/testify/require" @@ -26,3 +27,36 @@ func TestExtractListUserKeyFromDeltaRequiresExactDeltaShape(t *testing.T) { require.False(t, IsListMetaDeltaKey([]byte("not-a-delta-key"))) require.Nil(t, ExtractListUserKeyFromDelta([]byte("not-a-delta-key"))) } + +func TestListMetaDeltaPrefixDoesNotOverlapBaseMetaKeys(t *testing.T) { + t.Parallel() + + fakeUserKey := []byte("fake-user") + userKey := deltaLookingListMetaUserKey(fakeUserKey, 42, 7) + baseMeta := ListMetaKey(userKey) + deltaKey := ListMetaDeltaKey(userKey, 42, 7) + + require.True(t, IsListMetaKey(baseMeta)) + require.False(t, IsListMetaDeltaKey(baseMeta)) + require.Equal(t, userKey, ExtractListUserKey(baseMeta)) + require.Nil(t, ExtractListUserKeyFromDelta(baseMeta)) + + require.False(t, IsListMetaKey(deltaKey)) + require.True(t, IsListMetaDeltaKey(deltaKey)) + require.Equal(t, userKey, ExtractListUserKeyFromDelta(deltaKey)) +} + +func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { + buf := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) + buf = append(buf, "d|"...) + var keyLen [4]byte + binary.BigEndian.PutUint32(keyLen[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. + buf = append(buf, keyLen[:]...) + buf = append(buf, fakeUserKey...) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + buf = append(buf, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(buf, seq[:]...) +} From eebdbf22f1ed4be376c4e5d1a11be9f7fd7db3af Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 12:32:29 +0900 Subject: [PATCH 060/162] Stabilize stream latency seed writes --- adapter/redis_compat_commands_stream_test.go | 37 ++++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/adapter/redis_compat_commands_stream_test.go b/adapter/redis_compat_commands_stream_test.go index bc78c846f..ff605d3c1 100644 --- a/adapter/redis_compat_commands_stream_test.go +++ b/adapter/redis_compat_commands_stream_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sort" + "strings" "testing" "time" @@ -286,13 +287,7 @@ func TestRedis_StreamXReadLatencyIsConstant(t *testing.T) { ) lastID := "" for i := range total { - id, err := rdb.XAdd(ctx, &redis.XAddArgs{ - Stream: "stream-lat", - ID: "*", - Values: []string{"i", fmt.Sprint(i)}, - }).Result() - require.NoError(t, err) - lastID = id + lastID = xaddStreamLatencySeed(t, ctx, rdb, fmt.Sprint(i)) } measure := func() time.Duration { @@ -348,6 +343,34 @@ func TestRedis_StreamXReadLatencyIsConstant(t *testing.T) { baseline, median, p95) } +func xaddStreamLatencySeed(t *testing.T, ctx context.Context, rdb *redis.Client, value string) string { + t.Helper() + + const maxAttempts = 10 + for attempt := range maxAttempts { + id, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: "stream-lat", + ID: "*", + Values: []string{"i", value}, + }).Result() + if err == nil { + return id + } + if attempt == maxAttempts-1 || !isRetryableStreamLatencySeedErr(err) { + require.NoError(t, err) + } + time.Sleep(time.Duration(attempt+1) * leaderChurnRetryInterval) + } + return "" +} + +func isRetryableStreamLatencySeedErr(err error) bool { + if err == nil || isTransientNotLeaderErr(err) { + return err != nil + } + return strings.Contains(strings.ToLower(err.Error()), "write conflict") +} + func TestRedis_StreamXTrimMaxLen(t *testing.T) { t.Parallel() nodes, _, _ := createNode(t, 3) From c29806025400c0896df536c8ab1eca6b2038284c Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 12:38:44 +0900 Subject: [PATCH 061/162] Enable split migration capability gate --- adapter/distribution_server.go | 2 +- main.go | 42 ++++++++++++++------ main_catalog_test.go | 72 +++++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 15 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 40e76ba03..481159753 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -108,7 +108,7 @@ const ( splitJobListCursorTerminalOff = 1 splitJobListCursorJobIDOff = splitJobListCursorTerminalOff + 8 splitJobListCursorEncodedBytes = splitJobListCursorJobIDOff + 8 - splitMigrationCapabilityV2 = "split_migration_v2" + splitMigrationCapabilityV2 = "cap_migration_v2" ) var ( diff --git a/main.go b/main.go index 628ced4e9..47c44af55 100644 --- a/main.go +++ b/main.go @@ -500,9 +500,8 @@ func run() error { startKeyVizFlusher(runCtx, eg, sampler) startKeyVizLeaderTermPublisher(runCtx, eg, sampler, runtimes) startMemoryWatchdog(runCtx, eg, cancel) - defaultRuntime := findDefaultGroupRuntime(runtimes, cfg.defaultGroup) splitMigrationGate := newSplitMigrationCapabilityGate( - splitMigrationCapabilityPeerSourceForRuntime(defaultRuntime), + splitMigrationCapabilityPeerSourceForRuntimes(runtimes), splitMigrationCapabilityProbeTimeout, nil, ) @@ -513,7 +512,9 @@ func run() error { adapter.WithDistributionActiveTimestampTracker(readTracker), adapter.WithDistributionKnownRaftGroups(shardGroupIDs(shardGroups)...), adapter.WithSplitMigrationCapabilityGate(splitMigrationGate), + adapter.WithSplitJobRunnerReady(), ) + defaultRuntime := findDefaultGroupRuntime(runtimes, cfg.defaultGroup) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) compactor := kv.NewFSMCompactor( fsmCompactionRuntimes(runtimes), @@ -682,17 +683,35 @@ type splitMigrationCapabilityPeerSource func(context.Context) ([]splitMigrationC type splitMigrationCapabilityProbe func(context.Context, string) error -func splitMigrationCapabilityPeerSourceForRuntime(rt *raftGroupRuntime) splitMigrationCapabilityPeerSource { +func splitMigrationCapabilityPeerSourceForRuntimes(runtimes []*raftGroupRuntime) splitMigrationCapabilityPeerSource { return func(ctx context.Context) ([]splitMigrationCapabilityPeer, error) { - engine := rt.snapshotEngine() - if engine == nil { - return nil, errors.New("default raft group engine is not configured") + if len(runtimes) == 0 { + return nil, errors.New("raft group runtimes are not configured") } - cfg, err := engine.Configuration(ctx) - if err != nil { - return nil, errors.Wrap(err, "default raft group configuration") + peers := make([]splitMigrationCapabilityPeer, 0) + seen := make(map[string]struct{}) + for _, rt := range runtimes { + if rt == nil { + return nil, errors.New("raft group runtime is not configured") + } + engine := rt.snapshotEngine() + if engine == nil { + return nil, errors.Errorf("raft group %d engine is not configured", rt.spec.id) + } + cfg, err := engine.Configuration(ctx) + if err != nil { + return nil, errors.Wrapf(err, "raft group %d configuration", rt.spec.id) + } + for _, peer := range splitMigrationCapabilityPeersFromConfiguration(cfg) { + key := peer.ID + "\x00" + peer.Address + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + peers = append(peers, peer) + } } - return splitMigrationCapabilityPeersFromConfiguration(cfg), nil + return peers, nil } } @@ -704,9 +723,6 @@ func splitMigrationCapabilityPeersFromConfiguration(cfg raftengine.Configuration peers := make([]splitMigrationCapabilityPeer, 0, len(servers)) seen := make(map[string]struct{}, len(servers)) for _, server := range servers { - if server.Suffrage == etcdraftengine.SuffrageLearner { - continue - } key := server.ID + "\x00" + server.Address if _, ok := seen[key]; ok { continue diff --git a/main_catalog_test.go b/main_catalog_test.go index 968a0fefb..497213d36 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -143,7 +143,7 @@ func TestSplitMigrationCapabilityGateFailsClosedWhenPeerSourceErrors(t *testing. require.ErrorContains(t, err, "peers are not available") } -func TestSplitMigrationCapabilityPeersFromConfigurationUsesCurrentVoters(t *testing.T) { +func TestSplitMigrationCapabilityPeersFromConfigurationUsesCurrentMembers(t *testing.T) { t.Parallel() cfg := raftengine.Configuration{Servers: []raftengine.Server{ @@ -155,10 +155,40 @@ func TestSplitMigrationCapabilityPeersFromConfigurationUsesCurrentVoters(t *test require.Equal(t, []splitMigrationCapabilityPeer{ {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, {ID: "n3", Address: "10.0.0.13:50051"}, }, splitMigrationCapabilityPeersFromConfiguration(cfg)) } +func TestSplitMigrationCapabilityPeerSourceForRuntimesChecksAllGroups(t *testing.T) { + t.Parallel() + + runtimes := []*raftGroupRuntime{ + { + spec: groupSpec{id: 1}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "learner", ID: "n2", Address: "10.0.0.12:50051"}, + }}}, + }, + { + spec: groupSpec{id: 2}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "voter", ID: "n3", Address: "10.0.0.13:50051"}, + }}}, + }, + } + + peers, err := splitMigrationCapabilityPeerSourceForRuntimes(runtimes)(context.Background()) + require.NoError(t, err) + require.Equal(t, []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + {ID: "n3", Address: "10.0.0.13:50051"}, + }, peers) +} + func staticSplitMigrationCapabilityPeerSource(peers []splitMigrationCapabilityPeer) splitMigrationCapabilityPeerSource { return func(context.Context) ([]splitMigrationCapabilityPeer, error) { out := make([]splitMigrationCapabilityPeer, len(peers)) @@ -166,3 +196,43 @@ func staticSplitMigrationCapabilityPeerSource(peers []splitMigrationCapabilityPe return out, nil } } + +type capabilityConfigEngine struct { + cfg raftengine.Configuration +} + +func (e capabilityConfigEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, nil +} + +func (e capabilityConfigEngine) ProposeAdmin(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, nil +} + +func (e capabilityConfigEngine) State() raftengine.State { + return raftengine.StateFollower +} + +func (e capabilityConfigEngine) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{} +} + +func (e capabilityConfigEngine) VerifyLeader(context.Context) error { + return nil +} + +func (e capabilityConfigEngine) LinearizableRead(context.Context) (uint64, error) { + return 0, nil +} + +func (e capabilityConfigEngine) Status() raftengine.Status { + return raftengine.Status{} +} + +func (e capabilityConfigEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return e.cfg, nil +} + +func (e capabilityConfigEngine) Close() error { + return nil +} From 05b72cfb633ecc6aa5843e02926f6dffcbc144e2 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 12:46:12 +0900 Subject: [PATCH 062/162] Bound migration export scan skips --- store/lsm_migration.go | 51 +++++++++++---- store/lsm_store.go | 11 ++++ store/migration_versions.go | 17 +++-- store/migration_versions_test.go | 105 ++++++++++++++++++++++++++++--- 4 files changed, 158 insertions(+), 26 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 109b44e22..828ba481d 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -110,8 +110,8 @@ func (s *pebbleStore) exportPebbleIteratorPosition( return true, true, nil } if pebbleExportCursorWholeKeySkipped(pos, userKey, commitTS) { - advancePebbleExportPastCurrentUserKey(iter, userKey) - return false, true, nil + done := advancePebbleExportPastCurrentUserKey(iter, opts, userKey, pos.tag, result) + return false, done, nil } if pebbleExportCursorEqual(pos, userKey, commitTS) { return true, true, nil @@ -138,8 +138,7 @@ func (s *pebbleStore) skipPebbleExportKeyOutsideRange( result *ExportVersionsResult, ) (bool, error) { if opts.StartKey != nil && bytes.Compare(userKey, opts.StartKey) < 0 { - advancePebbleExportPastCurrentUserKey(iter, userKey) - return true, nil + return true, skipPebbleExportWholeKey(iter, opts, userKey, commitTS, exportCursorTagSkippedKey, result) } if opts.EndKey == nil || bytes.Compare(userKey, opts.EndKey) < 0 { return false, nil @@ -147,15 +146,28 @@ func (s *pebbleStore) skipPebbleExportKeyOutsideRange( if pebbleExportCanStopAtEndKey(opts.StartKey, opts.EndKey, userKey) { return true, errExportReachedEnd } + return true, skipPebbleExportWholeKey(iter, opts, userKey, commitTS, exportCursorTagSkippedKey, result) +} + +func skipPebbleExportWholeKey( + iter *pebble.Iterator, + opts ExportVersionsOptions, + userKey []byte, + commitTS uint64, + tag byte, + result *ExportVersionsResult, +) error { rawValue := iter.Value() result.ScannedBytes += versionExportSize(userKey, len(rawValue)) - result.NextCursor = encodeExportCursor(userKey, commitTS, exportCursorTagSkippedKey) + result.NextCursor = encodeExportCursor(userKey, commitTS, tag) if finishExportIfLimited(opts, result) { result.Done = false - return true, errExportChunkFull + return errExportChunkFull } - advancePebbleExportPastCurrentUserKey(iter, userKey) - return true, nil + if !advancePebbleExportPastCurrentUserKey(iter, opts, userKey, tag, result) { + return errExportChunkFull + } + return nil } func (s *pebbleStore) skipPebbleExportVersionBelowMinTS( @@ -172,18 +184,31 @@ func (s *pebbleStore) skipPebbleExportVersionBelowMinTS( result.Done = false return false, false, nil } - advancePebbleExportPastCurrentUserKey(iter, userKey) - return false, true, nil + return false, advancePebbleExportPastCurrentUserKey(iter, opts, userKey, exportCursorTagPrunedKey, result), nil } -func advancePebbleExportPastCurrentUserKey(iter *pebble.Iterator, userKey []byte) { +func advancePebbleExportPastCurrentUserKey( + iter *pebble.Iterator, + opts ExportVersionsOptions, + userKey []byte, + tag byte, + result *ExportVersionsResult, +) bool { userKey = bytes.Clone(userKey) for iter.Next() { - currentUserKey, _ := decodeKeyView(iter.Key()) + currentUserKey, commitTS := decodeKeyView(iter.Key()) if !bytes.Equal(currentUserKey, userKey) { - return + return true + } + rawValue := iter.Value() + result.ScannedBytes += versionExportSize(currentUserKey, len(rawValue)) + result.NextCursor = encodeExportCursor(currentUserKey, commitTS, tag) + if finishExportIfLimited(opts, result) { + result.Done = false + return false } } + return true } func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { diff --git a/store/lsm_store.go b/store/lsm_store.go index ba787b94b..96bb16203 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -557,10 +557,21 @@ func isPebbleMetaKey(rawKey []byte) bool { } func isPebbleWriterRegistryKey(rawKey []byte) bool { + if !couldBePebbleWriterRegistryKey(rawKey) { + return false + } _, _, err := encryption.DecodeRegistryKey(rawKey) return err == nil } +func couldBePebbleWriterRegistryKey(rawKey []byte) bool { + const writerRegistrySuffixSize = 4 + 1 + 2 + prefix := encryption.WriterRegistryPrefix + return len(rawKey) == len(prefix)+writerRegistrySuffixSize && + bytes.HasPrefix(rawKey, prefix) && + rawKey[len(prefix)+4] == '|' +} + var errMVCCMetadataKeyCollision = errors.New("store: mvcc encoded key collides with reserved pebble metadata key") func encodePebbleUserVersionKey(key []byte, commitTS uint64) ([]byte, error) { diff --git a/store/migration_versions.go b/store/migration_versions.go index e5c047a64..33c718e11 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -104,21 +104,26 @@ func validateExportCursorRange(opts ExportVersionsOptions, pos exportCursorPosit if !pos.hasKey { return nil } - if opts.StartKey != nil && bytes.Compare(pos.key, opts.StartKey) < 0 { - return errors.WithStack(ErrInvalidExportCursor) - } if pos.tag == exportCursorTagSkippedKey { - if opts.EndKey == nil || bytes.Compare(pos.key, opts.EndKey) < 0 { + if !exportSkippedCursorOutsideRange(opts, pos.key) { return errors.WithStack(ErrInvalidExportCursor) } return nil } + if opts.StartKey != nil && bytes.Compare(pos.key, opts.StartKey) < 0 { + return errors.WithStack(ErrInvalidExportCursor) + } if opts.EndKey != nil && bytes.Compare(pos.key, opts.EndKey) >= 0 { return errors.WithStack(ErrInvalidExportCursor) } return nil } +func exportSkippedCursorOutsideRange(opts ExportVersionsOptions, key []byte) bool { + return (opts.StartKey != nil && bytes.Compare(key, opts.StartKey) < 0) || + (opts.EndKey != nil && bytes.Compare(key, opts.EndKey) >= 0) +} + func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOptions { if opts.EndKey != nil && len(opts.EndKey) == 0 { opts.EndKey = nil @@ -132,7 +137,9 @@ func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOp func exportUsesSparseScanBudget(opts ExportVersionsOptions) bool { return opts.AcceptKey != nil || opts.MaxCommitTSInclusive != 0 || - opts.MinCommitTSExclusive != 0 + opts.MinCommitTSExclusive != 0 || + opts.StartKey != nil || + opts.EndKey != nil } func isMigrationMetadataKey(rawKey []byte) bool { diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 7e76c5987..82209e4c4 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -269,14 +269,23 @@ func TestExportVersionsMinTSPruneCursorSkipsWholeKey(t *testing.T) { require.Empty(t, first.Versions) require.NotEmpty(t, first.NextCursor) - second, err := st.ExportVersions(ctx, ExportVersionsOptions{ - Cursor: first.NextCursor, - MinCommitTSExclusive: 10, - MaxVersions: 10, - }) - require.NoError(t, err) - require.True(t, second.Done) - require.Equal(t, []MVCCVersion{{Key: []byte("tail"), CommitTS: 20, Value: []byte("v20")}}, second.Versions) + cursor := first.NextCursor + for attempts := 0; attempts < 4; attempts++ { + next, err := st.ExportVersions(ctx, ExportVersionsOptions{ + Cursor: cursor, + MinCommitTSExclusive: 10, + MaxVersions: 10, + }) + require.NoError(t, err) + if next.Done { + require.Equal(t, []MVCCVersion{{Key: []byte("tail"), CommitTS: 20, Value: []byte("v20")}}, next.Versions) + return + } + require.Empty(t, next.Versions) + require.NotEmpty(t, next.NextCursor) + cursor = next.NextCursor + } + t.Fatal("export did not finish after bounded pruned-key cursor resumes") }) } @@ -616,6 +625,86 @@ func TestPebbleExportOutOfRangeEndSkipReturnsResumableCursor(t *testing.T) { require.Equal(t, []MVCCVersion{{Key: []byte{}, CommitTS: 20, Value: []byte("empty")}}, second.Versions) } +func TestPebbleExportOutOfRangeEndSkipChargesAllVersions(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-out-of-range-end-skip-versions-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("v10"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("v20"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("v30"), 30, 0)) + require.NoError(t, st.PutAt(ctx, nil, []byte("empty"), 40, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("a"), + MaxVersions: 10, + MaxScannedBytes: versionExportSize([]byte("b"), len("v30")) + 1, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.NotEmpty(t, first.NextCursor) + require.GreaterOrEqual(t, first.ScannedBytes, versionExportSize([]byte("b"), len("v30"))+1) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("a"), + Cursor: first.NextCursor, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte{}, CommitTS: 40, Value: []byte("empty")}}, second.Versions) +} + +func TestPebbleExportAppliesDefaultScanBudgetForRangeBoundSkip(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-range-bound-scan-budget-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + + value := bytes.Repeat([]byte("x"), defaultSparseExportMaxScannedBytes) + require.NoError(t, st.PutAt(ctx, []byte("b"), value, 10, 0)) + require.NoError(t, st.PutAt(ctx, nil, []byte("empty"), 20, 0)) + + first, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("a"), + MaxVersions: 10, + }) + require.NoError(t, err) + require.False(t, first.Done) + require.Empty(t, first.Versions) + require.NotEmpty(t, first.NextCursor) + require.GreaterOrEqual(t, first.ScannedBytes, uint64(defaultSparseExportMaxScannedBytes)) + + second, err := st.ExportVersions(ctx, ExportVersionsOptions{ + EndKey: []byte("a"), + Cursor: first.NextCursor, + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, second.Done) + require.Equal(t, []MVCCVersion{{Key: []byte{}, CommitTS: 20, Value: []byte("empty")}}, second.Versions) +} + +func TestPebbleWriterRegistryKeyShapeRejectsOrdinaryRows(t *testing.T) { + registryKey := encryption.RegistryKey(1, 2) + require.True(t, isPebbleWriterRegistryKey(registryKey)) + + ordinary := encodeKey([]byte("ordinary"), 10) + require.False(t, isPebbleWriterRegistryKey(ordinary)) + + malformed := bytes.Clone(registryKey) + malformed[len(encryption.WriterRegistryPrefix)+4] = '#' + require.False(t, isPebbleWriterRegistryKey(malformed)) +} + func TestImportVersionsIdempotencyAndMetadata(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() From 5280996f9057f53aca4be2a7a6ab22c59e377822 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 13:22:02 +0900 Subject: [PATCH 063/162] Fix migration routing edge cases --- adapter/redis_compat_helpers.go | 42 +++++++++++---- adapter/redis_delta_compactor.go | 36 +++++++++++++ adapter/redis_delta_compactor_test.go | 37 +++++++++++++ adapter/redis_list_dedup_test.go | 66 +++++++++++++++++++++++ adapter/redis_txn.go | 25 ++++----- distribution/migrator.go | 3 ++ distribution/migrator_export_plan_test.go | 16 ++++++ internal/backup/redis_list.go | 25 +++++++-- internal/backup/redis_list_test.go | 53 ++++++++++++++++++ kv/migrator_filter.go | 36 ++++++++++--- kv/shard_key_test.go | 38 +++++++++++++ kv/shard_store.go | 30 +++++++++-- kv/shard_store_test.go | 59 ++++++++++++++++++++ store/hash_helpers.go | 6 +++ store/list_helpers.go | 59 +++++++++++++++++++- store/list_helpers_test.go | 17 ++++++ store/set_helpers.go | 6 +++ store/zset_helpers.go | 12 +++++ 18 files changed, 528 insertions(+), 38 deletions(-) diff --git a/adapter/redis_compat_helpers.go b/adapter/redis_compat_helpers.go index ff10b6f92..486cbdea5 100644 --- a/adapter/redis_compat_helpers.go +++ b/adapter/redis_compat_helpers.go @@ -925,12 +925,14 @@ func (r *RedisServer) deleteListElems(ctx context.Context, key []byte, readTS ui } // Always delete the base meta key (no-op tombstone if it doesn't exist). elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: listMetaKey(key)}) - // Delete all delta keys (paginated). - deltaElems, err := r.scanAllDeltaElems(ctx, store.ListMetaDeltaScanPrefix(key), readTS) - if err != nil { - return nil, err + // Delete all delta keys (paginated), including the pre-upgrade prefix. + for _, deltaPrefix := range store.ListMetaDeltaScanPrefixes(key) { + deltaElems, err := r.scanAllDeltaElems(ctx, deltaPrefix, readTS) + if err != nil { + return nil, err + } + elems = append(elems, deltaElems...) } - elems = append(elems, deltaElems...) // Delete all claim keys (paginated). claimElems, err := r.scanAllDeltaElems(ctx, store.ListClaimScanPrefix(key), readTS) if err != nil { @@ -1284,6 +1286,30 @@ func (r *RedisServer) aggregateLenDeltas(ctx context.Context, prefix []byte, rea return sum, len(deltas) > 0, nil } +func (r *RedisServer) aggregateListMetaDeltas(ctx context.Context, key []byte, readTS uint64, applyDelta func(store.ListMetaDelta)) (int64, bool, error) { + var total int64 + var any bool + for _, prefix := range store.ListMetaDeltaScanPrefixes(key) { + lenSum, hasDeltas, err := r.aggregateLenDeltas(ctx, prefix, readTS, func(b []byte) (int64, error) { + if !store.IsListMetaDeltaValue(b) { + return 0, nil + } + d, unmarshalErr := store.UnmarshalListMetaDelta(b) + if unmarshalErr != nil { + return 0, errors.WithStack(unmarshalErr) + } + applyDelta(d) + return d.LenDelta, nil + }) + if err != nil { + return 0, false, err + } + total += lenSum + any = any || hasDeltas + } + return total, any, nil +} + // resolveListMeta aggregates the base list metadata with all uncompacted Delta keys // visible at readTS. Returns ErrDeltaScanTruncated if > MaxDeltaScanLimit deltas exist. func (r *RedisServer) resolveListMeta(ctx context.Context, key []byte, readTS uint64) (store.ListMeta, bool, error) { @@ -1295,15 +1321,13 @@ func (r *RedisServer) resolveListMeta(ctx context.Context, key []byte, readTS ui // 2. Scan and aggregate delta keys. // The closure also captures baseMeta to accumulate the list-specific HeadDelta. - prefix := store.ListMetaDeltaScanPrefix(key) - lenSum, hasDeltas, err := r.aggregateLenDeltas(ctx, prefix, readTS, func(b []byte) (int64, error) { - d, unmarshalErr := store.UnmarshalListMetaDelta(b) + lenSum, hasDeltas, err := r.aggregateListMetaDeltas(ctx, key, readTS, func(d store.ListMetaDelta) { baseMeta.Head += d.HeadDelta - return d.LenDelta, errors.WithStack(unmarshalErr) }) if err != nil { if errors.Is(err, ErrDeltaScanTruncated) { r.triggerUrgentCompaction("list", key) + r.triggerUrgentCompaction("list-legacy", key) } return store.ListMeta{}, false, err } diff --git a/adapter/redis_delta_compactor.go b/adapter/redis_delta_compactor.go index ceeaaaaf8..bed684450 100644 --- a/adapter/redis_delta_compactor.go +++ b/adapter/redis_delta_compactor.go @@ -259,6 +259,10 @@ func (c *DeltaCompactor) compactUrgentKeyBatch(ctx context.Context, req urgentCo if len(kvs) == 0 { return 0, true } + kvs = filterDeltaKVs(kvs, h.acceptDeltaKV) + if len(kvs) == 0 { + return 0, true + } elems, err := h.buildElems(ctx, req.userKey, kvs, readTS) if err != nil { @@ -397,6 +401,7 @@ type collectionDeltaHandler struct { typeName string prefix []byte extractUserKey func(key []byte) []byte + acceptDeltaKV func(*store.KVPair) bool // deltaKeyPrefixFn returns the prefix that covers all delta keys for a single // user key. Used by compactUrgentKey to perform a targeted single-key scan. deltaKeyPrefixFn func(userKey []byte) []byte @@ -407,6 +412,7 @@ type collectionDeltaHandler struct { func (c *DeltaCompactor) allHandlers() []collectionDeltaHandler { return []collectionDeltaHandler{ c.listHandler(), + c.legacyListHandler(), c.hashHandler(), c.setHandler(), c.zsetHandler(), @@ -466,6 +472,7 @@ func (c *DeltaCompactor) compactHandler(ctx context.Context, h collectionDeltaHa return err } + kvs = filterDeltaKVs(kvs, h.acceptDeltaKV) byKey, ukOrder := c.groupByUserKey(kvs, h.extractUserKey) lastScannedKey := c.splitGuardCursor(byKey, ukOrder, kvs, truncated) @@ -483,6 +490,19 @@ func (c *DeltaCompactor) compactHandler(ctx context.Context, h collectionDeltaHa // groupByUserKey groups KVPairs by their user key, returning both the map and // the unique user keys in lexicographic (scan) order. +func filterDeltaKVs(kvs []*store.KVPair, accept func(*store.KVPair) bool) []*store.KVPair { + if accept == nil || len(kvs) == 0 { + return kvs + } + out := kvs[:0] + for _, pair := range kvs { + if accept(pair) { + out = append(out, pair) + } + } + return out +} + func (c *DeltaCompactor) groupByUserKey(kvs []*store.KVPair, extractUserKey func([]byte) []byte) (map[string][]*store.KVPair, []string) { byKey := make(map[string][]*store.KVPair) var ukOrder []string @@ -628,11 +648,27 @@ func (c *DeltaCompactor) listHandler() collectionDeltaHandler { typeName: "list", prefix: []byte(store.ListMetaDeltaPrefix), extractUserKey: store.ExtractListUserKeyFromDelta, + acceptDeltaKV: isListMetaDeltaKV, deltaKeyPrefixFn: store.ListMetaDeltaScanPrefix, buildElems: c.buildListCompactElems, } } +func (c *DeltaCompactor) legacyListHandler() collectionDeltaHandler { + return collectionDeltaHandler{ + typeName: "list-legacy", + prefix: []byte(store.LegacyListMetaDeltaPrefix), + extractUserKey: store.ExtractLegacyListUserKeyFromDelta, + acceptDeltaKV: isListMetaDeltaKV, + deltaKeyPrefixFn: store.LegacyListMetaDeltaScanPrefix, + buildElems: c.buildListCompactElems, + } +} + +func isListMetaDeltaKV(pair *store.KVPair) bool { + return pair != nil && store.IsListMetaDeltaValue(pair.Value) +} + func (c *DeltaCompactor) buildListCompactElems(ctx context.Context, userKey []byte, deltaKVs []*store.KVPair, readTS uint64) ([]*kv.Elem[kv.OP], error) { // Read base metadata (may not exist if all state is in deltas). baseMeta, err := c.loadListBaseMeta(ctx, userKey, readTS) diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index c3f6f1518..e7c05a75c 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -92,6 +92,7 @@ func TestDeltaCompactor_RotatesHandlerAfterTimeout(t *testing.T) { wantPrefixes := []string{ store.ListMetaDeltaPrefix, + store.LegacyListMetaDeltaPrefix, store.HashMetaDeltaPrefix, store.SetMetaDeltaPrefix, store.ZSetMetaDeltaPrefix, @@ -147,6 +148,42 @@ func TestDeltaCompactor_ListDeltaFoldedIntoBaseMeta(t *testing.T) { } } +func TestDeltaCompactor_LegacyListDeltaFoldedIntoBaseMeta(t *testing.T) { + t.Parallel() + + st, c := newDeltaCompactorTestFixture(t) + ctx := context.Background() + userKey := []byte("legacy-list") + + baseMeta := store.ListMeta{Head: 5, Len: 2} + baseMeta.Tail = baseMeta.Head + baseMeta.Len + metaBytes, err := store.MarshalListMeta(baseMeta) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(userKey), metaBytes, 1, 0)) + + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: -1, LenDelta: 2}) + d1Key := legacyListMetaDeltaKey(userKey, 10, 0) + d2Key := legacyListMetaDeltaKey(userKey, 11, 0) + require.NoError(t, st.PutAt(ctx, d1Key, delta, 10, 0)) + require.NoError(t, st.PutAt(ctx, d2Key, delta, 11, 0)) + + require.NoError(t, c.SyncOnce(ctx)) + + readTS := st.LastCommitTS() + raw, err := st.GetAt(ctx, store.ListMetaKey(userKey), readTS) + require.NoError(t, err) + got, err := store.UnmarshalListMeta(raw) + require.NoError(t, err) + require.Equal(t, int64(3), got.Head) + require.Equal(t, int64(6), got.Len) + require.Equal(t, int64(9), got.Tail) + + for _, dk := range [][]byte{d1Key, d2Key} { + _, getErr := st.GetAt(ctx, dk, readTS) + require.ErrorIs(t, getErr, store.ErrKeyNotFound, "legacy delta key should be deleted after compaction: %s", dk) + } +} + func TestDeltaCompactor_ListBelowThresholdNotCompacted(t *testing.T) { t.Parallel() diff --git a/adapter/redis_list_dedup_test.go b/adapter/redis_list_dedup_test.go index 916a3621d..37ef7cb56 100644 --- a/adapter/redis_list_dedup_test.go +++ b/adapter/redis_list_dedup_test.go @@ -3,6 +3,7 @@ package adapter import ( "bytes" "context" + "encoding/binary" "testing" "github.com/bootjp/elastickv/kv" @@ -54,6 +55,71 @@ func newDedupTestCoordinator(st store.MVCCStore, ambiguousDispatch int, lands bo } } +func TestResolveListMetaReadsLegacyDeltaPrefix(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := []byte("legacy-list") + base, err := store.MarshalListMeta(store.ListMeta{Head: 10, Tail: 12, Len: 2}) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: -1, LenDelta: 3}) + require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, 2, 0), delta, 2, 0)) + + meta, exists, err := srv.resolveListMeta(ctx, key, 3) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, int64(9), meta.Head) + require.Equal(t, int64(5), meta.Len) + require.Equal(t, int64(14), meta.Tail) +} + +func TestResolveListMetaDoesNotTreatDeltaLookingMetaValueAsLegacyDelta(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := deltaLookingListMetaUserKey([]byte("embedded"), 7, 1) + base, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) + + meta, exists, err := srv.resolveListMeta(ctx, key, 2) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, int64(4), meta.Head) + require.Equal(t, int64(2), meta.Len) + require.Equal(t, int64(6), meta.Tail) +} + +func legacyListMetaDeltaKey(userKey []byte, commitTS uint64, seqInTxn uint32) []byte { + key := store.LegacyListMetaDeltaScanPrefix(userKey) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + key = append(key, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(key, seq[:]...) +} + +func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { + key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) + key = append(key, "d|"...) + var lenPrefix [4]byte + binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. + key = append(key, lenPrefix[:]...) + key = append(key, fakeUserKey...) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + key = append(key, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(key, seq[:]...) +} + func (c *dedupTestCoordinator) Dispatch(ctx context.Context, req *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { c.dispatches++ n := c.dispatches diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index 4abaf21d5..779851614 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -351,18 +351,19 @@ func (t *txnContext) loadListState(key []byte) (*listTxnState, error) { // truncation: if >MaxDeltaScanLimit deltas exist the transaction cannot // safely enumerate all of them for deletion, so we return ErrDeltaScanTruncated // and let the caller retry after the background compactor has caught up. - deltaPrefix := store.ListMetaDeltaScanPrefix(key) - deltaEnd := store.PrefixScanEnd(deltaPrefix) - deltaKVs, err := t.server.store.ScanAt(ctx, deltaPrefix, deltaEnd, store.MaxDeltaScanLimit+1, t.startTS) - if err != nil { - return nil, errors.WithStack(err) - } - if len(deltaKVs) > store.MaxDeltaScanLimit { - return nil, ErrDeltaScanTruncated - } - existingDeltas := make([][]byte, 0, len(deltaKVs)) - for _, kv := range deltaKVs { - existingDeltas = append(existingDeltas, kv.Key) + var existingDeltas [][]byte + for _, deltaPrefix := range store.ListMetaDeltaScanPrefixes(key) { + deltaEnd := store.PrefixScanEnd(deltaPrefix) + deltaKVs, err := t.server.store.ScanAt(ctx, deltaPrefix, deltaEnd, store.MaxDeltaScanLimit+1, t.startTS) + if err != nil { + return nil, errors.WithStack(err) + } + if len(deltaKVs) > store.MaxDeltaScanLimit { + return nil, ErrDeltaScanTruncated + } + for _, kv := range deltaKVs { + existingDeltas = append(existingDeltas, kv.Key) + } } st := &listTxnState{ diff --git a/distribution/migrator.go b/distribution/migrator.go index 3b33a5318..813e64cd1 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -262,6 +262,9 @@ func (b MigrationBracket) containsDecodedS3Route(rawKey, routeStart, routeEnd [] if !ok { return false } + if routeKeyInRange(rawKey, routeStart, routeEnd) { + return true + } bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) return rangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) } diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index ca04ec701..5aabe0496 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -204,6 +204,22 @@ func TestMigrationBracketContainsRoutedKeyForS3BucketAuxiliaryState(t *testing.T } } +func TestMigrationBracketContainsRoutedKeyForS3BucketRawRoute(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("m"), []byte("z")) + require.NoError(t, err) + byFamily := bracketsByFamily(brackets) + routeStart := []byte("!s3|") + + require.True(t, byFamily[MigrationFamilyS3BucketMeta].ContainsRoutedKey( + s3keys.BucketMetaKey("bucket-b"), routeStart, nil, s3keys.ExtractRouteKey, + )) + require.True(t, byFamily[MigrationFamilyS3BucketGeneration].ContainsRoutedKey( + s3keys.BucketGenerationKey("bucket-b"), routeStart, nil, s3keys.ExtractRouteKey, + )) +} + func TestMigrationBracketContainsRoutedKeyUsesObjectRoutes(t *testing.T) { t.Parallel() diff --git a/internal/backup/redis_list.go b/internal/backup/redis_list.go index 8b9b579a0..76ac3dbfd 100644 --- a/internal/backup/redis_list.go +++ b/internal/backup/redis_list.go @@ -42,11 +42,16 @@ import ( // source of truth and the // delta arithmetic is not // replayed at backup time. +// - !lst|meta|d|... -> legacy meta delta. This overlaps +// with base !lst|meta| keys whose user key begins with d|, so routing +// checks both the key prefix and the 16-byte delta value shape before +// dropping it as a delta. const ( - ListMetaPrefix = "!lst|meta|" - ListItemPrefix = "!lst|itm|" - ListMetaDeltaPrefix = "!lst|delta|" - ListClaimPrefix = "!lst|claim|" + ListMetaPrefix = "!lst|meta|" + ListItemPrefix = "!lst|itm|" + ListMetaDeltaPrefix = "!lst|delta|" + LegacyListMetaDeltaPrefix = "!lst|meta|d|" + ListClaimPrefix = "!lst|claim|" // listMetaBinarySize mirrors store/list_helpers.go (24 bytes: // Head(8) + Tail(8) + Len(8)). Re-declared here rather than @@ -57,6 +62,8 @@ const ( // listSeqBytes is the fixed width of the trailing sortable-int64 // sequence number in an !lst|itm| key. listSeqBytes = 8 + + listMetaDeltaBinarySize = 16 ) // ErrRedisInvalidListMeta is returned when an !lst|meta| value is not @@ -90,7 +97,7 @@ type redisListState struct { // records are the source of truth for the restored list contents and // the delta arithmetic does not need to be replayed at backup time. func (r *RedisDB) HandleListMeta(key, value []byte) error { - if bytes.HasPrefix(key, []byte(ListMetaDeltaPrefix)) { + if isListMetaDeltaRecord(key, value) { return nil } userKey, ok := parseListMetaKey(key) @@ -173,6 +180,14 @@ func parseListMetaKey(key []byte) ([]byte, bool) { return rest, true } +func isListMetaDeltaRecord(key, value []byte) bool { + if bytes.HasPrefix(key, []byte(ListMetaDeltaPrefix)) { + return true + } + return bytes.HasPrefix(key, []byte(LegacyListMetaDeltaPrefix)) && + len(value) == listMetaDeltaBinarySize +} + // parseListItemKey strips !lst|itm| and extracts (userKey, seq). The // list item key shape (mirror of store.ListItemKey) is // `prefix + userKey + sortableInt64(seq)`, with no userKey length diff --git a/internal/backup/redis_list_test.go b/internal/backup/redis_list_test.go index a41dc3ac2..1417400b6 100644 --- a/internal/backup/redis_list_test.go +++ b/internal/backup/redis_list_test.go @@ -58,6 +58,20 @@ func listMetaDeltaKey(userKey string, commitTS uint64, seqInTxn uint32) []byte { return append(out, seq[:]...) } +func legacyListMetaDeltaKey(userKey string, commitTS uint64, seqInTxn uint32) []byte { + out := []byte(LegacyListMetaDeltaPrefix) + var l [4]byte + binary.BigEndian.PutUint32(l[:], uint32(len(userKey))) //nolint:gosec + out = append(out, l[:]...) + out = append(out, userKey...) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + out = append(out, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(out, seq[:]...) +} + // listClaimKey mirrors store.ListClaimKey: // !lst|claim|. func listClaimKey(userKey string, seq int64) []byte { @@ -187,6 +201,45 @@ func TestRedisDB_ListEmptyListStillEmitsFile(t *testing.T) { } } +func TestRedisDB_ListLegacyDeltaIsSkippedButDeltaLookingMetaIsPreserved(t *testing.T) { + t.Parallel() + db, root := newRedisDB(t) + if err := db.HandleListMeta(legacyListMetaDeltaKey("q", 10, 0), make([]byte, listMetaDeltaBinarySize)); err != nil { + t.Fatal(err) + } + userKey := string(deltaLookingListMetaUserKeyForBackup([]byte("real"), 11, 1)) + if err := db.HandleListMeta(listMetaKey(userKey), listMetaValue(0, 1)); err != nil { + t.Fatal(err) + } + if err := db.HandleListItem(listItemKey(userKey, 0), []byte("v")); err != nil { + t.Fatal(err) + } + if err := db.Finalize(); err != nil { + t.Fatal(err) + } + + got := readListJSON(t, filepath.Join(root, "redis", "db_0", "lists", EncodeSegment([]byte(userKey))+".json")) + assertListItems(t, got, []any{"v"}) + if _, err := os.Stat(filepath.Join(root, "redis", "db_0", "lists", "q.json")); !os.IsNotExist(err) { + t.Fatalf("legacy delta must not emit q.json: stat err=%v", err) + } +} + +func deltaLookingListMetaUserKeyForBackup(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { + key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) + key = append(key, "d|"...) + var lenPrefix [4]byte + binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. + key = append(key, lenPrefix[:]...) + key = append(key, fakeUserKey...) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + key = append(key, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(key, seq[:]...) +} + // TestRedisDB_ListTTLInlinedFromScanIndex pins that !redis|ttl| records // for a list user key fold into the list's JSON `expire_at_ms` rather // than landing in a separate sidecar (the strings/HLL pattern). A diff --git a/kv/migrator_filter.go b/kv/migrator_filter.go index 19cbb23da..8a1da5526 100644 --- a/kv/migrator_filter.go +++ b/kv/migrator_filter.go @@ -10,20 +10,29 @@ import ( // rangeEnd nil or empty means +infinity, matching the route descriptor wire // convention. func RouteKeyFilter(rangeStart, rangeEnd []byte) func([]byte) bool { + return RouteKeyFilterForGroup(rangeStart, rangeEnd, 0, nil) +} + +// RouteKeyFilterForGroup returns the migration export predicate for a source +// route and group. Partition-resolved keyspaces such as HT-FIFO SQS are matched +// by resolver group instead of the byte-range route key. +func RouteKeyFilterForGroup(rangeStart, rangeEnd []byte, sourceGroupID uint64, resolver PartitionResolver) func([]byte) bool { start := bytes.Clone(rangeStart) end := bytes.Clone(rangeEnd) return func(rawKey []byte) bool { + if resolver != nil { + if gid, ok := resolver.ResolveGroup(rawKey); ok { + return gid == sourceGroupID + } + if resolver.RecognisesPartitionedKey(rawKey) { + return false + } + } if s3BucketAuxiliaryRouteInRange(rawKey, start, end) { return true } rkey := routeKey(rawKey) - if bytes.Compare(rkey, start) < 0 { - return false - } - if len(end) > 0 && bytes.Compare(rkey, end) >= 0 { - return false - } - return true + return keyInMigrationRouteRange(rkey, start, end) } } @@ -35,10 +44,23 @@ func s3BucketAuxiliaryRouteInRange(rawKey, routeStart, routeEnd []byte) bool { if !ok { return false } + if keyInMigrationRouteRange(rawKey, routeStart, routeEnd) { + return true + } bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) return migrationRouteRangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) } +func keyInMigrationRouteRange(key, routeStart, routeEnd []byte) bool { + if key == nil { + return false + } + if bytes.Compare(key, routeStart) < 0 { + return false + } + return len(routeEnd) == 0 || bytes.Compare(key, routeEnd) < 0 +} + func migrationRouteRangesIntersect(aStart, aEnd, bStart, bEnd []byte) bool { if len(aEnd) > 0 && bytes.Compare(aEnd, bStart) <= 0 { return false diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 93050bfd1..7c0589dcf 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -1,6 +1,7 @@ package kv import ( + "bytes" "encoding/base64" "encoding/binary" "testing" @@ -269,3 +270,40 @@ func TestRouteKeyFilterIncludesS3BucketAuxiliaryKeys(t *testing.T) { require.False(t, filter(s3keys.BucketMetaKey("bucket-c"))) require.False(t, filter(s3keys.BucketGenerationKey("bucket-c"))) } + +func TestRouteKeyFilterIncludesS3BucketAuxiliaryRawRoute(t *testing.T) { + t.Parallel() + + filter := RouteKeyFilter([]byte("!s3|"), nil) + + require.True(t, filter(s3keys.BucketMetaKey("bucket-b"))) + require.True(t, filter(s3keys.BucketGenerationKey("bucket-b"))) +} + +func TestRouteKeyFilterForGroupUsesPartitionResolver(t *testing.T) { + t.Parallel() + + partitionedKey := []byte(sqsMsgDataPrefix + sqsPartitionMarker + "orders|partition-0|message") + resolver := &migrationFilterPartitionResolver{ + groups: map[string]uint64{string(partitionedKey): 42}, + } + + require.True(t, RouteKeyFilterForGroup(nil, nil, 42, resolver)(partitionedKey)) + require.False(t, RouteKeyFilterForGroup(nil, nil, 7, resolver)(partitionedKey)) + require.False(t, RouteKeyFilterForGroup(nil, nil, 42, resolver)( + []byte(sqsMsgDataPrefix+sqsPartitionMarker+"orders|unknown-partition"), + )) +} + +type migrationFilterPartitionResolver struct { + groups map[string]uint64 +} + +func (r *migrationFilterPartitionResolver) ResolveGroup(key []byte) (uint64, bool) { + gid, ok := r.groups[string(key)] + return gid, ok +} + +func (r *migrationFilterPartitionResolver) RecognisesPartitionedKey(key []byte) bool { + return bytes.HasPrefix(key, []byte(sqsMsgDataPrefix+sqsPartitionMarker)) +} diff --git a/kv/shard_store.go b/kv/shard_store.go index 175c81224..dbfab104c 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -260,9 +260,9 @@ func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Rou if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { return s.engine.GetIntersectingRoutes(routeStart, routeEnd), false } - // For internal list keys, shard routing is based on the logical user key - // rather than the raw key prefix. - if userKey := store.ExtractListUserKey(start); userKey != nil { + // For internal wide-column keys, shard routing is based on the logical + // user key rather than the raw key prefix. + if userKey := scanRouteUserKey(start); userKey != nil { route, ok := s.engine.GetRoute(userKey) if !ok { return []distribution.Route{}, false @@ -281,6 +281,30 @@ func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Rou return routes, true } +func scanRouteUserKey(start []byte) []byte { + for _, extract := range scanRouteUserKeyExtractors { + if userKey := extract(start); userKey != nil { + return userKey + } + } + return nil +} + +var scanRouteUserKeyExtractors = []func([]byte) []byte{ + store.ExtractListUserKey, + store.ExtractListUserKeyFromDeltaScanPrefix, + store.ExtractLegacyListUserKeyFromDeltaScanPrefix, + store.ExtractListUserKeyFromClaimScanPrefix, + store.ExtractHashUserKeyFromField, + store.ExtractHashUserKeyFromDeltaScanPrefix, + store.ExtractSetUserKeyFromMember, + store.ExtractSetUserKeyFromDeltaScanPrefix, + store.ExtractZSetUserKeyFromMember, + store.ExtractZSetUserKeyFromScore, + store.ExtractZSetUserKeyFromScoreScanPrefix, + store.ExtractZSetUserKeyFromDeltaScanPrefix, +} + func (s *ShardStore) scanRoutesAt(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) for _, route := range routes { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 4f0da38b7..14ee7ce67 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -71,6 +71,65 @@ func TestShardStoreScanAt_RoutesListItemScansByUserKey(t *testing.T) { require.Equal(t, k2, kvs[2].Key) } +func TestShardStoreScanAt_RoutesListDeltaScansByUserKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := newTwoRouteShardStoreForScanTest() + userKey := []byte("x") // routes to group 2; raw !lst|delta| prefix would route to group 1. + deltaKey := store.ListMetaDeltaKey(userKey, 10, 1) + deltaValue := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) + require.NoError(t, st.PutAt(ctx, deltaKey, deltaValue, 1, 0)) + + start := store.ListMetaDeltaScanPrefix(userKey) + kvs, err := st.ScanAt(ctx, start, store.PrefixScanEnd(start), 10, ^uint64(0)) + require.NoError(t, err) + require.Len(t, kvs, 1) + require.Equal(t, deltaKey, kvs[0].Key) +} + +func TestShardStoreScanAt_RoutesWideColumnScansByUserKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + key []byte + scanStart []byte + }{ + {name: "hash field", key: store.HashFieldKey([]byte("x"), []byte("f")), scanStart: store.HashFieldScanPrefix([]byte("x"))}, + {name: "hash delta", key: store.HashMetaDeltaKey([]byte("x"), 10, 0), scanStart: store.HashMetaDeltaScanPrefix([]byte("x"))}, + {name: "set member", key: store.SetMemberKey([]byte("x"), []byte("m")), scanStart: store.SetMemberScanPrefix([]byte("x"))}, + {name: "set delta", key: store.SetMetaDeltaKey([]byte("x"), 10, 0), scanStart: store.SetMetaDeltaScanPrefix([]byte("x"))}, + {name: "zset member", key: store.ZSetMemberKey([]byte("x"), []byte("m")), scanStart: store.ZSetMemberScanPrefix([]byte("x"))}, + {name: "zset score", key: store.ZSetScoreKey([]byte("x"), 1.5, []byte("m")), scanStart: store.ZSetScoreScanPrefix([]byte("x"))}, + {name: "zset delta", key: store.ZSetMetaDeltaKey([]byte("x"), 10, 0), scanStart: store.ZSetMetaDeltaScanPrefix([]byte("x"))}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + st := newTwoRouteShardStoreForScanTest() + require.NoError(t, st.PutAt(ctx, tc.key, []byte("v"), 1, 0)) + + kvs, err := st.ScanAt(ctx, tc.scanStart, store.PrefixScanEnd(tc.scanStart), 10, ^uint64(0)) + require.NoError(t, err) + require.Len(t, kvs, 1) + require.Equal(t, tc.key, kvs[0].Key) + }) + } +} + +func newTwoRouteShardStoreForScanTest() *ShardStore { + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), []byte("m"), 1) + engine.UpdateRoute([]byte("m"), nil, 2) + + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + return NewShardStore(engine, groups) +} + func TestShardStoreScanGroupAt_UsesExplicitGroup(t *testing.T) { t.Parallel() diff --git a/store/hash_helpers.go b/store/hash_helpers.go index 56c3268d5..4dc773016 100644 --- a/store/hash_helpers.go +++ b/store/hash_helpers.go @@ -162,3 +162,9 @@ func ExtractHashUserKeyFromField(key []byte) []byte { func ExtractHashUserKeyFromDelta(key []byte) []byte { return extractWideColumnUserKey(key, []byte(HashMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) } + +// ExtractHashUserKeyFromDeltaScanPrefix extracts the user key from a hash +// metadata delta scan start/prefix. +func ExtractHashUserKeyFromDeltaScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(HashMetaDeltaPrefix), 0, false) +} diff --git a/store/list_helpers.go b/store/list_helpers.go index 896ae230a..5e4b4d274 100644 --- a/store/list_helpers.go +++ b/store/list_helpers.go @@ -14,6 +14,11 @@ const ( // Layout: !lst|delta| ListMetaDeltaPrefix = "!lst|delta|" + // LegacyListMetaDeltaPrefix is the pre-upgrade list metadata delta prefix. + // Writers use ListMetaDeltaPrefix, but readers and compactors keep scanning + // this prefix until old uncompacted deltas have been drained. + LegacyListMetaDeltaPrefix = "!lst|meta|d|" + // ListClaimPrefix is the prefix for list claim keys used by POP operations. // Layout: !lst|claim| ListClaimPrefix = "!lst|claim|" @@ -85,8 +90,27 @@ func ListMetaDeltaKey(userKey []byte, commitTS uint64, seqInTxn uint32) []byte { // ListMetaDeltaScanPrefix returns the prefix used to scan all delta keys for a userKey. func ListMetaDeltaScanPrefix(userKey []byte) []byte { - buf := make([]byte, 0, len(ListMetaDeltaPrefix)+wideColKeyLenSize+len(userKey)) - buf = append(buf, ListMetaDeltaPrefix...) + return listMetaDeltaScanPrefixFor(ListMetaDeltaPrefix, userKey) +} + +// LegacyListMetaDeltaScanPrefix returns the pre-upgrade delta scan prefix for a userKey. +func LegacyListMetaDeltaScanPrefix(userKey []byte) []byte { + return listMetaDeltaScanPrefixFor(LegacyListMetaDeltaPrefix, userKey) +} + +// ListMetaDeltaScanPrefixes returns all prefixes that may contain visible list +// metadata deltas for userKey. New writers emit only ListMetaDeltaPrefix, but +// upgrade reads must include LegacyListMetaDeltaPrefix until compaction drains it. +func ListMetaDeltaScanPrefixes(userKey []byte) [][]byte { + return [][]byte{ + ListMetaDeltaScanPrefix(userKey), + LegacyListMetaDeltaScanPrefix(userKey), + } +} + +func listMetaDeltaScanPrefixFor(prefix string, userKey []byte) []byte { + buf := make([]byte, 0, len(prefix)+wideColKeyLenSize+len(userKey)) + buf = append(buf, prefix...) var kl [4]byte binary.BigEndian.PutUint32(kl[:], uint32(len(userKey))) //nolint:gosec // len is bounded by max slice size buf = append(buf, kl[:]...) @@ -134,11 +158,42 @@ func ExtractListUserKeyFromDelta(key []byte) []byte { return extractWideColumnUserKey(key, []byte(ListMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) } +// ExtractLegacyListUserKeyFromDelta extracts the user key from an old +// !lst|meta|d| delta key. Callers that only have a key must be careful: this +// legacy layout overlaps with base !lst|meta| keys whose user key begins with +// d|. Prefer using it when the associated value is known to be a delta value. +func ExtractLegacyListUserKeyFromDelta(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(LegacyListMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) +} + +// ExtractListUserKeyFromDeltaScanPrefix extracts the user key from a new-list +// delta scan start/prefix. +func ExtractListUserKeyFromDeltaScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(ListMetaDeltaPrefix), 0, false) +} + +// ExtractLegacyListUserKeyFromDeltaScanPrefix extracts the user key from a +// legacy-list delta scan start/prefix. +func ExtractLegacyListUserKeyFromDeltaScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(LegacyListMetaDeltaPrefix), 0, false) +} + // ExtractListUserKeyFromClaim extracts the logical user key from a list claim key. func ExtractListUserKeyFromClaim(key []byte) []byte { return extractWideColumnUserKey(key, []byte(ListClaimPrefix), sortableInt64Bytes, true) } +// ExtractListUserKeyFromClaimScanPrefix extracts the user key from a list claim +// scan start/prefix. +func ExtractListUserKeyFromClaimScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(ListClaimPrefix), 0, false) +} + +// IsListMetaDeltaValue reports whether value has the fixed delta encoding. +func IsListMetaDeltaValue(value []byte) bool { + return len(value) == listDeltaSizeBytes +} + func extractWideColumnUserKey(key, prefix []byte, suffixLen uint64, exactLen bool) []byte { if !bytes.HasPrefix(key, prefix) { return nil diff --git a/store/list_helpers_test.go b/store/list_helpers_test.go index efd8c8559..dc2916f38 100644 --- a/store/list_helpers_test.go +++ b/store/list_helpers_test.go @@ -46,6 +46,23 @@ func TestListMetaDeltaPrefixDoesNotOverlapBaseMetaKeys(t *testing.T) { require.Equal(t, userKey, ExtractListUserKeyFromDelta(deltaKey)) } +func TestLegacyListMetaDeltaHelpersScanOldPrefixWithoutReclassifyingNewDelta(t *testing.T) { + t.Parallel() + + userKey := []byte("list") + legacyPrefix := LegacyListMetaDeltaScanPrefix(userKey) + legacyKey := append(append([]byte(nil), legacyPrefix...), make([]byte, deltaKeyTSSize+deltaKeySeqSize)...) + + require.Equal(t, userKey, ExtractLegacyListUserKeyFromDeltaScanPrefix(legacyPrefix)) + require.Equal(t, userKey, ExtractLegacyListUserKeyFromDelta(legacyKey)) + require.False(t, IsListMetaDeltaKey(legacyKey), "legacy prefix is ambiguous with base !lst|meta| keys") + require.True(t, IsListMetaDeltaValue(MarshalListMetaDelta(ListMetaDelta{HeadDelta: 1, LenDelta: 2}))) + + metaValue, err := MarshalListMeta(ListMeta{Head: 1, Tail: 2, Len: 1}) + require.NoError(t, err) + require.False(t, IsListMetaDeltaValue(metaValue)) +} + func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { buf := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) buf = append(buf, "d|"...) diff --git a/store/set_helpers.go b/store/set_helpers.go index ff8822f52..1f911769d 100644 --- a/store/set_helpers.go +++ b/store/set_helpers.go @@ -162,3 +162,9 @@ func ExtractSetUserKeyFromMember(key []byte) []byte { func ExtractSetUserKeyFromDelta(key []byte) []byte { return extractWideColumnUserKey(key, []byte(SetMetaDeltaPrefix), deltaKeyTSSize+deltaKeySeqSize, true) } + +// ExtractSetUserKeyFromDeltaScanPrefix extracts the user key from a set +// metadata delta scan start/prefix. +func ExtractSetUserKeyFromDeltaScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(SetMetaDeltaPrefix), 0, false) +} diff --git a/store/zset_helpers.go b/store/zset_helpers.go index 8bed4b8a0..618490479 100644 --- a/store/zset_helpers.go +++ b/store/zset_helpers.go @@ -281,3 +281,15 @@ func ExtractZSetUserKeyFromMember(key []byte) []byte { func ExtractZSetUserKeyFromScore(key []byte) []byte { return extractWideColumnUserKey(key, []byte(ZSetScorePrefix), zsetMetaSizeBytes, false) } + +// ExtractZSetUserKeyFromScoreScanPrefix extracts the user key from a zset +// score-index scan start/prefix that does not yet include the sortable score. +func ExtractZSetUserKeyFromScoreScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(ZSetScorePrefix), 0, false) +} + +// ExtractZSetUserKeyFromDeltaScanPrefix extracts the user key from a zset +// metadata delta scan start/prefix. +func ExtractZSetUserKeyFromDeltaScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, []byte(ZSetMetaDeltaPrefix), 0, false) +} From a653226303e2111412b02099c7361adf9316153d Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 13:37:42 +0900 Subject: [PATCH 064/162] Fence routed migration exports --- kv/leader_routed_store.go | 5 +++++ kv/leader_routed_store_test.go | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/kv/leader_routed_store.go b/kv/leader_routed_store.go index 3f6004242..f1dc32cd5 100644 --- a/kv/leader_routed_store.go +++ b/kv/leader_routed_store.go @@ -507,6 +507,11 @@ func (s *LeaderRoutedStore) ExportVersions(ctx context.Context, opts store.Expor if s == nil || s.local == nil { return store.ExportVersionsResult{}, errors.WithStack(store.ErrNotSupported) } + if s.coordinator != nil { + if _, err := s.coordinator.LinearizableRead(ctx); err != nil { + return store.ExportVersionsResult{}, errors.WithStack(err) + } + } result, err := s.local.ExportVersions(ctx, opts) return result, errors.WithStack(err) } diff --git a/kv/leader_routed_store_test.go b/kv/leader_routed_store_test.go index 1d83ead89..ae8c4498f 100644 --- a/kv/leader_routed_store_test.go +++ b/kv/leader_routed_store_test.go @@ -320,3 +320,39 @@ func TestLeaderRoutedStore_GlobalLastCommitTS_FallsBackWhenNoLeader(t *testing.T ts := s.GlobalLastCommitTS(ctx) require.Equal(t, uint64(7), ts) } + +func TestLeaderRoutedStore_ExportVersionsUsesLinearizableFence(t *testing.T) { + t.Parallel() + + ctx := context.Background() + local := store.NewMVCCStore() + require.NoError(t, local.PutAt(ctx, []byte("k"), []byte("v"), 10, 0)) + + coord := &stubLeaderCoordinator{isLeader: true, clock: NewHLC()} + s := NewLeaderRoutedStore(local, coord) + t.Cleanup(func() { _ = s.Close() }) + + result, err := s.ExportVersions(ctx, store.ExportVersionsOptions{MaxVersions: 10}) + require.NoError(t, err) + require.Len(t, result.Versions, 1) + require.Equal(t, []byte("k"), result.Versions[0].Key) + require.Equal(t, uint64(10), result.Versions[0].CommitTS) + require.Equal(t, 1, coord.linearizableCalls) +} + +func TestLeaderRoutedStore_ExportVersionsFailsClosedWithoutFence(t *testing.T) { + t.Parallel() + + ctx := context.Background() + local := store.NewMVCCStore() + require.NoError(t, local.PutAt(ctx, []byte("k"), []byte("v"), 10, 0)) + + coord := &stubLeaderCoordinator{isLeader: false, clock: NewHLC()} + s := NewLeaderRoutedStore(local, coord) + t.Cleanup(func() { _ = s.Close() }) + + result, err := s.ExportVersions(ctx, store.ExportVersionsOptions{MaxVersions: 10}) + require.ErrorIs(t, err, ErrLeaderNotFound) + require.Empty(t, result.Versions) + require.Equal(t, 1, coord.linearizableCalls) +} From e7467d581dbfc42c751656e998513987a9bfa30d Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 13:55:01 +0900 Subject: [PATCH 065/162] Route legacy list deltas and stream scans --- adapter/redis_compat_helpers.go | 104 ++++++++++++++++++++------ adapter/redis_delta_compactor_test.go | 4 +- adapter/redis_list_dedup_test.go | 92 +++++++++++++++++++++-- kv/shard_key.go | 16 ++-- kv/shard_key_test.go | 23 +++--- kv/shard_store.go | 4 +- kv/shard_store_test.go | 34 ++++++--- store/stream_helpers.go | 12 +++ store/stream_helpers_test.go | 23 ++++++ 9 files changed, 252 insertions(+), 60 deletions(-) diff --git a/adapter/redis_compat_helpers.go b/adapter/redis_compat_helpers.go index 486cbdea5..b6955cbc1 100644 --- a/adapter/redis_compat_helpers.go +++ b/adapter/redis_compat_helpers.go @@ -267,16 +267,33 @@ func (r *RedisServer) probeListType(ctx context.Context, key []byte, readTS uint if metaExists { return redisTypeList, true, nil } - deltaPrefix := store.ListMetaDeltaScanPrefix(key) + for _, deltaPrefix := range store.ListMetaDeltaScanPrefixes(key) { + found, err := r.listMetaDeltaExistsAt(ctx, key, deltaPrefix, readTS) + if err != nil { + return redisTypeNone, false, err + } + if found { + return redisTypeList, true, nil + } + } + return redisTypeNone, false, nil +} + +func (r *RedisServer) listMetaDeltaExistsAt(ctx context.Context, key []byte, deltaPrefix []byte, readTS uint64) (bool, error) { deltaEnd := store.PrefixScanEnd(deltaPrefix) - deltaKVs, err := r.store.ScanAt(ctx, deltaPrefix, deltaEnd, 1, readTS) + deltaKVs, err := r.store.ScanAt(ctx, deltaPrefix, deltaEnd, store.MaxDeltaScanLimit, readTS) if err != nil { - return redisTypeNone, false, errors.WithStack(err) + return false, errors.WithStack(err) } - if len(deltaKVs) > 0 { - return redisTypeList, true, nil + if !isLegacyListMetaDeltaPrefix(deltaPrefix) { + return len(deltaKVs) > 0, nil } - return redisTypeNone, false, nil + for _, pair := range deltaKVs { + if legacyListDeltaPairForUserKey(pair, key) { + return true, nil + } + } + return false, nil } // probeLegacyCollectionTypes checks for single-blob hash/set/zset/stream @@ -927,7 +944,7 @@ func (r *RedisServer) deleteListElems(ctx context.Context, key []byte, readTS ui elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: listMetaKey(key)}) // Delete all delta keys (paginated), including the pre-upgrade prefix. for _, deltaPrefix := range store.ListMetaDeltaScanPrefixes(key) { - deltaElems, err := r.scanAllDeltaElems(ctx, deltaPrefix, readTS) + deltaElems, err := r.scanListDeltaDelElems(ctx, key, deltaPrefix, readTS) if err != nil { return nil, err } @@ -979,6 +996,24 @@ func (r *RedisServer) scanListItemDelElems(ctx context.Context, key []byte, read // MaxDeltaScanLimit entries. Total results are capped at maxWideColumnItems // to prevent unbounded memory growth if the compactor falls behind. func (r *RedisServer) scanAllDeltaElems(ctx context.Context, deltaPrefix []byte, readTS uint64) ([]*kv.Elem[kv.OP], error) { + return r.scanAllDeltaElemsFiltered(ctx, deltaPrefix, readTS, nil) +} + +func (r *RedisServer) scanListDeltaDelElems(ctx context.Context, key []byte, deltaPrefix []byte, readTS uint64) ([]*kv.Elem[kv.OP], error) { + if !isLegacyListMetaDeltaPrefix(deltaPrefix) { + return r.scanAllDeltaElems(ctx, deltaPrefix, readTS) + } + return r.scanAllDeltaElemsFiltered(ctx, deltaPrefix, readTS, func(pair *store.KVPair) bool { + return legacyListDeltaPairForUserKey(pair, key) + }) +} + +func (r *RedisServer) scanAllDeltaElemsFiltered( + ctx context.Context, + deltaPrefix []byte, + readTS uint64, + include func(*store.KVPair) bool, +) ([]*kv.Elem[kv.OP], error) { const cursorAdv = byte(0x00) // appended to advance past the last scanned key var elems []*kv.Elem[kv.OP] deltaEnd := store.PrefixScanEnd(deltaPrefix) @@ -988,12 +1023,14 @@ func (r *RedisServer) scanAllDeltaElems(ctx context.Context, deltaPrefix []byte, if scanErr != nil { return nil, errors.WithStack(scanErr) } - // Check before appending so len(elems) never exceeds maxWideColumnItems - // by more than one scan page. - if len(elems)+len(deltaKVs) > maxWideColumnItems { - return nil, errors.Wrapf(ErrCollectionTooLarge, "delta key count exceeds %d", maxWideColumnItems) - } for _, pair := range deltaKVs { + if include != nil && !include(pair) { + continue + } + // Check before appending so len(elems) never exceeds maxWideColumnItems. + if len(elems)+1 > maxWideColumnItems { + return nil, errors.Wrapf(ErrCollectionTooLarge, "delta key count exceeds %d", maxWideColumnItems) + } elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: pair.Key}) } if len(deltaKVs) < store.MaxDeltaScanLimit { @@ -1004,6 +1041,17 @@ func (r *RedisServer) scanAllDeltaElems(ctx context.Context, deltaPrefix []byte, return elems, nil } +func isLegacyListMetaDeltaPrefix(prefix []byte) bool { + return bytes.HasPrefix(prefix, []byte(store.LegacyListMetaDeltaPrefix)) +} + +func legacyListDeltaPairForUserKey(pair *store.KVPair, userKey []byte) bool { + if pair == nil || !store.IsListMetaDeltaValue(pair.Value) { + return false + } + return bytes.Equal(store.ExtractLegacyListUserKeyFromDelta(pair.Key), userKey) +} + // deleteWideColumnElems returns delete operations for all wide-column field/member keys, // the base meta key, and all delta keys for a collection identified by the given scan prefix, // meta key, and delta prefix. @@ -1262,7 +1310,12 @@ func minRedisInt(a, b int) int { // aggregateLenDeltas scans delta keys under prefix and sums the LenDelta values // via unmarshalDelta. Returns (sum, hasDeltas, error). // ErrDeltaScanTruncated is returned when the scan hits MaxDeltaScanLimit. -func (r *RedisServer) aggregateLenDeltas(ctx context.Context, prefix []byte, readTS uint64, unmarshalDelta func([]byte) (int64, error)) (int64, bool, error) { +func (r *RedisServer) aggregateLenDeltas( + ctx context.Context, + prefix []byte, + readTS uint64, + unmarshalDelta func(key []byte, value []byte) (int64, bool, error), +) (int64, bool, error) { end := store.PrefixScanEnd(prefix) // Scan one extra key beyond the limit so we can distinguish "exactly // MaxDeltaScanLimit results" (no truncation) from "more than MaxDeltaScanLimit @@ -1276,30 +1329,36 @@ func (r *RedisServer) aggregateLenDeltas(ctx context.Context, prefix []byte, rea return 0, false, ErrDeltaScanTruncated } var sum int64 + var any bool for _, d := range deltas { - delta, err := unmarshalDelta(d.Value) + delta, include, err := unmarshalDelta(d.Key, d.Value) if err != nil { return 0, false, errors.WithStack(err) } + if !include { + continue + } + any = true sum += delta } - return sum, len(deltas) > 0, nil + return sum, any, nil } func (r *RedisServer) aggregateListMetaDeltas(ctx context.Context, key []byte, readTS uint64, applyDelta func(store.ListMetaDelta)) (int64, bool, error) { var total int64 var any bool for _, prefix := range store.ListMetaDeltaScanPrefixes(key) { - lenSum, hasDeltas, err := r.aggregateLenDeltas(ctx, prefix, readTS, func(b []byte) (int64, error) { - if !store.IsListMetaDeltaValue(b) { - return 0, nil + legacy := isLegacyListMetaDeltaPrefix(prefix) + lenSum, hasDeltas, err := r.aggregateLenDeltas(ctx, prefix, readTS, func(deltaKey []byte, b []byte) (int64, bool, error) { + if legacy && (!store.IsListMetaDeltaValue(b) || !bytes.Equal(store.ExtractLegacyListUserKeyFromDelta(deltaKey), key)) { + return 0, false, nil } d, unmarshalErr := store.UnmarshalListMetaDelta(b) if unmarshalErr != nil { - return 0, errors.WithStack(unmarshalErr) + return 0, false, errors.WithStack(unmarshalErr) } applyDelta(d) - return d.LenDelta, nil + return d.LenDelta, true, nil }) if err != nil { return 0, false, err @@ -1369,7 +1428,10 @@ func (r *RedisServer) resolveCollectionLen( } } - deltaSum, hasDeltas, err := r.aggregateLenDeltas(ctx, deltaPrefix, readTS, unmarshalDelta) + deltaSum, hasDeltas, err := r.aggregateLenDeltas(ctx, deltaPrefix, readTS, func(_ []byte, value []byte) (int64, bool, error) { + delta, err := unmarshalDelta(value) + return delta, true, err + }) if err != nil { return 0, false, err } diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index e7c05a75c..f519de6d4 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -162,8 +162,8 @@ func TestDeltaCompactor_LegacyListDeltaFoldedIntoBaseMeta(t *testing.T) { require.NoError(t, st.PutAt(ctx, store.ListMetaKey(userKey), metaBytes, 1, 0)) delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: -1, LenDelta: 2}) - d1Key := legacyListMetaDeltaKey(userKey, 10, 0) - d2Key := legacyListMetaDeltaKey(userKey, 11, 0) + d1Key := legacyListMetaDeltaKey(userKey, 10) + d2Key := legacyListMetaDeltaKey(userKey, 11) require.NoError(t, st.PutAt(ctx, d1Key, delta, 10, 0)) require.NoError(t, st.PutAt(ctx, d2Key, delta, 11, 0)) diff --git a/adapter/redis_list_dedup_test.go b/adapter/redis_list_dedup_test.go index 37ef7cb56..548cb27ce 100644 --- a/adapter/redis_list_dedup_test.go +++ b/adapter/redis_list_dedup_test.go @@ -66,7 +66,7 @@ func TestResolveListMetaReadsLegacyDeltaPrefix(t *testing.T) { require.NoError(t, err) require.NoError(t, st.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: -1, LenDelta: 3}) - require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, 2, 0), delta, 2, 0)) + require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, 2), delta, 2, 0)) meta, exists, err := srv.resolveListMeta(ctx, key, 3) require.NoError(t, err) @@ -82,7 +82,7 @@ func TestResolveListMetaDoesNotTreatDeltaLookingMetaValueAsLegacyDelta(t *testin ctx := context.Background() st := store.NewMVCCStore() srv := &RedisServer{store: st} - key := deltaLookingListMetaUserKey([]byte("embedded"), 7, 1) + key := deltaLookingListMetaUserKey([]byte("embedded")) base, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) require.NoError(t, err) require.NoError(t, st.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) @@ -95,17 +95,95 @@ func TestResolveListMetaDoesNotTreatDeltaLookingMetaValueAsLegacyDelta(t *testin require.Equal(t, int64(6), meta.Tail) } -func legacyListMetaDeltaKey(userKey []byte, commitTS uint64, seqInTxn uint32) []byte { +func TestResolveListMetaIgnoresLegacyDeltaPrefixCollisionForMissingKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := []byte("legacy-collision") + collidingUserKey := deltaLookingListMetaUserKey(key) + base, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(collidingUserKey), base, 1, 0)) + + _, exists, err := srv.resolveListMeta(ctx, key, 2) + require.NoError(t, err) + require.False(t, exists) +} + +func TestProbeListTypeReadsLegacyDeltaPrefix(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := []byte("legacy-list-type") + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) + require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, 2), delta, 2, 0)) + + typ, found, err := srv.probeListType(ctx, key, 3) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, redisTypeList, typ) +} + +func TestProbeListTypeIgnoresLegacyDeltaPrefixCollision(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := []byte("legacy-list-type-collision") + collidingUserKey := deltaLookingListMetaUserKey(key) + base, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(collidingUserKey), base, 1, 0)) + + _, found, err := srv.probeListType(ctx, key, 2) + require.NoError(t, err) + require.False(t, found) +} + +func TestDeleteListElemsFiltersLegacyDeltaPrefixCollisions(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := []byte("legacy-list-delete") + collidingUserKey := deltaLookingListMetaUserKey(key) + base, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + collidingMetaKey := store.ListMetaKey(collidingUserKey) + require.NoError(t, st.PutAt(ctx, collidingMetaKey, base, 1, 0)) + deltaKey := legacyListMetaDeltaKey(key, 2) + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) + require.NoError(t, st.PutAt(ctx, deltaKey, delta, 2, 0)) + + elems, err := srv.deleteListElems(ctx, key, 3) + require.NoError(t, err) + deleted := make(map[string]struct{}, len(elems)) + for _, elem := range elems { + if elem.Op == kv.Del { + deleted[string(elem.Key)] = struct{}{} + } + } + require.Contains(t, deleted, string(deltaKey)) + require.NotContains(t, deleted, string(collidingMetaKey)) +} + +func legacyListMetaDeltaKey(userKey []byte, commitTS uint64) []byte { key := store.LegacyListMetaDeltaScanPrefix(userKey) var ts [8]byte binary.BigEndian.PutUint64(ts[:], commitTS) key = append(key, ts[:]...) var seq [4]byte - binary.BigEndian.PutUint32(seq[:], seqInTxn) + binary.BigEndian.PutUint32(seq[:], 0) return append(key, seq[:]...) } -func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { +func deltaLookingListMetaUserKey(fakeUserKey []byte) []byte { key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) key = append(key, "d|"...) var lenPrefix [4]byte @@ -113,10 +191,10 @@ func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn u key = append(key, lenPrefix[:]...) key = append(key, fakeUserKey...) var ts [8]byte - binary.BigEndian.PutUint64(ts[:], commitTS) + binary.BigEndian.PutUint64(ts[:], 7) key = append(key, ts[:]...) var seq [4]byte - binary.BigEndian.PutUint32(seq[:], seqInTxn) + binary.BigEndian.PutUint32(seq[:], 1) return append(key, seq[:]...) } diff --git a/kv/shard_key.go b/kv/shard_key.go index deb5fe468..ee38945a4 100644 --- a/kv/shard_key.go +++ b/kv/shard_key.go @@ -149,14 +149,16 @@ func dynamoRouteTableKey(tableSegment []byte) []byte { } func listRouteKey(key []byte) []byte { - switch { - case store.IsListMetaDeltaKey(key): - return store.ExtractListUserKeyFromDelta(key) - case store.IsListClaimKey(key): - return store.ExtractListUserKeyFromClaim(key) - default: - return nil + if userKey := store.ExtractListUserKeyFromDelta(key); userKey != nil { + return userKey + } + if userKey := store.ExtractLegacyListUserKeyFromDelta(key); userKey != nil { + return userKey + } + if userKey := store.ExtractListUserKeyFromClaim(key); userKey != nil { + return userKey } + return nil } func hashRouteKey(key []byte) []byte { diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 7c0589dcf..733ecac4a 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -125,18 +125,24 @@ func TestRouteKey_NormalizesCollectionMigrationFamilies(t *testing.T) { } } -func TestRouteKey_ListMetaKeyThatLooksLikeDeltaRoutesByRealListKey(t *testing.T) { +func TestRouteKey_ListMetaKeyThatLooksLikeNewDeltaRoutesByRealListKey(t *testing.T) { t.Parallel() - fakeUserKey := []byte("fake:user") - userKey := deltaLookingListMetaUserKey(fakeUserKey, 42, 7) + userKey := []byte(store.ListMetaDeltaPrefix + "fake:user") baseMeta := store.ListMetaKey(userKey) deltaKey := store.ListMetaDeltaKey(userKey, 10, 1) - require.Equal(t, userKey, routeKey(baseMeta), "base list metadata must not decode as a delta for %q", fakeUserKey) + require.Equal(t, userKey, routeKey(baseMeta), "base list metadata must not decode as a new delta") require.Equal(t, userKey, routeKey(deltaKey), "real list deltas must still route by the logical list key") } +func TestRouteKey_LegacyListDeltaRoutesByDecodedUserKey(t *testing.T) { + t.Parallel() + + userKey := []byte("legacy:list") + require.Equal(t, userKey, routeKey(legacyListMetaDeltaKey(userKey, 42, 7))) +} + func TestRouteKey_MalformedWideColumnKeysFallBackToRaw(t *testing.T) { t.Parallel() @@ -169,13 +175,8 @@ func malformedWideColumnKey(prefix string, suffixLen int) []byte { return key } -func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { - key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) - key = append(key, "d|"...) - var lenPrefix [4]byte - binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. - key = append(key, lenPrefix[:]...) - key = append(key, fakeUserKey...) +func legacyListMetaDeltaKey(userKey []byte, commitTS uint64, seqInTxn uint32) []byte { + key := store.LegacyListMetaDeltaScanPrefix(userKey) var ts [8]byte binary.BigEndian.PutUint64(ts[:], commitTS) key = append(key, ts[:]...) diff --git a/kv/shard_store.go b/kv/shard_store.go index dbfab104c..62fe0a79f 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -291,9 +291,9 @@ func scanRouteUserKey(start []byte) []byte { } var scanRouteUserKeyExtractors = []func([]byte) []byte{ - store.ExtractListUserKey, store.ExtractListUserKeyFromDeltaScanPrefix, store.ExtractLegacyListUserKeyFromDeltaScanPrefix, + store.ExtractListUserKey, store.ExtractListUserKeyFromClaimScanPrefix, store.ExtractHashUserKeyFromField, store.ExtractHashUserKeyFromDeltaScanPrefix, @@ -303,6 +303,8 @@ var scanRouteUserKeyExtractors = []func([]byte) []byte{ store.ExtractZSetUserKeyFromScore, store.ExtractZSetUserKeyFromScoreScanPrefix, store.ExtractZSetUserKeyFromDeltaScanPrefix, + store.ExtractStreamUserKeyFromMeta, + store.ExtractStreamUserKeyFromEntryScanPrefix, } func (s *ShardStore) scanRoutesAt(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool) ([]*store.KVPair, error) { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 14ee7ce67..75741bcdb 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -75,17 +75,27 @@ func TestShardStoreScanAt_RoutesListDeltaScansByUserKey(t *testing.T) { t.Parallel() ctx := context.Background() - st := newTwoRouteShardStoreForScanTest() - userKey := []byte("x") // routes to group 2; raw !lst|delta| prefix would route to group 1. - deltaKey := store.ListMetaDeltaKey(userKey, 10, 1) - deltaValue := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) - require.NoError(t, st.PutAt(ctx, deltaKey, deltaValue, 1, 0)) - - start := store.ListMetaDeltaScanPrefix(userKey) - kvs, err := st.ScanAt(ctx, start, store.PrefixScanEnd(start), 10, ^uint64(0)) - require.NoError(t, err) - require.Len(t, kvs, 1) - require.Equal(t, deltaKey, kvs[0].Key) + userKey := []byte("x") // routes to group 2; raw !lst|* prefixes route to group 1. + for _, tc := range []struct { + name string + key []byte + scanStart []byte + }{ + {name: "current", key: store.ListMetaDeltaKey(userKey, 10, 1), scanStart: store.ListMetaDeltaScanPrefix(userKey)}, + {name: "legacy", key: legacyListMetaDeltaKey(userKey, 10, 1), scanStart: store.LegacyListMetaDeltaScanPrefix(userKey)}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + st := newTwoRouteShardStoreForScanTest() + deltaValue := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) + require.NoError(t, st.PutAt(ctx, tc.key, deltaValue, 1, 0)) + + kvs, err := st.ScanAt(ctx, tc.scanStart, store.PrefixScanEnd(tc.scanStart), 10, ^uint64(0)) + require.NoError(t, err) + require.Len(t, kvs, 1) + require.Equal(t, tc.key, kvs[0].Key) + }) + } } func TestShardStoreScanAt_RoutesWideColumnScansByUserKey(t *testing.T) { @@ -104,6 +114,8 @@ func TestShardStoreScanAt_RoutesWideColumnScansByUserKey(t *testing.T) { {name: "zset member", key: store.ZSetMemberKey([]byte("x"), []byte("m")), scanStart: store.ZSetMemberScanPrefix([]byte("x"))}, {name: "zset score", key: store.ZSetScoreKey([]byte("x"), 1.5, []byte("m")), scanStart: store.ZSetScoreScanPrefix([]byte("x"))}, {name: "zset delta", key: store.ZSetMetaDeltaKey([]byte("x"), 10, 0), scanStart: store.ZSetMetaDeltaScanPrefix([]byte("x"))}, + {name: "stream meta", key: store.StreamMetaKey([]byte("x")), scanStart: store.StreamMetaKey([]byte("x"))}, + {name: "stream entry", key: store.StreamEntryKey([]byte("x"), 10, 0), scanStart: store.StreamEntryScanPrefix([]byte("x"))}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() diff --git a/store/stream_helpers.go b/store/stream_helpers.go index fb1eaca1c..67697a825 100644 --- a/store/stream_helpers.go +++ b/store/stream_helpers.go @@ -129,6 +129,9 @@ func IsStreamEntryKey(key []byte) bool { // math.MaxUint32 cannot wrap (uint32(wideColKeyLenSize)+ukLen) and pass a // false negative, which would then panic on the trimmed[lo:hi] slice below. func ExtractStreamUserKeyFromMeta(key []byte) []byte { + if !bytes.HasPrefix(key, streamMetaPrefixBytes) { + return nil + } trimmed := bytes.TrimPrefix(key, streamMetaPrefixBytes) if len(trimmed) < wideColKeyLenSize { return nil @@ -140,12 +143,21 @@ func ExtractStreamUserKeyFromMeta(key []byte) []byte { return trimmed[wideColKeyLenSize : wideColKeyLenSize+ukLen] } +// ExtractStreamUserKeyFromEntryScanPrefix extracts the logical user key from +// the prefix used to scan all entries for a stream. +func ExtractStreamUserKeyFromEntryScanPrefix(key []byte) []byte { + return extractWideColumnUserKey(key, streamEntryPrefixBytes, 0, false) +} + // ExtractStreamUserKeyFromEntry extracts the logical user key from a stream entry key. // // See ExtractStreamUserKeyFromMeta for the rationale of the uint64 bounds // check; the entry variant additionally has to account for the trailing // StreamIDBytes (16 bytes) suffix. func ExtractStreamUserKeyFromEntry(key []byte) []byte { + if !bytes.HasPrefix(key, streamEntryPrefixBytes) { + return nil + } trimmed := bytes.TrimPrefix(key, streamEntryPrefixBytes) if len(trimmed) < wideColKeyLenSize+StreamIDBytes { return nil diff --git a/store/stream_helpers_test.go b/store/stream_helpers_test.go index 5e469fabe..fd2285b7d 100644 --- a/store/stream_helpers_test.go +++ b/store/stream_helpers_test.go @@ -75,3 +75,26 @@ func TestExtractStreamUserKeyFromEntry_RoundTrip(t *testing.T) { t.Fatalf("round trip: want %q, got %q", want, got) } } + +func TestExtractStreamUserKeyFromEntryScanPrefix_RoundTrip(t *testing.T) { + t.Parallel() + want := []byte("entry-scan-user-key") + if got := ExtractStreamUserKeyFromEntryScanPrefix(StreamEntryScanPrefix(want)); !bytes.Equal(got, want) { + t.Fatalf("scan prefix round trip: want %q, got %q", want, got) + } +} + +func TestExtractStreamUserKeyRejectsForeignPrefixes(t *testing.T) { + t.Parallel() + + key := append([]byte{0, 0, 0, 1}, 'x') + if got := ExtractStreamUserKeyFromMeta(key); got != nil { + t.Fatalf("meta extractor accepted non-stream key: %q", got) + } + if got := ExtractStreamUserKeyFromEntry(key); got != nil { + t.Fatalf("entry extractor accepted non-stream key: %q", got) + } + if got := ExtractStreamUserKeyFromEntryScanPrefix(key); got != nil { + t.Fatalf("entry scan extractor accepted non-stream key: %q", got) + } +} From 70fb630f55ee3e4d3a77ae855c4ab21ec8efa3fc Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 14:05:24 +0900 Subject: [PATCH 066/162] Keep split migration capability fail closed --- adapter/distribution_server.go | 16 ++++--- main.go | 10 ++++- main_catalog_test.go | 80 ++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 7 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 481159753..e2683dcbc 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -108,7 +108,8 @@ const ( splitJobListCursorTerminalOff = 1 splitJobListCursorJobIDOff = splitJobListCursorTerminalOff + 8 splitJobListCursorEncodedBytes = splitJobListCursorJobIDOff + 8 - splitMigrationCapabilityV2 = "cap_migration_v2" + SplitMigrationCapabilityV2 = "cap_migration_v2" + splitMigrationCapabilityV2 = SplitMigrationCapabilityV2 ) var ( @@ -238,8 +239,7 @@ func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.St if err != nil { return nil, err } - readPin := s.pinReadTS(snapshot.ReadTS) - defer readPin.Release() + defer releaseReadPin(s.pinReadTS(snapshot.ReadTS)) parent, err := s.startSplitMigrationParent(ctx, snapshot, req) if err != nil { @@ -445,8 +445,7 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR if err != nil { return nil, err } - readPin := s.pinReadTS(snapshot.ReadTS) - defer readPin.Release() + defer releaseReadPin(s.pinReadTS(snapshot.ReadTS)) if err := validateExpectedCatalogVersion(snapshot.Version, req.GetExpectedCatalogVersion()); err != nil { return nil, err } @@ -492,6 +491,13 @@ func (s *DistributionServer) pinReadTS(ts uint64) *kv.ActiveTimestampToken { return s.readTracker.Pin(ts) } +func releaseReadPin(token *kv.ActiveTimestampToken) { + if token == nil { + return + } + token.Release() +} + func (s *DistributionServer) verifyCatalogLeader(ctx context.Context) error { if s.coordinator == nil { return grpcStatusError(codes.FailedPrecondition, errDistributionCoordinatorRequired.Error()) diff --git a/main.go b/main.go index 47c44af55..7d2111e8a 100644 --- a/main.go +++ b/main.go @@ -512,7 +512,6 @@ func run() error { adapter.WithDistributionActiveTimestampTracker(readTracker), adapter.WithDistributionKnownRaftGroups(shardGroupIDs(shardGroups)...), adapter.WithSplitMigrationCapabilityGate(splitMigrationGate), - adapter.WithSplitJobRunnerReady(), ) defaultRuntime := findDefaultGroupRuntime(runtimes, cfg.defaultGroup) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) @@ -702,7 +701,11 @@ func splitMigrationCapabilityPeerSourceForRuntimes(runtimes []*raftGroupRuntime) if err != nil { return nil, errors.Wrapf(err, "raft group %d configuration", rt.spec.id) } - for _, peer := range splitMigrationCapabilityPeersFromConfiguration(cfg) { + groupPeers := splitMigrationCapabilityPeersFromConfiguration(cfg) + if len(groupPeers) == 0 { + return nil, errors.Errorf("raft group %d configuration has no split migration capability peers", rt.spec.id) + } + for _, peer := range groupPeers { key := peer.ID + "\x00" + peer.Address if _, ok := seen[key]; ok { continue @@ -791,6 +794,9 @@ func probeSplitMigrationCapabilityPeer(ctx context.Context, address string) erro if resp == nil || !resp.GetMigrationCapable() { return errors.New("peer does not advertise split migration capability") } + if !slices.Contains(resp.GetCapabilities(), adapter.SplitMigrationCapabilityV2) { + return errors.Errorf("peer does not advertise %s", adapter.SplitMigrationCapabilityV2) + } return nil } diff --git a/main_catalog_test.go b/main_catalog_test.go index 497213d36..b4227a9ee 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -3,13 +3,17 @@ package main import ( "context" "errors" + "net" "testing" "time" + "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -189,6 +193,53 @@ func TestSplitMigrationCapabilityPeerSourceForRuntimesChecksAllGroups(t *testing }, peers) } +func TestSplitMigrationCapabilityPeerSourceForRuntimesFailsClosedOnEmptyGroup(t *testing.T) { + t.Parallel() + + runtimes := []*raftGroupRuntime{ + { + spec: groupSpec{id: 1}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + }}}, + }, + { + spec: groupSpec{id: 2}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{}}, + }, + } + + peers, err := splitMigrationCapabilityPeerSourceForRuntimes(runtimes)(context.Background()) + require.Error(t, err) + require.Nil(t, peers) + require.ErrorContains(t, err, "raft group 2") + require.ErrorContains(t, err, "no split migration capability peers") +} + +func TestProbeSplitMigrationCapabilityPeerRequiresDocumentedToken(t *testing.T) { + t.Parallel() + + addr := startSplitMigrationCapabilityTestServer(t, &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{"split_migration_v2"}, + }) + + err := probeSplitMigrationCapabilityPeer(context.Background(), addr) + require.Error(t, err) + require.ErrorContains(t, err, adapter.SplitMigrationCapabilityV2) +} + +func TestProbeSplitMigrationCapabilityPeerAcceptsDocumentedToken(t *testing.T) { + t.Parallel() + + addr := startSplitMigrationCapabilityTestServer(t, &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{adapter.SplitMigrationCapabilityV2}, + }) + + require.NoError(t, probeSplitMigrationCapabilityPeer(context.Background(), addr)) +} + func staticSplitMigrationCapabilityPeerSource(peers []splitMigrationCapabilityPeer) splitMigrationCapabilityPeerSource { return func(context.Context) ([]splitMigrationCapabilityPeer, error) { out := make([]splitMigrationCapabilityPeer, len(peers)) @@ -197,6 +248,35 @@ func staticSplitMigrationCapabilityPeerSource(peers []splitMigrationCapabilityPe } } +type splitMigrationCapabilityTestServer struct { + pb.UnimplementedDistributionServer + resp *pb.GetSplitMigrationCapabilityResponse +} + +func (s *splitMigrationCapabilityTestServer) GetSplitMigrationCapability(context.Context, *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + return s.resp, nil +} + +func startSplitMigrationCapabilityTestServer(t *testing.T, resp *pb.GetSplitMigrationCapabilityResponse) string { + t.Helper() + + var listenConfig net.ListenConfig + listener, err := listenConfig.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + grpcServer := grpc.NewServer() + pb.RegisterDistributionServer(grpcServer, &splitMigrationCapabilityTestServer{resp: resp}) + done := make(chan struct{}) + go func() { + defer close(done) + _ = grpcServer.Serve(listener) + }() + t.Cleanup(func() { + grpcServer.Stop() + <-done + }) + return listener.Addr().String() +} + type capabilityConfigEngine struct { cfg raftengine.Configuration } From 6892dd9aa2c17d5ffb251d92f0cc2190138ab0e1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 15:31:31 +0900 Subject: [PATCH 067/162] Skip txn locks in migration export --- store/lsm_migration.go | 3 +++ store/migration_versions.go | 7 +++++++ store/migration_versions_test.go | 14 ++++++++++++++ store/store.go | 4 ++++ 4 files changed, 28 insertions(+) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 828ba481d..1aab03419 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -264,6 +264,9 @@ func (s *pebbleStore) exportPebbleVersion( } func shouldExportPebbleVersion(opts ExportVersionsOptions, userKey []byte, commitTS uint64) bool { + if shouldSkipMigrationExportKey(userKey) { + return false + } if opts.AcceptKey != nil && !opts.AcceptKey(userKey) { return false } diff --git a/store/migration_versions.go b/store/migration_versions.go index 33c718e11..6eb5f65db 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -400,6 +400,9 @@ func finishExportIfLimited(opts ExportVersionsOptions, result *ExportVersionsRes } func appendMemoryExportVersion(opts ExportVersionsOptions, key []byte, version VersionedValue, result *ExportVersionsResult) byte { + if shouldSkipMigrationExportKey(key) { + return exportCursorTagScanned + } if opts.AcceptKey != nil && !opts.AcceptKey(key) { return exportCursorTagScanned } @@ -419,6 +422,10 @@ func appendMemoryExportVersion(opts ExportVersionsOptions, key []byte, version V return exportCursorTagEmitted } +func shouldSkipMigrationExportKey(key []byte) bool { + return bytes.HasPrefix(key, txnLockKeyPrefix) +} + func finishMemoryExportPosition(opts ExportVersionsOptions, key []byte, version VersionedValue, tag byte, result *ExportVersionsResult) bool { result.ScannedBytes += versionExportSize(key, len(version.Value)) result.NextCursor = encodeExportCursor(key, version.TS, tag) diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 82209e4c4..d76a10ee3 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -56,6 +56,20 @@ func TestExportVersionsPreservesRawVersionMetadata(t *testing.T) { }) } +func TestExportVersionsExcludesTxnLocks(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + lockKey := append(bytes.Clone(txnLockKeyPrefix), []byte("user")...) + require.NoError(t, st.PutAt(ctx, lockKey, []byte("lock"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("user"), []byte("value"), 20, 0)) + + result, err := st.ExportVersions(ctx, ExportVersionsOptions{MaxVersions: 10}) + require.NoError(t, err) + require.True(t, result.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("user"), CommitTS: 20, Value: []byte("value")}}, result.Versions) + }) +} + func TestExportVersionsCursorResumesWithinHotKey(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() diff --git a/store/store.go b/store/store.go index 78cb59354..04e943af8 100644 --- a/store/store.go +++ b/store/store.go @@ -15,6 +15,10 @@ import ( // single definition due to the store→kv import cycle. var txnInternalKeyPrefix = []byte("!txn|") +// txnLockKeyPrefix must match kv's txn lock namespace. Migration exports skip +// these in-flight lock records so a target range never imports a stale lock. +var txnLockKeyPrefix = []byte("!txn|lock|") + var ErrKeyNotFound = errors.New("not found") var ErrUnknownOp = errors.New("unknown op") var ErrNotSupported = errors.New("not supported") From fba756029aca56be2710683ca8cc467fb7e595b1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 16:09:23 +0900 Subject: [PATCH 068/162] Filter legacy list deltas by value --- adapter/redis_compat_helpers.go | 75 ++++++++++++++++++++------- adapter/redis_delta_compactor.go | 21 ++++---- adapter/redis_delta_compactor_test.go | 22 ++++++++ adapter/redis_list_dedup_test.go | 35 ++++++++++++- adapter/redis_txn.go | 23 ++++++-- adapter/redis_txn_test.go | 26 ++++++++++ kv/shard_key.go | 3 -- kv/shard_key_test.go | 25 ++++++++- kv/shard_store_test.go | 15 ++++-- 9 files changed, 199 insertions(+), 46 deletions(-) diff --git a/adapter/redis_compat_helpers.go b/adapter/redis_compat_helpers.go index b6955cbc1..b0f083e73 100644 --- a/adapter/redis_compat_helpers.go +++ b/adapter/redis_compat_helpers.go @@ -1041,6 +1041,39 @@ func (r *RedisServer) scanAllDeltaElemsFiltered( return elems, nil } +func scanAcceptedDeltaKVsAt( + ctx context.Context, + st store.MVCCStore, + prefix []byte, + limit int, + readTS uint64, + accept func(*store.KVPair) bool, +) ([]*store.KVPair, bool, error) { + const cursorAdv = byte(0x00) + end := store.PrefixScanEnd(prefix) + cursor := prefix + out := make([]*store.KVPair, 0, limit) + for { + rawKVs, err := st.ScanAt(ctx, cursor, end, limit+1, readTS) + if err != nil { + return nil, false, errors.WithStack(err) + } + for _, pair := range rawKVs { + if accept != nil && !accept(pair) { + continue + } + out = append(out, pair) + if len(out) > limit { + return out, true, nil + } + } + if len(rawKVs) < limit+1 { + return out, false, nil + } + cursor = append(bytes.Clone(rawKVs[len(rawKVs)-1].Key), cursorAdv) + } +} + func isLegacyListMetaDeltaPrefix(prefix []byte) bool { return bytes.HasPrefix(prefix, []byte(store.LegacyListMetaDeltaPrefix)) } @@ -1309,37 +1342,43 @@ func minRedisInt(a, b int) int { // aggregateLenDeltas scans delta keys under prefix and sums the LenDelta values // via unmarshalDelta. Returns (sum, hasDeltas, error). -// ErrDeltaScanTruncated is returned when the scan hits MaxDeltaScanLimit. +// ErrDeltaScanTruncated is returned when accepted deltas exceed MaxDeltaScanLimit. func (r *RedisServer) aggregateLenDeltas( ctx context.Context, prefix []byte, readTS uint64, unmarshalDelta func(key []byte, value []byte) (int64, bool, error), ) (int64, bool, error) { + const cursorAdv = byte(0x00) end := store.PrefixScanEnd(prefix) - // Scan one extra key beyond the limit so we can distinguish "exactly - // MaxDeltaScanLimit results" (no truncation) from "more than MaxDeltaScanLimit - // results" (truncated). Without the +1, a collection with exactly - // MaxDeltaScanLimit deltas would incorrectly trigger ErrDeltaScanTruncated. - deltas, err := r.store.ScanAt(ctx, prefix, end, store.MaxDeltaScanLimit+1, readTS) - if err != nil { - return 0, false, errors.WithStack(err) - } - if len(deltas) > store.MaxDeltaScanLimit { - return 0, false, ErrDeltaScanTruncated - } var sum int64 var any bool - for _, d := range deltas { - delta, include, err := unmarshalDelta(d.Key, d.Value) + included := 0 + cursor := prefix + for { + deltas, err := r.store.ScanAt(ctx, cursor, end, store.MaxDeltaScanLimit+1, readTS) if err != nil { return 0, false, errors.WithStack(err) } - if !include { - continue + for _, d := range deltas { + delta, include, err := unmarshalDelta(d.Key, d.Value) + if err != nil { + return 0, false, errors.WithStack(err) + } + if !include { + continue + } + included++ + if included > store.MaxDeltaScanLimit { + return 0, false, ErrDeltaScanTruncated + } + any = true + sum += delta + } + if len(deltas) < store.MaxDeltaScanLimit+1 { + break } - any = true - sum += delta + cursor = append(bytes.Clone(deltas[len(deltas)-1].Key), cursorAdv) } return sum, any, nil } diff --git a/adapter/redis_delta_compactor.go b/adapter/redis_delta_compactor.go index bed684450..e27a7736b 100644 --- a/adapter/redis_delta_compactor.go +++ b/adapter/redis_delta_compactor.go @@ -223,11 +223,10 @@ func (c *DeltaCompactor) compactUrgentKey(ctx context.Context, req urgentCompact defer cancel() prefix := h.deltaKeyPrefixFn(req.userKey) - end := store.PrefixScanEnd(prefix) totalCompacted, done := 0, false for !done { var n int - n, done = c.compactUrgentKeyBatch(tickCtx, req, h, prefix, end) + n, done = c.compactUrgentKeyBatch(tickCtx, req, h, prefix) totalCompacted += n if n == 0 { break @@ -244,13 +243,12 @@ func (c *DeltaCompactor) compactUrgentKey(ctx context.Context, req urgentCompact // Returns (n, done): n is the number of delta keys processed in this batch; // done is true when the remaining delta count is at or below MaxDeltaScanLimit // (the key is now readable). -func (c *DeltaCompactor) compactUrgentKeyBatch(ctx context.Context, req urgentCompactionRequest, h *collectionDeltaHandler, prefix, end []byte) (int, bool) { +func (c *DeltaCompactor) compactUrgentKeyBatch(ctx context.Context, req urgentCompactionRequest, h *collectionDeltaHandler, prefix []byte) (int, bool) { // Use a fresh readTS each iteration so we observe the committed state from // the previous compaction pass and do not re-scan already-deleted delta keys. readTS := snapshotTS(c.coord.Clock(), c.st) - // Scan one extra beyond MaxDeltaScanLimit to detect whether more remain. - kvs, err := c.st.ScanAt(ctx, prefix, end, store.MaxDeltaScanLimit+1, readTS) + kvs, truncated, err := scanAcceptedDeltaKVsAt(ctx, c.st, prefix, store.MaxDeltaScanLimit, readTS, h.acceptDeltaKV) if err != nil { c.logger.WarnContext(ctx, "delta compactor urgent: scan failed", "type", req.typeName, "key", string(req.userKey), "error", err) @@ -259,10 +257,6 @@ func (c *DeltaCompactor) compactUrgentKeyBatch(ctx context.Context, req urgentCo if len(kvs) == 0 { return 0, true } - kvs = filterDeltaKVs(kvs, h.acceptDeltaKV) - if len(kvs) == 0 { - return 0, true - } elems, err := h.buildElems(ctx, req.userKey, kvs, readTS) if err != nil { @@ -275,8 +269,7 @@ func (c *DeltaCompactor) compactUrgentKeyBatch(ctx context.Context, req urgentCo "type", req.typeName, "key", string(req.userKey), "error", err) return 0, true } - // Fewer than MaxDeltaScanLimit+1 results means no more deltas remain; key is readable. - return len(kvs), len(kvs) <= store.MaxDeltaScanLimit + return len(kvs), !truncated } // SyncOnce runs one compaction pass. The IsLeader() guard avoids the @@ -472,9 +465,13 @@ func (c *DeltaCompactor) compactHandler(ctx context.Context, h collectionDeltaHa return err } - kvs = filterDeltaKVs(kvs, h.acceptDeltaKV) + rawKVs := kvs + kvs = filterDeltaKVs(rawKVs, h.acceptDeltaKV) byKey, ukOrder := c.groupByUserKey(kvs, h.extractUserKey) lastScannedKey := c.splitGuardCursor(byKey, ukOrder, kvs, truncated) + if truncated && len(lastScannedKey) == 0 && len(kvs) == 0 && len(rawKVs) > 0 { + lastScannedKey = rawKVs[len(rawKVs)-1].Key + } allElems := c.buildBatchElems(ctx, h, byKey, ukOrder, readTS) diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index f519de6d4..07dd120cc 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -372,6 +372,28 @@ func TestDeltaCompactor_ListNoBaseMeta(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) } +func TestDeltaCompactor_LegacyListCursorAdvancesPastFilteredRawRows(t *testing.T) { + t.Parallel() + + st, c := newDeltaCompactorTestFixture(t) + ctx := context.Background() + meta, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + for i := uint64(1); i <= deltaCompactorTickScanLimit; i++ { + userKey := deltaLookingListMetaUserKeyAt([]byte("compactor-collision"), i, 0) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(userKey), meta, i, 0)) + } + + h := c.legacyListHandler() + readTS := st.LastCommitTS() + require.NoError(t, c.compactHandler(ctx, h, readTS)) + + c.cursorMu.Lock() + got := bytes.Clone(c.cursors[h.typeName]) + c.cursorMu.Unlock() + require.Equal(t, store.ListMetaKey(deltaLookingListMetaUserKeyAt([]byte("compactor-collision"), deltaCompactorTickScanLimit, 0)), got) +} + // TestDeltaCompactor_UrgentCompactionTriggeredByChannel verifies that a request // queued via TriggerUrgentCompaction is processed by the Run loop, compacting // the targeted key without waiting for the next regular tick. diff --git a/adapter/redis_list_dedup_test.go b/adapter/redis_list_dedup_test.go index 548cb27ce..d57e94509 100644 --- a/adapter/redis_list_dedup_test.go +++ b/adapter/redis_list_dedup_test.go @@ -112,6 +112,33 @@ func TestResolveListMetaIgnoresLegacyDeltaPrefixCollisionForMissingKey(t *testin require.False(t, exists) } +func TestResolveListMetaCountsOnlyAcceptedLegacyDeltasForTruncation(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := []byte("legacy-collision-window") + base, err := store.MarshalListMeta(store.ListMeta{Head: 0, Tail: 1, Len: 1}) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) + + collidingMeta, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + for i := uint64(2); i < uint64(store.MaxDeltaScanLimit+2); i++ { + collidingUserKey := deltaLookingListMetaUserKeyAt(key, i, 0) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(collidingUserKey), collidingMeta, i, 0)) + } + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) + deltaTS := uint64(store.MaxDeltaScanLimit + 2) + require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, deltaTS), delta, deltaTS, 0)) + + meta, exists, err := srv.resolveListMeta(ctx, key, deltaTS+1) + require.NoError(t, err) + require.True(t, exists) + require.Equal(t, int64(2), meta.Len) +} + func TestProbeListTypeReadsLegacyDeltaPrefix(t *testing.T) { t.Parallel() @@ -184,6 +211,10 @@ func legacyListMetaDeltaKey(userKey []byte, commitTS uint64) []byte { } func deltaLookingListMetaUserKey(fakeUserKey []byte) []byte { + return deltaLookingListMetaUserKeyAt(fakeUserKey, 7, 1) +} + +func deltaLookingListMetaUserKeyAt(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) key = append(key, "d|"...) var lenPrefix [4]byte @@ -191,10 +222,10 @@ func deltaLookingListMetaUserKey(fakeUserKey []byte) []byte { key = append(key, lenPrefix[:]...) key = append(key, fakeUserKey...) var ts [8]byte - binary.BigEndian.PutUint64(ts[:], 7) + binary.BigEndian.PutUint64(ts[:], commitTS) key = append(key, ts[:]...) var seq [4]byte - binary.BigEndian.PutUint32(seq[:], 1) + binary.BigEndian.PutUint32(seq[:], seqInTxn) return append(key, seq[:]...) } diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index 779851614..750011ca7 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -353,12 +353,18 @@ func (t *txnContext) loadListState(key []byte) (*listTxnState, error) { // and let the caller retry after the background compactor has caught up. var existingDeltas [][]byte for _, deltaPrefix := range store.ListMetaDeltaScanPrefixes(key) { - deltaEnd := store.PrefixScanEnd(deltaPrefix) - deltaKVs, err := t.server.store.ScanAt(ctx, deltaPrefix, deltaEnd, store.MaxDeltaScanLimit+1, t.startTS) + deltaKVs, truncated, err := scanAcceptedDeltaKVsAt( + ctx, + t.server.store, + deltaPrefix, + store.MaxDeltaScanLimit, + t.startTS, + listTxnDeltaFilter(key, deltaPrefix), + ) if err != nil { - return nil, errors.WithStack(err) + return nil, err } - if len(deltaKVs) > store.MaxDeltaScanLimit { + if truncated { return nil, ErrDeltaScanTruncated } for _, kv := range deltaKVs { @@ -391,6 +397,15 @@ func (t *txnContext) loadListState(key []byte) (*listTxnState, error) { return st, nil } +func listTxnDeltaFilter(key []byte, deltaPrefix []byte) func(*store.KVPair) bool { + if !isLegacyListMetaDeltaPrefix(deltaPrefix) { + return nil + } + return func(pair *store.KVPair) bool { + return legacyListDeltaPairForUserKey(pair, key) + } +} + func (t *txnContext) loadHashStateForFields(key []byte, fields [][]byte) (*hashTxnState, error) { k := string(key) if t.hashStates == nil { diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index ed96fcc04..3349c4b8b 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -41,6 +41,32 @@ func newRedisTxnTestContext(server *RedisServer) *txnContext { } } +func TestRedisTxnLoadListStateFiltersLegacyDeltaPrefixCollisions(t *testing.T) { + t.Parallel() + + server, st := newRedisStorageMigrationTestServer(t) + ctx := context.Background() + key := []byte("txn-legacy-list-delete") + base, err := store.MarshalListMeta(store.ListMeta{Head: 0, Tail: 1, Len: 1}) + require.NoError(t, err) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) + + collidingMeta, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + collidingMetaKey := store.ListMetaKey(deltaLookingListMetaUserKey(key)) + require.NoError(t, st.PutAt(ctx, collidingMetaKey, collidingMeta, 2, 0)) + + deltaKey := legacyListMetaDeltaKey(key, 3) + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) + require.NoError(t, st.PutAt(ctx, deltaKey, delta, 3, 0)) + + txn := newRedisTxnTestContext(server) + txn.startTS = 4 + stState, err := txn.loadListState(key) + require.NoError(t, err) + require.Equal(t, [][]byte{deltaKey}, stState.existingDeltas) +} + func elemKeysContain(elems []*kv.Elem[kv.OP], want []byte) bool { for _, elem := range elems { if elem != nil && string(elem.Key) == string(want) { diff --git a/kv/shard_key.go b/kv/shard_key.go index ee38945a4..07366f51a 100644 --- a/kv/shard_key.go +++ b/kv/shard_key.go @@ -152,9 +152,6 @@ func listRouteKey(key []byte) []byte { if userKey := store.ExtractListUserKeyFromDelta(key); userKey != nil { return userKey } - if userKey := store.ExtractLegacyListUserKeyFromDelta(key); userKey != nil { - return userKey - } if userKey := store.ExtractListUserKeyFromClaim(key); userKey != nil { return userKey } diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 733ecac4a..9e64c8419 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -136,11 +136,17 @@ func TestRouteKey_ListMetaKeyThatLooksLikeNewDeltaRoutesByRealListKey(t *testing require.Equal(t, userKey, routeKey(deltaKey), "real list deltas must still route by the logical list key") } -func TestRouteKey_LegacyListDeltaRoutesByDecodedUserKey(t *testing.T) { +func TestRouteKey_LegacyListDeltaKeyOnlyUsesBaseMetaRoute(t *testing.T) { t.Parallel() userKey := []byte("legacy:list") - require.Equal(t, userKey, routeKey(legacyListMetaDeltaKey(userKey, 42, 7))) + raw := legacyListMetaDeltaKey(userKey, 42, 7) + require.Equal(t, store.ExtractListUserKey(raw), routeKey(raw)) + require.NotEqual(t, userKey, routeKey(raw), "key-only routing must not decode ambiguous legacy deltas") + + collidingUserKey := deltaLookingListMetaUserKey(userKey, 42, 7) + collidingMeta := store.ListMetaKey(collidingUserKey) + require.Equal(t, collidingUserKey, routeKey(collidingMeta)) } func TestRouteKey_MalformedWideColumnKeysFallBackToRaw(t *testing.T) { @@ -185,6 +191,21 @@ func legacyListMetaDeltaKey(userKey []byte, commitTS uint64, seqInTxn uint32) [] return append(key, seq[:]...) } +func deltaLookingListMetaUserKey(fakeUserKey []byte, commitTS uint64, seqInTxn uint32) []byte { + key := make([]byte, 0, len("d|")+4+len(fakeUserKey)+8+4) + key = append(key, "d|"...) + var lenPrefix [4]byte + binary.BigEndian.PutUint32(lenPrefix[:], uint32(len(fakeUserKey))) //nolint:gosec // test data is small. + key = append(key, lenPrefix[:]...) + key = append(key, fakeUserKey...) + var ts [8]byte + binary.BigEndian.PutUint64(ts[:], commitTS) + key = append(key, ts[:]...) + var seq [4]byte + binary.BigEndian.PutUint32(seq[:], seqInTxn) + return append(key, seq[:]...) +} + func TestRouteKey_NormalizesTxnSuccessMarkerByLockedKey(t *testing.T) { t.Parallel() diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 5c81750ed..c57b48bf0 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -78,18 +78,23 @@ func TestShardStoreScanAt_RoutesListDeltaScansByUserKey(t *testing.T) { ctx := context.Background() userKey := []byte("x") // routes to group 2; raw !lst|* prefixes route to group 1. for _, tc := range []struct { - name string - key []byte - scanStart []byte + name string + key []byte + scanStart []byte + legacyRouting bool }{ {name: "current", key: store.ListMetaDeltaKey(userKey, 10, 1), scanStart: store.ListMetaDeltaScanPrefix(userKey)}, - {name: "legacy", key: legacyListMetaDeltaKey(userKey, 10, 1), scanStart: store.LegacyListMetaDeltaScanPrefix(userKey)}, + {name: "legacy", key: legacyListMetaDeltaKey(userKey, 10, 1), scanStart: store.LegacyListMetaDeltaScanPrefix(userKey), legacyRouting: true}, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() st := newTwoRouteShardStoreForScanTest() deltaValue := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) - require.NoError(t, st.PutAt(ctx, tc.key, deltaValue, 1, 0)) + if tc.legacyRouting { + require.NoError(t, st.groups[2].Store.PutAt(ctx, tc.key, deltaValue, 1, 0)) + } else { + require.NoError(t, st.PutAt(ctx, tc.key, deltaValue, 1, 0)) + } kvs, err := st.ScanAt(ctx, tc.scanStart, store.PrefixScanEnd(tc.scanStart), 10, ^uint64(0)) require.NoError(t, err) From 9f80fa9d2eae49581353890b149b4ebb6a9ca019 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 16:47:42 +0900 Subject: [PATCH 069/162] migration: tighten readiness guard publication --- kv/sharded_coordinator.go | 2 +- kv/sharded_coordinator_txn_test.go | 10 +++++++--- store/migration_readiness.go | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index a31e02e6d..04083d352 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1317,7 +1317,7 @@ type preparedGroup struct { } func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, groupedReadKeys map[uint64][][]byte, observedRouteVersion uint64) ([]preparedGroup, error) { - prepareMeta := txnMetaMutation(primaryKey, defaultTxnLockTTLms, 0) + prepareMeta := txnMetaMutation(primaryKey, defaultTxnLockTTLms, commitTS) prepared := make([]preparedGroup, 0, len(gids)) for _, gid := range gids { diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 964f2713f..3316fc847 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -144,8 +144,8 @@ func TestShardedCoordinatorDispatchTxn_CrossShardPhasesAndCommitIndex(t *testing require.Equal(t, []byte("b"), prepareMeta2.PrimaryKey) require.Equal(t, defaultTxnLockTTLms, prepareMeta1.LockTTLms) require.Equal(t, defaultTxnLockTTLms, prepareMeta2.LockTTLms) - require.Zero(t, prepareMeta1.CommitTS) - require.Zero(t, prepareMeta2.CommitTS) + require.Greater(t, prepareMeta1.CommitTS, startTS) + require.Equal(t, prepareMeta1.CommitTS, prepareMeta2.CommitTS) require.Equal(t, pb.Phase_COMMIT, g1Commit.Phase) require.Equal(t, pb.Phase_COMMIT, g2Commit.Phase) @@ -164,7 +164,7 @@ func TestShardedCoordinatorDispatchTxn_CrossShardPhasesAndCommitIndex(t *testing require.Equal(t, []byte("b"), commitMeta2.PrimaryKey) require.Zero(t, commitMeta1.LockTTLms) require.Zero(t, commitMeta2.LockTTLms) - require.Greater(t, commitMeta1.CommitTS, startTS) + require.Equal(t, prepareMeta1.CommitTS, commitMeta1.CommitTS) require.Equal(t, commitMeta1.CommitTS, commitMeta2.CommitTS) } @@ -244,6 +244,10 @@ func TestShardedCoordinatorDispatchTxn_UsesProvidedCommitTS(t *testing.T) { commitMeta2 := requestTxnMeta(t, g2Txn.requests[1]) require.Equal(t, commitTS, commitMeta1.CommitTS) require.Equal(t, commitTS, commitMeta2.CommitTS) + prepareMeta1 := requestTxnMeta(t, g1Txn.requests[0]) + prepareMeta2 := requestTxnMeta(t, g2Txn.requests[0]) + require.Equal(t, commitTS, prepareMeta1.CommitTS) + require.Equal(t, commitTS, prepareMeta2.CommitTS) } func TestCommitSecondaryWithRetry_RetriesAndSucceeds(t *testing.T) { diff --git a/store/migration_readiness.go b/store/migration_readiness.go index e0c83a1df..d26d25d36 100644 --- a/store/migration_readiness.go +++ b/store/migration_readiness.go @@ -52,11 +52,11 @@ func (s *pebbleStore) ApplyTargetStagedReadiness(_ context.Context, state Target } s.dbMu.RLock() defer s.dbMu.RUnlock() + s.mtx.Lock() + defer s.mtx.Unlock() if err := s.db.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), s.directApplyWriteOpts()); err != nil { return errors.WithStack(err) } - s.mtx.Lock() - defer s.mtx.Unlock() s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, state) return nil } From 5acb33f4261e3445b275eb221d4039fbada82252 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 18:01:29 +0900 Subject: [PATCH 070/162] migration: fence S3 bucket metadata writes --- kv/fsm.go | 9 +++-- kv/fsm_migration_fence_test.go | 40 +++++++++++++++++++++++ kv/migrator_filter.go | 18 +++++++--- kv/sharded_coordinator.go | 23 ++++++++++--- kv/sharded_coordinator_del_prefix_test.go | 31 ++++++++++++++++++ 5 files changed, 109 insertions(+), 12 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 5cd95d05d..a9ff4cea5 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -563,10 +563,13 @@ func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { return nil } rkey := routeKey(key) - if !snap.WriteFencedForKey(rkey) { - return nil + if snap.WriteFencedForKey(rkey) { + return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", key, rkey) + } + if start, end, ok := s3BucketAuxiliaryRouteRange(key); ok && snap.WriteFencedIntersects(start, end) { + return errors.Wrapf(ErrRouteWriteFenced, "key %q route range [%q,%q)", key, start, end) } - return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", key, rkey) + return nil } func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index e47e71f95..7aff2b4f4 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -22,6 +23,24 @@ func newWriteFencedFSM(t *testing.T) *kvFSM { return newComposed1FSM(t, engine, 1) } +func s3BucketAuxiliaryFenceRoutes(bucket string, rawGroupID, fencedGroupID uint64) []distribution.RouteDescriptor { + start := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + end := prefixScanEnd(start) + return []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: start, GroupID: rawGroupID, State: distribution.RouteStateActive}, + {RouteID: 2, Start: start, End: end, GroupID: fencedGroupID, State: distribution.RouteStateWriteFenced}, + {RouteID: 3, Start: end, End: nil, GroupID: rawGroupID, State: distribution.RouteStateActive}, + } +} + +func newS3BucketAuxiliaryWriteFencedFSM(t *testing.T, bucket string) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, s3BucketAuxiliaryFenceRoutes(bucket, 1, 1)) + return newComposed1FSM(t, engine, 1) +} + func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() @@ -35,6 +54,27 @@ func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } +func TestFSMRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + fsm := newS3BucketAuxiliaryWriteFencedFSM(t, bucket) + + for _, key := range [][]byte{ + s3keys.BucketMetaKey(bucket), + s3keys.BucketGenerationKey(bucket), + } { + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + _, getErr := fsm.store.GetAt(ctx, key, ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) + } +} + func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { t.Parallel() diff --git a/kv/migrator_filter.go b/kv/migrator_filter.go index 8a1da5526..acbf1a796 100644 --- a/kv/migrator_filter.go +++ b/kv/migrator_filter.go @@ -37,18 +37,26 @@ func RouteKeyFilterForGroup(rangeStart, rangeEnd []byte, sourceGroupID uint64, r } func s3BucketAuxiliaryRouteInRange(rawKey, routeStart, routeEnd []byte) bool { - bucket, ok := s3keys.ParseBucketMetaKey(rawKey) - if !ok { - bucket, ok = s3keys.ParseBucketGenerationKey(rawKey) - } + bucketRouteStart, bucketRouteEnd, ok := s3BucketAuxiliaryRouteRange(rawKey) if !ok { return false } if keyInMigrationRouteRange(rawKey, routeStart, routeEnd) { return true } + return migrationRouteRangesIntersect(routeStart, routeEnd, bucketRouteStart, bucketRouteEnd) +} + +func s3BucketAuxiliaryRouteRange(rawKey []byte) ([]byte, []byte, bool) { + bucket, ok := s3keys.ParseBucketMetaKey(rawKey) + if !ok { + bucket, ok = s3keys.ParseBucketGenerationKey(rawKey) + } + if !ok { + return nil, nil, false + } bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) - return migrationRouteRangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) + return bucketRouteStart, prefixScanEnd(bucketRouteStart), true } func keyInMigrationRouteRange(key, routeStart, routeEnd []byte) bool { diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 7ed2ccb7c..f345f2e63 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1057,11 +1057,26 @@ func (c *ShardedCoordinator) rejectWriteFencedPointElems(elems []*Elem[OP]) erro if elem == nil || len(elem.Key) == 0 { continue } - route, ok := c.engine.GetRoute(routeKey(elem.Key)) - if !ok || route.State != distribution.RouteStateWriteFenced { - continue + if err := c.rejectWriteFencedPointKey(elem.Key); err != nil { + return err + } + } + return nil +} + +func (c *ShardedCoordinator) rejectWriteFencedPointKey(key []byte) error { + rkey := routeKey(key) + if route, ok := c.engine.GetRoute(rkey); ok && route.State == distribution.RouteStateWriteFenced { + return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", key, rkey) + } + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return nil + } + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if route.State == distribution.RouteStateWriteFenced { + return errors.Wrapf(ErrRouteWriteFenced, "key %q route range [%q,%q)", key, start, end) } - return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", elem.Key, routeKey(elem.Key)) } return nil } diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 4b83dc131..5cc0f845c 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -146,6 +147,36 @@ func TestShardedCoordinatorRejectsPointWriteOnWriteFencedRoute(t *testing.T) { require.Empty(t, g2Txn.requests, "coordinator must reject before proposing to the fenced shard") } +func TestShardedCoordinatorRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { + t.Parallel() + + const bucket = "bucket-a" + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: s3BucketAuxiliaryFenceRoutes(bucket, 1, 2), + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + for _, key := range [][]byte{ + s3keys.BucketMetaKey(bucket), + s3keys.BucketGenerationKey(bucket), + } { + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: key, Value: []byte("v")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteFenced) + require.Empty(t, g1Txn.requests, "coordinator must reject before proposing to the raw-key shard") + require.Empty(t, g2Txn.requests, "coordinator must reject before proposing to the fenced shard") + } +} + func TestShardedCoordinatorRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { t.Parallel() From 14aa9e9631ee4b51bc9a88c6f8d2642dd3b65f15 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 18:16:57 +0900 Subject: [PATCH 071/162] migration: guard read-only shard validation --- kv/sharded_coordinator.go | 5 +++ kv/sharded_coordinator_txn_test.go | 57 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 04083d352..bcb1d957e 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1979,7 +1979,12 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui if _, err := linearizableReadEngineCtx(ctx, engineForGroup(g)); err != nil { return errors.WithStack(err) } + readiness := &ShardStore{engine: c.engine, groups: c.groups} for _, key := range keys { + route := readiness.explicitGroupRouteForKey(gid, key) + if err := readiness.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } ts, exists, err := g.Store.LatestCommitTS(ctx, key) if err != nil { return errors.WithStack(err) diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 3316fc847..fdce86762 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -387,6 +387,7 @@ func TestGroupReadKeysByShardID_FailsClosedOnUnroutable(t *testing.T) { type stubMVCCStore struct { store.MVCCStore latestTS map[string]uint64 + readiness []store.TargetStagedReadinessState returnErr error } @@ -398,6 +399,10 @@ func (s *stubMVCCStore) LatestCommitTS(_ context.Context, key []byte) (uint64, b return ts, ok, nil } +func (s *stubMVCCStore) MigrationTargetReadinessStates(_ context.Context) ([]store.TargetStagedReadinessState, error) { + return s.readiness, nil +} + // noopEngine satisfies raftengine.Engine for unit tests. // LinearizableRead returns immediately (simulates an already-up-to-date FSM). type noopEngine struct{} @@ -484,6 +489,58 @@ func TestValidateReadOnlyShards_NoConflictWhenKeyUnchanged(t *testing.T) { require.NoError(t, err) } +func TestValidateReadOnlyShards_FailsClosedOnTargetReadiness(t *testing.T) { + t.Parallel() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + }, + }, + })) + + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{ + "x": 5, + }, + readiness: []store.TargetStagedReadinessState{ + { + JobID: 7, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }, + }, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + groupedReadKeys := map[uint64][][]byte{ + 2: {[]byte("x")}, + } + err := coord.validateReadOnlyShards(context.Background(), groupedReadKeys, []uint64{1}, 10) + require.Error(t, err) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestValidateReadOnlyShards_PropagatesStoreError(t *testing.T) { t.Parallel() engine := distribution.NewEngine() From 295cdf6bde424cf9727205e5a6322b973e2ad3ef Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 19:01:08 +0900 Subject: [PATCH 072/162] migration: close cross-group floor gaps --- adapter/internal.go | 67 +++++++++--- adapter/internal_migration_test.go | 52 ++++++++++ adapter/internal_test.go | 55 ++++++++++ kv/shard_store.go | 64 +++++++++++- kv/shard_store_test.go | 121 ++++++++++++++++++++++ kv/sharded_coordinator.go | 31 ++++-- kv/sharded_coordinator_del_prefix_test.go | 36 +++++++ 7 files changed, 401 insertions(+), 25 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index 515faf9aa..65bd16d95 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -287,12 +287,23 @@ func decodedS3BucketRouteFilter(family uint32, routeStart, routeEnd []byte) func if !ok { return false } - routeKey := s3keys.RouteKey(bucket, 0, "") - if len(routeStart) > 0 && bytes.Compare(routeKey, routeStart) < 0 { - return false - } - return len(routeEnd) == 0 || bytes.Compare(routeKey, routeEnd) < 0 + return decodedS3BucketRouteIntersects(bucket, routeStart, routeEnd) + } +} + +func decodedS3BucketRouteIntersects(bucket string, routeStart, routeEnd []byte) bool { + bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + return rangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) +} + +func rangesIntersect(aStart, aEnd, bStart, bEnd []byte) bool { + if len(aEnd) > 0 && bytes.Compare(aEnd, bStart) <= 0 { + return false } + if len(bEnd) > 0 && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true } func decodedS3BucketName(family uint32, rawKey []byte) (string, bool) { @@ -407,14 +418,13 @@ func (i *Internal) stampRawTimestamps(ctx context.Context, reqs []*pb.Request) e if r == nil { continue } - if r.Ts != 0 { - continue - } - ts, err := i.nextTimestamp(ctx, "stampRawTimestamps") - if err != nil { - return err + if r.Ts == 0 { + ts, err := i.nextTimestamp(ctx, "stampRawTimestamps") + if err != nil { + return err + } + r.Ts = ts } - r.Ts = ts if err := i.rejectWriteTimestampFloorMutations(r.Mutations, r.Ts); err != nil { return err } @@ -467,7 +477,10 @@ func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) e } } - return i.fillForwardedTxnCommitTS(ctx, reqs, startTS) + if err := i.fillForwardedTxnCommitTS(ctx, reqs, startTS); err != nil { + return err + } + return i.rejectWriteTimestampFloorTxnRequests(reqs) } func forwardedTxnStartTS(reqs []*pb.Request) uint64 { @@ -533,6 +546,34 @@ func (i *Internal) fillForwardedTxnCommitTS(ctx context.Context, reqs []*pb.Requ return nil } +func (i *Internal) rejectWriteTimestampFloorTxnRequests(reqs []*pb.Request) error { + for _, r := range reqs { + commitTS, ok := forwardedTxnRequestCommitTS(r) + if !ok { + continue + } + if err := i.rejectWriteTimestampFloorMutations(r.Mutations, commitTS); err != nil { + return err + } + } + return nil +} + +func forwardedTxnRequestCommitTS(r *pb.Request) (uint64, bool) { + if r == nil || (r.Phase != pb.Phase_COMMIT && r.Phase != pb.Phase_NONE) { + return 0, false + } + m, ok := forwardedTxnMetaMutation(r, []byte(kv.TxnMetaPrefix)) + if !ok { + return 0, false + } + meta, err := kv.DecodeTxnMeta(m.Value) + if err != nil || meta.CommitTS == 0 { + return 0, false + } + return meta.CommitTS, true +} + // forwardedTxnCommitTS allocates a commit timestamp for a forwarded // transaction, strictly greater than startTS. Pulled out of // fillForwardedTxnCommitTS so that function stays under cyclop. diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 57ca63494..2a403c963 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -225,6 +225,58 @@ func TestInternalExportRangeVersionsDecodedS3EmptyRouteEndIsUnbounded(t *testing }, stream.responses[0].GetVersions()) } +func TestInternalExportRangeVersionsIncludesS3BucketAuxiliaryForBucketRouteIntersection(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + + const bucket = "bucket-b" + for _, tc := range []struct { + name string + family uint32 + prefix string + key []byte + value []byte + }{ + { + name: "bucket meta", + family: distribution.MigrationFamilyS3BucketMeta, + prefix: s3keys.BucketMetaPrefix, + key: s3keys.BucketMetaKey(bucket), + value: []byte("meta"), + }, + { + name: "bucket generation", + family: distribution.MigrationFamilyS3BucketGeneration, + prefix: s3keys.BucketGenerationPrefix, + key: s3keys.BucketGenerationKey(bucket), + value: []byte("generation"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, st.PutAt(ctx, tc.key, tc.value, 10, 0)) + + stream := &captureExportRangeVersionsStream{ctx: ctx} + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + RouteStart: s3keys.RouteKey(bucket, 7, "m"), + RouteEnd: s3keys.RouteKey(bucket, 7, "z"), + KeyFamily: tc.family, + RangeStart: []byte(tc.prefix), + RangeEnd: testPrefixScanEnd([]byte(tc.prefix)), + MaxScannedBytes: 1 << 20, + }, stream) + require.NoError(t, err) + require.Len(t, stream.responses, 1) + require.Equal(t, []*pb.MVCCVersion{ + {Key: tc.key, CommitTs: 10, Value: tc.value, KeyFamily: tc.family}, + }, stream.responses[0].GetVersions()) + }) + } +} + func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() diff --git a/adapter/internal_test.go b/adapter/internal_test.go index a028aa65d..fe4a0c5ea 100644 --- a/adapter/internal_test.go +++ b/adapter/internal_test.go @@ -204,3 +204,58 @@ func TestStampRawTimestampsRejectsRouteWriteFloor(t *testing.T) { err := i.stampRawTimestamps(context.Background(), reqs) require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) } + +func TestStampRawTimestampsRejectsPreStampedRouteWriteFloor(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + i := &Internal{routeEngine: engine} + reqs := []*pb.Request{{ + Ts: 100, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}}, + }} + + err := i.stampRawTimestamps(context.Background(), reqs) + require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) +} + +func TestStampTxnTimestampsRejectsRouteWriteFloor(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + i := &Internal{routeEngine: engine} + reqs := []*pb.Request{{ + IsTxn: true, + Phase: pb.Phase_NONE, + Ts: 50, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("k"), CommitTS: 100})}, + {Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}, + }, + }} + + err := i.stampTxnTimestamps(context.Background(), reqs) + require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) +} diff --git a/kv/shard_store.go b/kv/shard_store.go index e5fb6e1bb..107be5a49 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -140,6 +140,12 @@ func (s *ShardStore) routeForExplicitGroupKey(groupID uint64, key []byte) (distr if s == nil || s.engine == nil { return fallback, nil } + if route, ok := s.stagedVisibilityRouteForS3BucketAuxiliaryKey(key); ok { + if route.GroupID == groupID { + return route, nil + } + return distribution.Route{}, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d key=%q", groupID, key) + } if route, ok := s.engine.GetRoute(routeKey(key)); ok { if route.GroupID == groupID { return route, nil @@ -1457,9 +1463,12 @@ func (s *ShardStore) scanRouteAtLeaderRouteFilter( kvs []*store.KVPair err error ) - if reverse { + switch { + case routeHasStagedVisibility(route): + kvs, err = s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) + case reverse: kvs, err = g.Store.ReverseScanAt(ctx, start, end, limit, ts) - } else { + default: kvs, err = g.Store.ScanAt(ctx, start, end, limit, ts) } if err != nil { @@ -1622,6 +1631,9 @@ func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commit if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } + if err := s.ensureS3BucketAuxiliaryWriteTimestampFloor(key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.PutAt(ctx, key, value, commitTS, expireAt)) } @@ -1633,6 +1645,9 @@ func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } + if err := s.ensureS3BucketAuxiliaryWriteTimestampFloor(key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.DeleteAt(ctx, key, commitTS)) } @@ -1644,6 +1659,9 @@ func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } + if err := s.ensureS3BucketAuxiliaryWriteTimestampFloor(key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.PutWithTTLAt(ctx, key, value, commitTS, expireAt)) } @@ -1655,6 +1673,9 @@ func (s *ShardStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } + if err := s.ensureS3BucketAuxiliaryWriteTimestampFloor(key, commitTS); err != nil { + return err + } return errors.WithStack(g.Store.ExpireAt(ctx, key, expireAt, commitTS)) } @@ -2434,6 +2455,25 @@ func (s *ShardStore) ensureMutationWriteTimestampFloors(mutations []*store.KVPai if err := ensureRouteWriteTimestampFloor(route, mut.Key, commitTS); err != nil { return err } + if err := s.ensureS3BucketAuxiliaryWriteTimestampFloor(mut.Key, commitTS); err != nil { + return err + } + } + return nil +} + +func (s *ShardStore) ensureS3BucketAuxiliaryWriteTimestampFloor(key []byte, commitTS uint64) error { + if s == nil || s.engine == nil || commitTS == 0 { + return nil + } + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return nil + } + for _, route := range s.engine.GetIntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) + } } return nil } @@ -2778,6 +2818,10 @@ func (s *ShardStore) groupForKey(key []byte) (*ShardGroup, bool) { } func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *ShardGroup, bool) { + if route, ok := s.stagedVisibilityRouteForS3BucketAuxiliaryKey(key); ok { + g, ok := s.groups[route.GroupID] + return route, g, ok + } route, ok := s.engine.GetRoute(routeKey(key)) if !ok { return distribution.Route{}, nil, false @@ -2786,6 +2830,22 @@ func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *Shard return route, g, ok } +func (s *ShardStore) stagedVisibilityRouteForS3BucketAuxiliaryKey(key []byte) (distribution.Route, bool) { + if s == nil || s.engine == nil { + return distribution.Route{}, false + } + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return distribution.Route{}, false + } + for _, route := range s.engine.GetIntersectingRoutes(start, end) { + if routeHasStagedVisibility(route) { + return route, true + } + } + return distribution.Route{}, false +} + func (s *ShardStore) proxyRawGet(ctx context.Context, g *ShardGroup, key []byte, ts uint64, groupID uint64, readRouteVersion uint64) ([]byte, error) { engine := engineForGroup(g) if engine == nil { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 20cd9ba9e..c4d2615d0 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -101,6 +101,100 @@ func TestShardStoreGetAt_MergesStagedVisibilityPebbleExactKey(t *testing.T) { require.Equal(t, []byte("staged-new"), got) } +func TestShardStoreGetAt_MergesStagedVisibilityForS3BucketAuxiliary(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 2, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{2: group}) + + for _, tc := range []struct { + name string + key []byte + value []byte + }{ + {name: "bucket meta", key: s3keys.BucketMetaKey(bucket), value: []byte("meta")}, + {name: "bucket generation", key: s3keys.BucketGenerationKey(bucket), value: []byte("generation")}, + } { + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, tc.key), tc.value, 20, 0)) + + got, err := st.GetAt(ctx, tc.key, 25) + require.NoError(t, err) + require.Equal(t, tc.value, got) + }) + } +} + +func TestShardStoreRejectsS3BucketAuxiliaryWriteAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte(""), + End: routeStart, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: routeStart, + End: routeEnd, + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 3, + Start: routeEnd, + End: nil, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + }, + })) + st := NewShardStore(engine, map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + }) + + for _, key := range [][]byte{ + s3keys.BucketMetaKey(bucket), + s3keys.BucketGenerationKey(bucket), + } { + require.ErrorIs(t, st.PutAt(ctx, key, []byte("v"), 100, 0), ErrRouteWriteTimestampTooLow) + require.ErrorIs(t, st.ApplyMutations(ctx, []*store.KVPairMutation{{Op: store.OpTypePut, Key: key, Value: []byte("v")}}, nil, 90, 100), ErrRouteWriteTimestampTooLow) + } +} + func TestShardStoreStagedVisibilityReadTSCompacted(t *testing.T) { t.Parallel() @@ -161,6 +255,33 @@ func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { require.Equal(t, uint64(40), ts) } +func TestShardStoreRouteFilteredLeaderScanUsesStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged-b"), 20, 0)) + route, _, ok := st.routeAndGroupForKey([]byte("b")) + require.True(t, ok) + + filtered, cursorKVs, err := st.scanRouteAtLeaderRouteFilter( + ctx, + group, + route, + []byte("a"), + []byte("z"), + 10, + 10, + 25, + false, + []byte("b"), + []byte("c"), + ) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: []byte("b"), Value: []byte("staged-b")}}, filtered) + require.Equal(t, []*store.KVPair{{Key: []byte("b"), Value: []byte("staged-b")}}, cursorKVs) +} + func TestShardStoreExplicitGroupReads_MergeStagedVisibility(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 0fab4636c..edbac1140 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1110,12 +1110,26 @@ func (c *ShardedCoordinator) rejectWriteTimestampFloorPointElems(elems []*Elem[O if elem == nil || len(elem.Key) == 0 { continue } - rkey := routeKey(elem.Key) - route, ok := c.engine.GetRoute(rkey) - if !ok || route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { - continue + if err := c.rejectWriteTimestampFloorPointKey(elem.Key, commitTS); err != nil { + return err + } + } + return nil +} + +func (c *ShardedCoordinator) rejectWriteTimestampFloorPointKey(key []byte, commitTS uint64) error { + rkey := routeKey(key) + if route, ok := c.engine.GetRoute(rkey); ok && route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, rkey, commitTS, route.MinWriteTSExclusive) + } + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return nil + } + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) } - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", elem.Key, rkey, commitTS, route.MinWriteTSExclusive) } return nil } @@ -2195,12 +2209,9 @@ func (c *ShardedCoordinator) rejectWriteTimestampFloorMutations(muts []*pb.Mutat if mut == nil || len(mut.Key) == 0 { continue } - rkey := routeKey(mut.Key) - route, ok := c.engine.GetRoute(rkey) - if !ok || route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { - continue + if err := c.rejectWriteTimestampFloorPointKey(mut.Key, commitTS); err != nil { + return err } - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", mut.Key, rkey, commitTS, route.MinWriteTSExclusive) } return nil } diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 3ffc0a4b9..c6aa9192f 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -227,6 +227,42 @@ func TestShardedCoordinatorRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute( } } +func TestShardedCoordinatorRejectsS3BucketAuxiliaryPointWriteAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + const bucket = "bucket-a" + start := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + end := prefixScanEnd(start) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: start, GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: start, End: end, GroupID: 2, State: distribution.RouteStateActive, MinWriteTSExclusive: ^uint64(0)}, + {RouteID: 3, Start: end, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + for _, key := range [][]byte{ + s3keys.BucketMetaKey(bucket), + s3keys.BucketGenerationKey(bucket), + } { + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: key, Value: []byte("v")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + require.Empty(t, g1Txn.requests, "coordinator must reject before proposing to the raw-key shard") + require.Empty(t, g2Txn.requests, "coordinator must reject before proposing to the floor-fenced shard") + } +} + func TestShardedCoordinatorRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { t.Parallel() From b1381f3afb46d4a7c05f56427b539b94c35c0fa2 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 19:13:57 +0900 Subject: [PATCH 073/162] migration: validate promotion resume state --- adapter/internal.go | 2 +- adapter/internal_migration_test.go | 21 ++++++++++++- store/migration_versions.go | 31 ++++++++++++++++++- store/migration_versions_test.go | 48 ++++++++++++++++++++++++++++++ store/mvcc_store.go | 1 + store/mvcc_store_snapshot_test.go | 46 ++++++++++++++++++++++++++++ 6 files changed, 146 insertions(+), 3 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index 019abec82..7227296bf 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -253,7 +253,7 @@ func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { func validatePromoteStagedVersionsRequest(req *pb.PromoteStagedVersionsRequest) error { prefix := distribution.MigrationStagedDataKeyPrefix(req.GetJobId()) - if err := store.ValidateExportCursorForRange(req.GetCursor(), prefix, store.PrefixScanEnd(prefix)); err != nil { + if err := store.ValidatePromotionCursorForRange(req.GetCursor(), prefix, store.PrefixScanEnd(prefix)); err != nil { if errors.Is(err, store.ErrInvalidExportCursor) { return errors.WithStack(status.Error(codes.InvalidArgument, store.ErrInvalidExportCursor.Error())) } diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index c1cf5df04..542a8b394 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -60,6 +60,13 @@ type captureExportRangeVersionsStream struct { responses []*pb.ExportRangeVersionsResponse } +const ( + testExportCursorTagEmitted byte = iota + testExportCursorTagScanned + testExportCursorTagPrunedKey + testExportCursorTagSkippedKey +) + func (s *captureExportRangeVersionsStream) Context() context.Context { if s.ctx != nil { return s.ctx @@ -312,7 +319,19 @@ func TestInternalPromoteStagedVersionsRejectsInvalidCursorBeforePropose(t *testi {name: "malformed cursor", cursor: []byte{0xff}}, { name: "cursor outside job staged prefix", - cursor: encodeTestExportCursor(distribution.MigrationStagedDataKey(8, []byte("k")), 30, 0), + cursor: encodeTestExportCursor(distribution.MigrationStagedDataKey(8, []byte("k")), 30, testExportCursorTagEmitted), + }, + { + name: "scanned cursor inside staged prefix", + cursor: encodeTestExportCursor(distribution.MigrationStagedDataKey(7, []byte("k")), 31, testExportCursorTagScanned), + }, + { + name: "pruned-key cursor inside staged prefix", + cursor: encodeTestExportCursor(distribution.MigrationStagedDataKey(7, []byte("k")), 32, testExportCursorTagPrunedKey), + }, + { + name: "skipped-key cursor inside staged prefix", + cursor: encodeTestExportCursor(distribution.MigrationStagedDataKey(7, []byte("k")), 33, testExportCursorTagSkippedKey), }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/store/migration_versions.go b/store/migration_versions.go index df40b33fc..ef3ef8254 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -92,8 +92,20 @@ func decodeExportCursor(cursor []byte) (exportCursorPosition, error) { } // ValidateExportCursorForRange verifies that an export cursor decodes and -// resumes inside the supplied key interval. +// resumes inside the supplied key interval. Skipped-key cursors are accepted +// only when they describe a key outside the interval. func ValidateExportCursorForRange(cursor, startKey, endKey []byte) error { + pos, err := decodeExportCursor(cursor) + if err != nil { + return err + } + return validateExportCursorPositionForRange(pos, startKey, endKey) +} + +// ValidatePromotionCursorForRange verifies a promotion cursor before it is +// proposed to Raft. Promotion scans emit only accepted positions, so callers +// must not resume from sparse-scan-only cursor tags. +func ValidatePromotionCursorForRange(cursor, startKey, endKey []byte) error { pos, err := decodeExportCursor(cursor) if err != nil { return err @@ -101,6 +113,23 @@ func ValidateExportCursorForRange(cursor, startKey, endKey []byte) error { if !pos.hasKey { return nil } + if pos.tag != exportCursorTagEmitted { + return errors.WithStack(ErrInvalidExportCursor) + } + return validateExportCursorPositionForRange(pos, startKey, endKey) +} + +func validateExportCursorPositionForRange(pos exportCursorPosition, startKey, endKey []byte) error { + if !pos.hasKey { + return nil + } + if pos.tag == exportCursorTagSkippedKey { + opts := ExportVersionsOptions{StartKey: startKey, EndKey: endKey} + if !exportSkippedCursorOutsideRange(opts, pos.key) { + return errors.WithStack(ErrInvalidExportCursor) + } + return nil + } if startKey != nil && bytes.Compare(pos.key, startKey) < 0 { return errors.WithStack(ErrInvalidExportCursor) } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index b4c0b964d..34cffdfeb 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -405,6 +405,54 @@ func TestExportVersionsRejectsCursorOutsideRequestedRange(t *testing.T) { }) } +func TestValidateExportCursorForRangeRejectsSkippedCursorInsideRange(t *testing.T) { + t.Parallel() + + err := ValidateExportCursorForRange( + encodeExportCursor([]byte("stage|k"), 10, exportCursorTagSkippedKey), + []byte("stage|"), + PrefixScanEnd([]byte("stage|")), + ) + require.ErrorIs(t, err, ErrInvalidExportCursor) + + err = ValidateExportCursorForRange( + encodeExportCursor([]byte("outside|k"), 10, exportCursorTagSkippedKey), + []byte("stage|"), + PrefixScanEnd([]byte("stage|")), + ) + require.NoError(t, err) +} + +func TestValidatePromotionCursorForRangeAcceptsOnlyEmittedPositions(t *testing.T) { + t.Parallel() + + prefix := []byte("stage|") + key := []byte("stage|k") + for _, tc := range []struct { + name string + cursor []byte + wantErr bool + }{ + {name: "empty cursor"}, + {name: "emitted cursor", cursor: encodeExportCursor(key, 10, exportCursorTagEmitted)}, + {name: "scanned cursor", cursor: encodeExportCursor(key, 10, exportCursorTagScanned), wantErr: true}, + {name: "pruned-key cursor", cursor: encodeExportCursor(key, 10, exportCursorTagPrunedKey), wantErr: true}, + {name: "skipped-key cursor", cursor: encodeExportCursor(key, 10, exportCursorTagSkippedKey), wantErr: true}, + {name: "emitted cursor outside range", cursor: encodeExportCursor([]byte("other|k"), 10, exportCursorTagEmitted), wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + err := ValidatePromotionCursorForRange(tc.cursor, prefix, PrefixScanEnd(prefix)) + if tc.wantErr { + require.ErrorIs(t, err, ErrInvalidExportCursor) + return + } + require.NoError(t, err) + }) + } +} + func TestExportVersionsDoesNotTreatMigrationPrefixUserKeyAsMetadata(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 9590a1c44..88817d94d 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -938,6 +938,7 @@ func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { s.minRetainedTS = minRetainedTS s.migrationAcks = make(map[migrationAckID]migrationImportAck) s.migrationHLCFloors = make(map[uint64]uint64) + s.migrationPromotions = make(map[uint64]PromotionState) return nil } diff --git a/store/mvcc_store_snapshot_test.go b/store/mvcc_store_snapshot_test.go index b16cbe68b..3032a1cc3 100644 --- a/store/mvcc_store_snapshot_test.go +++ b/store/mvcc_store_snapshot_test.go @@ -61,6 +61,19 @@ func TestMVCCStore_RestoreClearsMigrationMetadata(t *testing.T) { ctx := context.Background() st := newTestMVCCStore(t) + promoter, ok := any(st).(MigrationPromoter) + require.True(t, ok) + stateReader, ok := any(st).(MigrationPromotionStateReader) + require.True(t, ok) + + prefix := []byte("stage|") + stage := func(raw string) []byte { + return append([]byte("stage|"), []byte(raw)...) + } + targetKey := func(staged []byte) ([]byte, bool) { + return bytes.TrimPrefix(staged, prefix), bytes.HasPrefix(staged, prefix) + } + require.NoError(t, st.PutAt(ctx, []byte("base"), []byte("v1"), 10, 0)) snap, err := st.Snapshot() @@ -80,13 +93,46 @@ func TestMVCCStore_RestoreClearsMigrationMetadata(t *testing.T) { require.NoError(t, err) require.Equal(t, uint64(50), floor) + require.NoError(t, st.PutAt(ctx, stage("stale"), []byte("old"), 70, 0)) + promoted, err := promoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 7, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, promoted.Done) + state, ok, err := stateReader.MigrationPromotionState(ctx, 7) + require.NoError(t, err) + require.True(t, ok) + require.True(t, state.Done) + require.NoError(t, st.Restore(bytes.NewReader(raw))) floor, err = st.MigrationHLCFloor(ctx, 7) require.NoError(t, err) require.Zero(t, floor) + _, ok, err = stateReader.MigrationPromotionState(ctx, 7) + require.NoError(t, err) + require.False(t, ok) _, err = st.GetAt(ctx, []byte("imported"), 50) require.ErrorIs(t, err, ErrKeyNotFound) + require.NoError(t, st.PutAt(ctx, stage("fresh"), []byte("new"), 80, 0)) + promoted, err = promoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 7, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, promoted.Done) + require.Equal(t, uint64(1), promoted.PromotedRows) + got, err := st.GetAt(ctx, []byte("fresh"), 80) + require.NoError(t, err) + require.Equal(t, []byte("new"), got) + res, err := st.ImportVersions(ctx, ImportVersionsOptions{ JobID: 7, BracketID: 3, From dddda075967b645d5e6eafb11bd49bae4f250c57 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 19:16:17 +0900 Subject: [PATCH 074/162] migration: validate staged readiness conflicts --- kv/fsm.go | 152 +++++++++++++++++++++++++---- kv/fsm_migration_fence_test.go | 65 ++++++++++++ kv/sharded_coordinator.go | 5 +- kv/sharded_coordinator_txn_test.go | 53 ++++++++++ store/lsm_migration.go | 2 +- store/migration_versions.go | 9 +- store/migration_versions_test.go | 10 +- 7 files changed, 268 insertions(+), 28 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index e4f1650fd..15d32f4bb 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -719,34 +719,70 @@ func (f *kvFSM) verifyTargetReadinessForRange(ctx context.Context, start []byte, } func (f *kvFSM) verifyTargetReadinessForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) error { + _, err := f.targetReadyRoutesForRouteRange(ctx, routeStart, routeEnd) + return err +} + +func (f *kvFSM) targetReadyRoutesForRange(ctx context.Context, start []byte, end []byte) ([]distribution.Route, error) { + routeStart, routeEnd := readinessRouteRange(start, end) + return f.targetReadyRoutesForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) targetReadyRoutesForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) ([]distribution.Route, error) { + routes, catalogVersion, proof := f.currentShardRoutesForRouteRange(routeStart, routeEnd) + reader, ok := f.store.(store.MigrationTargetReadinessReader) if !ok { - return nil + return routes, nil } states, err := reader.MigrationTargetReadinessStates(ctx) if err != nil { - return errors.WithStack(err) + return nil, errors.WithStack(err) } if len(states) == 0 { - return nil + return routes, nil } + if targetReadinessStatesSatisfied(states, routes, routeStart, routeEnd, f.shardGroupID, catalogVersion, proof) { + return routes, nil + } + return nil, errors.WithStack(ErrRouteCutoverPending) +} - var ( - snap RouteSnapshot - proof bool - ) - if f.routes != nil { - snap, proof = f.routes.Current() +func (f *kvFSM) currentShardRoutesForRouteRange(routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if f.routes == nil { + return nil, 0, false + } + snap, ok := f.routes.Current() + if !ok { + return nil, 0, false } + routes := make([]distribution.Route, 0) + for _, route := range snap.IntersectingRoutes(routeStart, routeEnd) { + if route.GroupID == f.shardGroupID { + routes = append(routes, route) + } + } + return routes, snap.Version(), true +} + +func targetReadinessStatesSatisfied( + states []store.TargetStagedReadinessState, + routes []distribution.Route, + routeStart []byte, + routeEnd []byte, + groupID uint64, + catalogVersion uint64, + proof bool, +) bool { for _, ready := range states { if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { continue } - if !proof || !routesSatisfyTargetReadiness(snap.IntersectingRoutes(routeStart, routeEnd), ready, f.shardGroupID, snap.Version()) { - return errors.WithStack(ErrRouteCutoverPending) + if !proof || !routesSatisfyTargetReadiness(routes, ready, groupID, catalogVersion) { + return false } } - return nil + return true } func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState, groupID uint64, catalogVersion uint64) bool { @@ -1114,7 +1150,7 @@ func (f *kvFSM) validateConflicts(ctx context.Context, muts []*pb.Mutation, star } seen[keyStr] = struct{}{} - latest, exists, err := f.store.LatestCommitTS(ctx, mut.Key) + latest, exists, err := f.latestCommitTSForTargetReadyKey(ctx, mut.Key) if err != nil { return errors.WithStack(err) } @@ -1125,6 +1161,64 @@ func (f *kvFSM) validateConflicts(ctx context.Context, muts []*pb.Mutation, star return nil } +func (f *kvFSM) validateReadConflicts(ctx context.Context, keys [][]byte, startTS uint64) error { + seen := make(map[string]struct{}, len(keys)) + for _, key := range keys { + if len(key) == 0 || isTxnInternalKey(key) { + continue + } + keyStr := string(key) + if _, ok := seen[keyStr]; ok { + continue + } + seen[keyStr] = struct{}{} + + latest, exists, err := f.latestCommitTSForTargetReadyKey(ctx, key) + if err != nil { + return errors.WithStack(err) + } + if exists && latest > startTS { + return errors.WithStack(store.NewWriteConflictError(key)) + } + } + return nil +} + +func (f *kvFSM) latestCommitTSForTargetReadyKey(ctx context.Context, key []byte) (uint64, bool, error) { + routes, err := f.targetReadyRoutesForRange(ctx, key, nextScanCursor(key)) + if err != nil { + return 0, false, err + } + liveTS, liveExists, err := f.store.LatestCommitTS(ctx, key) + if err != nil { + return 0, false, errors.WithStack(err) + } + latest, exists := liveTS, liveExists + for _, route := range routes { + if !routeHasStagedVisibility(route) { + continue + } + stagedTS, stagedExists, err := f.store.LatestCommitTS(ctx, distribution.MigrationStagedDataKey(route.MigrationJobID, key)) + if err != nil { + return 0, false, errors.WithStack(err) + } + if stagedExists && (!exists || stagedTS > latest) { + latest, exists = stagedTS, true + } + } + return latest, exists, nil +} + +func (f *kvFSM) validateTxnConflicts(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, startTS uint64) error { + if err := f.validateConflicts(ctx, muts, startTS); err != nil { + return errors.WithStack(err) + } + if err := f.validateReadConflicts(ctx, readKeys, startTS); err != nil { + return errors.WithStack(err) + } + return nil +} + func uniqueMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { if len(muts) == 0 { return []*pb.Mutation{}, nil @@ -1177,8 +1271,8 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - if err := f.validateConflicts(ctx, uniq, startTS); err != nil { - return errors.WithStack(err) + if err := f.validateTxnConflicts(ctx, uniq, r.ReadKeys, startTS); err != nil { + return err } expireAt := txnLockExpireAt(meta.LockTTLms) @@ -1243,12 +1337,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsNotFenced(ctx, muts, commitTS) - if err != nil { - return err - } - - storeMuts, err := f.buildOnePhaseStoreMutations(ctx, uniq) + uniq, storeMuts, err := f.onePhaseStoreMutations(ctx, muts, r.ReadKeys, startTS, commitTS) if err != nil { return err } @@ -1259,6 +1348,27 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } +func (f *kvFSM) onePhaseStoreMutations( + ctx context.Context, + muts []*pb.Mutation, + readKeys [][]byte, + startTS uint64, + commitTS uint64, +) ([]*pb.Mutation, []*store.KVPairMutation, error) { + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, commitTS) + if err != nil { + return nil, nil, err + } + if err := f.validateTxnConflicts(ctx, uniq, readKeys, startTS); err != nil { + return nil, nil, err + } + storeMuts, err := f.buildOnePhaseStoreMutations(ctx, uniq) + if err != nil { + return nil, nil, err + } + return uniq, storeMuts, nil +} + func (f *kvFSM) uniqueMutationsNotFenced(ctx context.Context, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 82c2d5410..65113c949 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -439,6 +439,71 @@ func TestFSMPrepareTxnChecksReadKeysForTargetReadiness(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } +func TestFSMPrepareTxnChecksStagedVisibilityWriteConflicts(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + require.NoError(t, fsm.store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged"), 20, 0)) + + err := fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + CommitTS: 120, + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, + }, + }, 10) + require.ErrorIs(t, err, store.ErrWriteConflict) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("b")), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMOnePhaseTxnChecksStagedVisibilityReadConflicts(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + require.NoError(t, fsm.store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("n")), []byte("staged"), 20, 0)) + + req := onePhaseReq(10, 120, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 120) + require.ErrorIs(t, err, store.ErrWriteConflict) + + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + func TestFSMPrepareUsesCommitTSForRouteFloorWhenPresent(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index bcb1d957e..603360f7f 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1982,10 +1982,7 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui readiness := &ShardStore{engine: c.engine, groups: c.groups} for _, key := range keys { route := readiness.explicitGroupRouteForKey(gid, key) - if err := readiness.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { - return err - } - ts, exists, err := g.Store.LatestCommitTS(ctx, key) + ts, exists, err := readiness.localLatestCommitTS(ctx, g, route, key) if err != nil { return errors.WithStack(err) } diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index fdce86762..759f100d0 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -541,6 +541,59 @@ func TestValidateReadOnlyShards_FailsClosedOnTargetReadiness(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestValidateReadOnlyShards_ChecksStagedVisibilityLatestCommitTS(t *testing.T) { + t.Parallel() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 2, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 7, + }, + }, + })) + + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{ + "x": 5, + string(distribution.MigrationStagedDataKey(7, []byte("x"))): 20, + }, + readiness: []store.TargetStagedReadinessState{ + { + JobID: 7, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + Armed: true, + }, + }, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + groupedReadKeys := map[uint64][][]byte{ + 2: {[]byte("x")}, + } + err := coord.validateReadOnlyShards(context.Background(), groupedReadKeys, []uint64{1}, 10) + require.ErrorIs(t, err, store.ErrWriteConflict) +} + func TestValidateReadOnlyShards_PropagatesStoreError(t *testing.T) { t.Parallel() engine := distribution.NewEngine() diff --git a/store/lsm_migration.go b/store/lsm_migration.go index f60f5ae02..424b3f633 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -24,7 +24,7 @@ func (s *pebbleStore) exportVersionsLocked(ctx context.Context, opts ExportVersi if opts.MaxVersions <= 0 { return ExportVersionsResult{Done: true}, nil } - if readTSCompacted(opts.ReadTS, s.effectiveMinRetainedTS()) { + if readTSCompacted(exportRetentionReadTS(opts), s.effectiveMinRetainedTS()) { return ExportVersionsResult{}, ErrReadTSCompacted } diff --git a/store/migration_versions.go b/store/migration_versions.go index 4c5f4edb8..4d2989833 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -235,12 +235,19 @@ func (s *mvccStore) exportVersionsLocked(ctx context.Context, opts ExportVersion } func (s *mvccStore) checkExportReadTSLocked(opts ExportVersionsOptions) error { - if readTSCompacted(opts.ReadTS, s.minRetainedTS) { + if readTSCompacted(exportRetentionReadTS(opts), s.minRetainedTS) { return ErrReadTSCompacted } return nil } +func exportRetentionReadTS(opts ExportVersionsOptions) uint64 { + if opts.ReadTS != 0 { + return opts.ReadTS + } + return opts.MaxCommitTSInclusive +} + var errExportReachedEnd = errors.New("export reached end") var errExportChunkFull = errors.New("export chunk full") diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 2fb9ee72c..384c176c5 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -70,12 +70,20 @@ func TestExportVersionsReadTSPreservesCompactionErrors(t *testing.T) { }) require.ErrorIs(t, err, ErrReadTSCompacted) - result, err := st.ExportVersions(ctx, ExportVersionsOptions{ + _, err = st.ExportVersions(ctx, ExportVersionsOptions{ StartKey: []byte("k"), EndKey: []byte("l"), MaxCommitTSInclusive: 15, MaxVersions: 10, }) + require.ErrorIs(t, err, ErrReadTSCompacted) + + result, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 25, + MaxVersions: 10, + }) require.NoError(t, err) require.Len(t, result.Versions, 1) }) From 84704f947f5691a5875aea41f4e295d19e08419c Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 19:32:35 +0900 Subject: [PATCH 075/162] Close legacy list export gaps --- adapter/redis_compat_helpers.go | 22 +++++++--- adapter/redis_delta_compactor.go | 53 ++++++++++++++++++----- adapter/redis_delta_compactor_test.go | 48 ++++++++++++++++++++ adapter/redis_list_dedup_test.go | 18 ++++++++ adapter/redis_txn.go | 5 +++ adapter/redis_txn_test.go | 19 ++++++++ distribution/migrator.go | 7 ++- distribution/migrator_export_plan_test.go | 23 ++++++++++ kv/shard_store.go | 43 ++++++++++++++---- kv/shard_store_test.go | 20 +++++++++ kv/sharded_coordinator.go | 12 +++-- kv/sharded_coordinator_txn_test.go | 21 +++++++++ kv/transcoder.go | 3 ++ store/lsm_migration.go | 7 ++- store/migration_versions_test.go | 29 +++++++++++++ store/store.go | 5 ++- 16 files changed, 302 insertions(+), 33 deletions(-) diff --git a/adapter/redis_compat_helpers.go b/adapter/redis_compat_helpers.go index e24bffb0a..de96882aa 100644 --- a/adapter/redis_compat_helpers.go +++ b/adapter/redis_compat_helpers.go @@ -1357,13 +1357,13 @@ func (r *RedisServer) aggregateLenDeltas( ctx context.Context, prefix []byte, readTS uint64, + scanCap *deltaScanCap, unmarshalDelta func(key []byte, value []byte) (int64, bool, error), ) (int64, bool, error) { const cursorAdv = byte(0x00) end := store.PrefixScanEnd(prefix) var sum int64 var any bool - included := 0 cursor := prefix for { deltas, err := r.store.ScanAt(ctx, cursor, end, store.MaxDeltaScanLimit+1, readTS) @@ -1378,8 +1378,10 @@ func (r *RedisServer) aggregateLenDeltas( if !include { continue } - included++ - if included > store.MaxDeltaScanLimit { + if scanCap == nil { + scanCap = &deltaScanCap{} + } + if !scanCap.accept() { return 0, false, ErrDeltaScanTruncated } any = true @@ -1393,12 +1395,22 @@ func (r *RedisServer) aggregateLenDeltas( return sum, any, nil } +type deltaScanCap struct { + accepted int +} + +func (c *deltaScanCap) accept() bool { + c.accepted++ + return c.accepted <= store.MaxDeltaScanLimit +} + func (r *RedisServer) aggregateListMetaDeltas(ctx context.Context, key []byte, readTS uint64, applyDelta func(store.ListMetaDelta)) (int64, bool, error) { var total int64 var any bool + scanCap := &deltaScanCap{} for _, prefix := range store.ListMetaDeltaScanPrefixes(key) { legacy := isLegacyListMetaDeltaPrefix(prefix) - lenSum, hasDeltas, err := r.aggregateLenDeltas(ctx, prefix, readTS, func(deltaKey []byte, b []byte) (int64, bool, error) { + lenSum, hasDeltas, err := r.aggregateLenDeltas(ctx, prefix, readTS, scanCap, func(deltaKey []byte, b []byte) (int64, bool, error) { if legacy && (!store.IsListMetaDeltaValue(b) || !bytes.Equal(store.ExtractLegacyListUserKeyFromDelta(deltaKey), key)) { return 0, false, nil } @@ -1477,7 +1489,7 @@ func (r *RedisServer) resolveCollectionLen( } } - deltaSum, hasDeltas, err := r.aggregateLenDeltas(ctx, deltaPrefix, readTS, func(_ []byte, value []byte) (int64, bool, error) { + deltaSum, hasDeltas, err := r.aggregateLenDeltas(ctx, deltaPrefix, readTS, nil, func(_ []byte, value []byte) (int64, bool, error) { delta, err := unmarshalDelta(value) return delta, true, err }) diff --git a/adapter/redis_delta_compactor.go b/adapter/redis_delta_compactor.go index 4d9a35415..69a13c413 100644 --- a/adapter/redis_delta_compactor.go +++ b/adapter/redis_delta_compactor.go @@ -468,17 +468,8 @@ func (c *DeltaCompactor) compactHandler(ctx context.Context, h collectionDeltaHa rawKVs := kvs kvs = filterDeltaKVs(rawKVs, h.acceptDeltaKV) byKey, ukOrder := c.groupByUserKey(kvs, h.extractUserKey) - var lastAcceptedKey []byte - if len(kvs) > 0 { - lastAcceptedKey = kvs[len(kvs)-1].Key - } lastScannedKey := c.splitGuardCursor(byKey, ukOrder, kvs, truncated) - if truncated && len(rawKVs) > 0 { - if len(kvs) == 0 || - (!bytes.Equal(lastScannedKey, lastAcceptedKey) && rawPageHasRejectedTailAfter(rawKVs, lastAcceptedKey)) { - lastScannedKey = rawKVs[len(rawKVs)-1].Key - } - } + lastScannedKey = deltaCompactorCursorAfterFiltering(rawKVs, kvs, lastScannedKey, truncated) allElems := c.buildBatchElems(ctx, h, byKey, ukOrder, readTS) @@ -492,6 +483,28 @@ func (c *DeltaCompactor) compactHandler(ctx context.Context, h collectionDeltaHa return nil } +func deltaCompactorCursorAfterFiltering(rawKVs, acceptedKVs []*store.KVPair, lastScannedKey []byte, truncated bool) []byte { + if !truncated || len(rawKVs) == 0 { + return lastScannedKey + } + if len(acceptedKVs) == 0 { + return rawKVs[len(rawKVs)-1].Key + } + lastAcceptedKey := acceptedKVs[len(acceptedKVs)-1].Key + switch { + case len(lastScannedKey) == 0: + if prev := rawKeyBeforeAcceptedTail(rawKVs, lastAcceptedKey); len(prev) > 0 { + return prev + } + if rawPageHasRejectedTailAfter(rawKVs, lastAcceptedKey) { + return rawKVs[len(rawKVs)-1].Key + } + case !bytes.Equal(lastScannedKey, lastAcceptedKey) && rawPageHasRejectedTailAfter(rawKVs, lastAcceptedKey): + return rawKVs[len(rawKVs)-1].Key + } + return lastScannedKey +} + func rawPageHasRejectedTailAfter(rawKVs []*store.KVPair, acceptedKey []byte) bool { if len(rawKVs) == 0 || len(acceptedKey) == 0 { return false @@ -504,6 +517,22 @@ func rawPageHasRejectedTailAfter(rawKVs []*store.KVPair, acceptedKey []byte) boo return false } +func rawKeyBeforeAcceptedTail(rawKVs []*store.KVPair, acceptedKey []byte) []byte { + if len(rawKVs) < 2 || len(acceptedKey) == 0 { + return nil + } + last := len(rawKVs) - 1 + if rawKVs[last] == nil || !bytes.Equal(rawKVs[last].Key, acceptedKey) { + return nil + } + for i := last - 1; i >= 0; i-- { + if rawKVs[i] != nil { + return rawKVs[i].Key + } + } + return nil +} + // groupByUserKey groups KVPairs by their user key, returning both the map and // the unique user keys in lexicographic (scan) order. func filterDeltaKVs(kvs []*store.KVPair, accept func(*store.KVPair) bool) []*store.KVPair { @@ -729,7 +758,7 @@ func (c *DeltaCompactor) buildListCompactElems(ctx context.Context, userKey []by elems := make([]*kv.Elem[kv.OP], 0, 1+len(deltaKVs)+len(claimElems)) elems = append(elems, metaElem) for _, d := range deltaKVs { - elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: bytes.Clone(d.Key)}) + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: bytes.Clone(d.Key), GroupID: d.RouteGroupID}) } return append(elems, claimElems...), nil } @@ -894,7 +923,7 @@ func foldSimpleLenDeltas( elems := make([]*kv.Elem[kv.OP], 0, 1+len(deltaKVs)) elems = append(elems, metaElem) for _, d := range deltaKVs { - elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: bytes.Clone(d.Key)}) + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: bytes.Clone(d.Key), GroupID: d.RouteGroupID}) } return elems, nil } diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index 12ab9513a..137884419 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -423,6 +423,54 @@ func TestDeltaCompactor_LegacyListCursorAdvancesPastFilteredTailAfterBacktrack(t require.NoError(t, err) } +func TestDeltaCompactor_LegacyListCursorAdvancesPastRejectedPrefixBeforeAcceptedTail(t *testing.T) { + t.Parallel() + + st, c := newDeltaCompactorTestFixture(t) + ctx := context.Background() + userKey := []byte("compactor-tail-after-prefix") + meta, err := store.MarshalListMeta(store.ListMeta{Head: 4, Tail: 6, Len: 2}) + require.NoError(t, err) + for i := uint64(1); i < deltaCompactorTickScanLimit; i++ { + collidingUserKey := deltaLookingListMetaUserKeyAt(userKey, i, 0) + require.NoError(t, st.PutAt(ctx, store.ListMetaKey(collidingUserKey), meta, i, 0)) + } + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) + acceptedKey := legacyListMetaDeltaKey(userKey, deltaCompactorTickScanLimit) + require.NoError(t, st.PutAt(ctx, acceptedKey, delta, deltaCompactorTickScanLimit, 0)) + + h := c.legacyListHandler() + readTS := st.LastCommitTS() + require.NoError(t, c.compactHandler(ctx, h, readTS)) + + c.cursorMu.Lock() + got := bytes.Clone(c.cursors[h.typeName]) + c.cursorMu.Unlock() + require.Equal(t, store.ListMetaKey(deltaLookingListMetaUserKeyAt(userKey, deltaCompactorTickScanLimit-1, 0)), got) + _, err = st.GetAt(ctx, acceptedKey, readTS) + require.NoError(t, err) +} + +func TestDeltaCompactor_ListDeltaDeletesPreserveScanRouteGroup(t *testing.T) { + t.Parallel() + + _, c := newDeltaCompactorTestFixture(t) + ctx := context.Background() + userKey := []byte("compactor-route-group") + delta := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) + d1 := legacyListMetaDeltaKey(userKey, 1) + d2 := legacyListMetaDeltaKey(userKey, 2) + + elems, err := c.buildListCompactElems(ctx, userKey, []*store.KVPair{ + {Key: d1, Value: delta, RouteGroupID: 7}, + {Key: d2, Value: delta, RouteGroupID: 7}, + }, 3) + require.NoError(t, err) + require.Zero(t, elems[0].GroupID) + require.Equal(t, uint64(7), requireElemByKey(t, elems, d1).GroupID) + require.Equal(t, uint64(7), requireElemByKey(t, elems, d2).GroupID) +} + // TestDeltaCompactor_UrgentCompactionTriggeredByChannel verifies that a request // queued via TriggerUrgentCompaction is processed by the Run loop, compacting // the targeted key without waiting for the next regular tick. diff --git a/adapter/redis_list_dedup_test.go b/adapter/redis_list_dedup_test.go index 64575e93f..442eb0cde 100644 --- a/adapter/redis_list_dedup_test.go +++ b/adapter/redis_list_dedup_test.go @@ -76,6 +76,24 @@ func TestResolveListMetaReadsLegacyDeltaPrefix(t *testing.T) { require.Equal(t, int64(14), meta.Tail) } +func TestResolveListMetaEnforcesDeltaLimitAcrossCurrentAndLegacyPrefixes(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + srv := &RedisServer{store: st} + key := []byte("list-delta-cap") + delta := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) + for i := uint64(1); i <= uint64(store.MaxDeltaScanLimit); i++ { + require.NoError(t, st.PutAt(ctx, store.ListMetaDeltaKey(key, i, 0), delta, i, 0)) + } + legacyTS := uint64(store.MaxDeltaScanLimit + 1) + require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, legacyTS), delta, legacyTS, 0)) + + _, _, err := srv.resolveListMeta(ctx, key, legacyTS) + require.ErrorIs(t, err, ErrDeltaScanTruncated) +} + func TestResolveListMetaDoesNotTreatDeltaLookingMetaValueAsLegacyDelta(t *testing.T) { t.Parallel() diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index 750011ca7..24417fa54 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -352,6 +352,7 @@ func (t *txnContext) loadListState(key []byte) (*listTxnState, error) { // safely enumerate all of them for deletion, so we return ErrDeltaScanTruncated // and let the caller retry after the background compactor has caught up. var existingDeltas [][]byte + acceptedDeltas := 0 for _, deltaPrefix := range store.ListMetaDeltaScanPrefixes(key) { deltaKVs, truncated, err := scanAcceptedDeltaKVsAt( ctx, @@ -368,6 +369,10 @@ func (t *txnContext) loadListState(key []byte) (*listTxnState, error) { return nil, ErrDeltaScanTruncated } for _, kv := range deltaKVs { + acceptedDeltas++ + if acceptedDeltas > store.MaxDeltaScanLimit { + return nil, ErrDeltaScanTruncated + } existingDeltas = append(existingDeltas, kv.Key) } } diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index 3349c4b8b..9a5c647ff 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -67,6 +67,25 @@ func TestRedisTxnLoadListStateFiltersLegacyDeltaPrefixCollisions(t *testing.T) { require.Equal(t, [][]byte{deltaKey}, stState.existingDeltas) } +func TestRedisTxnLoadListStateEnforcesDeltaLimitAcrossCurrentAndLegacyPrefixes(t *testing.T) { + t.Parallel() + + server, st := newRedisStorageMigrationTestServer(t) + ctx := context.Background() + key := []byte("txn-list-delta-cap") + delta := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) + for i := uint64(1); i <= uint64(store.MaxDeltaScanLimit); i++ { + require.NoError(t, st.PutAt(ctx, store.ListMetaDeltaKey(key, i, 0), delta, i, 0)) + } + legacyTS := uint64(store.MaxDeltaScanLimit + 1) + require.NoError(t, st.PutAt(ctx, legacyListMetaDeltaKey(key, legacyTS), delta, legacyTS, 0)) + + txn := newRedisTxnTestContext(server) + txn.startTS = legacyTS + _, err := txn.loadListState(key) + require.ErrorIs(t, err, ErrDeltaScanTruncated) +} + func elemKeysContain(elems []*kv.Elem[kv.OP], want []byte) bool { for _, elem := range elems { if elem != nil && string(elem.Key) == string(want) { diff --git a/distribution/migrator.go b/distribution/migrator.go index a70bd554f..097bfbe2e 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -240,7 +240,7 @@ func (b MigrationBracket) ContainsRoutedKey(rawKey, routeStart, routeEnd []byte, return b.containsDecodedS3Route(rawKey, routeStart, routeEnd) } if b.Family == MigrationFamilyLegacyListMetaDelta { - return routeKeyInRange(store.ExtractLegacyListUserKeyFromDelta(rawKey), routeStart, routeEnd) + return b.containsLegacyListMetaDeltaRoute(rawKey, routeStart, routeEnd) } if !b.RequiresRouteKeyCheck { return true @@ -251,6 +251,11 @@ func (b MigrationBracket) ContainsRoutedKey(rawKey, routeStart, routeEnd []byte, return routeKeyInRange(routeKey(rawKey), routeStart, routeEnd) } +func (b MigrationBracket) containsLegacyListMetaDeltaRoute(rawKey, routeStart, routeEnd []byte) bool { + return routeKeyInRange(store.ExtractLegacyListUserKeyFromDelta(rawKey), routeStart, routeEnd) || + routeKeyInRange(store.ExtractListUserKey(rawKey), routeStart, routeEnd) +} + func (b MigrationBracket) containsFamilyShape(rawKey []byte) bool { switch b.Family { case MigrationFamilyListMetaDelta: diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index 06489eee6..d87ed277d 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -295,6 +295,29 @@ func TestMigrationBracketContainsRoutedKeyUsesLegacyListDeltaUserKey(t *testing. )) } +func TestMigrationBracketContainsRoutedKeyRoutesAmbiguousLegacyListMetaByBaseKey(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("a"), []byte("z")) + require.NoError(t, err) + legacy := bracketsByFamily(brackets)[MigrationFamilyLegacyListMetaDelta] + userKey := deltaLookingListMetaUserKey([]byte("target-list"), 10, 0) + raw := store.ListMetaKey(userKey) + + require.True(t, legacy.ContainsRoutedKey( + raw, + []byte("d|"), + []byte("d}"), + store.ExtractListUserKey, + )) + require.False(t, legacy.ContainsRoutedKey( + raw, + []byte("zzz"), + nil, + store.ExtractListUserKey, + )) +} + func TestMigrationKnownInternalPrefixesAreConcreteOnly(t *testing.T) { t.Parallel() diff --git a/kv/shard_store.go b/kv/shard_store.go index def64dba5..81b21e28d 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -306,6 +306,9 @@ func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Rou if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { return s.engine.GetIntersectingRoutes(routeStart, routeEnd), false } + if isBroadLegacyListDeltaScan(start) { + return s.engine.GetIntersectingRoutes(nil, nil), false + } if routes, ok := s.routesForLegacyListDeltaScan(start, end); ok { return routes, false } @@ -350,6 +353,15 @@ func (s *ShardStore) routesForLegacyListDeltaScan(start []byte, end []byte) ([]d return routes, true } +func isBroadLegacyListDeltaScan(start []byte) bool { + prefix := []byte(store.LegacyListMetaDeltaPrefix) + if !bytes.HasPrefix(start, prefix) { + return false + } + logicalUserKey := store.ExtractLegacyListUserKeyFromDeltaScanPrefix(start) + return logicalUserKey == nil || !bytes.Equal(start, store.LegacyListMetaDeltaScanPrefix(logicalUserKey)) +} + func scanRouteUserKey(start []byte) []byte { for _, extract := range scanRouteUserKeyExtractors { if userKey := extract(start); userKey != nil { @@ -614,11 +626,12 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceRouteFilterPage( if err != nil { return nil, nil, errors.WithStack(err) } - return filterTxnInternalKVs(kvs), kvs, nil + return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID), markScanRouteGroup(kvs, route.GroupID), nil } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.scanRouteAtLeaderRouteFilter(ctx, g, start, end, limit, visibleLimit, ts, reverse, routeStart, routeEnd) + kvs, cursorKVs, err := s.scanRouteAtLeaderRouteFilter(ctx, g, start, end, limit, visibleLimit, ts, reverse, routeStart, routeEnd) + return markScanRouteGroup(kvs, route.GroupID), markScanRouteGroup(cursorKVs, route.GroupID), err } groupID := proxyScanGroupID(route, explicitGroup, routeStart, routeEnd) @@ -627,7 +640,7 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceRouteFilterPage( return nil, nil, err } filtered := filterTxnInternalKVs(kvs) - return filtered, kvs, nil + return markScanRouteGroup(filtered, route.GroupID), markScanRouteGroup(kvs, route.GroupID), nil } func proxyScanGroupID(route distribution.Route, explicitGroup bool, routeStart []byte, routeEnd []byte) uint64 { @@ -660,11 +673,12 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceOnce( if err != nil { return nil, errors.WithStack(err) } - return filterTxnInternalKVs(kvs), nil + return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID), nil } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.scanRouteAtLeader(ctx, g, start, end, limit, ts, reverse) + kvs, err := s.scanRouteAtLeader(ctx, g, start, end, limit, ts, reverse) + return markScanRouteGroup(kvs, route.GroupID), err } groupID := proxyScanGroupID(route, explicitGroup, routeStart, routeEnd) @@ -674,7 +688,7 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceOnce( } // The leader's RawScanAt is expected to perform lock resolution and filtering // via ShardStore.ScanAt, so avoid N+1 proxy gets here. - return filterTxnInternalKVs(kvs), nil + return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID), nil } const routeFilteredScanBatchMin = 128 @@ -794,11 +808,12 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( if err != nil { return nil, limitReached, errors.WithStack(err) } - return filterTxnInternalKVs(kvs), limitReached, nil + return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID), limitReached, nil } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - return s.scanRouteAtLeaderPhysicalLimit(ctx, g, start, end, visibleLimit, physicalLimit, ts, reverse) + kvs, limitReached, err := s.scanRouteAtLeaderPhysicalLimit(ctx, g, start, end, visibleLimit, physicalLimit, ts, reverse) + return markScanRouteGroup(kvs, route.GroupID), limitReached, err } // RawScanAt cannot enforce physicalLimit, so report truncation and let @@ -1631,6 +1646,18 @@ func filterTxnInternalKVs(kvs []*store.KVPair) []*store.KVPair { return out } +func markScanRouteGroup(kvs []*store.KVPair, groupID uint64) []*store.KVPair { + if groupID == 0 { + return kvs + } + for _, kvp := range kvs { + if kvp != nil { + kvp.RouteGroupID = groupID + } + } + return kvs +} + type txnStatus int const ( diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 8a86b7307..33cccaad0 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -104,6 +104,26 @@ func TestShardStoreScanAt_RoutesListDeltaScansByUserKey(t *testing.T) { } } +func TestShardStoreScanAt_BroadLegacyListDeltaScansAllRoutes(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := newTwoRouteShardStoreForScanTest() + deltaValue := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) + leftKey := legacyListMetaDeltaKey([]byte("left-list"), 10, 1) + rightKey := legacyListMetaDeltaKey([]byte("right-list"), 11, 1) + require.NoError(t, st.groups[1].Store.PutAt(ctx, leftKey, deltaValue, 1, 0)) + require.NoError(t, st.groups[2].Store.PutAt(ctx, rightKey, deltaValue, 1, 0)) + + kvs, err := st.ScanAt(ctx, []byte(store.LegacyListMetaDeltaPrefix), store.PrefixScanEnd([]byte(store.LegacyListMetaDeltaPrefix)), 10, ^uint64(0)) + require.NoError(t, err) + require.Len(t, kvs, 2) + require.Equal(t, leftKey, kvs[0].Key) + require.Equal(t, uint64(1), kvs[0].RouteGroupID) + require.Equal(t, rightKey, kvs[1].Key) + require.Equal(t, uint64(2), kvs[1].RouteGroupID) +} + func TestShardStoreScanAt_RoutesWideColumnScansByUserKey(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index b8b67dba5..980066724 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -2034,9 +2034,15 @@ func (c *ShardedCoordinator) groupMutations(reqs []*Elem[OP], label keyviz.Label return nil, nil, ErrInvalidRequest } mut := elemToMutation(req) - gid, ok := c.router.ResolveGroup(mut.Key) - if !ok { - return nil, nil, errors.Wrapf(ErrInvalidRequest, "no route for key %q", mut.Key) + gid := req.GroupID + if gid == 0 { + var ok bool + gid, ok = c.router.ResolveGroup(mut.Key) + if !ok { + return nil, nil, errors.Wrapf(ErrInvalidRequest, "no route for key %q", mut.Key) + } + } else if _, ok := c.groups[gid]; !ok { + return nil, nil, errors.Wrapf(ErrInvalidRequest, "no shard group %d for key %q", gid, mut.Key) } // Engine RouteID for keyviz observation; partition-resolved // keys observe under the !sqs|route|global RouteID until diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 964f2713f..44f745176 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -8,6 +8,7 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/keyviz" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -55,6 +56,26 @@ func cloneTxnRequest(req *pb.Request) *pb.Request { return request } +func TestShardedCoordinatorGroupMutationsUsesExplicitElemGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), []byte("m"), 1) + engine.UpdateRoute([]byte("m"), nil, 2) + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {}, + }, 1, NewHLC(), nil) + + grouped, gids, err := coord.groupMutations([]*Elem[OP]{ + {Op: Del, Key: []byte("a-key"), GroupID: 2}, + }, keyviz.Label("")) + require.NoError(t, err) + require.Equal(t, []uint64{2}, gids) + require.Len(t, grouped[2], 1) + require.Equal(t, []byte("a-key"), grouped[2][0].Key) +} + func requestTxnMeta(t *testing.T, req *pb.Request) TxnMeta { t.Helper() require.NotNil(t, req) diff --git a/kv/transcoder.go b/kv/transcoder.go index afc52bed3..1df987a92 100644 --- a/kv/transcoder.go +++ b/kv/transcoder.go @@ -19,6 +19,9 @@ type Elem[T OP] struct { Op T Key []byte Value []byte + // GroupID optionally pins this mutation to a shard group. Zero preserves + // normal key-based routing. + GroupID uint64 } // OperationGroup is a group of operations that should be executed atomically. diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 1aab03419..5a6d22f55 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -212,12 +212,15 @@ func advancePebbleExportPastCurrentUserKey( } func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { - if len(startKey) == 0 { + if startKey == nil { return false } for prefixLen := 1; prefixLen <= len(userKey); prefixLen++ { prefix := userKey[:prefixLen] - if bytes.Compare(prefix, startKey) >= 0 && bytes.Compare(prefix, endKey) < 0 { + if len(startKey) != 0 && bytes.Compare(prefix, startKey) < 0 { + continue + } + if bytes.Compare(prefix, endKey) < 0 { return false } } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index d76a10ee3..9035dd692 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -606,6 +606,35 @@ func TestPebbleExportStopsAtEndKeyWhenNoLaterInRangeKeyCanTrail(t *testing.T) { require.Zero(t, result.ScannedBytes) } +func TestPebbleExportStopsLeadingRangeWithExplicitEmptyStartAtEndKey(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-leading-end-key-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + st, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + ps, ok := st.(*pebbleStore) + require.True(t, ok) + require.NoError(t, ps.PutAt(ctx, []byte("b"), []byte("later"), 10, 0)) + + iter, err := ps.db.NewIter(nil) + require.NoError(t, err) + defer func() { require.NoError(t, iter.Close()) }() + require.True(t, iter.SeekGE(encodeKey([]byte("b"), math.MaxUint64))) + + result := newExportVersionsResult(10) + advance, done, err := ps.exportPebbleIteratorPosition(ctx, iter, ExportVersionsOptions{ + StartKey: []byte{}, + EndKey: []byte("b"), + }, exportCursorPosition{}, &result) + require.ErrorIs(t, err, errExportReachedEnd) + require.False(t, advance) + require.True(t, done) + require.Empty(t, result.Versions) + require.Zero(t, result.ScannedBytes) +} + func TestPebbleExportOutOfRangeEndSkipReturnsResumableCursor(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-out-of-range-end-skip-*") diff --git a/store/store.go b/store/store.go index 04e943af8..bc80d6293 100644 --- a/store/store.go +++ b/store/store.go @@ -65,8 +65,9 @@ func (e *WriteConflictError) Unwrap() error { } type KVPair struct { - Key []byte - Value []byte + Key []byte + Value []byte + RouteGroupID uint64 } // MVCCVersion is a raw committed MVCC version for range migration. From 650c45072f0207f7705446244c9e0a4565125b26 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 19:41:26 +0900 Subject: [PATCH 076/162] migration: route staged s3 auxiliaries --- adapter/internal.go | 34 ++++++++++++-- adapter/internal_migration_test.go | 49 ++++++++++++++++++++ adapter/internal_test.go | 35 ++++++++++++++ kv/fsm.go | 5 ++ kv/shard_router.go | 19 ++++++++ kv/sharded_coordinator.go | 56 +++++++++++++++++------ kv/sharded_coordinator_del_prefix_test.go | 39 ++++++++++++++++ kv/sharded_coordinator_txn_test.go | 25 ++++++++++ 8 files changed, 243 insertions(+), 19 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index 65bd16d95..85b7caf29 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -181,6 +181,9 @@ func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeV if req == nil { return nil, errors.WithStack(status.Error(codes.InvalidArgument, "import range versions request is nil")) } + if err := validateImportRangeVersionsRequest(req); err != nil { + return nil, err + } if i.migrationProposer == nil { return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration import proposer is not configured")) } @@ -197,6 +200,16 @@ func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeV return &pb.ImportRangeVersionsResponse{AckedCursor: result.AckedCursor}, nil } +func validateImportRangeVersionsRequest(req *pb.ImportRangeVersionsRequest) error { + if req.GetJobId() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "import range versions job_id is required")) + } + if req.GetBracketId() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "import range versions bracket_id is required")) + } + return nil +} + func (i *Internal) verifyInternalLeader(ctx context.Context) error { if i.leader.State() != raftengine.StateLeader { return errors.WithStack(ErrNotLeader) @@ -282,15 +295,24 @@ func migrationFamilyRequiresDecodedS3(family uint32) bool { } func decodedS3BucketRouteFilter(family uint32, routeStart, routeEnd []byte) func([]byte) bool { + allowRawRouteMatch := !s3BucketRouteBounds(routeStart, routeEnd) return func(rawKey []byte) bool { bucket, ok := decodedS3BucketName(family, rawKey) if !ok { return false } + if allowRawRouteMatch && kv.RouteKeyFilter(routeStart, routeEnd)(rawKey) { + return true + } return decodedS3BucketRouteIntersects(bucket, routeStart, routeEnd) } } +func s3BucketRouteBounds(routeStart, routeEnd []byte) bool { + return bytes.HasPrefix(routeStart, []byte(s3keys.RoutePrefix)) || + bytes.HasPrefix(routeEnd, []byte(s3keys.RoutePrefix)) +} + func decodedS3BucketRouteIntersects(bucket string, routeStart, routeEnd []byte) bool { bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) return rangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) @@ -438,11 +460,11 @@ func (i *Internal) rejectWriteTimestampFloorMutations(muts []*pb.Mutation, commi } routes := i.routeEngine.Stats() for _, mut := range muts { - if mut == nil || len(mut.Key) == 0 { + if mut == nil || (len(mut.Key) == 0 && mut.GetOp() != pb.Op_DEL_PREFIX) { continue } for _, route := range routes { - if routeWriteTimestampFloorApplies(route, mut.Key, commitTS) { + if routeWriteTimestampFloorApplies(route, mut, commitTS) { return errors.Wrapf(kv.ErrRouteWriteTimestampTooLow, "key %q commit_ts=%d floor=%d", mut.Key, commitTS, route.MinWriteTSExclusive) } } @@ -450,11 +472,15 @@ func (i *Internal) rejectWriteTimestampFloorMutations(muts []*pb.Mutation, commi return nil } -func routeWriteTimestampFloorApplies(route distribution.Route, key []byte, commitTS uint64) bool { +func routeWriteTimestampFloorApplies(route distribution.Route, mut *pb.Mutation, commitTS uint64) bool { if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { return false } - return kv.RouteKeyFilter(route.Start, route.End)(key) + if mut.GetOp() == pb.Op_DEL_PREFIX { + start, end := kv.RoutePrefixRange(mut.GetKey()) + return rangesIntersect(route.Start, route.End, start, end) + } + return kv.RouteKeyFilter(route.Start, route.End)(mut.GetKey()) } func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) error { diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 2a403c963..05e74e76a 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -277,6 +277,33 @@ func TestInternalExportRangeVersionsIncludesS3BucketAuxiliaryForBucketRouteInter } } +func TestInternalExportRangeVersionsPreservesS3BucketRawRouteMatches(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + + key := s3keys.BucketMetaKey("bucket-raw") + require.NoError(t, st.PutAt(ctx, key, []byte("meta"), 10, 0)) + + stream := &captureExportRangeVersionsStream{ctx: ctx} + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + RouteStart: []byte("!s3|"), + RouteEnd: nil, + KeyFamily: distribution.MigrationFamilyS3BucketMeta, + RangeStart: []byte(s3keys.BucketMetaPrefix), + RangeEnd: testPrefixScanEnd([]byte(s3keys.BucketMetaPrefix)), + MaxScannedBytes: 1 << 20, + }, stream) + require.NoError(t, err) + require.Len(t, stream.responses, 1) + require.Equal(t, []*pb.MVCCVersion{ + {Key: key, CommitTs: 10, Value: []byte("meta"), KeyFamily: distribution.MigrationFamilyS3BucketMeta}, + }, stream.responses[0].GetVersions()) +} + func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() @@ -315,3 +342,25 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { require.Equal(t, uint64(30), floor) require.GreaterOrEqual(t, clock.Current(), uint64(30)) } + +func TestInternalImportRangeVersionsRejectsMissingIdentifiers(t *testing.T) { + t.Parallel() + + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalMigrationProposer(&applyingMigrationProposer{}), + ) + + _, err := internal.ImportRangeVersions(context.Background(), &pb.ImportRangeVersionsRequest{ + BracketId: 1, + BatchSeq: 1, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + + _, err = internal.ImportRangeVersions(context.Background(), &pb.ImportRangeVersionsRequest{ + JobId: 1, + BatchSeq: 1, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} diff --git a/adapter/internal_test.go b/adapter/internal_test.go index fe4a0c5ea..f7e922505 100644 --- a/adapter/internal_test.go +++ b/adapter/internal_test.go @@ -230,6 +230,41 @@ func TestStampRawTimestampsRejectsPreStampedRouteWriteFloor(t *testing.T) { require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) } +func TestStampRawTimestampsRejectsPreStampedDelPrefixRouteWriteFloor(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 0, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + }, + })) + i := &Internal{routeEngine: engine} + reqs := []*pb.Request{{ + Ts: 100, + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: nil}}, + }} + + err := i.stampRawTimestamps(context.Background(), reqs) + require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) +} + func TestStampTxnTimestampsRejectsRouteWriteFloor(t *testing.T) { t.Parallel() diff --git a/kv/fsm.go b/kv/fsm.go index 181ae515a..771f359c5 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -624,6 +624,11 @@ func routePrefixRange(prefix []byte) ([]byte, []byte) { return start, prefixScanEnd(start) } +// RoutePrefixRange maps a raw key prefix to the routed key range it may touch. +func RoutePrefixRange(prefix []byte) ([]byte, []byte) { + return routePrefixRange(prefix) +} + func routeKeyspaceWideRawPrefix(prefix []byte) bool { if !rawPrefixMayContainRouteMappedKeys(prefix) { return false diff --git a/kv/shard_router.go b/kv/shard_router.go index 4f6cd094c..4016fa452 100644 --- a/kv/shard_router.go +++ b/kv/shard_router.go @@ -134,6 +134,9 @@ func (s *ShardRouter) ResolveGroup(rawKey []byte) (uint64, bool) { return 0, false } } + if route, ok := s.stagedVisibilityRouteForS3BucketAuxiliaryKey(rawKey); ok { + return route.GroupID, true + } // Engine routes against the user-key view of the byte-range // space; routeKey may rewrite SQS / DynamoDB / Redis-internal // keys to a stable per-table or per-namespace route key so the @@ -145,6 +148,22 @@ func (s *ShardRouter) ResolveGroup(rawKey []byte) (uint64, bool) { return route.GroupID, true } +func (s *ShardRouter) stagedVisibilityRouteForS3BucketAuxiliaryKey(rawKey []byte) (distribution.Route, bool) { + if s == nil || s.engine == nil { + return distribution.Route{}, false + } + start, end, ok := s3BucketAuxiliaryRouteRange(rawKey) + if !ok { + return distribution.Route{}, false + } + for _, route := range s.engine.GetIntersectingRoutes(start, end) { + if routeHasStagedVisibility(route) { + return route, true + } + } + return distribution.Route{}, false +} + // Register associates a raft group ID with its transactional manager and store. func (s *ShardRouter) Register(group uint64, tm Transactional, st store.MVCCStore) { s.mu.Lock() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index edbac1140..d943302ad 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1947,7 +1947,7 @@ func (c *ShardedCoordinator) groupForKey(key []byte) (*ShardGroup, bool) { // catalog RouteID for !sqs|route|global. Partition-aware keyviz // is a Phase 3.D follow-up. func (c *ShardedCoordinator) routeAndGroupForKey(key []byte) (uint64, *ShardGroup, bool) { - gid, ok := c.router.ResolveGroup(key) + gid, routeID, ok := c.resolveGroupAndRouteForKey(key) if !ok { return 0, nil, false } @@ -1955,21 +1955,48 @@ func (c *ShardedCoordinator) routeAndGroupForKey(key []byte) (uint64, *ShardGrou if !ok { return 0, nil, false } - var routeID uint64 - if route, found := c.engine.GetRoute(routeKey(key)); found { - routeID = route.RouteID - } return routeID, g, true } func (c *ShardedCoordinator) engineGroupIDForKey(key []byte) uint64 { - gid, ok := c.router.ResolveGroup(key) + gid, _, ok := c.resolveGroupAndRouteForKey(key) if !ok { return 0 } return gid } +func (c *ShardedCoordinator) resolveGroupAndRouteForKey(key []byte) (uint64, uint64, bool) { + if route, ok := c.stagedVisibilityRouteForS3BucketAuxiliaryKey(key); ok { + return route.GroupID, route.RouteID, true + } + gid, ok := c.router.ResolveGroup(key) + if !ok { + return 0, 0, false + } + var routeID uint64 + if route, found := c.engine.GetRoute(routeKey(key)); found { + routeID = route.RouteID + } + return gid, routeID, true +} + +func (c *ShardedCoordinator) stagedVisibilityRouteForS3BucketAuxiliaryKey(key []byte) (distribution.Route, bool) { + if c == nil || c.engine == nil { + return distribution.Route{}, false + } + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return distribution.Route{}, false + } + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if routeHasStagedVisibility(route) { + return route, true + } + } + return distribution.Route{}, false +} + // EngineGroupIDForKey reports the Raft group ID that owns key, or 0 when // the key cannot be routed. Callers that batch lease checks across many // keys use it to collapse keys sharing a group into a single lease read @@ -2001,7 +2028,7 @@ func (c *ShardedCoordinator) groupReadKeysByShardID(readKeys [][]byte) (map[uint } grouped := make(map[uint64][][]byte) for _, key := range readKeys { - gid, ok := c.router.ResolveGroup(key) + gid, _, ok := c.resolveGroupAndRouteForKey(key) if !ok || gid == 0 { return nil, errors.Wrapf(ErrInvalidRequest, "no route for txn read key %q — recognised-but-"+ @@ -2051,6 +2078,12 @@ func (c *ShardedCoordinator) stagedVisibilityReadKeyAlias(gid uint64, key []byte if _, _, ok := distribution.MigrationStagedDataKeyParts(key); ok { return nil, false } + if route, ok := c.stagedVisibilityRouteForS3BucketAuxiliaryKey(key); ok { + if route.GroupID != gid { + return nil, false + } + return distribution.MigrationStagedDataKey(route.MigrationJobID, key), true + } route, ok := c.engine.GetRoute(routeKey(key)) if !ok || route.GroupID != gid || !routeHasStagedVisibility(route) { return nil, false @@ -2308,17 +2341,10 @@ func (c *ShardedCoordinator) groupMutations(reqs []*Elem[OP], label keyviz.Label return nil, nil, ErrInvalidRequest } mut := elemToMutation(req) - gid, ok := c.router.ResolveGroup(mut.Key) + gid, routeID, ok := c.resolveGroupAndRouteForKey(mut.Key) if !ok { return nil, nil, errors.Wrapf(ErrInvalidRequest, "no route for key %q", mut.Key) } - // Engine RouteID for keyviz observation; partition-resolved - // keys observe under the !sqs|route|global RouteID until - // partition-aware keyviz lands. - var routeID uint64 - if route, found := c.engine.GetRoute(routeKey(mut.Key)); found { - routeID = route.RouteID - } c.observeMutation(routeID, mut, label) grouped[gid] = append(grouped[gid], mut) } diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index c6aa9192f..26408e36b 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -227,6 +227,45 @@ func TestShardedCoordinatorRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute( } } +func s3BucketAuxiliaryStagedRoutes(bucket string, rawGroupID, stagedGroupID uint64) []distribution.RouteDescriptor { + routes := s3BucketAuxiliaryFenceRoutes(bucket, rawGroupID, stagedGroupID) + routes[1].State = distribution.RouteStateActive + routes[1].StagedVisibilityActive = true + routes[1].MigrationJobID = 9 + return routes +} + +func TestShardedCoordinatorRoutesS3BucketAuxiliaryWriteToStagedOwner(t *testing.T) { + t.Parallel() + + const bucket = "bucket-a" + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: s3BucketAuxiliaryStagedRoutes(bucket, 1, 2), + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{responses: []*TransactionResponse{{CommitIndex: 22}}} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + key := s3keys.BucketMetaKey(bucket) + route, ok := coord.stagedVisibilityRouteForS3BucketAuxiliaryKey(key) + require.True(t, ok) + require.Equal(t, uint64(2), route.GroupID) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: key, Value: []byte("meta")}}, + }) + require.NoError(t, err) + require.Empty(t, g1Txn.requests) + require.Len(t, g2Txn.requests, 1) + require.Equal(t, key, g2Txn.requests[0].Mutations[0].Key) +} + func TestShardedCoordinatorRejectsS3BucketAuxiliaryPointWriteAtMigrationTimestampFloor(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index e40318e00..ee8cce672 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -8,6 +8,7 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -547,6 +548,30 @@ func TestGroupReadKeysByShardID_FailsClosedOnUnroutable(t *testing.T) { require.ErrorIs(t, err, ErrInvalidRequest) } +func TestGroupReadKeysByShardID_RoutesS3BucketAuxiliaryToStagedOwner(t *testing.T) { + t.Parallel() + + const bucket = "bucket-a" + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: s3BucketAuxiliaryStagedRoutes(bucket, 1, 2), + })) + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {}, + }, 1, NewHLC(), nil) + + key := s3keys.BucketMetaKey(bucket) + grouped, err := coord.groupReadKeysByShardID([][]byte{key}) + require.NoError(t, err) + require.Empty(t, grouped[1]) + require.Equal(t, [][]byte{ + key, + distribution.MigrationStagedDataKey(9, key), + }, grouped[2]) +} + // --------------------------------------------------------------------------- // validateReadOnlyShards // --------------------------------------------------------------------------- From 19a28e908e5ebcff0b8c04172f5bd0a3297a4ff5 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 20:00:53 +0900 Subject: [PATCH 077/162] kv: keep floor checks out of raft replay --- kv/shard_store.go | 12 ------------ kv/shard_store_test.go | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/kv/shard_store.go b/kv/shard_store.go index 107be5a49..750e4b08b 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -2409,9 +2409,6 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. if err != nil || group == nil { return err } - if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { - return err - } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) @@ -2425,9 +2422,6 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor if err != nil || group == nil { return err } - if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { - return err - } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) @@ -2579,9 +2573,6 @@ func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS ui // DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { - if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { - return err - } for _, g := range s.groups { if g == nil || g.Store == nil { continue @@ -2608,9 +2599,6 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl // is the receiver only when an aggregate (admin / coordinator) path // is replaying a global FLUSHALL, which is not raft-applied. func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { - if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { - return err - } for _, g := range s.groups { if g == nil || g.Store == nil { continue diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index c4d2615d0..99a82eda0 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -548,6 +548,22 @@ func TestShardStoreRejectsWritesAtMigrationTimestampFloor(t *testing.T) { require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0)) } +func TestShardStoreRaftApplySkipsMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, _ := newStagedVisibilityShardStore(t) + + require.NoError(t, st.ApplyMutationsRaft(ctx, []*store.KVPairMutation{ + {Op: store.OpTypePut, Key: []byte("k-raft"), Value: []byte("v")}, + }, nil, 90, 100)) + require.NoError(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ + {Op: store.OpTypePut, Key: []byte("k-raft-at"), Value: []byte("v")}, + }, nil, 90, 100, 1)) + require.NoError(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100)) + require.NoError(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2)) +} + func TestShardStoreScanAt_IncludesListKeysAcrossShards(t *testing.T) { t.Parallel() From 40c2f527b3fd6d10d22e633112d703de373c6f6a Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 20:04:06 +0900 Subject: [PATCH 078/162] store: ignore stale promotion cursors --- store/migration_promote.go | 11 ++++--- store/migration_versions_test.go | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/store/migration_promote.go b/store/migration_promote.go index 8ee56e862..bfcd1af44 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -124,10 +124,11 @@ func (s *mvccStore) promotionStateAndCursorLocked(opts PromoteVersionsOptions) ( if opts.JobID == 0 { return PromotionState{}, opts.Cursor } - state := clonePromotionState(s.migrationPromotions[opts.JobID]) - if len(state.Cursor) == 0 { - return state, opts.Cursor + state, ok := s.migrationPromotions[opts.JobID] + if !ok { + return PromotionState{}, nil } + state = clonePromotionState(state) return state, state.Cursor } @@ -318,8 +319,8 @@ func (s *pebbleStore) pebblePromotionStateAndCursor(opts PromoteVersionsOptions) if err != nil { return PromotionState{}, nil, err } - if !ok || len(state.Cursor) == 0 { - return state, opts.Cursor, nil + if !ok { + return PromotionState{}, nil, nil } return state, state.Cursor, nil } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 34cffdfeb..c83ab6ca9 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -934,6 +934,62 @@ func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { }) } +func TestPromoteVersionsIgnoresClientCursorWhenStateMissing(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + promoter, ok := st.(MigrationPromoter) + require.True(t, ok) + stateReader, ok := st.(MigrationPromotionStateReader) + require.True(t, ok) + + stage := func(raw string) []byte { + return append([]byte("stage|"), []byte(raw)...) + } + targetKey := func(staged []byte) ([]byte, bool) { + return bytes.TrimPrefix(staged, []byte("stage|")), bytes.HasPrefix(staged, []byte("stage|")) + } + prefix := []byte("stage|") + + require.NoError(t, st.PutAt(ctx, stage("a"), []byte("a10"), 10, 0)) + require.NoError(t, st.PutAt(ctx, stage("z"), []byte("z20"), 20, 0)) + staleCursor := encodeExportCursor(stage("m"), 1, exportCursorTagEmitted) + + result, err := promoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 202, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + Cursor: staleCursor, + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, result.Done) + require.Equal(t, uint64(2), result.PromotedRows) + require.Equal(t, uint64(2), result.TotalPromotedRows) + + state, ok, err := stateReader.MigrationPromotionState(ctx, 202) + require.NoError(t, err) + require.True(t, ok) + require.True(t, state.Done) + require.Equal(t, uint64(2), state.PromotedRows) + + got, err := st.GetAt(ctx, []byte("a"), 10) + require.NoError(t, err) + require.Equal(t, []byte("a10"), got) + got, err = st.GetAt(ctx, []byte("z"), 20) + require.NoError(t, err) + require.Equal(t, []byte("z20"), got) + stagedLeft, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, stagedLeft.Done) + require.Empty(t, stagedLeft.Versions) + }) +} + func TestPebblePromoteVersionsAdvancesLastCommitTS(t *testing.T) { ctx := context.Background() dir := t.TempDir() From f1aeeeb396903a904e62fdfee9203cfc11aaafe3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 20:07:14 +0900 Subject: [PATCH 079/162] Guard split fence retry paths --- adapter/distribution_server.go | 39 ++++++++++--- adapter/distribution_server_test.go | 79 ++++++++++++++++++++++++++- adapter/dynamodb_transact.go | 2 +- adapter/redis_retry.go | 2 +- adapter/redis_retry_test.go | 1 + adapter/retryable_write_fence_test.go | 16 ++++++ adapter/s3.go | 2 +- distribution/engine.go | 15 +++-- distribution/engine_test.go | 15 +++++ 9 files changed, 150 insertions(+), 21 deletions(-) create mode 100644 adapter/retryable_write_fence_test.go diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index fa2b73a7d..0fcacba58 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -10,6 +10,7 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -162,7 +163,8 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR if err := validateSplitKey(parent, splitKey); err != nil { return nil, err } - if err := s.rejectLiveSplitJobOverlap(ctx, snapshot, parent); err != nil { + splitJobReadKeys, err := s.splitJobOverlapReadKeys(ctx, snapshot, parent) + if err != nil { return nil, err } @@ -172,7 +174,7 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR } left, right := splitCatalogRoutes(parent, splitKey, leftID, rightID) - saved, err := s.saveSplitResultViaCoordinator(ctx, snapshot.ReadTS, req.GetExpectedCatalogVersion(), parent.RouteID, left, right) + saved, err := s.saveSplitResultViaCoordinator(ctx, snapshot.ReadTS, req.GetExpectedCatalogVersion(), parent.RouteID, splitJobReadKeys, left, right) if err != nil { return nil, err } @@ -213,6 +215,7 @@ func (s *DistributionServer) saveSplitResultViaCoordinator( readTS uint64, expectedVersion uint64, parentID uint64, + readKeys [][]byte, left distribution.RouteDescriptor, right distribution.RouteDescriptor, ) (distribution.CatalogSnapshot, error) { @@ -230,10 +233,14 @@ func (s *DistributionServer) saveSplitResultViaCoordinator( return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "build split mutations: %v", err) } if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ - Elems: ops, - IsTxn: true, - StartTS: readTS, + Elems: ops, + IsTxn: true, + StartTS: readTS, + ReadKeys: readKeys, }); err != nil { + if errors.Is(err, store.ErrWriteConflict) { + return distribution.CatalogSnapshot{}, grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) + } return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "commit split mutations: %v", err) } return s.loadCatalogSnapshotAtLeastVersion(ctx, nextVersion) @@ -380,22 +387,36 @@ func validateSplitKey(parent distribution.RouteDescriptor, splitKey []byte) erro return nil } -func (s *DistributionServer) rejectLiveSplitJobOverlap(ctx context.Context, snapshot distribution.CatalogSnapshot, parent distribution.RouteDescriptor) error { +func (s *DistributionServer) splitJobOverlapReadKeys(ctx context.Context, snapshot distribution.CatalogSnapshot, parent distribution.RouteDescriptor) ([][]byte, error) { jobs, err := s.catalog.ListSplitJobsAt(ctx, snapshot.ReadTS) if err != nil { - return grpcStatusErrorf(codes.Internal, "load split jobs: %v", err) + return nil, grpcStatusErrorf(codes.Internal, "load split jobs: %v", err) } + readKeys := splitJobReadFenceKeys(jobs) for _, job := range jobs { if !splitJobIsLive(job) { continue } for _, interval := range liveSplitJobIntervals(job, snapshot.Routes) { if routeRangeIntersects(parent.Start, parent.End, interval.start, interval.end) { - return grpcStatusError(codes.Aborted, distribution.ErrSplitJobOverlap.Error()) + return nil, grpcStatusError(codes.Aborted, distribution.ErrSplitJobOverlap.Error()) } } } - return nil + return readKeys, nil +} + +func splitJobReadFenceKeys(jobs []distribution.SplitJob) [][]byte { + readKeys := make([][]byte, 0, len(jobs)+1) + readKeys = append(readKeys, distribution.CatalogNextSplitJobIDKey()) + for _, job := range jobs { + if job.TerminalAtMs > 0 { + readKeys = append(readKeys, distribution.CatalogSplitJobHistoryKey(job.TerminalAtMs, job.JobID)) + continue + } + readKeys = append(readKeys, distribution.CatalogSplitJobKey(job.JobID)) + } + return readKeys } func splitJobIsLive(job distribution.SplitJob) bool { diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 392cb581b..e69171029 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -1,6 +1,7 @@ package adapter import ( + "bytes" "context" "testing" "time" @@ -338,6 +339,47 @@ func TestDistributionServerSplitRange_AllowsDisjointRouteWhileSplitJobLive(t *te require.NoError(t, err) require.Equal(t, uint64(2), resp.CatalogVersion) require.Equal(t, 1, coordinator.dispatchCalls) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogNextSplitJobIDKey()) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogSplitJobKey(10)) +} + +func TestDistributionServerSplitRange_ConflictsWhenSplitJobCreatedAfterOverlapScan(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + coordinator.beforeApply = func(ctx context.Context, _ store.MVCCStore) error { + return catalog.CreateSplitJob(ctx, distribution.SplitJob{ + JobID: 10, + SourceRouteID: 1, + SplitKey: []byte("g"), + TargetGroupID: 8, + Phase: distribution.SplitJobPhaseBackfill, + }) + } + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + _, err = s.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("c"), + }) + require.Error(t, err) + require.Equal(t, codes.Aborted, status.Code(err)) + require.ErrorContains(t, err, errDistributionCatalogConflict.Error()) + require.Equal(t, 1, coordinator.dispatchCalls) + + snapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version, snapshot.Version) } func TestDistributionServerSplitRange_UsesCoordinatorForCatalogWrites(t *testing.T) { @@ -669,6 +711,8 @@ type distributionCoordinatorStub struct { leader bool nextTS uint64 lastStartTS uint64 + lastReadKeys [][]byte + beforeApply func(context.Context, store.MVCCStore) error afterDispatch func(context.Context, store.MVCCStore, uint64) error asyncApplyDone chan error asyncApplyDelay time.Duration @@ -689,6 +733,8 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope s.dispatchCalls++ startTS, commitTS := s.nextTimestamps(reqs.StartTS) s.lastStartTS = startTS + readKeys := cloneDistributionReadKeys(reqs.ReadKeys) + s.lastReadKeys = readKeys mutations, err := coordinatorStubMutations(reqs.Elems) if err != nil { @@ -699,14 +745,14 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope delay := s.asyncApplyDelay go func() { time.Sleep(delay) - err := s.applyDispatch(ctx, mutations, startTS, commitTS) + err := s.applyDispatch(ctx, mutations, readKeys, startTS, commitTS) if done != nil { done <- err } }() return &kv.CoordinateResponse{CommitIndex: commitTS}, nil } - if err := s.applyDispatch(ctx, mutations, startTS, commitTS); err != nil { + if err := s.applyDispatch(ctx, mutations, readKeys, startTS, commitTS); err != nil { return nil, err } return &kv.CoordinateResponse{CommitIndex: commitTS}, nil @@ -737,10 +783,16 @@ func (s *distributionCoordinatorStub) nextTimestamps(startTS uint64) (uint64, ui func (s *distributionCoordinatorStub) applyDispatch( ctx context.Context, mutations []*store.KVPairMutation, + readKeys [][]byte, startTS uint64, commitTS uint64, ) error { - if err := s.store.ApplyMutations(ctx, mutations, nil, startTS, commitTS); err != nil { + if s.beforeApply != nil { + if err := s.beforeApply(ctx, s.store); err != nil { + return err + } + } + if err := s.store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS); err != nil { return err } if s.afterDispatch != nil { @@ -751,6 +803,27 @@ func (s *distributionCoordinatorStub) applyDispatch( return nil } +func cloneDistributionReadKeys(in [][]byte) [][]byte { + if len(in) == 0 { + return nil + } + out := make([][]byte, len(in)) + for i := range in { + out[i] = distribution.CloneBytes(in[i]) + } + return out +} + +func requireReadKeysContain(t *testing.T, readKeys [][]byte, want []byte) { + t.Helper() + for _, key := range readKeys { + if bytes.Equal(key, want) { + return + } + } + t.Fatalf("expected read keys to contain %q, got %q", want, readKeys) +} + func coordinatorStubMutations(elems []*kv.Elem[kv.OP]) ([]*store.KVPairMutation, error) { mutations := make([]*store.KVPairMutation, 0, len(elems)) for _, elem := range elems { diff --git a/adapter/dynamodb_transact.go b/adapter/dynamodb_transact.go index 3f58a688a..7ccaaac59 100644 --- a/adapter/dynamodb_transact.go +++ b/adapter/dynamodb_transact.go @@ -1105,7 +1105,7 @@ func (d *DynamoDBServer) resolveTransactTableSchema(ctx context.Context, cache m } func isRetryableTransactWriteError(err error) bool { - return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) + return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || errors.Is(err, kv.ErrRouteWriteFenced) } func waitTransactRetryBackoff(ctx context.Context, delay time.Duration) error { diff --git a/adapter/redis_retry.go b/adapter/redis_retry.go index 206bc0930..0c38ddf41 100644 --- a/adapter/redis_retry.go +++ b/adapter/redis_retry.go @@ -41,7 +41,7 @@ var ( ) func isRetryableRedisTxnErr(err error) bool { - return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) + return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || errors.Is(err, kv.ErrRouteWriteFenced) } func retryPolicyForRedisTxnErr(err error) redisTxnRetryPolicy { diff --git a/adapter/redis_retry_test.go b/adapter/redis_retry_test.go index 7c1ce8491..b74a73277 100644 --- a/adapter/redis_retry_test.go +++ b/adapter/redis_retry_test.go @@ -353,6 +353,7 @@ func TestRetryPolicyForRedisTxnErr(t *testing.T) { require.Equal(t, redisWriteConflictRetryPolicy, retryPolicyForRedisTxnErr(store.ErrWriteConflict)) require.Equal(t, redisTxnLockedRetryPolicy, retryPolicyForRedisTxnErr(kv.ErrTxnLocked)) + require.Equal(t, redisWriteConflictRetryPolicy, retryPolicyForRedisTxnErr(kv.ErrRouteWriteFenced)) } // TestZCard_LegacyBlobZSet verifies that ZCARD inside a Lua script returns the diff --git a/adapter/retryable_write_fence_test.go b/adapter/retryable_write_fence_test.go new file mode 100644 index 000000000..fb9f9a5e0 --- /dev/null +++ b/adapter/retryable_write_fence_test.go @@ -0,0 +1,16 @@ +package adapter + +import ( + "testing" + + "github.com/bootjp/elastickv/kv" + "github.com/stretchr/testify/require" +) + +func TestWriteFenceErrorsAreAdapterRetryable(t *testing.T) { + t.Parallel() + + require.True(t, isRetryableRedisTxnErr(kv.ErrRouteWriteFenced)) + require.True(t, isRetryableS3MutationErr(kv.ErrRouteWriteFenced)) + require.True(t, isRetryableTransactWriteError(kv.ErrRouteWriteFenced)) +} diff --git a/adapter/s3.go b/adapter/s3.go index e267d7ccf..c01165c31 100644 --- a/adapter/s3.go +++ b/adapter/s3.go @@ -2459,7 +2459,7 @@ func (s *S3Server) nextTxnCommitTS(ctx context.Context, startTS uint64) (uint64, } func isRetryableS3MutationErr(err error) bool { - return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) + return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || errors.Is(err, kv.ErrRouteWriteFenced) } func waitS3RetryBackoff(ctx context.Context, delay time.Duration) bool { diff --git a/distribution/engine.go b/distribution/engine.go index 972bc2c4d..96d9a4c8e 100644 --- a/distribution/engine.go +++ b/distribution/engine.go @@ -424,12 +424,15 @@ func (e *Engine) GetIntersectingRoutes(start, end []byte) []Route { func cloneRoute(r Route) Route { return Route{ - RouteID: r.RouteID, - Start: CloneBytes(r.Start), - End: CloneBytes(r.End), - GroupID: r.GroupID, - State: r.State, - Load: r.Load, + RouteID: r.RouteID, + Start: CloneBytes(r.Start), + End: CloneBytes(r.End), + GroupID: r.GroupID, + State: r.State, + StagedVisibilityActive: r.StagedVisibilityActive, + MigrationJobID: r.MigrationJobID, + MinWriteTSExclusive: r.MinWriteTSExclusive, + Load: r.Load, } } diff --git a/distribution/engine_test.go b/distribution/engine_test.go index c464ba3fd..b777cb0f3 100644 --- a/distribution/engine_test.go +++ b/distribution/engine_test.go @@ -126,6 +126,21 @@ func TestEngineApplySnapshot_PreservesMigrationRouteFields(t *testing.T) { t.Fatalf("expected 1 intersecting route, got %d", len(intersections)) } requireMigrationRouteFields(t, "GetIntersectingRoutes", intersections[0]) + + snapshot, ok := e.Current() + if !ok { + t.Fatal("expected current history snapshot") + } + historyRoute, ok := snapshot.RouteOf([]byte("m")) + if !ok { + t.Fatal("expected history route") + } + requireMigrationRouteFields(t, "RouteHistorySnapshot.RouteOf", historyRoute) + historyIntersections := snapshot.IntersectingRoutes([]byte("b"), []byte("c")) + if len(historyIntersections) != 1 { + t.Fatalf("expected 1 history intersecting route, got %d", len(historyIntersections)) + } + requireMigrationRouteFields(t, "RouteHistorySnapshot.IntersectingRoutes", historyIntersections[0]) } func requireMigrationRouteFields(t *testing.T, label string, route Route) { From 9d9e051a7f0e1432b6e4cb0258b7c9a29cd8debc Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 20:17:34 +0900 Subject: [PATCH 080/162] Fix promotion completion merge compatibility --- distribution/migration_promotion_complete.go | 2 +- distribution/migration_promotion_complete_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go index 23473d6e5..88ee72b14 100644 --- a/distribution/migration_promotion_complete.go +++ b/distribution/migration_promotion_complete.go @@ -177,7 +177,7 @@ func (s *CatalogStore) buildPromotionCompleteMutations( nextVersion: expectedVersion + 1, routes: completion.Routes, } - mutations, err := s.buildSaveMutations(ctx, plan) + mutations, err := s.buildSaveMutations(ctx, &plan) if err != nil { return savePlan{}, nil, 0, err } diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go index f10870a51..b04045bfd 100644 --- a/distribution/migration_promotion_complete_test.go +++ b/distribution/migration_promotion_complete_test.go @@ -63,7 +63,7 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionCommitsRouteAndJobTogether(t t.Parallel() ctx := context.Background() - cs := NewCatalogStore(store.NewMVCCStore()) + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) require.NoError(t, err) job := promotionCompleteTestJob() @@ -96,7 +96,7 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDesc t.Parallel() ctx := context.Background() - cs := NewCatalogStore(store.NewMVCCStore()) + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) routes := promotionCompleteTestRoutes() routes[1].StagedVisibilityActive = false routes[1].MigrationJobID = 0 @@ -122,7 +122,7 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionRejectsStaleInputs(t *testin t.Parallel() ctx := context.Background() - cs := NewCatalogStore(store.NewMVCCStore()) + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) require.NoError(t, err) job := promotionCompleteTestJob() From 2d57ae9f4e3f5dd78e2b9c8bed677ba83c4cba79 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 21:12:49 +0900 Subject: [PATCH 081/162] Fail closed read-only shard readiness checks --- kv/sharded_coordinator.go | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index ab947913e..cb2868d6f 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -2159,6 +2159,9 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return errors.WithStack(err) } for _, key := range keys { + if err := c.verifyTargetReadinessForReadKeyOnShard(ctx, gid, g, key); err != nil { + return err + } ts, exists, err := c.latestCommitTSForReadKeyOnShard(ctx, gid, g, key) if err != nil { return errors.WithStack(err) @@ -2170,6 +2173,43 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return nil } +func (c *ShardedCoordinator) verifyTargetReadinessForReadKeyOnShard(ctx context.Context, gid uint64, g *ShardGroup, key []byte) error { + reader, ok := g.Store.(store.MigrationTargetReadinessReader) + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + if len(states) == 0 { + return nil + } + routeStart, routeEnd := readinessRouteRange(key, nextScanCursor(key)) + routes, catalogVersion, proof := c.currentShardRoutesForRouteRange(gid, routeStart, routeEnd) + if targetReadinessStatesSatisfied(states, routes, routeStart, routeEnd, gid, catalogVersion, proof) { + return nil + } + return errors.WithStack(ErrRouteCutoverPending) +} + +func (c *ShardedCoordinator) currentShardRoutesForRouteRange(gid uint64, routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if c == nil || c.engine == nil { + return nil, 0, false + } + snap, ok := c.engine.Current() + if !ok { + return nil, 0, false + } + routes := make([]distribution.Route, 0) + for _, route := range snap.IntersectingRoutes(routeStart, routeEnd) { + if route.GroupID == gid { + routes = append(routes, route) + } + } + return routes, snap.Version(), true +} + func (c *ShardedCoordinator) latestCommitTSForReadKeyOnShard(ctx context.Context, gid uint64, g *ShardGroup, key []byte) (uint64, bool, error) { liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) if err != nil { From 8d111618e5b0caa8d451f48676b160c45b6b83d9 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 23:19:21 +0900 Subject: [PATCH 082/162] Guard snapshot spooling disk headroom --- internal/raftengine/etcd/snapshot_spool.go | 70 +++++++++++++++++-- .../etcd/snapshot_spool_space_other.go | 9 +++ .../etcd/snapshot_spool_space_unix.go | 30 ++++++++ .../raftengine/etcd/snapshot_spool_test.go | 23 ++++++ 4 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 internal/raftengine/etcd/snapshot_spool_space_other.go create mode 100644 internal/raftengine/etcd/snapshot_spool_space_unix.go diff --git a/internal/raftengine/etcd/snapshot_spool.go b/internal/raftengine/etcd/snapshot_spool.go index 4065c4526..40afbead3 100644 --- a/internal/raftengine/etcd/snapshot_spool.go +++ b/internal/raftengine/etcd/snapshot_spool.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "io" "log/slog" + "math" "os" "path/filepath" "strconv" @@ -30,6 +31,10 @@ const defaultMaxSnapshotPayloadBytes int64 = 16 << 30 // 16 GiB const maxSnapshotPayloadBytesEnvVar = "ELASTICKV_RAFT_MAX_SNAPSHOT_PAYLOAD_BYTES" +const defaultSnapshotSpoolMinFreeBytes int64 = 2 << 30 // 2 GiB + +const snapshotSpoolMinFreeBytesEnvVar = "ELASTICKV_RAFT_SNAPSHOT_SPOOL_MIN_FREE_BYTES" + // resolveMaxSnapshotPayloadBytes evaluates the env override once per spool // creation. Snapshots are infrequent enough that one Getenv + ParseInt per // spool is invisible in profiles, and resolving at construction means tests @@ -48,8 +53,24 @@ func resolveMaxSnapshotPayloadBytes() int64 { return n } +func resolveSnapshotSpoolMinFreeBytes() int64 { + v := strings.TrimSpace(os.Getenv(snapshotSpoolMinFreeBytesEnvVar)) + if v == "" { + return defaultSnapshotSpoolMinFreeBytes + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil || n < 0 { + slog.Warn("invalid ELASTICKV_RAFT_SNAPSHOT_SPOOL_MIN_FREE_BYTES; using default", + "value", v, "default_bytes", defaultSnapshotSpoolMinFreeBytes) + return defaultSnapshotSpoolMinFreeBytes + } + return n +} + var errSnapshotPayloadTooLarge = errors.New("etcd raft snapshot payload exceeds limit") +var errSnapshotSpoolDiskHeadroom = errors.New("etcd raft snapshot spool insufficient disk headroom") + // snapshotSyncDir indirects fsync-on-directory through a package var so a // fault-injection test can simulate "rename succeeded but the fsync that // would persist the directory entry failed" — that's the partial-failure @@ -57,13 +78,16 @@ var errSnapshotPayloadTooLarge = errors.New("etcd raft snapshot payload exceeds // and without an injection seam there's no portable way to reproduce it. var snapshotSyncDir = syncDir +var snapshotSpoolAvailableBytes = snapshotSpoolAvailableBytesFS + const snapshotSpoolPattern = "elastickv-etcd-snapshot-*" type snapshotSpool struct { - file *os.File - path string - size int64 - maxSize int64 + file *os.File + path string + size int64 + maxSize int64 + minFreeBytes int64 } func newSnapshotSpool(dir string) (*snapshotSpool, error) { @@ -72,9 +96,10 @@ func newSnapshotSpool(dir string) (*snapshotSpool, error) { return nil, errors.WithStack(err) } return &snapshotSpool{ - file: file, - path: file.Name(), - maxSize: resolveMaxSnapshotPayloadBytes(), + file: file, + path: file.Name(), + maxSize: resolveMaxSnapshotPayloadBytes(), + minFreeBytes: resolveSnapshotSpoolMinFreeBytes(), }, nil } @@ -87,6 +112,9 @@ func (s *snapshotSpool) Write(p []byte) (int, error) { if int64(len(p)) > s.maxSize-s.size { return 0, errors.Wrapf(errSnapshotPayloadTooLarge, "adding %d bytes to current %d would exceed limit %d", len(p), s.size, s.maxSize) } + if err := s.checkDiskHeadroom(len(p)); err != nil { + return 0, err + } n, err := s.file.Write(p) s.size += int64(n) if err != nil { @@ -95,6 +123,31 @@ func (s *snapshotSpool) Write(p []byte) (int, error) { return n, nil } +func (s *snapshotSpool) checkDiskHeadroom(writeBytes int) error { + if writeBytes <= 0 { + return nil + } + available, err := snapshotSpoolAvailableBytes(filepath.Dir(s.path)) + if err != nil { + return errors.WithStack(err) + } + if available < 0 { + available = 0 + } + if s.minFreeBytes > math.MaxInt64-int64(writeBytes) { + return errors.Wrapf(errSnapshotSpoolDiskHeadroom, + "write_bytes=%d min_free_bytes=%d available_bytes=%d", + writeBytes, s.minFreeBytes, available) + } + required := s.minFreeBytes + int64(writeBytes) + if available < required { + return errors.Wrapf(errSnapshotSpoolDiskHeadroom, + "write_bytes=%d min_free_bytes=%d available_bytes=%d", + writeBytes, s.minFreeBytes, available) + } + return nil +} + func (s *snapshotSpool) Bytes() ([]byte, error) { if _, err := s.file.Seek(0, io.SeekStart); err != nil { return nil, errors.WithStack(err) @@ -160,6 +213,9 @@ func (s *snapshotSpool) FinalizeAsFSMFile(fsmSnapDir string, index uint64, crc32 // so Close() is a no-op. Without the per-step clear, Close() // would attempt os.Remove(s.path) and surface a misleading // ErrNotExist that buries the real syncDir error. + if err := s.checkDiskHeadroom(fsmFooterSize); err != nil { + return err + } if err := binary.Write(s.file, binary.BigEndian, crc32c); err != nil { return errors.WithStack(err) } diff --git a/internal/raftengine/etcd/snapshot_spool_space_other.go b/internal/raftengine/etcd/snapshot_spool_space_other.go new file mode 100644 index 000000000..0e5a1ba33 --- /dev/null +++ b/internal/raftengine/etcd/snapshot_spool_space_other.go @@ -0,0 +1,9 @@ +//go:build !(aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris) + +package etcd + +import "math" + +func snapshotSpoolAvailableBytesFS(string) (int64, error) { + return math.MaxInt64, nil +} diff --git a/internal/raftengine/etcd/snapshot_spool_space_unix.go b/internal/raftengine/etcd/snapshot_spool_space_unix.go new file mode 100644 index 000000000..194d46e86 --- /dev/null +++ b/internal/raftengine/etcd/snapshot_spool_space_unix.go @@ -0,0 +1,30 @@ +//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris + +package etcd + +import ( + "math" + + "github.com/cockroachdb/errors" + "golang.org/x/sys/unix" +) + +func snapshotSpoolAvailableBytesFS(path string) (int64, error) { + var st unix.Statfs_t + if err := unix.Statfs(path, &st); err != nil { + return 0, errors.WithStack(err) + } + if st.Bsize <= 0 { + return 0, nil + } + blockSize := uint64(st.Bsize) + availableBlocks := st.Bavail + if availableBlocks > uint64(math.MaxInt64)/blockSize { + return math.MaxInt64, nil + } + availableBytes := availableBlocks * blockSize + if availableBytes > uint64(math.MaxInt64) { + return math.MaxInt64, nil + } + return int64(availableBytes), nil +} diff --git a/internal/raftengine/etcd/snapshot_spool_test.go b/internal/raftengine/etcd/snapshot_spool_test.go index c8ed62474..01f9edf70 100644 --- a/internal/raftengine/etcd/snapshot_spool_test.go +++ b/internal/raftengine/etcd/snapshot_spool_test.go @@ -95,6 +95,29 @@ func TestSnapshotSpool_OverrideInvalidFallsBack(t *testing.T) { require.Equal(t, defaultMaxSnapshotPayloadBytes, spool.maxSize) } +func TestSnapshotSpoolRejectsWhenReserveWouldBeConsumed(t *testing.T) { + t.Setenv(snapshotSpoolMinFreeBytesEnvVar, "1024") + originalAvailable := snapshotSpoolAvailableBytes + snapshotSpoolAvailableBytes = func(string) (int64, error) { + return 1024, nil + } + t.Cleanup(func() { snapshotSpoolAvailableBytes = originalAvailable }) + + spool, err := newSnapshotSpool(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { _ = spool.Close() }) + + n, err := spool.Write([]byte("x")) + require.Zero(t, n) + require.Error(t, err) + require.True(t, errors.Is(err, errSnapshotSpoolDiskHeadroom), "got %v", err) + require.Zero(t, spool.size) + + info, statErr := spool.file.Stat() + require.NoError(t, statErr) + require.Zero(t, info.Size(), "headroom rejection must happen before writing bytes") +} + // TestFinalizeAsFSMFile_PostFinalizeCloseIsNoop pins the gemini-medium // review on PR #747: after a successful FinalizeAsFSMFile, the deferred // caller-side spool.Close() must NOT attempt to remove the renamed file From 0215c389fcf1d43f027b83418dd76e36df9870be Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 23:24:18 +0900 Subject: [PATCH 083/162] Raise snapshot spool headroom reserve --- internal/raftengine/etcd/snapshot_spool.go | 8 +++++++- internal/raftengine/etcd/snapshot_spool_test.go | 10 ++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/raftengine/etcd/snapshot_spool.go b/internal/raftengine/etcd/snapshot_spool.go index 40afbead3..c6e569a63 100644 --- a/internal/raftengine/etcd/snapshot_spool.go +++ b/internal/raftengine/etcd/snapshot_spool.go @@ -31,7 +31,13 @@ const defaultMaxSnapshotPayloadBytes int64 = 16 << 30 // 16 GiB const maxSnapshotPayloadBytesEnvVar = "ELASTICKV_RAFT_MAX_SNAPSHOT_PAYLOAD_BYTES" -const defaultSnapshotSpoolMinFreeBytes int64 = 2 << 30 // 2 GiB +// Keep one full max-size snapshot worth of headroom after receive-side spooling. +// Applying a token snapshot restores the FSM from the completed .fsm into a new +// local store directory, so a node can transiently need old snapshot + incoming +// .fsm + restored store bytes. A small reserve lets the spool complete and only +// fails later in restore, which can leave the host near-full until startup +// orphan cleanup runs. +const defaultSnapshotSpoolMinFreeBytes = defaultMaxSnapshotPayloadBytes const snapshotSpoolMinFreeBytesEnvVar = "ELASTICKV_RAFT_SNAPSHOT_SPOOL_MIN_FREE_BYTES" diff --git a/internal/raftengine/etcd/snapshot_spool_test.go b/internal/raftengine/etcd/snapshot_spool_test.go index 01f9edf70..21e8c2425 100644 --- a/internal/raftengine/etcd/snapshot_spool_test.go +++ b/internal/raftengine/etcd/snapshot_spool_test.go @@ -22,6 +22,7 @@ func TestSnapshotSpool_DefaultCapAcceptsRealisticFSM(t *testing.T) { if testing.Short() { t.Skip("skipping: writes 1.5 GiB to a temp file") } + t.Setenv(snapshotSpoolMinFreeBytesEnvVar, "0") dir := t.TempDir() spool, err := newSnapshotSpool(dir) require.NoError(t, err) @@ -66,6 +67,7 @@ func TestSnapshotSpool_DefaultCapAcceptsRealisticFSM(t *testing.T) { func TestSnapshotSpool_OverrideViaEnv(t *testing.T) { const spoolCap = int64(4096) t.Setenv(maxSnapshotPayloadBytesEnvVar, strconv.FormatInt(spoolCap, 10)) + t.Setenv(snapshotSpoolMinFreeBytesEnvVar, "0") spool, err := newSnapshotSpool(t.TempDir()) require.NoError(t, err) @@ -95,6 +97,14 @@ func TestSnapshotSpool_OverrideInvalidFallsBack(t *testing.T) { require.Equal(t, defaultMaxSnapshotPayloadBytes, spool.maxSize) } +func TestSnapshotSpoolDefaultReserveTracksPayloadCap(t *testing.T) { + spool, err := newSnapshotSpool(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { _ = spool.Close() }) + + require.Equal(t, defaultMaxSnapshotPayloadBytes, spool.minFreeBytes) +} + func TestSnapshotSpoolRejectsWhenReserveWouldBeConsumed(t *testing.T) { t.Setenv(snapshotSpoolMinFreeBytesEnvVar, "1024") originalAvailable := snapshotSpoolAvailableBytes From 774779b109682ecffeb16c8fe4108d92b3153917 Mon Sep 17 00:00:00 2001 From: bootjp Date: Tue, 14 Jul 2026 23:53:42 +0900 Subject: [PATCH 084/162] Fix raft startup and snapshot spool guards --- adapter/distribution_server.go | 6 +- adapter/distribution_server_test.go | 36 ++++++++ internal/raftengine/etcd/engine.go | 19 +++- internal/raftengine/etcd/engine_test.go | 90 +++++++++++++++++++ internal/raftengine/etcd/grpc_transport.go | 2 +- internal/raftengine/etcd/snapshot_spool.go | 35 ++++---- .../etcd/snapshot_spool_space_other.go | 2 +- .../etcd/snapshot_spool_space_unix.go | 2 +- .../raftengine/etcd/snapshot_spool_test.go | 26 +++++- 9 files changed, 193 insertions(+), 25 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 0fcacba58..3d3276f5c 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -410,11 +410,9 @@ func splitJobReadFenceKeys(jobs []distribution.SplitJob) [][]byte { readKeys := make([][]byte, 0, len(jobs)+1) readKeys = append(readKeys, distribution.CatalogNextSplitJobIDKey()) for _, job := range jobs { - if job.TerminalAtMs > 0 { - readKeys = append(readKeys, distribution.CatalogSplitJobHistoryKey(job.TerminalAtMs, job.JobID)) - continue + if splitJobIsLive(job) { + readKeys = append(readKeys, distribution.CatalogSplitJobKey(job.JobID)) } - readKeys = append(readKeys, distribution.CatalogSplitJobKey(job.JobID)) } return readKeys } diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index e69171029..f7cc59f17 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -343,6 +343,33 @@ func TestDistributionServerSplitRange_AllowsDisjointRouteWhileSplitJobLive(t *te requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogSplitJobKey(10)) } +func TestSplitJobReadFenceKeysExcludesTerminalHistory(t *testing.T) { + t.Parallel() + + readKeys := splitJobReadFenceKeys([]distribution.SplitJob{ + { + JobID: 10, + Phase: distribution.SplitJobPhaseBackfill, + }, + { + JobID: 11, + Phase: distribution.SplitJobPhaseDone, + TerminalAtMs: 1000, + }, + { + JobID: 12, + Phase: distribution.SplitJobPhaseAbandoned, + TerminalAtMs: 1001, + }, + }) + + require.Len(t, readKeys, 2) + requireReadKeysContain(t, readKeys, distribution.CatalogNextSplitJobIDKey()) + requireReadKeysContain(t, readKeys, distribution.CatalogSplitJobKey(10)) + requireReadKeysNotContain(t, readKeys, distribution.CatalogSplitJobHistoryKey(1000, 11)) + requireReadKeysNotContain(t, readKeys, distribution.CatalogSplitJobHistoryKey(1001, 12)) +} + func TestDistributionServerSplitRange_ConflictsWhenSplitJobCreatedAfterOverlapScan(t *testing.T) { t.Parallel() @@ -824,6 +851,15 @@ func requireReadKeysContain(t *testing.T, readKeys [][]byte, want []byte) { t.Fatalf("expected read keys to contain %q, got %q", want, readKeys) } +func requireReadKeysNotContain(t *testing.T, readKeys [][]byte, want []byte) { + t.Helper() + for _, key := range readKeys { + if bytes.Equal(key, want) { + t.Fatalf("expected read keys not to contain %q, got %q", want, readKeys) + } + } +} + func coordinatorStubMutations(elems []*kv.Elem[kv.OP]) ([]*store.KVPairMutation, error) { mutations := make([]*store.KVPairMutation, 0, len(elems)) for _, elem := range elems { diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index df65236b7..7b88b06a8 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -351,10 +351,15 @@ type Engine struct { snapshotStopCh chan struct{} closeCh chan struct{} doneCh chan struct{} - startedCh chan struct{} + // openCh is closed when run starts and callers may bind public/raft + // listeners. startedCh remains the stronger gate for processing inbound + // raft messages after the startup Ready drain has completed. + openCh chan struct{} + startedCh chan struct{} leaderReady chan struct{} leaderOnce sync.Once + openOnce sync.Once startOnce sync.Once closeOnce sync.Once dispatchOnce sync.Once @@ -656,6 +661,7 @@ func Open(ctx context.Context, cfg OpenConfig) (*Engine, error) { dispatchReportCh: make(chan dispatchReport, inboundQueueCap), closeCh: make(chan struct{}), doneCh: make(chan struct{}), + openCh: make(chan struct{}), startedCh: make(chan struct{}), leaderReady: make(chan struct{}), config: configurationFromConfState(peerMap, confStateValue(prepared.disk.LocalSnap.GetMetadata().GetConfState())), @@ -1685,6 +1691,7 @@ func (e *Engine) run() { ticker := time.NewTicker(e.tickInterval) defer ticker.Stop() + e.markOpen() if err := e.startup(); err != nil { e.fail(err) return @@ -3582,10 +3589,20 @@ func (e *Engine) markStarted() { e.startOnce.Do(func() { close(e.startedCh) }) } +func (e *Engine) markOpen() { + if e.openCh == nil { + return + } + e.openOnce.Do(func() { close(e.openCh) }) +} + func (e *Engine) openReady(waitForLeader bool) <-chan struct{} { if waitForLeader { return e.leaderReady } + if e.openCh != nil { + return e.openCh + } return e.startedCh } diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 00aa807c2..4139728ff 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -108,6 +108,12 @@ type blockingSnapshotStateMachine struct { release chan struct{} } +type blockingApplyStateMachine struct { + started chan struct{} + release chan struct{} + startOnce sync.Once +} + type blockingSnapshot struct { started chan struct{} release chan struct{} @@ -141,6 +147,21 @@ func (s *blockingSnapshotStateMachine) Apply(data []byte) any { return string(data) } +func (s *blockingApplyStateMachine) Apply(data []byte) any { + s.startOnce.Do(func() { close(s.started) }) + <-s.release + return string(data) +} + +func (s *blockingApplyStateMachine) Snapshot() (Snapshot, error) { + return &testSnapshot{}, nil +} + +func (s *blockingApplyStateMachine) Restore(r io.Reader) error { + _, err := io.Copy(io.Discard, r) + return err +} + func (s *blockingSnapshotStateMachine) Snapshot() (Snapshot, error) { return &blockingSnapshot{ started: s.started, @@ -1409,6 +1430,75 @@ func TestOpenRestoresLegacySnapshotState(t *testing.T) { require.Equal(t, [][]byte{[]byte("snap"), []byte("tail")}, fsm.Applied()) } +func TestOpenMultiNodeReturnsBeforeCommittedTailDrain(t *testing.T) { + dir := t.TempDir() + peers := []Peer{ + {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, + {NodeID: 2, ID: "n2", Address: "127.0.0.1:7002"}, + } + require.NoError(t, saveStateFile(stateFilePath(dir), persistedState{ + HardState: testHardState(2, 2), + Snapshot: raftTestSnapshot(1, 1, []uint64{1, 2}, mustEncodeSnapshotData(t, nil)), + Entries: []raftpb.Entry{{ + Type: entryTypePtr(raftpb.EntryNormal), + Term: uint64Ptr(2), + Index: uint64Ptr(2), + Data: encodeProposalEnvelope(1, []byte("tail")), + }}, + })) + + fsm := &blockingApplyStateMachine{ + started: make(chan struct{}), + release: make(chan struct{}), + } + type openResult struct { + engine *Engine + err error + } + done := make(chan openResult, 1) + go func() { + engine, err := Open(context.Background(), OpenConfig{ + NodeID: 1, + LocalID: "n1", + LocalAddress: "127.0.0.1:7001", + DataDir: dir, + Peers: peers, + StateMachine: fsm, + }) + done <- openResult{engine: engine, err: err} + }() + + var result openResult + select { + case result = <-done: + case <-time.After(time.Second): + t.Fatal("multi-node Open blocked on committed tail drain before returning") + } + require.NoError(t, result.err) + require.NotNil(t, result.engine) + defer func() { + require.NoError(t, result.engine.Close()) + }() + + select { + case <-fsm.started: + case <-time.After(time.Second): + t.Fatal("startup committed tail was not being applied") + } + select { + case <-result.engine.startedCh: + t.Fatal("engine marked started before startup committed tail drain completed") + default: + } + + close(fsm.release) + select { + case <-result.engine.startedCh: + case <-time.After(time.Second): + t.Fatal("engine did not mark started after committed tail drain completed") + } +} + func TestOpenMultiNodeReplicatesOverGRPCTransport(t *testing.T) { nodes, peers := newTransportTestNodes(t, 3) startTransportTestServers(nodes, peers) diff --git a/internal/raftengine/etcd/grpc_transport.go b/internal/raftengine/etcd/grpc_transport.go index d3ef7d572..32d82e83e 100644 --- a/internal/raftengine/etcd/grpc_transport.go +++ b/internal/raftengine/etcd/grpc_transport.go @@ -1231,7 +1231,7 @@ func (t *GRPCTransport) receiveSnapshotStream(stream pb.EtcdRaft_SendSnapshotSer if err != nil { return raftpb.Message{}, err } - spool, err := newSnapshotSpool(spoolPlacement) + spool, err := newReceiveSnapshotSpool(spoolPlacement) if err != nil { return raftpb.Message{}, err } diff --git a/internal/raftengine/etcd/snapshot_spool.go b/internal/raftengine/etcd/snapshot_spool.go index c6e569a63..cb1b3bc8e 100644 --- a/internal/raftengine/etcd/snapshot_spool.go +++ b/internal/raftengine/etcd/snapshot_spool.go @@ -31,14 +31,6 @@ const defaultMaxSnapshotPayloadBytes int64 = 16 << 30 // 16 GiB const maxSnapshotPayloadBytesEnvVar = "ELASTICKV_RAFT_MAX_SNAPSHOT_PAYLOAD_BYTES" -// Keep one full max-size snapshot worth of headroom after receive-side spooling. -// Applying a token snapshot restores the FSM from the completed .fsm into a new -// local store directory, so a node can transiently need old snapshot + incoming -// .fsm + restored store bytes. A small reserve lets the spool complete and only -// fails later in restore, which can leave the host near-full until startup -// orphan cleanup runs. -const defaultSnapshotSpoolMinFreeBytes = defaultMaxSnapshotPayloadBytes - const snapshotSpoolMinFreeBytesEnvVar = "ELASTICKV_RAFT_SNAPSHOT_SPOOL_MIN_FREE_BYTES" // resolveMaxSnapshotPayloadBytes evaluates the env override once per spool @@ -59,16 +51,16 @@ func resolveMaxSnapshotPayloadBytes() int64 { return n } -func resolveSnapshotSpoolMinFreeBytes() int64 { +func resolveSnapshotSpoolMinFreeBytes(defaultBytes int64) int64 { v := strings.TrimSpace(os.Getenv(snapshotSpoolMinFreeBytesEnvVar)) if v == "" { - return defaultSnapshotSpoolMinFreeBytes + return defaultBytes } n, err := strconv.ParseInt(v, 10, 64) if err != nil || n < 0 { slog.Warn("invalid ELASTICKV_RAFT_SNAPSHOT_SPOOL_MIN_FREE_BYTES; using default", - "value", v, "default_bytes", defaultSnapshotSpoolMinFreeBytes) - return defaultSnapshotSpoolMinFreeBytes + "value", v, "default_bytes", defaultBytes) + return defaultBytes } return n } @@ -97,6 +89,19 @@ type snapshotSpool struct { } func newSnapshotSpool(dir string) (*snapshotSpool, error) { + return newSnapshotSpoolWithLimits(dir, resolveMaxSnapshotPayloadBytes(), 0) +} + +func newReceiveSnapshotSpool(dir string) (*snapshotSpool, error) { + maxSize := resolveMaxSnapshotPayloadBytes() + // Keep one full max-size snapshot worth of headroom after receive-side + // spooling. Applying a token snapshot restores the FSM from the completed + // .fsm into a new local store directory, so a node can transiently need old + // snapshot + incoming .fsm + restored store bytes. + return newSnapshotSpoolWithLimits(dir, maxSize, resolveSnapshotSpoolMinFreeBytes(maxSize)) +} + +func newSnapshotSpoolWithLimits(dir string, maxSize, minFreeBytes int64) (*snapshotSpool, error) { file, err := os.CreateTemp(dir, snapshotSpoolPattern) if err != nil { return nil, errors.WithStack(err) @@ -104,8 +109,8 @@ func newSnapshotSpool(dir string) (*snapshotSpool, error) { return &snapshotSpool{ file: file, path: file.Name(), - maxSize: resolveMaxSnapshotPayloadBytes(), - minFreeBytes: resolveSnapshotSpoolMinFreeBytes(), + maxSize: maxSize, + minFreeBytes: minFreeBytes, }, nil } @@ -130,7 +135,7 @@ func (s *snapshotSpool) Write(p []byte) (int, error) { } func (s *snapshotSpool) checkDiskHeadroom(writeBytes int) error { - if writeBytes <= 0 { + if writeBytes <= 0 || s.minFreeBytes <= 0 { return nil } available, err := snapshotSpoolAvailableBytes(filepath.Dir(s.path)) diff --git a/internal/raftengine/etcd/snapshot_spool_space_other.go b/internal/raftengine/etcd/snapshot_spool_space_other.go index 0e5a1ba33..f9d733c4a 100644 --- a/internal/raftengine/etcd/snapshot_spool_space_other.go +++ b/internal/raftengine/etcd/snapshot_spool_space_other.go @@ -1,4 +1,4 @@ -//go:build !(aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris) +//go:build !(aix || darwin || linux) package etcd diff --git a/internal/raftengine/etcd/snapshot_spool_space_unix.go b/internal/raftengine/etcd/snapshot_spool_space_unix.go index 194d46e86..a2c5db370 100644 --- a/internal/raftengine/etcd/snapshot_spool_space_unix.go +++ b/internal/raftengine/etcd/snapshot_spool_space_unix.go @@ -1,4 +1,4 @@ -//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris +//go:build aix || darwin || linux package etcd diff --git a/internal/raftengine/etcd/snapshot_spool_test.go b/internal/raftengine/etcd/snapshot_spool_test.go index 21e8c2425..bed5fd223 100644 --- a/internal/raftengine/etcd/snapshot_spool_test.go +++ b/internal/raftengine/etcd/snapshot_spool_test.go @@ -98,11 +98,33 @@ func TestSnapshotSpool_OverrideInvalidFallsBack(t *testing.T) { } func TestSnapshotSpoolDefaultReserveTracksPayloadCap(t *testing.T) { + const spoolCap = int64(4096) + t.Setenv(maxSnapshotPayloadBytesEnvVar, strconv.FormatInt(spoolCap, 10)) + + spool, err := newReceiveSnapshotSpool(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { _ = spool.Close() }) + + require.Equal(t, spoolCap, spool.maxSize) + require.Equal(t, spoolCap, spool.minFreeBytes) +} + +func TestSnapshotSpoolMaterializeDoesNotReserveDiskHeadroom(t *testing.T) { + t.Setenv(snapshotSpoolMinFreeBytesEnvVar, "1024") + originalAvailable := snapshotSpoolAvailableBytes + snapshotSpoolAvailableBytes = func(string) (int64, error) { + return 0, nil + } + t.Cleanup(func() { snapshotSpoolAvailableBytes = originalAvailable }) + spool, err := newSnapshotSpool(t.TempDir()) require.NoError(t, err) t.Cleanup(func() { _ = spool.Close() }) - require.Equal(t, defaultMaxSnapshotPayloadBytes, spool.minFreeBytes) + n, err := spool.Write([]byte("x")) + require.NoError(t, err) + require.Equal(t, 1, n) + require.Equal(t, int64(1), spool.size) } func TestSnapshotSpoolRejectsWhenReserveWouldBeConsumed(t *testing.T) { @@ -113,7 +135,7 @@ func TestSnapshotSpoolRejectsWhenReserveWouldBeConsumed(t *testing.T) { } t.Cleanup(func() { snapshotSpoolAvailableBytes = originalAvailable }) - spool, err := newSnapshotSpool(t.TempDir()) + spool, err := newReceiveSnapshotSpool(t.TempDir()) require.NoError(t, err) t.Cleanup(func() { _ = spool.Close() }) From 46722f6a436c72cb84e9ba8e1b1db921784a6fde Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 00:04:17 +0900 Subject: [PATCH 085/162] Handle route fences in adapter retries --- adapter/redis_exec_dedup_test.go | 25 +++++++++++ adapter/redis_list_dedup_test.go | 56 +++++++++++++++++++---- adapter/redis_lists.go | 17 ++++--- adapter/redis_retry.go | 4 ++ adapter/redis_txn.go | 20 ++++----- adapter/s3.go | 9 ++-- adapter/s3_admin.go | 29 ++++++------ adapter/s3_admin_test.go | 76 ++++++++++++++++++++++++++++++++ 8 files changed, 188 insertions(+), 48 deletions(-) diff --git a/adapter/redis_exec_dedup_test.go b/adapter/redis_exec_dedup_test.go index 40bc6956b..ea59634b4 100644 --- a/adapter/redis_exec_dedup_test.go +++ b/adapter/redis_exec_dedup_test.go @@ -47,6 +47,31 @@ func TestExecDedup_LandedPriorAttempt_ReturnsCachedResults(t *testing.T) { require.Equal(t, []byte("v1"), val) } +func TestExecDedup_RouteFenceRetryPreservesPriorProbe(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + coord := newDedupTestCoordinator(st, 1, true) + coord.routeFenceAtDispatch = 2 + srv := &RedisServer{store: st, coordinator: coord, scriptCache: map[string]string{}, onePhaseTxnDedup: true} + + queue := []redcon.Command{ + {Args: [][]byte{[]byte(cmdSet), []byte("k"), []byte("v1")}}, + } + results, err := srv.runTransaction(queue) + require.NoError(t, err) + require.Len(t, results, 1) + require.Equal(t, "OK", results[0].str) + require.Equal(t, 3, coord.dispatches) + require.Equal(t, 1, coord.probeNoOps, "route-fenced reuse must not replace the prior landed probe") + + rawVal, err := st.GetAt(ctx, redisStrKey([]byte("k")), snapshotTS(coord.Clock(), st)) + require.NoError(t, err) + val, _, err := decodeRedisStr(rawVal) + require.NoError(t, err) + require.Equal(t, []byte("v1"), val) +} + // TestExecDedup_PriorAttemptDidNotLand_Applies covers the truncated case for // MULTI/EXEC: attempt 1 errored without committing (OCC-style pre-reject), // so the probe misses and the reuse applies the same write set at a fresh diff --git a/adapter/redis_list_dedup_test.go b/adapter/redis_list_dedup_test.go index 64575e93f..5e5c46303 100644 --- a/adapter/redis_list_dedup_test.go +++ b/adapter/redis_list_dedup_test.go @@ -39,8 +39,12 @@ type dedupTestCoordinator struct { // WITHOUT applying — an ambiguous retryable error distinct from // WriteConflict, exercising the "advance pending.commitTS and retry" branch. txnLockedAtDispatch int - dispatches int - probeNoOps int + // routeFenceAtDispatch makes the named dispatch return ErrRouteWriteFenced + // before the FSM dedup probe or apply. It is retryable but cannot be + // treated as an ambiguous landing. + routeFenceAtDispatch int + dispatches int + probeNoOps int // beforeDispatch, if set, runs at the start of each Dispatch with the // 1-based dispatch number — lets a test inject a concurrent commit // between the adapter's attempts. @@ -258,16 +262,14 @@ func (c *dedupTestCoordinator) Dispatch(ctx context.Context, req *kv.OperationGr if c.beforeDispatch != nil { c.beforeDispatch(n) } + if c.shouldRouteFence(n) { + return nil, kv.ErrRouteWriteFenced + } if handled, resp, err := c.maybeProbe(ctx, req); handled { return resp, err } - if n == c.ambiguousDispatch && !c.ambiguousLands { - // OCC-style pre-reject: nothing is written, definitely did not land. - return nil, store.ErrWriteConflict - } - if n == c.txnLockedAtDispatch { - // Ambiguous lock error, nothing written: definitely did not land. - return nil, kv.ErrTxnLocked + if err := c.preApplyError(n); err != nil { + return nil, err } resp, err := c.occAdapterCoordinator.Dispatch(ctx, req) if err != nil { @@ -287,6 +289,21 @@ func (c *dedupTestCoordinator) Dispatch(ctx context.Context, req *kv.OperationGr return resp, nil } +func (c *dedupTestCoordinator) shouldRouteFence(dispatch int) bool { + return dispatch == c.routeFenceAtDispatch +} + +func (c *dedupTestCoordinator) preApplyError(dispatch int) error { + switch { + case dispatch == c.ambiguousDispatch && !c.ambiguousLands: + return store.ErrWriteConflict + case dispatch == c.txnLockedAtDispatch: + return kv.ErrTxnLocked + default: + return nil + } +} + // maybeProbe mimics handleOnePhaseTxnRequest's exact-ts dedup check. It // returns handled=true when the probe owns the response (hit → no-op success // or probe error), and handled=false to fall through to the normal apply. @@ -365,6 +382,27 @@ func TestListPushDedup_LandedPriorAttempt_NoDuplicate(t *testing.T) { require.Equal(t, []byte("v"), val) } +func TestListPushDedup_RouteFenceRetryPreservesPriorProbe(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + coord := newDedupTestCoordinator(st, 1, true) + coord.routeFenceAtDispatch = 2 + srv := &RedisServer{store: st, coordinator: coord, scriptCache: map[string]string{}, onePhaseTxnDedup: true} + + key := []byte("mylist") + n, err := srv.listRPush(ctx, key, [][]byte{[]byte("v")}) + require.NoError(t, err) + require.Equal(t, int64(1), n) + require.Equal(t, 3, coord.dispatches, "attempt 1 landed, route-fenced reuse, then dedup probe retry") + require.Equal(t, 1, coord.probeNoOps, "route-fenced reuse must not replace the prior landed probe") + + readTS := snapshotTS(coord.Clock(), st) + meta, _, err := srv.resolveListMeta(ctx, key, readTS) + require.NoError(t, err) + require.Equal(t, int64(1), meta.Len, "route-fence retry must not append a duplicate") +} + // TestListPushDedup_PriorAttemptDidNotLand_Applies covers the truncated case: // attempt 1 errored without committing, so the probe misses and the reuse // applies the same write set at a fresh commit_ts. The element lands exactly diff --git a/adapter/redis_lists.go b/adapter/redis_lists.go index 4380073cd..e36cde008 100644 --- a/adapter/redis_lists.go +++ b/adapter/redis_lists.go @@ -243,13 +243,10 @@ func (r *RedisServer) dispatchListPushReuse(ctx context.Context, key []byte, pen // iteration recomputes from a fresh meta read. return 0, true, errors.WithStack(dispErr) } - // Still ambiguous (lock / other retryable): this reuse may itself - // have landed, so the next retry must probe THIS commit_ts. Only - // advance pending.commitTS if retryRedisWrite will actually loop - // (non-retryable errors escape to the client; pending is then - // discarded with the goroutine, so the update is wasted and the - // stale value would be misleading if some future caller reads it). - if isRetryableRedisTxnErr(dispErr) { + // Still ambiguous (lock / other retryable): this reuse may itself have + // landed, so the next retry must probe THIS commit_ts. Route-fence + // rejections are retryable but pre-apply, so keep the older witness. + if shouldPreserveRedisTxnAttempt(dispErr) { pending.commitTS = commitTS } return 0, false, errors.WithStack(dispErr) @@ -411,7 +408,9 @@ func (r *RedisServer) listPushCoreWithDedup(ctx context.Context, key []byte, val return nil } // Only remember the attempt for reuse if retryRedisWrite will actually - // loop — i.e. the error is one of WriteConflict / TxnLocked. For + // loop and the attempt may have landed. Route-fence rejections are + // retryable but happen before this write set can apply, so preserving + // that commitTS would overwrite an older ambiguous witness. For // errors that escape the loop (transient-leader, context deadline, // FSM apply error, etc.), `pending` would be discarded with the // goroutine, and recording it would mislead a future reader about @@ -419,7 +418,7 @@ func (r *RedisServer) listPushCoreWithDedup(ctx context.Context, key []byte, val // retryRedisWrite's retry predicate; ambiguous errors that escape // to the client are a separate problem space (cross-request // idempotency cache) and out of scope for this design. - if isRetryableRedisTxnErr(dispErr) { + if shouldPreserveRedisTxnAttempt(dispErr) { pending = &reusableListPush{ ops: ops, startTS: startTS, diff --git a/adapter/redis_retry.go b/adapter/redis_retry.go index 0c38ddf41..d6f13f0e5 100644 --- a/adapter/redis_retry.go +++ b/adapter/redis_retry.go @@ -44,6 +44,10 @@ func isRetryableRedisTxnErr(err error) bool { return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || errors.Is(err, kv.ErrRouteWriteFenced) } +func shouldPreserveRedisTxnAttempt(err error) bool { + return isRetryableRedisTxnErr(err) && !errors.Is(err, kv.ErrRouteWriteFenced) +} + func retryPolicyForRedisTxnErr(err error) redisTxnRetryPolicy { if errors.Is(err, kv.ErrTxnLocked) { return redisTxnLockedRetryPolicy diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index 750011ca7..53b937ed2 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -2216,12 +2216,10 @@ func (r *RedisServer) dispatchExecReuse(ctx context.Context, pending *reusableEx // iteration rebuilds from a fresh snapshot. return nil, true, errors.WithStack(dispErr) } - // Still ambiguous (lock / other retryable): the reuse may itself - // have landed, so the next retry must probe THIS commit_ts. Only - // advance pending.commitTS if retryRedisWrite will actually loop - // (non-retryable errors escape to the client; pending is then - // discarded with the goroutine). - if isRetryableRedisTxnErr(dispErr) { + // Still ambiguous (lock / other retryable): the reuse may itself have + // landed, so the next retry must probe THIS commit_ts. Route-fence + // rejections are retryable but pre-apply, so keep the older witness. + if shouldPreserveRedisTxnAttempt(dispErr) { pending.commitTS = commitTS } return nil, false, errors.WithStack(dispErr) @@ -2347,12 +2345,10 @@ func (r *RedisServer) firstExecAttempt(dispatchCtx context.Context, queue []redc } if _, dispErr := r.coordinator.Dispatch(prepared.ctx, group); dispErr != nil { // Only remember the attempt for reuse if retryRedisWrite will - // actually loop. Mirrors listPushCoreWithDedup's gating - // rationale — errors that escape the loop (transient-leader, - // context deadline, FSM apply error) leave pending pointing at - // state wasted with the goroutine; ambiguous errors that - // escape to the client are out of scope for this loop. - if isRetryableRedisTxnErr(dispErr) { + // actually loop and the attempt may have landed. Mirrors + // listPushCoreWithDedup's gating rationale; route-fence + // rejections are retryable but pre-apply. + if shouldPreserveRedisTxnAttempt(dispErr) { return nil, &reusableExecTxn{ elems: prepared.elems, startTS: txn.startTS, diff --git a/adapter/s3.go b/adapter/s3.go index c01165c31..d08f0e4d6 100644 --- a/adapter/s3.go +++ b/adapter/s3.go @@ -728,9 +728,12 @@ func (s *S3Server) deleteBucket(w http.ResponseWriter, r *http.Request, bucket s writeS3MutationError(w, err, bucket, "") return } - // Phase 2: best-effort DEL_PREFIX safety net. See - // AdminDeleteBucket / runBucketDeleteSafetyNet for the contract. - s.runBucketDeleteSafetyNet(r.Context(), bucket, deletedGeneration) + // Phase 2: DEL_PREFIX safety net. See AdminDeleteBucket / + // runBucketDeleteSafetyNet for the contract. + if err := s.runBucketDeleteSafetyNet(r.Context(), bucket, deletedGeneration); err != nil { + writeS3MutationError(w, err, bucket, "") + return + } w.WriteHeader(http.StatusNoContent) } diff --git a/adapter/s3_admin.go b/adapter/s3_admin.go index a9eb8915a..ccc65bd1f 100644 --- a/adapter/s3_admin.go +++ b/adapter/s3_admin.go @@ -423,12 +423,7 @@ func (s *S3Server) AdminDeleteBucket(ctx context.Context, principal AdminPrincip if err != nil { return err //nolint:wrapcheck // sentinel errors propagate as-is. } - // Phase 2: best-effort safety-net DEL_PREFIX. Outside the - // retryS3Mutation closure because retrying after Phase 1 - // committed would 404 at loadBucketMetaAt; we want the error - // (if any) logged but not propagated to the operator. - s.runBucketDeleteSafetyNet(ctx, name, deletedGeneration) - return nil + return s.runBucketDeleteSafetyNet(ctx, name, deletedGeneration) } // adminDeleteBucketTxnBody is the per-attempt body retryS3Mutation @@ -501,22 +496,26 @@ func bucketDeleteSafetyNetElems(bucket string, generation uint64) []*kv.Elem[kv. } } -// runBucketDeleteSafetyNet runs the Phase-2 DEL_PREFIX dispatch -// and swallows transport / cluster errors after logging — the -// caller has already deleted the bucket meta and the operator- -// visible state is consistent with that. Shared between admin and -// SigV4 paths. -func (s *S3Server) runBucketDeleteSafetyNet(ctx context.Context, bucket string, generation uint64) { - if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ - Elems: bucketDeleteSafetyNetElems(bucket, generation), - }); err != nil { +// runBucketDeleteSafetyNet runs the Phase-2 DEL_PREFIX dispatch. Phase 1 has +// already deleted the bucket meta, so this helper retries transient fencing in +// place instead of re-entering the full delete transaction. +func (s *S3Server) runBucketDeleteSafetyNet(ctx context.Context, bucket string, generation uint64) error { + err := s.retryS3Mutation(ctx, func() error { + _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: bucketDeleteSafetyNetElems(bucket, generation), + }) + return errors.WithStack(err) + }) + if err != nil { slog.WarnContext(ctx, "bucket delete safety-net DEL_PREFIX failed; bucket meta is gone but orphan sweep incomplete", slog.String("bucket", bucket), slog.Uint64("generation", generation), slog.String("error", err.Error()), ) + return err } + return nil } // adminCanonicalACL normalises an empty input to the canned diff --git a/adapter/s3_admin_test.go b/adapter/s3_admin_test.go index 3a83677f2..23d4549fd 100644 --- a/adapter/s3_admin_test.go +++ b/adapter/s3_admin_test.go @@ -264,6 +264,56 @@ func TestS3Server_AdminDeleteBucket_HappyPath(t *testing.T) { require.False(t, exists) } +func TestS3Server_AdminDeleteBucket_RetriesSafetyNetRouteFence(t *testing.T) { + t.Parallel() + + st := store.NewMVCCStore() + coord := &bucketDeleteSafetyNetFenceCoordinator{ + localAdapterCoordinator: newLocalAdapterCoordinator(st), + failuresRemaining: 1, + } + server := NewS3Server(nil, "", st, coord, nil) + ctx := context.Background() + + summary, err := server.AdminCreateBucket(ctx, + fullAdminBucketsPrincipal(), "to-delete", s3AclPrivate) + require.NoError(t, err) + orphan := s3keys.RouteKey("to-delete", summary.Generation, "orphan") + _, err = coord.localAdapterCoordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{{Op: kv.Put, Key: orphan, Value: []byte("orphan")}}, + }) + require.NoError(t, err) + + err = server.AdminDeleteBucket(ctx, + fullAdminBucketsPrincipal(), "to-delete") + require.NoError(t, err) + require.Equal(t, 2, coord.safetyNetCalls) + + _, err = st.GetAt(ctx, orphan, snapshotTS(coord.Clock(), st)) + require.ErrorIs(t, err, store.ErrKeyNotFound, "safety-net retry must sweep the orphan before acknowledging delete") +} + +func TestS3Server_AdminDeleteBucket_PropagatesPersistentSafetyNetRouteFence(t *testing.T) { + t.Parallel() + + st := store.NewMVCCStore() + coord := &bucketDeleteSafetyNetFenceCoordinator{ + localAdapterCoordinator: newLocalAdapterCoordinator(st), + failuresRemaining: s3TxnRetryMaxAttempts, + } + server := NewS3Server(nil, "", st, coord, nil) + ctx := context.Background() + + _, err := server.AdminCreateBucket(ctx, + fullAdminBucketsPrincipal(), "to-delete", s3AclPrivate) + require.NoError(t, err) + + err = server.AdminDeleteBucket(ctx, + fullAdminBucketsPrincipal(), "to-delete") + require.ErrorIs(t, err, kv.ErrRouteWriteFenced) + require.Equal(t, s3TxnRetryMaxAttempts, coord.safetyNetCalls) +} + func TestS3Server_AdminDeleteBucket_MissingBucket(t *testing.T) { t.Parallel() @@ -275,6 +325,32 @@ func TestS3Server_AdminDeleteBucket_MissingBucket(t *testing.T) { require.ErrorIs(t, err, ErrAdminBucketNotFound) } +type bucketDeleteSafetyNetFenceCoordinator struct { + *localAdapterCoordinator + failuresRemaining int + safetyNetCalls int +} + +func (c *bucketDeleteSafetyNetFenceCoordinator) Dispatch(ctx context.Context, req *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { + if req != nil && operationGroupHasDelPrefix(req.Elems) { + c.safetyNetCalls++ + if c.failuresRemaining > 0 { + c.failuresRemaining-- + return nil, kv.ErrRouteWriteFenced + } + } + return c.localAdapterCoordinator.Dispatch(ctx, req) +} + +func operationGroupHasDelPrefix(elems []*kv.Elem[kv.OP]) bool { + for _, elem := range elems { + if elem != nil && elem.Op == kv.DelPrefix { + return true + } + } + return false +} + func TestS3Server_AdminDeleteBucket_RejectsReadOnly(t *testing.T) { t.Parallel() From 70cf6f86bed772630fe224e4eda41e56fd1daf12 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 02:21:02 +0900 Subject: [PATCH 086/162] Stabilize raft snapshot dispatch and maintenance gates --- ...4_27_implemented_keyviz_cluster_fanout.md} | 11 +++- ...04_28_implemented_keyviz_adapter_labels.md | 2 +- .../2026_04_29_proposed_logical_backup.md | 2 +- ...ial_hotspot_split_milestone3_automation.md | 2 +- .../2026_06_23_proposed_scaling_roadmap.md | 2 +- internal/admin/keyviz_fanout.go | 2 +- internal/admin/keyviz_handler.go | 2 +- .../raftengine/etcd/dispatch_report_test.go | 33 ++++++++++++ internal/raftengine/etcd/engine.go | 50 +++++++++++++------ main.go | 49 ++++++++++++++---- main_maintenance_env_test.go | 33 ++++++++++++ scripts/rolling-update.env.example | 2 +- 12 files changed, 157 insertions(+), 33 deletions(-) rename docs/design/{2026_04_27_proposed_keyviz_cluster_fanout.md => 2026_04_27_implemented_keyviz_cluster_fanout.md} (98%) create mode 100644 main_maintenance_env_test.go diff --git a/docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md b/docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md similarity index 98% rename from docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md rename to docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md index 6e1b8f762..0ceafb300 100644 --- a/docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md +++ b/docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md @@ -1,5 +1,5 @@ --- -status: proposed +status: implemented phase: 2-C parent_design: docs/admin_ui_key_visualizer_design.md date: 2026-04-27 @@ -47,6 +47,15 @@ operator-visible value (cluster-wide heatmap, degraded-node banner) without requiring the full proto extension §9.1 calls for. +## 1.1 Implementation status + +Implemented in `internal/admin/keyviz_fanout.go`, +`internal/admin/keyviz_handler.go`, `web/admin/src/pages/KeyViz.tsx`, +and `main.go`'s `--keyvizFanoutNodes` / `--keyvizFanoutTimeout` +flags. Tests cover fan-out merge and degraded-node handling in +`internal/admin/keyviz_fanout_test.go` and the admin KeyViz UI +suite. + ## 2. Scope ### 2.1 In scope diff --git a/docs/design/2026_04_28_implemented_keyviz_adapter_labels.md b/docs/design/2026_04_28_implemented_keyviz_adapter_labels.md index 831f8cd18..b68dca189 100644 --- a/docs/design/2026_04_28_implemented_keyviz_adapter_labels.md +++ b/docs/design/2026_04_28_implemented_keyviz_adapter_labels.md @@ -587,7 +587,7 @@ from PR #694: Claude bot critical, Gemini high.) The fan-out aggregator's per-cell merge key gains the label: - Phase 2-C (current): `(bucketID, raftGroupID, leaderTerm, - windowStart)` per design `2026_04_27_proposed_keyviz_cluster_fanout.md` + windowStart)` per design `2026_04_27_implemented_keyviz_cluster_fanout.md` §4. - With labels: same tuple — but `bucketID` itself now carries the label via the §5 composite (`route:1:dynamo`). The merge key diff --git a/docs/design/2026_04_29_proposed_logical_backup.md b/docs/design/2026_04_29_proposed_logical_backup.md index 24c033cd8..39350d188 100644 --- a/docs/design/2026_04_29_proposed_logical_backup.md +++ b/docs/design/2026_04_29_proposed_logical_backup.md @@ -1570,7 +1570,7 @@ Scope: out of this proposal; mentioned only to draw the boundary. - Concurrent multi-cluster fan-out (one logical backup spanning shards on different physical clusters) — depends on the `Distribution` control plane being fan-out-aware (see - `2026_04_27_proposed_keyviz_cluster_fanout.md`). + `2026_04_27_implemented_keyviz_cluster_fanout.md`). ## Required Tests diff --git a/docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md b/docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md index 2f8a1c799..89c2e85a4 100644 --- a/docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md +++ b/docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md @@ -161,7 +161,7 @@ Options considered: **Follower-forwarded write caveat (codex P2).** `ShardedCoordinator.observeMutation` runs on the node that received the client request before `ShardRouter.Commit` forwards (`kv/sharded_coordinator.go:1795-1844`), and the follower's `Internal.Forward` handler calls `transactionManager.Commit` directly (`adapter/internal.go:52`) rather than re-entering the leader's `ShardedCoordinator` observation hook. Therefore even a route whose shard group is led by the default-group leader can be **under-observed** if most clients connect through followers. M3 does not treat "missing local rows" as evidence of low load: local evidence can promote a split, but absence of local evidence only means "unknown" and is fail-closed for target selection (§6.2) and conservative for candidacy (fewer splits). Full coverage for follower-forwarded writes needs follower reports / fan-out into the same detector interface and is tracked under OQ-5. -**Why not the keyviz cluster fan-out** (`docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md`): that aggregates across nodes for the admin heatmap UI and dedupes write samples by `(groupID, leaderTerm)`. Pulling cross-node data in would add latency and a dependency on the fan-out being configured, so M3 stays leader-local and fail-closed on unknown load. If a future milestone wants complete cluster-wide scoring, including follower-forwarded writes, it can read the same fan-out aggregate behind the same detector interface (§5) — recorded as OQ-5. +**Why not the keyviz cluster fan-out** (`docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md`): that aggregates across nodes for the admin heatmap UI and dedupes write samples by `(groupID, leaderTerm)`. Pulling cross-node data in would add latency and a dependency on the fan-out being configured, so M3 stays leader-local and fail-closed on unknown load. If a future milestone wants complete cluster-wide scoring, including follower-forwarded writes, it can read the same fan-out aggregate behind the same detector interface (§5) — recorded as OQ-5. **Known limitation — multi-leader (multi-group) deployments.** Leader-local consumption is only authoritative for routes whose shard group is led by the **same node** that runs the detector — i.e. the default-group leader (where the scheduler must run, §6.1). In a multi-group cluster (`--raftGroups` with G1, G2, …), the default-group leader is **not necessarily** the leader of G1/G2/…: each node runs a single `ShardedCoordinator` and `MemSampler` and only `Observe`s traffic it *receives* (`kv/sharded_coordinator.go:1795-1824`). Concretely, in a 3-node cluster where node A leads the default group and node B leads G1, **node A's sampler has no data for routes in G1 served through node B**, so the detector on A cannot score or split them. This is **broader than "heavy follower-forwarded reads"**: it is a structural gap for any route whose group leader differs from the default-group leader, not just an edge case of read forwarding. diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index b51827adb..d30405c20 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -260,7 +260,7 @@ memory each group's private cache/memtable pins. - **keyviz** — the per-route load sampler is wired and allocation-free on the write hot path (`observeMutation`, `kv/sharded_coordinator.go:1841-1846`), with proposed extensions for cluster fan-out - (`2026_04_27_proposed_keyviz_cluster_fanout.md`), + (`2026_04_27_implemented_keyviz_cluster_fanout.md`), subrange sampling (`2026_05_25_implemented_keyviz_subrange_sampling.md`), hot-key top-K (`2026_05_28_implemented_keyviz_hot_key_topk.md`), and per-cell conflict (implemented). It is the detection signal M3 reuses. Current diff --git a/internal/admin/keyviz_fanout.go b/internal/admin/keyviz_fanout.go index 472c2fc3f..6e3dfa7af 100644 --- a/internal/admin/keyviz_fanout.go +++ b/internal/admin/keyviz_fanout.go @@ -71,7 +71,7 @@ const keyVizMergeBucketHint = 64 // a stable row order; Responded counts ok=true entries; Expected is // the configured peer count plus self. // -// See docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md 5. +// See docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md 5. type FanoutResult struct { Nodes []FanoutNodeStatus `json:"nodes"` Responded int `json:"responded"` diff --git a/internal/admin/keyviz_handler.go b/internal/admin/keyviz_handler.go index d4192d693..db54f812b 100644 --- a/internal/admin/keyviz_handler.go +++ b/internal/admin/keyviz_handler.go @@ -56,7 +56,7 @@ const keyVizRowBudgetCap = 1024 // // Fanout is non-nil when the handler is configured for cluster-wide // fan-out (Phase 2-C): it carries per-node status so the SPA can -// surface degraded responses inline (see design 2026_04_27_proposed_keyviz_cluster_fanout.md). +// surface degraded responses inline (see design 2026_04_27_implemented_keyviz_cluster_fanout.md). // The field is omitted from the wire form when fan-out is disabled // so old clients keep working unchanged. type KeyVizMatrix struct { diff --git a/internal/raftengine/etcd/dispatch_report_test.go b/internal/raftengine/etcd/dispatch_report_test.go index d0ba97049..dfd1dfd67 100644 --- a/internal/raftengine/etcd/dispatch_report_test.go +++ b/internal/raftengine/etcd/dispatch_report_test.go @@ -33,6 +33,39 @@ func TestPostDispatchReport_DeliversWhenChannelHasSpace(t *testing.T) { } } +func TestReportSuccessfulDispatchReportsSnapshotFinish(t *testing.T) { + t.Parallel() + e := &Engine{ + dispatchReportCh: make(chan dispatchReport, 1), + closeCh: make(chan struct{}), + } + + e.reportSuccessfulDispatch(raftpb.Message{Type: messageTypePtr(raftpb.MsgSnap), To: uint64Ptr(2)}) + + select { + case got := <-e.dispatchReportCh: + require.Equal(t, dispatchReport{to: 2, msgType: raftpb.MsgSnap, snapshotFinish: true}, got) + default: + t.Fatal("expected successful MsgSnap dispatch to report SnapshotFinish input") + } +} + +func TestReportSuccessfulDispatchIgnoresRegularMessage(t *testing.T) { + t.Parallel() + e := &Engine{ + dispatchReportCh: make(chan dispatchReport, 1), + closeCh: make(chan struct{}), + } + + e.reportSuccessfulDispatch(raftpb.Message{Type: messageTypePtr(raftpb.MsgApp), To: uint64Ptr(2)}) + + select { + case got := <-e.dispatchReportCh: + t.Fatalf("unexpected dispatch report for regular message: %+v", got) + default: + } +} + // TestPostDispatchReport_DropsWhenChannelFull asserts the non-blocking // contract: dispatch workers must not stall because the event loop is busy. // The worst case is an eventually-consistent gap that raft will fix on the diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 7b88b06a8..57fbee77b 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -1808,13 +1808,14 @@ func (e *Engine) tryReceivePriorityStep() (raftpb.Message, bool) { } } -// dispatchReport is posted by the dispatch workers when a transport send -// to a peer fails; the engine goroutine drains these and informs etcd/raft -// via rawNode so follower Progress leaves StateReplicate / StateSnapshot on -// unreachable peers and does not silently stall. +// dispatchReport is posted by the dispatch workers after a transport send +// completes or fails; the engine goroutine drains these and informs etcd/raft +// via rawNode so follower Progress leaves StateReplicate / StateSnapshot and +// does not silently stall. type dispatchReport struct { - to uint64 - msgType raftpb.MessageType + to uint64 + msgType raftpb.MessageType + snapshotFinish bool } func (e *Engine) handleDispatchReport(report dispatchReport) { @@ -1827,19 +1828,23 @@ func (e *Engine) handleDispatchReport(report dispatchReport) { // peer from StateReplicate to StateProbe so the next heartbeat response // drives a fresh sendAppend attempt. if report.msgType == raftpb.MsgSnap { - e.rawNode.ReportSnapshot(report.to, etcdraft.SnapshotFailure) + status := etcdraft.SnapshotFailure + if report.snapshotFinish { + status = etcdraft.SnapshotFinish + } + e.rawNode.ReportSnapshot(report.to, status) return } e.rawNode.ReportUnreachable(report.to) } -// postDispatchReport delivers a dispatch failure to the event loop without -// blocking the caller. Dispatch workers use it for transport failures, and the -// event loop uses it for local queue drops before transport. If the channel is -// full (unlikely — the buffer is sized to MaxInflightMsg), the report is -// dropped and logged; this is acceptable because raft will retry on the next -// tick and we only need eventual consistency between transport state and -// Progress state. +// postDispatchReport delivers a dispatch outcome to the event loop without +// blocking the caller. Dispatch workers use it for transport completion or +// failures, and the event loop uses it for local queue drops before transport. +// If the channel is full (unlikely — the buffer is sized to MaxInflightMsg), +// the report is dropped and logged; this is acceptable because raft will retry +// on the next tick and we only need eventual consistency between transport +// state and Progress state. func (e *Engine) postDispatchReport(report dispatchReport) { select { case e.dispatchReportCh <- report: @@ -4423,7 +4428,11 @@ func (e *Engine) handleDispatchRequest(ctx context.Context, req dispatchRequest) if err := req.Close(); err != nil { slog.Error("etcd raft dispatch: failed to close request", "err", err) } - if dispatchErr == nil || errors.Is(dispatchErr, ctx.Err()) { + if dispatchErr == nil { + e.reportSuccessfulDispatch(req.msg) + return + } + if errors.Is(dispatchErr, ctx.Err()) { return } code := dispatchErrorCodeOf(dispatchErr) @@ -4445,6 +4454,17 @@ func (e *Engine) handleDispatchRequest(ctx context.Context, req dispatchRequest) e.postDispatchReport(dispatchReport{to: req.msg.GetTo(), msgType: req.msg.GetType()}) } +func (e *Engine) reportSuccessfulDispatch(msg raftpb.Message) { + if msg.GetType() != raftpb.MsgSnap { + return + } + e.postDispatchReport(dispatchReport{ + to: msg.GetTo(), + msgType: msg.GetType(), + snapshotFinish: true, + }) +} + func (e *Engine) stopDispatchWorkers() { e.dispatchOnce.Do(func() { if e.dispatchCancel != nil { diff --git a/main.go b/main.go index 530dbf4b7..236fcb0c7 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,9 @@ const ( etcdMaxSizePerMsg = 1 << 20 etcdMaxInflightMsg = 1024 defaultTSOBatchSize = 256 + + lockResolverEnabledEnv = "ELASTICKV_LOCK_RESOLVER_ENABLED" + fsmCompactorEnabledEnv = "ELASTICKV_FSM_COMPACTOR_ENABLED" ) func newRaftFactory(engineType raftEngineType, coldStartObs raftengine.ColdStartObserver) (raftengine.Factory, error) { @@ -85,6 +88,18 @@ func durationToTicks(timeout time.Duration, tick time.Duration, min int) int { return ticks } +func optionalBoolEnv(name string, def bool) bool { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return def + } + v, err := strconv.ParseBool(raw) + if err != nil { + return def + } + return v +} + var ( myAddr = flag.String("address", "localhost:50051", "TCP host+port for this node") redisAddr = flag.String("redisAddress", "localhost:6379", "TCP host+port for redis") @@ -236,7 +251,7 @@ var ( // HTTP endpoints (host:port or scheme://host:port). When set, // the admin keyviz handler aggregates the local matrix with // peer responses; when empty, behaviour is unchanged - // (single-node view). See docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md. + // (single-node view). See docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md. keyvizFanoutNodes = flag.String("keyvizFanoutNodes", "", "Comma-separated peer admin endpoints (host:port) for keyviz cluster-wide fan-out; empty disables") keyvizFanoutTimeout = flag.Duration("keyvizFanoutTimeout", keyvizFanoutDefaultTimeout, "Per-peer timeout for keyviz fan-out HTTP calls") ) @@ -445,8 +460,7 @@ func run() error { } }) cleanup.Add(cancel) - lockResolver := kv.NewLockResolver(shardStore, shardGroups, nil) - cleanup.Add(func() { lockResolver.Close() }) + startLockResolverIfEnabled(shardStore, shardGroups, &cleanup) sampler := buildKeyVizSampler() coordinate := kv.NewShardedCoordinator(cfg.engine, shardGroups, cfg.defaultGroup, clock, shardStore). WithLeaseReadObserver(metricsRegistry.LeaseReadObserver()). @@ -505,13 +519,7 @@ func run() error { adapter.WithDistributionActiveTimestampTracker(readTracker), ) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) - compactor := kv.NewFSMCompactor( - fsmCompactionRuntimes(runtimes), - kv.WithFSMCompactorActiveTimestampTracker(readTracker), - ) - eg.Go(func() error { - return compactor.Run(runCtx) - }) + startFSMCompactorIfEnabled(runCtx, eg, runtimes, readTracker) // Stage 7c §3.1: build the encryption-aware // MembershipChangeInterceptor here where the concrete @@ -2029,6 +2037,27 @@ func writeConflictMonitorSources(runtimes []*raftGroupRuntime) []monitoring.Writ return out } +func startLockResolverIfEnabled(ss *kv.ShardStore, groups map[uint64]*kv.ShardGroup, cleanup *internalutil.CleanupStack) { + if !optionalBoolEnv(lockResolverEnabledEnv, true) { + return + } + lockResolver := kv.NewLockResolver(ss, groups, nil) + cleanup.Add(func() { lockResolver.Close() }) +} + +func startFSMCompactorIfEnabled(ctx context.Context, eg *errgroup.Group, runtimes []*raftGroupRuntime, readTracker *kv.ActiveTimestampTracker) { + if !optionalBoolEnv(fsmCompactorEnabledEnv, true) { + return + } + compactor := kv.NewFSMCompactor( + fsmCompactionRuntimes(runtimes), + kv.WithFSMCompactorActiveTimestampTracker(readTracker), + ) + eg.Go(func() error { + return compactor.Run(ctx) + }) +} + func fsmCompactionRuntimes(runtimes []*raftGroupRuntime) []kv.FSMCompactRuntime { out := make([]kv.FSMCompactRuntime, 0, len(runtimes)) for _, runtime := range runtimes { diff --git a/main_maintenance_env_test.go b/main_maintenance_env_test.go new file mode 100644 index 000000000..bee78cdeb --- /dev/null +++ b/main_maintenance_env_test.go @@ -0,0 +1,33 @@ +package main + +import "testing" + +func TestOptionalBoolEnv(t *testing.T) { + t.Setenv("ELASTICKV_TEST_BOOL", "") + if got := optionalBoolEnv("ELASTICKV_TEST_BOOL", true); !got { + t.Fatal("empty env should use default true") + } + if got := optionalBoolEnv("ELASTICKV_TEST_BOOL", false); got { + t.Fatal("empty env should use default false") + } + + for _, tc := range []struct { + name string + raw string + want bool + }{ + {name: "true", raw: "true", want: true}, + {name: "one", raw: "1", want: true}, + {name: "false", raw: "false", want: false}, + {name: "zero", raw: "0", want: false}, + {name: "trimmed", raw: " false ", want: false}, + {name: "invalid uses default", raw: "disabled", want: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("ELASTICKV_TEST_BOOL", tc.raw) + if got := optionalBoolEnv("ELASTICKV_TEST_BOOL", true); got != tc.want { + t.Fatalf("optionalBoolEnv()=%v, want %v", got, tc.want) + } + }) + } +} diff --git a/scripts/rolling-update.env.example b/scripts/rolling-update.env.example index 6ee72dd60..f71cfee29 100644 --- a/scripts/rolling-update.env.example +++ b/scripts/rolling-update.env.example @@ -122,7 +122,7 @@ ADMIN_ENABLED="false" # role allow-lists must be configured cluster-wide. Peers without # --adminEnabled expose an unauthenticated keyviz endpoint and # respond unconditionally. -# See docs/design/2026_04_27_proposed_keyviz_cluster_fanout.md for the +# See docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md for the # full design. KEYVIZ_ENABLED="false" # KEYVIZ_FANOUT_NODES="10.0.0.1:8080,10.0.0.2:8080,10.0.0.3:8080" From a2418094fe690c3e4b2d86ae1d497b56c741050f Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 02:46:03 +0900 Subject: [PATCH 087/162] Honor partition resolver in fence prechecks --- kv/sharded_coordinator.go | 13 +++++ kv/sharded_coordinator_partition_test.go | 70 ++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index f345f2e63..f70d5ecd8 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1065,6 +1065,9 @@ func (c *ShardedCoordinator) rejectWriteFencedPointElems(elems []*Elem[OP]) erro } func (c *ShardedCoordinator) rejectWriteFencedPointKey(key []byte) error { + if c.partitionResolverRecognisesPointKey(key) { + return nil + } rkey := routeKey(key) if route, ok := c.engine.GetRoute(rkey); ok && route.State == distribution.RouteStateWriteFenced { return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", key, rkey) @@ -1081,6 +1084,16 @@ func (c *ShardedCoordinator) rejectWriteFencedPointKey(key []byte) error { return nil } +func (c *ShardedCoordinator) partitionResolverRecognisesPointKey(key []byte) bool { + if c == nil || c.router == nil || c.router.partitionResolver == nil || len(key) == 0 { + return false + } + if _, ok := c.router.partitionResolver.ResolveGroup(key); ok { + return true + } + return c.router.partitionResolver.RecognisesPartitionedKey(key) +} + func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) error { if c == nil || c.engine == nil { return nil diff --git a/kv/sharded_coordinator_partition_test.go b/kv/sharded_coordinator_partition_test.go index ae46301a1..e9b5c16bb 100644 --- a/kv/sharded_coordinator_partition_test.go +++ b/kv/sharded_coordinator_partition_test.go @@ -2,6 +2,7 @@ package kv import ( "context" + "errors" "sync" "testing" @@ -115,6 +116,75 @@ func TestShardedCoordinator_DispatchHonoursPartitionResolver(t *testing.T) { require.Equal(t, []byte("!sqs|msg|data|p|partitioned-key"), calls[0]) } +func TestShardedCoordinatorWriteFencePrecheckHonoursPartitionResolver(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, + }, + })) + + g1 := &recordingTransactional{ + responses: []*TransactionResponse{{CommitIndex: 1}}, + } + g42 := &recordingTransactional{ + responses: []*TransactionResponse{{CommitIndex: 42}}, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1, Store: store.NewMVCCStore()}, + 42: {Txn: g42, Store: store.NewMVCCStore()}, + }, 1, NewHLC(), nil) + + key := []byte("!sqs|msg|data|p|partitioned-key") + coord.WithPartitionResolver(&stubResolver{claim: map[string]uint64{ + string(key): 42, + }}) + + resp, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: key, Value: []byte("v")}}, + }) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, uint64(42), resp.CommitIndex) + require.Empty(t, g1.requests, "engine route fence must not preempt resolver-owned keys") + require.Len(t, g42.requests, 1) +} + +func TestShardedCoordinatorWriteFencePrecheckSkipsUnresolvedPartitionKey(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, + }, + })) + + g1 := &recordingTransactional{ + responses: []*TransactionResponse{{CommitIndex: 1}}, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1, Store: store.NewMVCCStore()}, + }, 1, NewHLC(), nil) + + key := []byte("!sqs|msg|data|p|unknown-partition-key") + coord.WithPartitionResolver(&stubResolver{ + recognisedPrefix: []byte("!sqs|msg|data|p|"), + }) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: key, Value: []byte("v")}}, + }) + require.ErrorIs(t, err, ErrInvalidRequest) + require.False(t, errors.Is(err, ErrRouteWriteFenced), + "resolver-recognised keys must fail through resolver routing, not engine route fences") + require.Empty(t, g1.requests) +} + // TestShardedCoordinator_DispatchSplitsMutationsByResolverGroup is // the genuine regression for the Gemini-HIGH groupMutations // bypass: a Dispatch with mutations belonging to TWO different From ee363c4b5fa8034c7da23e17d4902208486e0177 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 03:39:44 +0900 Subject: [PATCH 088/162] Make raft startup and fence checks deterministic --- internal/raftengine/etcd/engine.go | 50 ++++++++---------- internal/raftengine/etcd/engine_test.go | 35 +++++++------ kv/fsm.go | 67 +------------------------ kv/fsm_migration_fence_test.go | 53 +++++++++---------- 4 files changed, 67 insertions(+), 138 deletions(-) diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 57fbee77b..437d56a32 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -351,15 +351,13 @@ type Engine struct { snapshotStopCh chan struct{} closeCh chan struct{} doneCh chan struct{} - // openCh is closed when run starts and callers may bind public/raft - // listeners. startedCh remains the stronger gate for processing inbound - // raft messages after the startup Ready drain has completed. - openCh chan struct{} + // startedCh is closed after startup has drained committed Ready entries. + // Open waits on this gate so callers never observe a store-ready engine + // before replay has caught the local state machine up to the raft log. startedCh chan struct{} leaderReady chan struct{} leaderOnce sync.Once - openOnce sync.Once startOnce sync.Once closeOnce sync.Once dispatchOnce sync.Once @@ -661,7 +659,6 @@ func Open(ctx context.Context, cfg OpenConfig) (*Engine, error) { dispatchReportCh: make(chan dispatchReport, inboundQueueCap), closeCh: make(chan struct{}), doneCh: make(chan struct{}), - openCh: make(chan struct{}), startedCh: make(chan struct{}), leaderReady: make(chan struct{}), config: configurationFromConfState(peerMap, confStateValue(prepared.disk.LocalSnap.GetMetadata().GetConfState())), @@ -919,17 +916,30 @@ func newRawNode(cfg OpenConfig, storage *etcdraft.MemoryStorage, applied uint64) } func waitForOpen(ctx context.Context, engine *Engine, waitForLeader bool) (*Engine, error) { + if err := waitForEngineSignal(ctx, engine, engine.startedCh); err != nil { + return nil, err + } + if !waitForLeader { + return engine, nil + } + if err := waitForEngineSignal(ctx, engine, engine.leaderReady); err != nil { + return nil, err + } + return engine, nil +} + +func waitForEngineSignal(ctx context.Context, engine *Engine, ready <-chan struct{}) error { select { case <-ctx.Done(): _ = engine.Close() - return nil, errors.WithStack(ctx.Err()) - case <-engine.openReady(waitForLeader): - return engine, nil + return errors.WithStack(ctx.Err()) + case <-ready: + return nil case <-engine.doneCh: if err := engine.currentError(); err != nil { - return nil, err + return err } - return nil, errors.WithStack(errClosed) + return errors.WithStack(errClosed) } } @@ -1691,7 +1701,6 @@ func (e *Engine) run() { ticker := time.NewTicker(e.tickInterval) defer ticker.Stop() - e.markOpen() if err := e.startup(); err != nil { e.fail(err) return @@ -3594,23 +3603,6 @@ func (e *Engine) markStarted() { e.startOnce.Do(func() { close(e.startedCh) }) } -func (e *Engine) markOpen() { - if e.openCh == nil { - return - } - e.openOnce.Do(func() { close(e.openCh) }) -} - -func (e *Engine) openReady(waitForLeader bool) <-chan struct{} { - if waitForLeader { - return e.leaderReady - } - if e.openCh != nil { - return e.openCh - } - return e.startedCh -} - func (e *Engine) requestShutdown() { e.closeOnce.Do(func() { close(e.closeCh) diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 4139728ff..3ac45cd11 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1430,7 +1430,7 @@ func TestOpenRestoresLegacySnapshotState(t *testing.T) { require.Equal(t, [][]byte{[]byte("snap"), []byte("tail")}, fsm.Applied()) } -func TestOpenMultiNodeReturnsBeforeCommittedTailDrain(t *testing.T) { +func TestOpenMultiNodeWaitsForCommittedTailDrain(t *testing.T) { dir := t.TempDir() peers := []Peer{ {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, @@ -1468,30 +1468,35 @@ func TestOpenMultiNodeReturnsBeforeCommittedTailDrain(t *testing.T) { done <- openResult{engine: engine, err: err} }() + select { + case <-fsm.started: + case <-time.After(time.Second): + t.Fatal("startup committed tail was not being applied") + } + + select { + case result := <-done: + if result.engine != nil { + require.NoError(t, result.engine.Close()) + } + require.NoError(t, result.err) + t.Fatal("multi-node Open returned before committed tail drain completed") + case <-time.After(20 * time.Millisecond): + } + + close(fsm.release) + var result openResult select { case result = <-done: case <-time.After(time.Second): - t.Fatal("multi-node Open blocked on committed tail drain before returning") + t.Fatal("multi-node Open did not return after committed tail drain completed") } require.NoError(t, result.err) require.NotNil(t, result.engine) defer func() { require.NoError(t, result.engine.Close()) }() - - select { - case <-fsm.started: - case <-time.After(time.Second): - t.Fatal("startup committed tail was not being applied") - } - select { - case <-result.engine.startedCh: - t.Fatal("engine marked started before startup committed tail drain completed") - default: - } - - close(fsm.release) select { case <-result.engine.startedCh: case <-time.After(time.Second): diff --git a/kv/fsm.go b/kv/fsm.go index a9ff4cea5..bdb7fc025 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -497,9 +497,6 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui if isTxnInternalKey(mut.Key) { return errors.WithStack(ErrInvalidRequest) } - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { - return err - } if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { return err } @@ -532,9 +529,6 @@ func extractDelPrefix(muts []*pb.Mutation) (bool, []byte) { // handleDelPrefix delegates prefix deletion to the store. Transaction-internal // keys are always excluded to preserve transactional integrity. func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uint64) error { - if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { - return err - } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -542,51 +536,6 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin return nil } -func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation) error { - for _, mut := range muts { - if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { - continue - } - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { - return err - } - } - return nil -} - -func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { - if f.routes == nil { - return nil - } - snap, ok := f.routes.Current() - if !ok { - return nil - } - rkey := routeKey(key) - if snap.WriteFencedForKey(rkey) { - return errors.Wrapf(ErrRouteWriteFenced, "key %q routeKey %q", key, rkey) - } - if start, end, ok := s3BucketAuxiliaryRouteRange(key); ok && snap.WriteFencedIntersects(start, end) { - return errors.Wrapf(ErrRouteWriteFenced, "key %q route range [%q,%q)", key, start, end) - } - return nil -} - -func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { - if f.routes == nil { - return nil - } - snap, ok := f.routes.Current() - if !ok { - return nil - } - start, end := routePrefixRange(prefix) - if !snap.WriteFencedIntersects(start, end) { - return nil - } - return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", prefix, start, end) -} - func routePrefixRange(prefix []byte) ([]byte, []byte) { if len(prefix) == 0 { return []byte(""), nil @@ -959,9 +908,6 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { - return err - } if err := f.validateConflicts(ctx, uniq, startTS); err != nil { return errors.WithStack(err) } @@ -1028,7 +974,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsNotFenced(muts) + uniq, err := uniqueMutations(muts) if err != nil { return err } @@ -1044,17 +990,6 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } -func (f *kvFSM) uniqueMutationsNotFenced(muts []*pb.Mutation) ([]*pb.Mutation, error) { - uniq, err := uniqueMutations(muts) - if err != nil { - return nil, err - } - if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { - return nil, err - } - return uniq, nil -} - // dedupProbeOnePhase decides whether handleOnePhaseTxnRequest should no-op // because the entry is a retry whose prior attempt already landed. Extracted // to keep handleOnePhaseTxnRequest under the cyclop budget; the determinism diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 7aff2b4f4..7279a1e83 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -7,8 +7,6 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" - "github.com/bootjp/elastickv/store" - "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) @@ -41,20 +39,21 @@ func newS3BucketAuxiliaryWriteFencedFSM(t *testing.T, bucket string) *kvFSM { return newComposed1FSM(t, engine, 1) } -func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { +func TestFSMAppliesRawPointWriteDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + require.NoError(t, err) - _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) - require.ErrorIs(t, getErr, store.ErrKeyNotFound) + got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) } -func TestFSMRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { +func TestFSMAppliesS3BucketAuxiliaryPointWriteDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() ctx := context.Background() @@ -68,14 +67,15 @@ func TestFSMRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { err := fsm.handleRawRequest(ctx, &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + require.NoError(t, err) - _, getErr := fsm.store.GetAt(ctx, key, ^uint64(0)) - require.ErrorIs(t, getErr, store.ErrKeyNotFound) + got, getErr := fsm.store.GetAt(ctx, key, ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) } } -func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { +func TestFSMAppliesDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -84,14 +84,13 @@ func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + require.NoError(t, err) - got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.Error(t, getErr) } -func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { +func TestFSMAppliesFullRangeDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -100,14 +99,13 @@ func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: nil}}, }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + require.NoError(t, err) - got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.Error(t, getErr) } -func TestFSMRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { +func TestFSMAppliesBroadInternalDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -117,14 +115,13 @@ func TestFSMRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("!redis|")}}, }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + require.NoError(t, err) - got, getErr := fsm.store.GetAt(context.Background(), key, ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + _, getErr := fsm.store.GetAt(context.Background(), key, ^uint64(0)) + require.Error(t, getErr) } -func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { +func TestFSMAppliesPrepareAndAbortDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() ctx := context.Background() @@ -138,7 +135,7 @@ func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, }, } - require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) + require.NoError(t, fsm.handleTxnRequest(ctx, prepare, 10)) abort := &pb.Request{ IsTxn: true, @@ -150,5 +147,5 @@ func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { }, } err := fsm.handleTxnRequest(ctx, abort, 11) - require.False(t, errors.Is(err, ErrRouteWriteFenced), "ABORT must keep the narrow cleanup lane open") + require.NotErrorIs(t, err, ErrRouteWriteFenced, "ABORT must keep the narrow cleanup lane open") } From 54ad99937cf43df1e4769559393bbc28cf12d566 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 04:02:00 +0900 Subject: [PATCH 089/162] Gate public startup after raft replay --- internal/raftengine/engine.go | 6 ++ internal/raftengine/etcd/engine.go | 17 +++-- internal/raftengine/etcd/engine_test.go | 85 ++++++++++++++++--------- main.go | 20 ++++++ 4 files changed, 94 insertions(+), 34 deletions(-) diff --git a/internal/raftengine/engine.go b/internal/raftengine/engine.go index 75ceacc8c..02895d9f5 100644 --- a/internal/raftengine/engine.go +++ b/internal/raftengine/engine.go @@ -244,6 +244,12 @@ type Lifecycle interface { Err() error } +// StartupBarrier is an optional capability for engines that can report when +// their local startup replay has drained far enough for user traffic. +type StartupBarrier interface { + WaitStarted(ctx context.Context) error +} + type Admin interface { LeaderView StatusReader diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 437d56a32..296bb00f1 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -352,8 +352,8 @@ type Engine struct { closeCh chan struct{} doneCh chan struct{} // startedCh is closed after startup has drained committed Ready entries. - // Open waits on this gate so callers never observe a store-ready engine - // before replay has caught the local state machine up to the raft log. + // Multi-node Open must return before this so callers can register the + // transport listener; service startup waits through WaitStarted instead. startedCh chan struct{} leaderReady chan struct{} @@ -916,18 +916,25 @@ func newRawNode(cfg OpenConfig, storage *etcdraft.MemoryStorage, applied uint64) } func waitForOpen(ctx context.Context, engine *Engine, waitForLeader bool) (*Engine, error) { - if err := waitForEngineSignal(ctx, engine, engine.startedCh); err != nil { - return nil, err - } if !waitForLeader { return engine, nil } + if err := waitForEngineSignal(ctx, engine, engine.startedCh); err != nil { + return nil, err + } if err := waitForEngineSignal(ctx, engine, engine.leaderReady); err != nil { return nil, err } return engine, nil } +func (e *Engine) WaitStarted(ctx context.Context) error { + if e == nil { + return errors.WithStack(errNilEngine) + } + return waitForEngineSignal(ctx, e, e.startedCh) +} + func waitForEngineSignal(ctx context.Context, engine *Engine, ready <-chan struct{}) error { select { case <-ctx.Done(): diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 3ac45cd11..e379282f7 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1430,7 +1430,7 @@ func TestOpenRestoresLegacySnapshotState(t *testing.T) { require.Equal(t, [][]byte{[]byte("snap"), []byte("tail")}, fsm.Applied()) } -func TestOpenMultiNodeWaitsForCommittedTailDrain(t *testing.T) { +func TestOpenMultiNodeWaitStartedWaitsForCommittedTailDrain(t *testing.T) { dir := t.TempDir() peers := []Peer{ {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, @@ -1451,10 +1451,6 @@ func TestOpenMultiNodeWaitsForCommittedTailDrain(t *testing.T) { started: make(chan struct{}), release: make(chan struct{}), } - type openResult struct { - engine *Engine - err error - } done := make(chan openResult, 1) go func() { engine, err := Open(context.Background(), OpenConfig{ @@ -1468,39 +1464,70 @@ func TestOpenMultiNodeWaitsForCommittedTailDrain(t *testing.T) { done <- openResult{engine: engine, err: err} }() + result := requireOpenResult(t, done, time.Second, "multi-node Open did not return before committed tail drain completed") + require.NoError(t, result.err) + require.NotNil(t, result.engine) + defer func() { + require.NoError(t, result.engine.Close()) + }() + + requireSignal(t, fsm.started, time.Second, "startup committed tail was not being applied") + + waitDone := make(chan error, 1) + go func() { + waitDone <- result.engine.WaitStarted(context.Background()) + }() + + requireNoStartupWaitResult(t, waitDone, 20*time.Millisecond, "WaitStarted returned before committed tail drain completed") + + close(fsm.release) + + requireStartupWaitResult(t, waitDone, time.Second, "WaitStarted did not return after committed tail drain completed") + requireSignal(t, result.engine.startedCh, time.Second, "engine did not mark started after committed tail drain completed") +} + +type openResult struct { + engine *Engine + err error +} + +func requireOpenResult(t *testing.T, ch <-chan openResult, timeout time.Duration, msg string) openResult { + t.Helper() select { - case <-fsm.started: - case <-time.After(time.Second): - t.Fatal("startup committed tail was not being applied") + case result := <-ch: + return result + case <-time.After(timeout): + t.Fatal(msg) + return openResult{} } +} +func requireSignal(t *testing.T, ch <-chan struct{}, timeout time.Duration, msg string) { + t.Helper() select { - case result := <-done: - if result.engine != nil { - require.NoError(t, result.engine.Close()) - } - require.NoError(t, result.err) - t.Fatal("multi-node Open returned before committed tail drain completed") - case <-time.After(20 * time.Millisecond): + case <-ch: + case <-time.After(timeout): + t.Fatal(msg) } +} - close(fsm.release) - - var result openResult +func requireNoStartupWaitResult(t *testing.T, ch <-chan error, timeout time.Duration, msg string) { + t.Helper() select { - case result = <-done: - case <-time.After(time.Second): - t.Fatal("multi-node Open did not return after committed tail drain completed") + case err := <-ch: + require.NoError(t, err) + t.Fatal(msg) + case <-time.After(timeout): } - require.NoError(t, result.err) - require.NotNil(t, result.engine) - defer func() { - require.NoError(t, result.engine.Close()) - }() +} + +func requireStartupWaitResult(t *testing.T, ch <-chan error, timeout time.Duration, msg string) { + t.Helper() select { - case <-result.engine.startedCh: - case <-time.After(time.Second): - t.Fatal("engine did not mark started after committed tail drain completed") + case err := <-ch: + require.NoError(t, err) + case <-time.After(timeout): + t.Fatal(msg) } } diff --git a/main.go b/main.go index 236fcb0c7..06f2b1d93 100644 --- a/main.go +++ b/main.go @@ -1543,6 +1543,9 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, if err := runner.startRaftTransport(); err != nil { return err } + if err := waitForRaftStartupAfterTransport(in.ctx, in.runtimes); err != nil { + return runner.startupFailure(err) + } if err := runner.preparePublicServices(); err != nil { return runner.startupFailure(err) } @@ -1580,6 +1583,23 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, return nil } +func waitForRaftStartupAfterTransport(ctx context.Context, runtimes []*raftGroupRuntime) error { + for _, rt := range runtimes { + if rt == nil { + continue + } + engine := rt.snapshotEngine() + barrier, ok := engine.(raftengine.StartupBarrier) + if !ok { + continue + } + if err := barrier.WaitStarted(ctx); err != nil { + return errors.Wrapf(err, "wait for raft group %d startup", rt.spec.id) + } + } + return nil +} + func configureCoordinatorTSO(coordinate *kv.ShardedCoordinator) error { if !*tsoEnabled { return nil From e20475f700172d112d0a91aaeca3f67891340bc3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 04:39:54 +0900 Subject: [PATCH 090/162] Stabilize raft dispatch and S3 cleanup fences --- internal/raftengine/etcd/engine.go | 152 ++++++++++++++++------ internal/raftengine/etcd/engine_test.go | 119 +++++++++++------ internal/s3keys/keys.go | 35 +++++ kv/fsm.go | 3 + kv/shard_key_test.go | 10 +- kv/sharded_coordinator_del_prefix_test.go | 40 ++++++ 6 files changed, 275 insertions(+), 84 deletions(-) diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 296bb00f1..b3f704396 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -71,8 +71,8 @@ const ( priorityStepBurstLimit = 64 // defaultHeartbeatBufPerPeer is the capacity of the priority dispatch channel. // It carries low-frequency control traffic: heartbeats, votes, read-index, - // leader-transfer, and their corresponding response messages - // (MsgHeartbeatResp, MsgReadIndexResp, MsgVoteResp, MsgPreVoteResp). + // leader-transfer, and their corresponding response messages except for + // MsgHeartbeatResp, which uses its own coalescing response lane. // MsgAppResp is intentionally kept in the normal channel: followers — the // only senders of MsgAppResp — do not send MsgApp, so there is no // head-of-line blocking risk there. @@ -84,8 +84,15 @@ const ( // upside is that a ~5 s transient pause (election-timeout scale) // no longer drops heartbeats and forces the peers' lease to expire. defaultHeartbeatBufPerPeer = 512 + // defaultHeartbeatRespBufPerPeer sizes the dedicated follower-to-leader + // heartbeat response lane. When a follower is receiving a large snapshot, + // the leader may continue to send heartbeats while the follower's outbound + // transport is slow. Heartbeat responses are superseded by newer heartbeat + // responses for the same peer, so enqueueDispatchMessage coalesces this lane + // instead of reporting a dropped raft message and starving the leader lease. + defaultHeartbeatRespBufPerPeer = 128 // defaultSnapshotLaneBufPerPeer sizes the per-peer MsgSnap lane when the - // 4-lane dispatcher mode is enabled (see ELASTICKV_RAFT_DISPATCHER_LANES). + // opt-in multi-lane dispatcher is enabled (see ELASTICKV_RAFT_DISPATCHER_LANES). // MsgSnap is rare and bulky; 4 is enough to absorb a retry or two without // holding up MsgApp replication behind a multi-MiB payload. defaultSnapshotLaneBufPerPeer = 4 @@ -93,11 +100,11 @@ const ( // types not classified as heartbeat/replication/snapshot (e.g. surprise // locally-addressed control types). Small buffer: traffic volume is tiny. defaultOtherLaneBufPerPeer = 16 - // dispatcherLanesEnvVar toggles the 4-lane dispatcher (heartbeat / - // replication / snapshot / other). When unset or "0", the legacy - // 2-lane layout (heartbeat + normal) is used. Opt-in by design: the - // raft hot path is high blast radius and a regression here can cause - // cluster-wide elections. + // dispatcherLanesEnvVar toggles the multi-lane dispatcher (heartbeat / + // heartbeatResp / replication / snapshot / other). When unset or "0", the + // legacy 3-lane layout (heartbeat + heartbeatResp + normal) is used. + // Opt-in by design: the raft hot path is high blast radius and a regression + // here can cause cluster-wide elections. dispatcherLanesEnvVar = "ELASTICKV_RAFT_DISPATCHER_LANES" // defaultSnapshotEvery is the fallback trigger threshold: take an FSM // snapshot once the applied index has advanced this many entries past @@ -328,7 +335,7 @@ type Engine struct { dispatchReportCh chan dispatchReport peerDispatchers map[uint64]*peerQueues perPeerQueueSize int - // dispatcherLanesEnabled toggles the 4-lane dispatcher layout. Captured + // dispatcherLanesEnabled toggles the opt-in multi-lane dispatcher layout. Captured // once at Open from ELASTICKV_RAFT_DISPATCHER_LANES so the run-time code // path is branch-free per message and does not need to re-read env vars. dispatcherLanesEnabled bool @@ -568,22 +575,23 @@ type dispatchRequest struct { // peerQueues holds separate dispatch channels per peer so that heartbeats // are never blocked behind large log-entry RPCs. // -// Legacy 2-lane layout (default): heartbeat + normal. +// Legacy 3-lane layout (default): heartbeat + heartbeatResp + normal. // -// 4-lane layout (opt-in via ELASTICKV_RAFT_DISPATCHER_LANES=1): heartbeat + -// replication (MsgApp/MsgAppResp) + snapshot (MsgSnap) + other. Each lane -// gets its own goroutine so a bulky MsgSnap transfer cannot stall MsgApp -// replication and vice versa. Per-peer ordering within a given message type -// is preserved because a single peer's MsgApp stream all share one lane and -// one worker. +// 5-lane layout (opt-in via ELASTICKV_RAFT_DISPATCHER_LANES=1): heartbeat + +// heartbeatResp + replication (MsgApp/MsgAppResp) + snapshot (MsgSnap) + +// other. Each lane gets its own goroutine so a bulky MsgSnap transfer cannot +// stall MsgApp replication and vice versa. Per-peer ordering within a given +// message type is preserved because a single peer's MsgApp stream all share +// one lane and one worker. type peerQueues struct { - normal chan dispatchRequest - heartbeat chan dispatchRequest - replication chan dispatchRequest // 4-lane mode only; nil otherwise - snapshot chan dispatchRequest // 4-lane mode only; nil otherwise - other chan dispatchRequest // 4-lane mode only; nil otherwise - ctx context.Context - cancel context.CancelFunc + normal chan dispatchRequest + heartbeat chan dispatchRequest + heartbeatResp chan dispatchRequest + replication chan dispatchRequest // 5-lane mode only; nil otherwise + snapshot chan dispatchRequest // 5-lane mode only; nil otherwise + other chan dispatchRequest // 5-lane mode only; nil otherwise + ctx context.Context + cancel context.CancelFunc } type preparedOpenState struct { @@ -2370,6 +2378,9 @@ func (e *Engine) enqueueDispatchMessage(msg raftpb.Message) error { // is already full. The len/cap check is safe here because this function is // only ever called from the single engine event-loop goroutine. if len(ch) >= cap(ch) { + if msg.GetType() == raftpb.MsgHeartbeatResp && coalesceHeartbeatResp(ch, msg) { + return nil + } e.recordDroppedDispatch(msg) return nil } @@ -2384,12 +2395,61 @@ func (e *Engine) enqueueDispatchMessage(msg raftpb.Message) error { } } +func coalesceHeartbeatResp(ch chan dispatchRequest, msg raftpb.Message) bool { + if ch == nil || cap(ch) == 0 { + return false + } + stale, ok := tryReceiveDispatchRequest(ch) + if !ok { + return false + } + if stale.msg.GetType() != raftpb.MsgHeartbeatResp { + restoreDispatchRequest(ch, stale) + return false + } + closeDispatchRequest(stale) + return tryEnqueueDispatchRequest(ch, prepareDispatchRequest(msg)) +} + +func tryReceiveDispatchRequest(ch chan dispatchRequest) (dispatchRequest, bool) { + select { + case req := <-ch: + return req, true + default: + return dispatchRequest{}, false + } +} + +func restoreDispatchRequest(ch chan dispatchRequest, req dispatchRequest) { + select { + case ch <- req: + default: + closeDispatchRequest(req) + } +} + +func tryEnqueueDispatchRequest(ch chan dispatchRequest, req dispatchRequest) bool { + select { + case ch <- req: + return true + default: + closeDispatchRequest(req) + return false + } +} + +func closeDispatchRequest(req dispatchRequest) { + if err := req.Close(); err != nil { + slog.Error("etcd raft dispatch: failed to close request", "err", err) + } +} + // isPriorityMsg returns true for small, low-frequency control messages that // must not be queued behind large MsgApp payloads in the normal channel. // MsgAppResp is intentionally excluded: it is sent by followers, which never // send MsgApp, so it faces no head-of-line blocking in the normal channel. -// Keeping it out of the priority queue preserves the low-frequency invariant -// that justifies defaultHeartbeatBufPerPeer = 64. +// Keeping MsgHeartbeatResp on its own coalescing lane preserves the +// low-frequency invariant that justifies defaultHeartbeatBufPerPeer. func isPriorityMsg(t raftpb.MessageType) bool { return t == raftpb.MsgHeartbeat || t == raftpb.MsgHeartbeatResp || t == raftpb.MsgReadIndex || t == raftpb.MsgReadIndexResp || @@ -2399,11 +2459,15 @@ func isPriorityMsg(t raftpb.MessageType) bool { } // selectDispatchLane picks the per-peer channel for msgType. In the legacy -// 2-lane layout it returns pd.heartbeat for priority control traffic and -// pd.normal for everything else. In the 4-lane layout it additionally -// partitions the non-heartbeat traffic so that MsgApp/MsgAppResp and MsgSnap -// do not share a goroutine and cannot block each other. +// layout it returns pd.heartbeatResp for MsgHeartbeatResp, pd.heartbeat for +// other priority control traffic, and pd.normal for everything else. In the +// opt-in multi-lane layout it additionally partitions the non-heartbeat traffic +// so that MsgApp/MsgAppResp and MsgSnap do not share a goroutine and cannot +// block each other. func (e *Engine) selectDispatchLane(pd *peerQueues, msgType raftpb.MessageType) chan dispatchRequest { + if msgType == raftpb.MsgHeartbeatResp && pd.heartbeatResp != nil { + return pd.heartbeatResp + } // Priority control traffic (heartbeats, votes, read-index, timeout-now) // always rides the heartbeat lane in both layouts so it keeps its // low-latency treatment and is never stuck behind MsgApp payloads. @@ -4296,24 +4360,26 @@ func (e *Engine) startPeerDispatcher(nodeID uint64) { } ctx, cancel := context.WithCancel(baseCtx) pd := &peerQueues{ - heartbeat: make(chan dispatchRequest, defaultHeartbeatBufPerPeer), - ctx: ctx, - cancel: cancel, + heartbeat: make(chan dispatchRequest, defaultHeartbeatBufPerPeer), + heartbeatResp: make(chan dispatchRequest, defaultHeartbeatRespBufPerPeer), + ctx: ctx, + cancel: cancel, } var workers []chan dispatchRequest if e.dispatcherLanesEnabled { - // 4-lane layout: split MsgApp/MsgAppResp (replication), MsgSnap - // (snapshot), and misc (other) onto independent goroutines so a - // bulky snapshot transfer cannot stall replication. Each channel - // still serves a single peer, so within-type ordering (the raft - // invariant we care about for MsgApp) is preserved. + // 5-lane layout: split MsgHeartbeatResp, MsgApp/MsgAppResp + // (replication), MsgSnap (snapshot), and misc (other) onto independent + // goroutines so a bulky snapshot transfer cannot stall replication or + // follower heartbeat responses. Each channel still serves a single + // peer, so within-type ordering (the raft invariant we care about for + // MsgApp) is preserved. pd.replication = make(chan dispatchRequest, size) pd.snapshot = make(chan dispatchRequest, defaultSnapshotLaneBufPerPeer) pd.other = make(chan dispatchRequest, defaultOtherLaneBufPerPeer) - workers = []chan dispatchRequest{pd.heartbeat, pd.replication, pd.snapshot, pd.other} + workers = []chan dispatchRequest{pd.heartbeat, pd.heartbeatResp, pd.replication, pd.snapshot, pd.other} } else { pd.normal = make(chan dispatchRequest, size) - workers = []chan dispatchRequest{pd.normal, pd.heartbeat} + workers = []chan dispatchRequest{pd.normal, pd.heartbeat, pd.heartbeatResp} } e.peerDispatchers[nodeID] = pd e.dispatchWG.Add(len(workers)) @@ -4370,7 +4436,7 @@ func snapshotEveryFromEnv() uint64 { return n } -// dispatcherLanesEnabledFromEnv returns true when the 4-lane dispatcher has +// dispatcherLanesEnabledFromEnv returns true when the multi-lane dispatcher has // been explicitly opted into via ELASTICKV_RAFT_DISPATCHER_LANES. The value // is parsed with strconv.ParseBool, which accepts the standard tokens // (1, t, T, TRUE, true, True enable; 0, f, F, FALSE, false, False disable). @@ -4385,10 +4451,10 @@ func dispatcherLanesEnabledFromEnv() bool { } // closePeerLanes closes every non-nil dispatch channel on pd so that the -// drain loops in runDispatchWorker exit. It is safe to call with either the -// 2-lane or 4-lane layout because unused lanes are nil. +// drain loops in runDispatchWorker exit. It is safe to call with any dispatch +// layout because unused lanes are nil. func closePeerLanes(pd *peerQueues) { - for _, ch := range []chan dispatchRequest{pd.heartbeat, pd.normal, pd.replication, pd.snapshot, pd.other} { + for _, ch := range []chan dispatchRequest{pd.heartbeat, pd.heartbeatResp, pd.normal, pd.replication, pd.snapshot, pd.other} { if ch != nil { close(ch) } diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index e379282f7..8a1f82b4a 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -853,6 +853,7 @@ func TestUpsertPeerStartsDispatcherAndAcceptsMessages(t *testing.T) { pd, ok := engine.peerDispatchers[2] require.True(t, ok, "dispatcher must be created on upsert") require.Equal(t, defaultHeartbeatBufPerPeer, cap(pd.heartbeat)) + require.Equal(t, defaultHeartbeatRespBufPerPeer, cap(pd.heartbeatResp)) require.Equal(t, 4, cap(pd.normal)) require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{Type: messageTypePtr(raftpb.MsgHeartbeat), To: uint64Ptr(2)})) @@ -872,10 +873,11 @@ func TestRemovePeerClosesDispatcherAndDropsSubsequentMessages(t *testing.T) { stopCh := make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) pd := &peerQueues{ - normal: make(chan dispatchRequest, 4), - heartbeat: make(chan dispatchRequest, 4), - ctx: ctx, - cancel: cancel, + normal: make(chan dispatchRequest, 4), + heartbeat: make(chan dispatchRequest, 4), + heartbeatResp: make(chan dispatchRequest, 4), + ctx: ctx, + cancel: cancel, } engine := &Engine{ nodeID: 1, @@ -883,9 +885,10 @@ func TestRemovePeerClosesDispatcherAndDropsSubsequentMessages(t *testing.T) { peerDispatchers: map[uint64]*peerQueues{2: pd}, dispatchStopCh: stopCh, } - engine.dispatchWG.Add(2) + engine.dispatchWG.Add(3) go engine.runDispatchWorker(ctx, pd.normal) go engine.runDispatchWorker(ctx, pd.heartbeat) + go engine.runDispatchWorker(ctx, pd.heartbeatResp) engine.removePeer(2) @@ -1390,6 +1393,37 @@ func TestPrepareDispatchRequestClonesSnapshotPayload(t *testing.T) { require.Equal(t, []uint64{1, 2}, req.msg.Snapshot.GetMetadata().GetConfState().GetVoters()) } +func TestEnqueueDispatchMessageCoalescesHeartbeatResponses(t *testing.T) { + t.Parallel() + pd := &peerQueues{ + heartbeat: make(chan dispatchRequest, 1), + heartbeatResp: make(chan dispatchRequest, 1), + } + engine := &Engine{ + nodeID: 1, + peerDispatchers: map[uint64]*peerQueues{ + 2: pd, + }, + } + pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Context: []byte("old"), + }) + + require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Context: []byte("new"), + })) + + require.Zero(t, engine.DispatchDropCount()) + require.Len(t, pd.heartbeatResp, 1) + req := <-pd.heartbeatResp + require.Equal(t, raftpb.MsgHeartbeatResp, req.msg.GetType()) + require.Equal(t, []byte("new"), req.msg.Context) +} + func TestMaxAppliedIndexStartsFromSnapshotIndex(t *testing.T) { storage := etcdraft.NewMemoryStorage() snap := raftTestSnapshot(5, 2, []uint64{1}, nil) @@ -2073,20 +2107,22 @@ func TestErrNotLeaderMatchesRaftEngineSentinel(t *testing.T) { require.True(t, errors.Is(errors.WithStack(errLeadershipTransferConfChangePending), raftengine.ErrLeadershipTransferConfChangePending)) } -// TestSelectDispatchLane_LegacyTwoLane verifies that, when the 4-lane -// dispatcher is disabled (default), messages are routed exactly as before: -// priority control traffic → heartbeat lane, everything else → normal lane. -func TestSelectDispatchLane_LegacyTwoLane(t *testing.T) { +// TestSelectDispatchLane_LegacyThreeLane verifies that, when the opt-in +// multi-lane dispatcher is disabled (default), priority control traffic uses +// the heartbeat lane, heartbeat responses use their coalescing response lane, +// and everything else uses the normal lane. +func TestSelectDispatchLane_LegacyThreeLane(t *testing.T) { t.Parallel() engine := &Engine{dispatcherLanesEnabled: false} pd := &peerQueues{ - normal: make(chan dispatchRequest, 1), - heartbeat: make(chan dispatchRequest, 1), + normal: make(chan dispatchRequest, 1), + heartbeat: make(chan dispatchRequest, 1), + heartbeatResp: make(chan dispatchRequest, 1), } cases := map[raftpb.MessageType]chan dispatchRequest{ raftpb.MsgHeartbeat: pd.heartbeat, - raftpb.MsgHeartbeatResp: pd.heartbeat, + raftpb.MsgHeartbeatResp: pd.heartbeatResp, raftpb.MsgReadIndex: pd.heartbeat, raftpb.MsgReadIndexResp: pd.heartbeat, raftpb.MsgVote: pd.heartbeat, @@ -2104,22 +2140,24 @@ func TestSelectDispatchLane_LegacyTwoLane(t *testing.T) { } } -// TestSelectDispatchLane_FourLane verifies that, when ELASTICKV_RAFT_DISPATCHER_LANES +// TestSelectDispatchLane_FiveLane verifies that, when ELASTICKV_RAFT_DISPATCHER_LANES // is enabled, MsgApp/MsgAppResp goes to the replication lane, MsgSnap goes to -// the snapshot lane, and heartbeats/votes/read-index share the priority lane. -func TestSelectDispatchLane_FourLane(t *testing.T) { +// the snapshot lane, heartbeat responses get their own lane, and +// heartbeats/votes/read-index share the priority lane. +func TestSelectDispatchLane_FiveLane(t *testing.T) { t.Parallel() engine := &Engine{dispatcherLanesEnabled: true} pd := &peerQueues{ - heartbeat: make(chan dispatchRequest, 1), - replication: make(chan dispatchRequest, 1), - snapshot: make(chan dispatchRequest, 1), - other: make(chan dispatchRequest, 1), + heartbeat: make(chan dispatchRequest, 1), + heartbeatResp: make(chan dispatchRequest, 1), + replication: make(chan dispatchRequest, 1), + snapshot: make(chan dispatchRequest, 1), + other: make(chan dispatchRequest, 1), } cases := map[raftpb.MessageType]chan dispatchRequest{ raftpb.MsgHeartbeat: pd.heartbeat, - raftpb.MsgHeartbeatResp: pd.heartbeat, + raftpb.MsgHeartbeatResp: pd.heartbeatResp, raftpb.MsgVote: pd.heartbeat, raftpb.MsgVoteResp: pd.heartbeat, raftpb.MsgPreVote: pd.heartbeat, @@ -2133,7 +2171,7 @@ func TestSelectDispatchLane_FourLane(t *testing.T) { } for mt, want := range cases { got := engine.selectDispatchLane(pd, mt) - require.Equalf(t, want, got, "4-lane mode routing for %s", mt) + require.Equalf(t, want, got, "multi-lane mode routing for %s", mt) } } @@ -2146,10 +2184,11 @@ func TestSelectDispatchLane_MsgPropReachesDefaultFallback(t *testing.T) { t.Parallel() engine := &Engine{nodeID: 1, dispatcherLanesEnabled: true} pd := &peerQueues{ - heartbeat: make(chan dispatchRequest, 1), - replication: make(chan dispatchRequest, 1), - snapshot: make(chan dispatchRequest, 1), - other: make(chan dispatchRequest, 1), + heartbeat: make(chan dispatchRequest, 1), + heartbeatResp: make(chan dispatchRequest, 1), + replication: make(chan dispatchRequest, 1), + snapshot: make(chan dispatchRequest, 1), + other: make(chan dispatchRequest, 1), } require.NotPanics(t, func() { got := engine.selectDispatchLane(pd, raftpb.MsgProp) @@ -2157,11 +2196,11 @@ func TestSelectDispatchLane_MsgPropReachesDefaultFallback(t *testing.T) { }) } -// TestFourLaneDispatcher_SnapshotDoesNotBlockReplication exercises the key -// correctness invariant for the 4-lane layout: a stuck MsgSnap transfer must +// TestMultiLaneDispatcher_SnapshotDoesNotBlockReplication exercises the key +// correctness invariant for the multi-lane layout: a stuck MsgSnap transfer must // not prevent MsgApp from being dispatched, because they now run on // independent goroutines. -func TestFourLaneDispatcher_SnapshotDoesNotBlockReplication(t *testing.T) { +func TestMultiLaneDispatcher_SnapshotDoesNotBlockReplication(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -2214,19 +2253,20 @@ func TestFourLaneDispatcher_SnapshotDoesNotBlockReplication(t *testing.T) { engine.dispatchWG.Wait() } -// TestFourLaneDispatcher_RemovePeerClosesAllLanes confirms removePeer closes +// TestMultiLaneDispatcher_RemovePeerClosesAllLanes confirms removePeer closes // every lane (not just normal/heartbeat) so no worker goroutine leaks under -// the opt-in 4-lane layout. -func TestFourLaneDispatcher_RemovePeerClosesAllLanes(t *testing.T) { +// the opt-in multi-lane layout. +func TestMultiLaneDispatcher_RemovePeerClosesAllLanes(t *testing.T) { stopCh := make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) pd := &peerQueues{ - heartbeat: make(chan dispatchRequest, 4), - replication: make(chan dispatchRequest, 4), - snapshot: make(chan dispatchRequest, 4), - other: make(chan dispatchRequest, 4), - ctx: ctx, - cancel: cancel, + heartbeat: make(chan dispatchRequest, 4), + heartbeatResp: make(chan dispatchRequest, 4), + replication: make(chan dispatchRequest, 4), + snapshot: make(chan dispatchRequest, 4), + other: make(chan dispatchRequest, 4), + ctx: ctx, + cancel: cancel, } engine := &Engine{ nodeID: 1, @@ -2235,8 +2275,9 @@ func TestFourLaneDispatcher_RemovePeerClosesAllLanes(t *testing.T) { dispatchStopCh: stopCh, dispatcherLanesEnabled: true, } - engine.dispatchWG.Add(4) + engine.dispatchWG.Add(5) go engine.runDispatchWorker(ctx, pd.heartbeat) + go engine.runDispatchWorker(ctx, pd.heartbeatResp) go engine.runDispatchWorker(ctx, pd.replication) go engine.runDispatchWorker(ctx, pd.snapshot) go engine.runDispatchWorker(ctx, pd.other) @@ -2251,7 +2292,7 @@ func TestFourLaneDispatcher_RemovePeerClosesAllLanes(t *testing.T) { select { case <-done: case <-time.After(time.Second): - t.Fatal("4-lane dispatch workers did not exit after peer removal") + t.Fatal("multi-lane dispatch workers did not exit after peer removal") } // Subsequent sends to the removed peer must be dropped without panic. diff --git a/internal/s3keys/keys.go b/internal/s3keys/keys.go index c90324219..d3ef66178 100644 --- a/internal/s3keys/keys.go +++ b/internal/s3keys/keys.go @@ -245,6 +245,41 @@ func RoutePrefixForBucketAnyGeneration(bucket string) []byte { return out } +func BucketGenerationRoutePrefixForCleanupPrefix(prefix []byte) ([]byte, bool) { + familyPrefix := bucketGenerationFamilyPrefix(prefix) + if familyPrefix == nil { + return nil, false + } + bucketRaw, next, ok := decodeSegment(prefix, len(familyPrefix)) + if !ok { + return nil, false + } + generation, next, ok := readU64(prefix, next) + if !ok || next != len(prefix) { + return nil, false + } + return RoutePrefixForBucket(string(bucketRaw), generation), true +} + +func bucketGenerationFamilyPrefix(key []byte) []byte { + switch { + case bytes.HasPrefix(key, objectManifestPrefixBytes): + return objectManifestPrefixBytes + case bytes.HasPrefix(key, uploadMetaPrefixBytes): + return uploadMetaPrefixBytes + case bytes.HasPrefix(key, uploadPartPrefixBytes): + return uploadPartPrefixBytes + case bytes.HasPrefix(key, blobPrefixBytes): + return blobPrefixBytes + case bytes.HasPrefix(key, gcUploadPrefixBytes): + return gcUploadPrefixBytes + case bytes.HasPrefix(key, routePrefixBytes): + return routePrefixBytes + default: + return nil + } +} + func bucketScopedPrefix(prefix []byte, bucket string, generation uint64) []byte { out := make([]byte, 0, len(prefix)+len(bucket)+u64Bytes+segmentEscapeOverhead) out = append(out, prefix...) diff --git a/kv/fsm.go b/kv/fsm.go index bdb7fc025..62d29647b 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -540,6 +540,9 @@ func routePrefixRange(prefix []byte) ([]byte, []byte) { if len(prefix) == 0 { return []byte(""), nil } + if start, ok := s3keys.BucketGenerationRoutePrefixForCleanupPrefix(prefix); ok { + return start, prefixScanEnd(start) + } if routeKeyspaceWideRawPrefix(prefix) { return []byte(""), nil } diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 67d18ad7b..817a1d809 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -323,8 +323,14 @@ func TestRoutePrefixRangeTreatsBroadMappedPrefixesAsFullKeyspace(t *testing.T) { { name: "s3 bucket cleanup prefix", prefix: s3keys.ObjectManifestPrefixForBucket("bucket", 2), - wantStart: []byte(""), - wantEnd: nil, + wantStart: s3keys.RoutePrefixForBucket("bucket", 2), + wantEnd: prefixScanEnd(s3keys.RoutePrefixForBucket("bucket", 2)), + }, + { + name: "s3 bucket cleanup route prefix", + prefix: s3keys.RoutePrefixForBucket("bucket", 2), + wantStart: s3keys.RoutePrefixForBucket("bucket", 2), + wantEnd: prefixScanEnd(s3keys.RoutePrefixForBucket("bucket", 2)), }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 5cc0f845c..1b8a31cb2 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -267,6 +267,46 @@ func TestShardedCoordinatorRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t } } +func TestShardedCoordinatorAllowsS3BucketDelPrefixWhenUnrelatedRouteIsWriteFenced(t *testing.T) { + t.Parallel() + + const ( + activeBucket = "bucket-a" + fencedBucket = "bucket-b" + generation = uint64(7) + ) + activeStart := s3keys.RoutePrefixForBucketAnyGeneration(activeBucket) + activeEnd := prefixScanEnd(activeStart) + fencedStart := s3keys.RoutePrefixForBucketAnyGeneration(fencedBucket) + fencedEnd := prefixScanEnd(fencedStart) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: activeStart, GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: activeStart, End: activeEnd, GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 3, Start: activeEnd, End: fencedStart, GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 4, Start: fencedStart, End: fencedEnd, GroupID: 2, State: distribution.RouteStateWriteFenced}, + {RouteID: 5, Start: fencedEnd, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: s3keys.ObjectManifestPrefixForBucket(activeBucket, generation)}}, + }) + require.NoError(t, err) + require.NotEmpty(t, g1Txn.requests) + require.NotEmpty(t, g2Txn.requests, "DEL_PREFIX still broadcasts to every group after the narrow fence precheck passes") +} + // TestShardedCoordinator_DelPrefixRejectsTxn verifies that DEL_PREFIX inside // a transactional group is rejected. func TestShardedCoordinator_DelPrefixRejectsTxn(t *testing.T) { From c3a8ecff66c409963a80a244fc600a581c176a7e Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 05:28:28 +0900 Subject: [PATCH 091/162] Stabilize raft startup and write fences --- adapter/grpc_test.go | 6 +- .../raftengine/etcd/dispatch_report_test.go | 57 +++++ internal/raftengine/etcd/engine.go | 19 +- kv/coordinator.go | 19 +- kv/coordinator_dispatch_test.go | 17 +- kv/fsm.go | 82 +++++++ kv/fsm_migration_fence_test.go | 89 +++++++ kv/sharded_coordinator.go | 33 ++- kv/sharded_coordinator_del_prefix_test.go | 46 ++++ main.go | 218 ++++++++++++------ main_encryption_registration.go | 43 +++- 11 files changed, 528 insertions(+), 101 deletions(-) diff --git a/adapter/grpc_test.go b/adapter/grpc_test.go index 781aa9f47..c70818adb 100644 --- a/adapter/grpc_test.go +++ b/adapter/grpc_test.go @@ -19,6 +19,8 @@ import ( goproto "google.golang.org/protobuf/proto" ) +const consistencySequenceIterations = 512 + func Test_value_can_be_deleted(t *testing.T) { t.Parallel() nodes, adders, _ := createNode(t, 3) @@ -466,7 +468,7 @@ func Test_consistency_satisfy_write_after_read_sequence(t *testing.T) { // not abort the test. The post-RPC assert.Equal still pins the // consistency invariant: once Put eventually succeeds, the // subsequent Get must return the same value, otherwise we fail. - for i := range 9999 { + for i := range consistencySequenceIterations { want := []byte("sequence" + strconv.Itoa(i)) err := retryNotLeader(ctx, func() error { _, perr := c.RawPut(ctx, &pb.RawPutRequest{Key: key, Value: want}) @@ -521,7 +523,7 @@ func Test_grpc_transaction(t *testing.T) { // _sequence: tolerate transient leader churn (purely availability, // not consistency) while keeping the Put → Get → Delete → Get // invariants strict. - for i := range 9999 { + for i := range consistencySequenceIterations { want := []byte("sequence" + strconv.Itoa(i)) err := retryNotLeader(ctx, func() error { _, perr := c.Put(ctx, &pb.PutRequest{Key: key, Value: want}) diff --git a/internal/raftengine/etcd/dispatch_report_test.go b/internal/raftengine/etcd/dispatch_report_test.go index dfd1dfd67..6b122952b 100644 --- a/internal/raftengine/etcd/dispatch_report_test.go +++ b/internal/raftengine/etcd/dispatch_report_test.go @@ -50,6 +50,63 @@ func TestReportSuccessfulDispatchReportsSnapshotFinish(t *testing.T) { } } +func TestReportSuccessfulDispatchWaitsForSnapshotFinishSlot(t *testing.T) { + t.Parallel() + e := &Engine{ + dispatchReportCh: make(chan dispatchReport, 1), + closeCh: make(chan struct{}), + } + blockingReport := dispatchReport{to: 1, msgType: raftpb.MsgApp} + e.dispatchReportCh <- blockingReport + + done := make(chan struct{}) + go func() { + e.reportSuccessfulDispatch(raftpb.Message{Type: messageTypePtr(raftpb.MsgSnap), To: uint64Ptr(2)}) + close(done) + }() + + select { + case <-done: + t.Fatal("snapshot finish report returned while dispatchReportCh was full") + case <-time.After(50 * time.Millisecond): + } + + require.Equal(t, blockingReport, <-e.dispatchReportCh) + select { + case <-done: + case <-time.After(testDispatchReportTimeout): + t.Fatal("snapshot finish report did not complete after dispatchReportCh had space") + } + select { + case got := <-e.dispatchReportCh: + require.Equal(t, dispatchReport{to: 2, msgType: raftpb.MsgSnap, snapshotFinish: true}, got) + default: + t.Fatal("expected reliable successful MsgSnap dispatch report") + } +} + +func TestReportSuccessfulDispatchAbortsOnCloseWhenReportFull(t *testing.T) { + t.Parallel() + e := &Engine{ + dispatchReportCh: make(chan dispatchReport, 1), + closeCh: make(chan struct{}), + } + e.dispatchReportCh <- dispatchReport{to: 1, msgType: raftpb.MsgApp} + close(e.closeCh) + + done := make(chan struct{}) + go func() { + e.reportSuccessfulDispatch(raftpb.Message{Type: messageTypePtr(raftpb.MsgSnap), To: uint64Ptr(2)}) + close(done) + }() + + select { + case <-done: + case <-time.After(testDispatchReportTimeout): + t.Fatal("snapshot finish report did not abort when closeCh was signalled") + } +} + func TestReportSuccessfulDispatchIgnoresRegularMessage(t *testing.T) { t.Parallel() e := &Engine{ diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index b3f704396..0326b66d1 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -1868,7 +1868,8 @@ func (e *Engine) handleDispatchReport(report dispatchReport) { // If the channel is full (unlikely — the buffer is sized to MaxInflightMsg), // the report is dropped and logged; this is acceptable because raft will retry // on the next tick and we only need eventual consistency between transport -// state and Progress state. +// state and Progress state. Successful MsgSnap reports are the exception; see +// postReliableDispatchReport. func (e *Engine) postDispatchReport(report dispatchReport) { select { case e.dispatchReportCh <- report: @@ -1881,6 +1882,20 @@ func (e *Engine) postDispatchReport(report dispatchReport) { } } +// postReliableDispatchReport delivers a dispatch outcome that must not be +// dropped. SnapshotFinish is not eventually consistent with ordinary +// unreachable reports: if it is lost, raft can keep the follower's Progress in +// StateSnapshot after the follower accepted the snapshot. +func (e *Engine) postReliableDispatchReport(report dispatchReport) { + if e.dispatchReportCh == nil { + return + } + select { + case e.dispatchReportCh <- report: + case <-e.closeCh: + } +} + func (e *Engine) handleProposal(req proposalRequest) { if err := contextErr(req.ctx); err != nil { req.done <- proposalResult{err: err} @@ -4523,7 +4538,7 @@ func (e *Engine) reportSuccessfulDispatch(msg raftpb.Message) { if msg.GetType() != raftpb.MsgSnap { return } - e.postDispatchReport(dispatchReport{ + e.postReliableDispatchReport(dispatchReport{ to: msg.GetTo(), msgType: msg.GetType(), snapshotFinish: true, diff --git a/kv/coordinator.go b/kv/coordinator.go index f1b8a5627..174c3b636 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -1145,12 +1145,13 @@ func (c *Coordinate) dispatchRaw(ctx context.Context, req []*Elem[OP]) (*Coordin // The returned Request is structurally identical to the pre-stamping // shape the leader's stampRawTimestamps already handles for Ts == 0 // (see adapter/internal.go). -func (c *Coordinate) toRawRequest(req *Elem[OP]) *pb.Request { +func (c *Coordinate) toRawRequest(req *Elem[OP], observedRouteVersion uint64) *pb.Request { switch req.Op { case Put: return &pb.Request{ - IsTxn: false, - Phase: pb.Phase_NONE, + IsTxn: false, + Phase: pb.Phase_NONE, + ObservedRouteVersion: observedRouteVersion, Mutations: []*pb.Mutation{ { Op: pb.Op_PUT, @@ -1162,8 +1163,9 @@ func (c *Coordinate) toRawRequest(req *Elem[OP]) *pb.Request { case Del: return &pb.Request{ - IsTxn: false, - Phase: pb.Phase_NONE, + IsTxn: false, + Phase: pb.Phase_NONE, + ObservedRouteVersion: observedRouteVersion, Mutations: []*pb.Mutation{ { Op: pb.Op_DEL, @@ -1174,8 +1176,9 @@ func (c *Coordinate) toRawRequest(req *Elem[OP]) *pb.Request { case DelPrefix: return &pb.Request{ - IsTxn: false, - Phase: pb.Phase_NONE, + IsTxn: false, + Phase: pb.Phase_NONE, + ObservedRouteVersion: observedRouteVersion, Mutations: []*pb.Mutation{ { Op: pb.Op_DEL_PREFIX, @@ -1238,7 +1241,7 @@ func (c *Coordinate) buildRedirectRequests(reqs *OperationGroup[OP]) ([]*pb.Requ if !reqs.IsTxn { requests := make([]*pb.Request, 0, len(reqs.Elems)) for _, req := range reqs.Elems { - requests = append(requests, c.toRawRequest(req)) + requests = append(requests, c.toRawRequest(req, reqs.ObservedRouteVersion)) } return requests, nil } diff --git a/kv/coordinator_dispatch_test.go b/kv/coordinator_dispatch_test.go index a559c6405..ae1ffdf30 100644 --- a/kv/coordinator_dispatch_test.go +++ b/kv/coordinator_dispatch_test.go @@ -213,7 +213,7 @@ func TestToRawRequestLeavesTsForLeaderStamping(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - r := c.toRawRequest(tc.req) + r := c.toRawRequest(tc.req, 0) require.NotNil(t, r) require.Equal(t, uint64(0), r.Ts, "forwarded raw requests must arrive with Ts==0 so the leader's stampRawTimestamps assigns the canonical ts (HLC leader-only invariant + HLC-4 (iii) fence)") @@ -221,6 +221,21 @@ func TestToRawRequestLeavesTsForLeaderStamping(t *testing.T) { } } +func TestBuildRedirectRequests_PreservesRawObservedRouteVersion(t *testing.T) { + t.Parallel() + + c := &Coordinate{} + got, err := c.buildRedirectRequests(&OperationGroup[OP]{ + ObservedRouteVersion: 17, + Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("k"), Value: []byte("v")}, + }, + }) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, uint64(17), got[0].GetObservedRouteVersion()) +} + // TestBuildRedirectRequestsSurvivesStaleFollowerCeiling exercises the // follower's redirect path end-to-end with an expired ceiling: it // confirms the follower hands off a Ts==0 request to the leader diff --git a/kv/fsm.go b/kv/fsm.go index 62d29647b..bc5da0b36 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -482,6 +482,9 @@ func (f *kvFSM) handleRequest(ctx context.Context, r *pb.Request, commitTS uint6 } func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS uint64) error { + if err := f.verifyWriteFence(r); err != nil { + return err + } // DEL_PREFIX mutations are handled by the store's DeletePrefixAt which // scans and writes tombstones locally. A DEL_PREFIX request must be the // sole mutation in a request (enforced by the coordinator's toRawRequest). @@ -720,6 +723,9 @@ func (f *kvFSM) IsVolatileOnlyPayload(payload []byte) bool { } func (f *kvFSM) handleTxnRequest(ctx context.Context, r *pb.Request, commitTS uint64) error { + if err := f.verifyWriteFence(r); err != nil { + return err + } if err := f.verifyComposed1(r); err != nil { return err } @@ -817,6 +823,82 @@ func (f *kvFSM) verifyComposed1(r *pb.Request) error { return f.verifyOwnerFromSnapshot(r.GetMutations(), currentSnap, currentSnap.Version(), "current") } +func (f *kvFSM) verifyWriteFence(r *pb.Request) error { + if requestBypassesWriteFence(r) { + return nil + } + observedVer := r.GetObservedRouteVersion() + if !f.writeFenceHistoryReady(observedVer) { + return nil + } + observedSnap, ok := f.routes.SnapshotAt(observedVer) + if !ok { + return errors.WithStack(ErrComposed1VersionGCd) + } + if err := verifyWriteFenceFromSnapshot(r.GetMutations(), observedSnap, observedVer, "observed"); err != nil { + return err + } + + currentSnap, ok := f.routes.Current() + if !ok || currentSnap.Version() == observedSnap.Version() { + return nil + } + return verifyWriteFenceFromSnapshot(r.GetMutations(), currentSnap, currentSnap.Version(), "current") +} + +func requestBypassesWriteFence(r *pb.Request) bool { + if !r.GetIsTxn() { + return false + } + switch r.GetPhase() { + case pb.Phase_COMMIT, pb.Phase_ABORT: + return true + case pb.Phase_NONE, pb.Phase_PREPARE: + return false + } + return false +} + +func (f *kvFSM) writeFenceHistoryReady(observedVer uint64) bool { + return f.routes != nil && f.shardGroupID != 0 && observedVer != 0 +} + +func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, snap RouteSnapshot, snapVer uint64, phase string) error { + for _, mut := range mutations { + if mut == nil { + continue + } + if isTxnInternalKey(mut.Key) { + continue + } + if mut.GetOp() == pb.Op_DEL_PREFIX { + start, end := routePrefixRange(mut.Key) + if snap.WriteFencedIntersects(start, end) { + return errors.Wrapf(ErrRouteWriteFenced, + "%s-version v=%d: prefix %q route range [%q,%q)", + phase, snapVer, mut.Key, start, end) + } + continue + } + if len(mut.Key) == 0 { + continue + } + rKey := routeKey(mut.Key) + if snap.WriteFencedForKey(rKey) { + return errors.Wrapf(ErrRouteWriteFenced, + "%s-version v=%d: key %q routeKey %q", + phase, snapVer, mut.Key, rKey) + } + start, end, ok := s3BucketAuxiliaryRouteRange(mut.Key) + if ok && snap.WriteFencedIntersects(start, end) { + return errors.Wrapf(ErrRouteWriteFenced, + "%s-version v=%d: key %q route range [%q,%q)", + phase, snapVer, mut.Key, start, end) + } + } + return nil +} + // verifyOwnerFromSnapshot is the shared per-mutation owner-check // loop used by verifyComposed1's observed-version and current- // version passes. `phase` is the diagnostic label ("observed" / diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 7279a1e83..959358513 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -53,6 +53,37 @@ func TestFSMAppliesRawPointWriteDespiteLocalWriteFencedRoute(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMRejectsObservedWriteFencedRawPointWrite(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + ObservedRouteVersion: 1, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + +func TestFSMRejectsCurrentWriteFenceAfterObservedActiveRawPointWrite(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, + }) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + ObservedRouteVersion: 1, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMAppliesS3BucketAuxiliaryPointWriteDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() @@ -75,6 +106,21 @@ func TestFSMAppliesS3BucketAuxiliaryPointWriteDespiteLocalWriteFencedRoute(t *te } } +func TestFSMRejectsObservedWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { + t.Parallel() + + const bucket = "bucket-a" + fsm := newS3BucketAuxiliaryWriteFencedFSM(t, bucket) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + ObservedRouteVersion: 1, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: s3keys.BucketGenerationKey(bucket), Value: []byte("v")}, + }, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMAppliesDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() @@ -90,6 +136,19 @@ func TestFSMAppliesDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { require.Error(t, getErr) } +func TestFSMRejectsObservedWriteFencedDelPrefix(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("z"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + ObservedRouteVersion: 1, + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMAppliesFullRangeDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { t.Parallel() @@ -149,3 +208,33 @@ func TestFSMAppliesPrepareAndAbortDespiteLocalWriteFencedRoute(t *testing.T) { err := fsm.handleTxnRequest(ctx, abort, 11) require.NotErrorIs(t, err, ErrRouteWriteFenced, "ABORT must keep the narrow cleanup lane open") } + +func TestFSMRejectsObservedWriteFencedPrepareButAllowsAbort(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFencedFSM(t) + prepare := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + ObservedRouteVersion: 1, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: 11, + ObservedRouteVersion: 1, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 11})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + require.NotErrorIs(t, fsm.handleTxnRequest(ctx, abort, 11), ErrRouteWriteFenced) +} diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index f70d5ecd8..e8bfb5e80 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -700,6 +700,7 @@ func (c *ShardedCoordinator) Dispatch(ctx context.Context, reqs *OperationGroup[ return nil, err } + c.maybeAutoPinRawObservedRouteVersion(reqs) if resp, handled, err := c.dispatchBeforeShardRouting(ctx, reqs); handled { return resp, err } @@ -746,7 +747,7 @@ func (c *ShardedCoordinator) dispatchBeforeShardRouting(ctx context.Context, req // span multiple shards (or be nil, meaning "all keys"). Broadcast the // operation to every shard group so each FSM scans locally. if hasDelPrefixElem(reqs.Elems) { - resp, err := c.dispatchDelPrefixBroadcast(ctx, reqs.IsTxn, reqs.Elems) + resp, err := c.dispatchDelPrefixBroadcast(ctx, reqs.IsTxn, reqs.Elems, reqs.ObservedRouteVersion) return resp, true, err } if err := c.rejectWriteFencedPointElems(reqs.Elems); err != nil { @@ -837,6 +838,16 @@ func (c *ShardedCoordinator) maybeAutoPinObservedRouteVersion(reqs *OperationGro reqs.ObservedRouteVersion = c.engine.Version() } +func (c *ShardedCoordinator) maybeAutoPinRawObservedRouteVersion(reqs *OperationGroup[OP]) { + if c.engine == nil || reqs == nil || reqs.IsTxn || reqs.ObservedRouteVersion != 0 { + return + } + if c.anyResolverClaimedKey(reqs.Elems) { + return + } + reqs.ObservedRouteVersion = c.engine.Version() +} + // anyResolverClaimedKey reports whether any element's key is // claimed by the partition resolver. Returns false when the // router has no resolver installed (the most common case) so the @@ -1021,7 +1032,7 @@ func validateDelPrefixOnly(elems []*Elem[OP]) error { // to every shard group. Each element becomes a separate pb.Request (the FSM's // extractDelPrefix processes only the first DEL_PREFIX mutation per request). // All requests are batched into a single Commit call per shard group. -func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isTxn bool, elems []*Elem[OP]) (*CoordinateResponse, error) { +func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isTxn bool, elems []*Elem[OP], observedRouteVersion uint64) (*CoordinateResponse, error) { if isTxn { return nil, errors.Wrap(ErrInvalidRequest, "DEL_PREFIX not supported in transactions") } @@ -1039,10 +1050,11 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT requests := make([]*pb.Request, 0, len(elems)) for _, elem := range elems { requests = append(requests, &pb.Request{ - IsTxn: false, - Phase: pb.Phase_NONE, - Ts: ts, - Mutations: []*pb.Mutation{elemToMutation(elem)}, + IsTxn: false, + Phase: pb.Phase_NONE, + Ts: ts, + Mutations: []*pb.Mutation{elemToMutation(elem)}, + ObservedRouteVersion: observedRouteVersion, }) } @@ -2013,10 +2025,11 @@ func (c *ShardedCoordinator) rawLogs(ctx context.Context, reqs *OperationGroup[O return nil, err } logs = append(logs, &pb.Request{ - IsTxn: false, - Phase: pb.Phase_NONE, - Ts: ts, - Mutations: grouped[gid], + IsTxn: false, + Phase: pb.Phase_NONE, + Ts: ts, + Mutations: grouped[gid], + ObservedRouteVersion: reqs.ObservedRouteVersion, }) } return logs, nil diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 1b8a31cb2..7b52fe105 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -122,6 +122,52 @@ func TestShardedCoordinator_DelPrefixBroadcastsToAllGroups(t *testing.T) { "same DEL_PREFIX element must use the same timestamp across shards") } +func TestShardedCoordinator_DelPrefixCarriesObservedRouteVersion(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: nil, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("user:")}}, + }) + require.NoError(t, err) + require.Len(t, txn.requests, 1) + require.Equal(t, uint64(7), txn.requests[0].GetObservedRouteVersion()) +} + +func TestShardedCoordinator_RawWriteCarriesObservedRouteVersion(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 9, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: nil, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: []byte("k"), Value: []byte("v")}}, + }) + require.NoError(t, err) + require.Len(t, txn.requests, 1) + require.Equal(t, uint64(9), txn.requests[0].GetObservedRouteVersion()) +} + func TestShardedCoordinatorRejectsPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 06f2b1d93..a630690d0 100644 --- a/main.go +++ b/main.go @@ -486,40 +486,6 @@ func run() error { cleanup.Add(leadershipRefusalDeregister) eg, runCtx := errgroup.WithContext(ctx) startRaftEngineLifecycleWatchers(runCtx, eg, runtimes) - // setupDistributionCatalog + the Stage 7a process-start registration - // gate are bundled so run() has a single startup-fault path: a - // registry-read / behind-epoch failure fails the process - // synchronously here, BEFORE the gRPC servers serve, so writes never - // run with no registration gate installed. - distCatalog, err := setupDistributionAndRegistration( - runCtx, eg, runtimes, cfg.engine, - coordinate, shardGroups[cfg.defaultGroup], encWiring, *raftId, *encryptionSidecarPath) - if err != nil { - cancel() - return err - } - // Seed AFTER setupDistributionCatalog so the sampler picks up the - // catalog-assigned RouteIDs. EnsureCatalogSnapshot inside - // setupDistributionCatalog applies a snapshot back into the engine - // with durable non-zero RouteIDs; seeding earlier would register - // the placeholder zero IDs from buildEngine and Observe would miss - // every dispatched mutation. - seedKeyVizRoutes(sampler, cfg.engine) - - eg.Go(func() error { - return runDistributionCatalogWatcher(runCtx, distCatalog, cfg.engine) - }) - startKeyVizFlusher(runCtx, eg, sampler) - startKeyVizLeaderTermPublisher(runCtx, eg, sampler, runtimes) - startMemoryWatchdog(runCtx, eg, cancel) - distServer := adapter.NewDistributionServer( - cfg.engine, - distCatalog, - adapter.WithDistributionCoordinator(coordinate), - adapter.WithDistributionActiveTimestampTracker(readTracker), - ) - startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) - startFSMCompactorIfEnabled(runCtx, eg, runtimes, readTracker) // Stage 7c §3.1: build the encryption-aware // MembershipChangeInterceptor here where the concrete @@ -543,16 +509,41 @@ func run() error { slog.Default(), ) cleanup.Add(rotateOnStartupDeregister) - if err := startServersAfterStartupRotation(waitRotateOnStartup, serversInput{ + + serverInput := serversInput{ ctx: runCtx, eg: eg, cancel: cancel, lc: &lc, runtimes: runtimes, shardGroups: shardGroups, bootstrapServers: bootstrapCfg.adminSeed(cfg.defaultGroup), shardStore: shardStore, coordinate: coordinate, - distServer: distServer, readTracker: readTracker, + readTracker: readTracker, metricsRegistry: metricsRegistry, cfg: cfg, redisApplyObserver: redisApplyObserver, encWiring: encWiring, keyvizSampler: sampler, encryptionConfChangeInterceptor: encryptionConfChangeInterceptor, + } + runtimeStartup, err := prepareDistributionRuntimeServer( + waitRotateOnStartup, serverInput, cfg.engine, runtimes, coordinate, readTracker) + if err != nil { + cancel() + return err + } + if err := startDistributionRuntimeAfterTransport(distributionRuntimeStartupInput{ + runCtx: runCtx, + eg: eg, + cancel: cancel, + runtimes: runtimes, + engine: cfg.engine, + coordinate: coordinate, + defaultGroup: shardGroups[cfg.defaultGroup], + encWiring: encWiring, + raftID: *raftId, + sidecarPath: *encryptionSidecarPath, + sampler: sampler, + clock: clock, + metricsRegistry: metricsRegistry, + readTracker: readTracker, + waitRotateOnStartup: waitRotateOnStartup, + prepared: runtimeStartup, }); err != nil { return err } @@ -1441,14 +1432,103 @@ type serversInput struct { encryptionConfChangeInterceptor internalraftadmin.MembershipChangeInterceptor } -// startServersAfterStartupRotation wires up the AdminServer, starts the -// per-group Raft listeners needed for quorum traffic, waits for any requested -// startup rotation, then opens the public service listeners. -func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, in serversInput) error { - adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler) +type preparedRuntimeServer struct { + runner *runtimeServerRunner + connCache *kv.GRPCConnCache +} + +type preparedDistributionRuntime struct { + catalog *distribution.CatalogStore + serverInput serversInput + preparedServer *preparedRuntimeServer +} + +type distributionRuntimeStartupInput struct { + runCtx context.Context + eg *errgroup.Group + cancel context.CancelFunc + runtimes []*raftGroupRuntime + engine *distribution.Engine + coordinate *kv.ShardedCoordinator + defaultGroup *kv.ShardGroup + encWiring encryptionWriteWiring + raftID string + sidecarPath string + sampler *keyviz.MemSampler + clock *kv.HLC + metricsRegistry *monitoring.Registry + readTracker *kv.ActiveTimestampTracker + waitRotateOnStartup startupRotationWaiter + prepared *preparedDistributionRuntime +} + +func prepareDistributionRuntimeServer( + waitRotateOnStartup startupRotationWaiter, + in serversInput, + engine *distribution.Engine, + runtimes []*raftGroupRuntime, + coordinate *kv.ShardedCoordinator, + readTracker *kv.ActiveTimestampTracker, +) (*preparedDistributionRuntime, error) { + distCatalog, err := distributionCatalogStoreForEngine(runtimes, engine) + if err != nil { + return nil, err + } + in.distServer = adapter.NewDistributionServer( + engine, + distCatalog, + adapter.WithDistributionCoordinator(coordinate), + adapter.WithDistributionActiveTimestampTracker(readTracker), + ) + preparedServer, err := prepareRuntimeServerRunner(waitRotateOnStartup, in) if err != nil { + return nil, err + } + return &preparedDistributionRuntime{ + catalog: distCatalog, + serverInput: in, + preparedServer: preparedServer, + }, nil +} + +func startDistributionRuntimeAfterTransport(in distributionRuntimeStartupInput) error { + prepared := in.prepared + if prepared == nil || prepared.preparedServer == nil || prepared.preparedServer.runner == nil { + return errors.New("distribution runtime server is not prepared") + } + if err := prepared.preparedServer.runner.startRaftTransport(); err != nil { return err } + if err := waitForRaftStartupAfterTransport(in.runCtx, in.runtimes); err != nil { + return prepared.preparedServer.runner.startupFailure(err) + } + // setupDistributionCatalog + the Stage 7a process-start registration + // gate are bundled so startup faults flow through one path. This now + // runs after raft transport is bound, but before public listeners serve. + if err := setupDistributionAndRegistration( + in.runCtx, in.eg, prepared.catalog, in.engine, + in.coordinate, in.defaultGroup, in.encWiring, in.raftID, in.sidecarPath); err != nil { + in.cancel() + return err + } + seedKeyVizRoutes(in.sampler, in.engine) + in.eg.Go(func() error { + return runDistributionCatalogWatcher(in.runCtx, prepared.catalog, in.engine) + }) + startKeyVizFlusher(in.runCtx, in.eg, in.sampler) + startKeyVizLeaderTermPublisher(in.runCtx, in.eg, in.sampler, in.runtimes) + startMemoryWatchdog(in.runCtx, in.eg, in.cancel) + startMonitoringCollectors(in.runCtx, in.metricsRegistry, in.runtimes, in.clock) + startFSMCompactorIfEnabled(in.runCtx, in.eg, in.runtimes, in.readTracker) + return startPreparedServerAfterStartupRotation( + in.waitRotateOnStartup, prepared.serverInput, prepared.preparedServer) +} + +func prepareRuntimeServerRunner(waitRotateOnStartup startupRotationWaiter, in serversInput) (*preparedRuntimeServer, error) { + adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler) + if err != nil { + return nil, err + } // roleStore + connCache are gated on *adminEnabled. With admin // disabled, building either is wasted work AND a security // regression risk: a non-empty -adminFullAccessKeys flag would @@ -1540,12 +1620,14 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, encryptionConfChangeInterceptor: in.encryptionConfChangeInterceptor, publicKVGate: publicKVGate, } - if err := runner.startRaftTransport(); err != nil { - return err - } - if err := waitForRaftStartupAfterTransport(in.ctx, in.runtimes); err != nil { - return runner.startupFailure(err) + return &preparedRuntimeServer{runner: &runner, connCache: connCache}, nil +} + +func startPreparedServerAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, in serversInput, prepared *preparedRuntimeServer) error { + if prepared == nil || prepared.runner == nil { + return errors.New("runtime server runner is not prepared") } + runner := prepared.runner if err := runner.preparePublicServices(); err != nil { return runner.startupFailure(err) } @@ -1565,17 +1647,19 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, Nodes: parseCSV(*keyvizFanoutNodes), Timeout: *keyvizFanoutTimeout, } - adminHTTP, err := prepareAdminFromFlags(in.ctx, in.lc, in.runtimes, runner.dynamoServer, runner.s3Server, runner.sqsServer, runner.coordinate, connCache, in.keyvizSampler, fanoutCfg) + adminHTTP, err := prepareAdminFromFlags(in.ctx, in.lc, in.runtimes, runner.dynamoServer, runner.s3Server, runner.sqsServer, runner.coordinate, prepared.connCache, in.keyvizSampler, fanoutCfg) if err != nil { return runner.startupFailure(err) } runner.adminHTTP = adminHTTP - publicKVGate.blockMutator = waitRotateOnStartup.BlockMutators + if runner.publicKVGate != nil { + runner.publicKVGate.blockMutator = waitRotateOnStartup.BlockMutators + } if err := waitRotateOnStartup.Wait(in.ctx); err != nil { return runner.startupFailure(errors.Wrap(err, "encryption rotate-on-startup: wait before serving")) } startHLCLeaseRenewal(in.ctx, in.eg, in.coordinate) - publicKVGate.markReady() + runner.publicKVGate.markReady() if err := runner.startPublicServices(); err != nil { return err } @@ -2405,8 +2489,7 @@ func distributionCatalogStoreForGroup(runtimes []*raftGroupRuntime, groupID uint return nil } -func setupDistributionCatalog( - ctx context.Context, +func distributionCatalogStoreForEngine( runtimes []*raftGroupRuntime, engine *distribution.Engine, ) (*distribution.CatalogStore, error) { @@ -2418,25 +2501,20 @@ func setupDistributionCatalog( if distCatalog == nil { return nil, errors.WithStack(errors.Newf("distribution catalog store is not available for group %d", catalogGroupID)) } - // EnsureCatalogSnapshot may Save through the direct (non-raft) write - // path. When the §7.1 storage envelope is active and this load's - // writer registration has not yet committed, that Save fails closed - // with store.ErrWriterNotRegistered (Stage 7a-2). retryUntilRegistered - // retries the bootstrap until the registration goroutine — armed - // before this call in setupDistributionAndRegistration — commits and - // the gate clears. The common cases (populated catalog → no-op Save, - // or pre-cutover → cleartext Save) never hit the gate and return on - // the first attempt. - // - // Idempotency requirement: the retry re-invokes EnsureCatalogSnapshot - // from scratch on each ErrWriterNotRegistered, so it MUST be - // re-entrant — on the populated-catalog path it is a version-unchanged - // no-op Save (no mutation, no nonce), so re-running it is safe. - if err := retryUntilRegistered(ctx, "distribution catalog bootstrap", func() error { - _, e := distribution.EnsureCatalogSnapshot(ctx, distCatalog, engine) - return errors.Wrap(e, "ensure catalog snapshot") - }); err != nil { - return nil, errors.Wrapf(err, "initialize distribution catalog") + return distCatalog, nil +} + +func setupDistributionCatalog( + ctx context.Context, + runtimes []*raftGroupRuntime, + engine *distribution.Engine, +) (*distribution.CatalogStore, error) { + distCatalog, err := distributionCatalogStoreForEngine(runtimes, engine) + if err != nil { + return nil, err + } + if err := ensureDistributionCatalogSnapshot(ctx, distCatalog, engine); err != nil { + return nil, err } return distCatalog, nil } diff --git a/main_encryption_registration.go b/main_encryption_registration.go index 7cdb53f9e..eb5aa68de 100644 --- a/main_encryption_registration.go +++ b/main_encryption_registration.go @@ -126,19 +126,19 @@ func retryUntilRegistered(ctx context.Context, what string, fn func() error) err func setupDistributionAndRegistration( runCtx context.Context, eg *errgroup.Group, - runtimes []*raftGroupRuntime, + distCatalog *distribution.CatalogStore, engine *distribution.Engine, coordinate *kv.ShardedCoordinator, defaultGroup *kv.ShardGroup, w encryptionWriteWiring, raftID string, sidecarPath string, -) (*distribution.CatalogStore, error) { +) error { if err := validateRaftRegistrationStartupEpoch(defaultGroup, w, raftID, sidecarPath); err != nil { - return nil, err + return err } if err := installProcessStartRegistrationGate(runCtx, eg, coordinate, defaultGroup, w, raftID); err != nil { - return nil, err + return err } installRaftRegistrationVerifier(defaultGroup, w, raftID) installRuntimeRaftRegistrationWatcher(runCtx, eg, coordinate, defaultGroup, w, raftID, sidecarPath) @@ -153,11 +153,38 @@ func setupDistributionAndRegistration( installRuntimeRegistrationWatcher(runCtx, eg, coordinate, defaultGroup, w, raftID) // Bootstrap + registration both run under runCtx so a shutdown // cancels the bounded retry rather than hanging. - distCatalog, err := setupDistributionCatalog(runCtx, runtimes, engine) - if err != nil { - return nil, err + return ensureDistributionCatalogSnapshot(runCtx, distCatalog, engine) +} + +func ensureDistributionCatalogSnapshot( + ctx context.Context, + distCatalog *distribution.CatalogStore, + engine *distribution.Engine, +) error { + if distCatalog == nil { + return errors.New("distribution catalog store is not available") + } + // EnsureCatalogSnapshot may Save through the direct (non-raft) write + // path. When the §7.1 storage envelope is active and this load's + // writer registration has not yet committed, that Save fails closed + // with store.ErrWriterNotRegistered (Stage 7a-2). retryUntilRegistered + // retries the bootstrap until the registration goroutine — armed + // before this call in setupDistributionAndRegistration — commits and + // the gate clears. The common cases (populated catalog → no-op Save, + // or pre-cutover → cleartext Save) never hit the gate and return on + // the first attempt. + // + // Idempotency requirement: the retry re-invokes EnsureCatalogSnapshot + // from scratch on each ErrWriterNotRegistered, so it MUST be + // re-entrant — on the populated-catalog path it is a version-unchanged + // no-op Save (no mutation, no nonce), so re-running it is safe. + if err := retryUntilRegistered(ctx, "distribution catalog bootstrap", func() error { + _, e := distribution.EnsureCatalogSnapshot(ctx, distCatalog, engine) + return errors.Wrap(e, "ensure catalog snapshot") + }); err != nil { + return errors.Wrapf(err, "initialize distribution catalog") } - return distCatalog, nil + return nil } func installRaftRegistrationVerifier(defaultGroup *kv.ShardGroup, w encryptionWriteWiring, raftID string) { From 61feed4ce354950d0eee65e3758906a271016ff9 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 05:44:57 +0900 Subject: [PATCH 092/162] Stabilize route-fence cleanup retries --- adapter/dynamodb_cleanup_retry_test.go | 42 ++++++++++++++++++ adapter/dynamodb_item_write.go | 13 +++--- adapter/dynamodb_onephase_dedup_test.go | 18 ++++++++ adapter/dynamodb_schema.go | 24 ++++++++++- adapter/dynamodb_transact.go | 4 ++ adapter/retryable_write_fence_test.go | 1 + adapter/s3.go | 11 ++++- adapter/s3_cleanup_retry_test.go | 57 +++++++++++++++++++++++++ internal/raftengine/etcd/engine.go | 4 ++ internal/raftengine/etcd/engine_test.go | 37 ++++++++++++++-- 10 files changed, 198 insertions(+), 13 deletions(-) create mode 100644 adapter/dynamodb_cleanup_retry_test.go create mode 100644 adapter/s3_cleanup_retry_test.go diff --git a/adapter/dynamodb_cleanup_retry_test.go b/adapter/dynamodb_cleanup_retry_test.go new file mode 100644 index 000000000..71b535e85 --- /dev/null +++ b/adapter/dynamodb_cleanup_retry_test.go @@ -0,0 +1,42 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestDynamoDBCleanupDeletedTableGeneration_RetriesRouteFence(t *testing.T) { + t.Parallel() + + st := store.NewMVCCStore() + coord := &cleanupRouteFenceCoordinator{ + localAdapterCoordinator: newLocalAdapterCoordinator(st), + failuresRemaining: 1, + } + server := NewDynamoDBServer(nil, st, coord) + + err := server.cleanupDeletedTableGeneration(context.Background(), "table-a", 7) + require.NoError(t, err) + require.Equal(t, 3, coord.calls, "first prefix is retried once, second prefix succeeds once") +} + +type cleanupRouteFenceCoordinator struct { + *localAdapterCoordinator + failuresRemaining int + calls int +} + +func (c *cleanupRouteFenceCoordinator) Dispatch(ctx context.Context, req *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { + if req != nil && operationGroupHasDelPrefix(req.Elems) { + c.calls++ + if c.failuresRemaining > 0 { + c.failuresRemaining-- + return nil, kv.ErrRouteWriteFenced + } + } + return c.localAdapterCoordinator.Dispatch(ctx, req) +} diff --git a/adapter/dynamodb_item_write.go b/adapter/dynamodb_item_write.go index a5aecda19..b3ad5678f 100644 --- a/adapter/dynamodb_item_write.go +++ b/adapter/dynamodb_item_write.go @@ -285,7 +285,7 @@ func (d *DynamoDBServer) itemWriteFirstAttempt( plan.req.CommitTS = commitTS if dispErr := d.commitItemWrite(ctx, plan.req); dispErr != nil { // dispErr is already wrapped by commitItemWrite; return it raw. - if isRetryableTransactWriteError(dispErr) { + if shouldPreserveTransactWriteAttempt(dispErr) { return nil, &reusableItemWrite{ plan: plan, commitTS: commitTS, @@ -319,10 +319,13 @@ func (d *DynamoDBServer) itemWriteReuseAttempt( return d.resolveReuseWriteConflict(ctx, tableName, pending, commitTS, dispErr) } if isRetryableTransactWriteError(dispErr) { - // Still ambiguous (e.g. TxnLocked): this reuse may itself have landed, - // so the next retry must probe THIS commit_ts. dispErr is already - // wrapped by commitItemWrite; return it raw. - pending.commitTS = commitTS + // Still ambiguous (e.g. TxnLocked): this reuse may itself have + // landed, so the next retry must probe THIS commit_ts. Route-fence + // rejections are retryable but happen before this write set can apply, + // so keep the older witness. + if shouldPreserveTransactWriteAttempt(dispErr) { + pending.commitTS = commitTS + } return nil, pending, dispErr } return nil, nil, dispErr diff --git a/adapter/dynamodb_onephase_dedup_test.go b/adapter/dynamodb_onephase_dedup_test.go index ce7b9c7d4..b28afe3bd 100644 --- a/adapter/dynamodb_onephase_dedup_test.go +++ b/adapter/dynamodb_onephase_dedup_test.go @@ -113,6 +113,24 @@ func TestItemWriteDedup_LandedPriorAttempt_NoDuplicate(t *testing.T) { require.Equal(t, 1, coord.probeNoOps, "the reuse must dedup via the FSM exact-ts probe") } +func TestItemWriteDedup_RouteFenceRetryPreservesPriorProbe(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + coord := newDedupTestCoordinator(st, 1, true) // dispatch 1 lands then errors + coord.routeFenceAtDispatch = 2 + schema, server := newDedupItemWriteServer(st, coord, true) + seedDedupItem(t, st, schema, "1", "2") + + plan, err := server.updateItemWithRetry(ctx, appendListInput()) + require.NoError(t, err) + require.NotNil(t, plan) + + require.Equal(t, []string{"1", "2", "3"}, readListValues(t, server, schema)) + require.Equal(t, 3, coord.dispatches, "attempt 1 landed, route-fenced reuse, then dedup probe retry") + require.Equal(t, 1, coord.probeNoOps, "route-fenced reuse must not replace the prior landed probe") +} + // TestItemWriteDedup_PriorAttemptDidNotLand_Applies: attempt 1 pre-rejects // (definitely did not commit); the reuse's probe misses, so it applies the // reused write set at a fresh commit_ts. One element, no duplicate. diff --git a/adapter/dynamodb_schema.go b/adapter/dynamodb_schema.go index 0ffa0f917..b3843b8d5 100644 --- a/adapter/dynamodb_schema.go +++ b/adapter/dynamodb_schema.go @@ -349,16 +349,36 @@ func (d *DynamoDBServer) cleanupDeletedTableGeneration(ctx context.Context, tabl // scans and writes tombstones locally, avoiding the enumerate-then-batch- // delete loop that previously required many Raft proposals. for _, prefix := range prefixes { + if err := d.dispatchDeletedTableCleanupPrefix(ctx, prefix); err != nil { + return err + } + } + return nil +} + +func (d *DynamoDBServer) dispatchDeletedTableCleanupPrefix(ctx context.Context, prefix []byte) error { + backoff := transactRetryInitialBackoff + deadline := time.Now().Add(transactRetryMaxDuration) + var lastErr error + for range transactRetryMaxAttempts { _, err := d.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ Elems: []*kv.Elem[kv.OP]{ {Op: kv.DelPrefix, Key: prefix}, }, }) - if err != nil { + if err == nil { + return nil + } + if !isRetryableTransactWriteError(err) { return errors.WithStack(err) } + lastErr = err + if waitErr := waitRetryWithDeadline(ctx, deadline, backoff); waitErr != nil { + return errors.Wrap(errors.Join(waitErr, lastErr), "dynamodb delete table cleanup retry canceled") + } + backoff = nextTransactRetryBackoff(backoff) } - return nil + return errors.WithStack(lastErr) } func (d *DynamoDBServer) dispatchDeleteBatch(ctx context.Context, keys [][]byte) error { diff --git a/adapter/dynamodb_transact.go b/adapter/dynamodb_transact.go index 7ccaaac59..9569b6053 100644 --- a/adapter/dynamodb_transact.go +++ b/adapter/dynamodb_transact.go @@ -1108,6 +1108,10 @@ func isRetryableTransactWriteError(err error) bool { return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || errors.Is(err, kv.ErrRouteWriteFenced) } +func shouldPreserveTransactWriteAttempt(err error) bool { + return isRetryableTransactWriteError(err) && !errors.Is(err, kv.ErrRouteWriteFenced) +} + func waitTransactRetryBackoff(ctx context.Context, delay time.Duration) error { timer := time.NewTimer(delay) defer timer.Stop() diff --git a/adapter/retryable_write_fence_test.go b/adapter/retryable_write_fence_test.go index fb9f9a5e0..e5b436c06 100644 --- a/adapter/retryable_write_fence_test.go +++ b/adapter/retryable_write_fence_test.go @@ -13,4 +13,5 @@ func TestWriteFenceErrorsAreAdapterRetryable(t *testing.T) { require.True(t, isRetryableRedisTxnErr(kv.ErrRouteWriteFenced)) require.True(t, isRetryableS3MutationErr(kv.ErrRouteWriteFenced)) require.True(t, isRetryableTransactWriteError(kv.ErrRouteWriteFenced)) + require.False(t, shouldPreserveTransactWriteAttempt(kv.ErrRouteWriteFenced)) } diff --git a/adapter/s3.go b/adapter/s3.go index d08f0e4d6..75cc2c074 100644 --- a/adapter/s3.go +++ b/adapter/s3.go @@ -1455,7 +1455,7 @@ func (s *S3Server) cleanupPartBlobsAsync(bucket string, generation uint64, objec if len(pending) == 0 { return } - if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{Elems: pending}); err != nil { + if err := s.dispatchS3CleanupBatch(ctx, pending); err != nil { slog.ErrorContext(ctx, "cleanupPartBlobsAsync: coordinator dispatch failed", "bucket", bucket, "object_key", objectKey, @@ -1519,7 +1519,7 @@ func (s *S3Server) deleteByPrefix(ctx context.Context, prefix []byte, bucket str for _, kvp := range kvs { pending = append(pending, &kv.Elem[kv.OP]{Op: kv.Del, Key: kvp.Key}) } - if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{Elems: pending}); err != nil { + if err := s.dispatchS3CleanupBatch(ctx, pending); err != nil { slog.ErrorContext(ctx, "deleteByPrefix: dispatch failed", "bucket", bucket, "generation", generation, "object_key", objectKey, "upload_id", uploadID, "err", err) @@ -1529,6 +1529,13 @@ func (s *S3Server) deleteByPrefix(ctx context.Context, prefix []byte, bucket str } } +func (s *S3Server) dispatchS3CleanupBatch(ctx context.Context, elems []*kv.Elem[kv.OP]) error { + return s.retryS3Mutation(ctx, func() error { + _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{Elems: elems}) + return errors.WithStack(err) + }) +} + func parseS3MaxParts(raw string) int { if strings.TrimSpace(raw) == "" { return s3ListPartsMaxParts diff --git a/adapter/s3_cleanup_retry_test.go b/adapter/s3_cleanup_retry_test.go new file mode 100644 index 000000000..c7591a90b --- /dev/null +++ b/adapter/s3_cleanup_retry_test.go @@ -0,0 +1,57 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/internal/s3keys" + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestS3DeleteByPrefix_RetriesRouteFence(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + coord := &s3CleanupRouteFenceCoordinator{ + localAdapterCoordinator: newLocalAdapterCoordinator(st), + failuresRemaining: 1, + } + server := NewS3Server(nil, "", st, coord, nil) + key := s3keys.BlobKey("bucket-a", 3, "object-a", "upload-a", 1, 0) + require.NoError(t, st.PutAt(ctx, key, []byte("blob"), 1, 0)) + + server.deleteByPrefix(ctx, s3keys.BlobPrefixForUpload("bucket-a", 3, "object-a", "upload-a"), "bucket-a", 3, "object-a", "upload-a") + + require.Equal(t, 2, coord.calls, "route-fenced cleanup batch should retry") + _, err := st.GetAt(ctx, key, snapshotTS(coord.Clock(), st)) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +type s3CleanupRouteFenceCoordinator struct { + *localAdapterCoordinator + failuresRemaining int + calls int +} + +func (c *s3CleanupRouteFenceCoordinator) Dispatch(ctx context.Context, req *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { + if req != nil && operationGroupHasDel(req.Elems) { + c.calls++ + if c.failuresRemaining > 0 { + c.failuresRemaining-- + return nil, kv.ErrRouteWriteFenced + } + } + return c.localAdapterCoordinator.Dispatch(ctx, req) +} + +func operationGroupHasDel(elems []*kv.Elem[kv.OP]) bool { + for _, elem := range elems { + if elem != nil && elem.Op == kv.Del { + return true + } + } + return false +} diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 0326b66d1..4df7ed243 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -2422,6 +2422,10 @@ func coalesceHeartbeatResp(ch chan dispatchRequest, msg raftpb.Message) bool { restoreDispatchRequest(ch, stale) return false } + if len(stale.msg.GetContext()) != 0 { + restoreDispatchRequest(ch, stale) + return false + } closeDispatchRequest(stale) return tryEnqueueDispatchRequest(ch, prepareDispatchRequest(msg)) } diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 8a1f82b4a..c4ecb1b73 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1393,7 +1393,7 @@ func TestPrepareDispatchRequestClonesSnapshotPayload(t *testing.T) { require.Equal(t, []uint64{1, 2}, req.msg.Snapshot.GetMetadata().GetConfState().GetVoters()) } -func TestEnqueueDispatchMessageCoalescesHeartbeatResponses(t *testing.T) { +func TestEnqueueDispatchMessageCoalescesPlainHeartbeatResponses(t *testing.T) { t.Parallel() pd := &peerQueues{ heartbeat: make(chan dispatchRequest, 1), @@ -1406,9 +1406,8 @@ func TestEnqueueDispatchMessageCoalescesHeartbeatResponses(t *testing.T) { }, } pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ - Type: messageTypePtr(raftpb.MsgHeartbeatResp), - To: uint64Ptr(2), - Context: []byte("old"), + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), }) require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{ @@ -1424,6 +1423,36 @@ func TestEnqueueDispatchMessageCoalescesHeartbeatResponses(t *testing.T) { require.Equal(t, []byte("new"), req.msg.Context) } +func TestEnqueueDispatchMessagePreservesReadIndexHeartbeatResponses(t *testing.T) { + t.Parallel() + pd := &peerQueues{ + heartbeat: make(chan dispatchRequest, 1), + heartbeatResp: make(chan dispatchRequest, 1), + } + engine := &Engine{ + nodeID: 1, + peerDispatchers: map[uint64]*peerQueues{ + 2: pd, + }, + } + pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Context: []byte("read-index"), + }) + + require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + })) + + require.Equal(t, uint64(1), engine.DispatchDropCount()) + require.Len(t, pd.heartbeatResp, 1) + req := <-pd.heartbeatResp + require.Equal(t, raftpb.MsgHeartbeatResp, req.msg.GetType()) + require.Equal(t, []byte("read-index"), req.msg.Context) +} + func TestMaxAppliedIndexStartsFromSnapshotIndex(t *testing.T) { storage := etcdraft.NewMemoryStorage() snap := raftTestSnapshot(5, 2, []uint64{1}, nil) From f373cd356880f53a1d4622e840ec05eab88efdd9 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 06:17:06 +0900 Subject: [PATCH 093/162] Keep heartbeat responses coalescing under read-index load --- internal/raftengine/etcd/engine.go | 40 +++++++++++----------- internal/raftengine/etcd/engine_test.go | 44 +++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 4df7ed243..f33257500 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -2414,20 +2414,30 @@ func coalesceHeartbeatResp(ch chan dispatchRequest, msg raftpb.Message) bool { if ch == nil || cap(ch) == 0 { return false } - stale, ok := tryReceiveDispatchRequest(ch) - if !ok { + n := len(ch) + if n == 0 { return false } - if stale.msg.GetType() != raftpb.MsgHeartbeatResp { - restoreDispatchRequest(ch, stale) - return false + + drained := make([]dispatchRequest, 0, n) + replaced := false + for i := 0; i < n; i++ { + req, ok := tryReceiveDispatchRequest(ch) + if !ok { + break + } + if !replaced && req.msg.GetType() == raftpb.MsgHeartbeatResp && len(req.msg.GetContext()) == 0 { + closeDispatchRequest(req) + drained = append(drained, prepareDispatchRequest(msg)) + replaced = true + continue + } + drained = append(drained, req) } - if len(stale.msg.GetContext()) != 0 { - restoreDispatchRequest(ch, stale) - return false + for _, req := range drained { + restoreDispatchRequest(ch, req) } - closeDispatchRequest(stale) - return tryEnqueueDispatchRequest(ch, prepareDispatchRequest(msg)) + return replaced } func tryReceiveDispatchRequest(ch chan dispatchRequest) (dispatchRequest, bool) { @@ -2447,16 +2457,6 @@ func restoreDispatchRequest(ch chan dispatchRequest, req dispatchRequest) { } } -func tryEnqueueDispatchRequest(ch chan dispatchRequest, req dispatchRequest) bool { - select { - case ch <- req: - return true - default: - closeDispatchRequest(req) - return false - } -} - func closeDispatchRequest(req dispatchRequest) { if err := req.Close(); err != nil { slog.Error("etcd raft dispatch: failed to close request", "err", err) diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index c4ecb1b73..87e8447df 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1453,6 +1453,50 @@ func TestEnqueueDispatchMessagePreservesReadIndexHeartbeatResponses(t *testing.T require.Equal(t, []byte("read-index"), req.msg.Context) } +func TestEnqueueDispatchMessageCoalescesPlainHeartbeatBehindReadIndex(t *testing.T) { + t.Parallel() + pd := &peerQueues{ + heartbeat: make(chan dispatchRequest, 1), + heartbeatResp: make(chan dispatchRequest, 3), + } + engine := &Engine{ + nodeID: 1, + peerDispatchers: map[uint64]*peerQueues{ + 2: pd, + }, + } + pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Context: []byte("read-index"), + }) + pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Index: uint64Ptr(10), + }) + pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Index: uint64Ptr(11), + }) + + require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Index: uint64Ptr(99), + })) + + require.Zero(t, engine.DispatchDropCount()) + require.Len(t, pd.heartbeatResp, 3) + req := <-pd.heartbeatResp + require.Equal(t, []byte("read-index"), req.msg.Context) + req = <-pd.heartbeatResp + require.Equal(t, uint64(99), req.msg.GetIndex()) + req = <-pd.heartbeatResp + require.Equal(t, uint64(11), req.msg.GetIndex()) +} + func TestMaxAppliedIndexStartsFromSnapshotIndex(t *testing.T) { storage := etcdraft.NewMemoryStorage() snap := raftTestSnapshot(5, 2, []uint64{1}, nil) From 8f821fe891294eae2c6b136ea21e6e652c2d5929 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 06:58:15 +0900 Subject: [PATCH 094/162] Stabilize raft snapshot recovery checkpoints --- .../raftengine/etcd/dispatch_report_test.go | 6 +- internal/raftengine/etcd/engine.go | 146 +++++++++++++----- .../etcd/engine_applied_index_test.go | 52 ++++++- internal/raftengine/etcd/engine_test.go | 94 ++++++++++- internal/raftengine/statemachine.go | 25 +-- 5 files changed, 264 insertions(+), 59 deletions(-) diff --git a/internal/raftengine/etcd/dispatch_report_test.go b/internal/raftengine/etcd/dispatch_report_test.go index 6b122952b..63c7fb631 100644 --- a/internal/raftengine/etcd/dispatch_report_test.go +++ b/internal/raftengine/etcd/dispatch_report_test.go @@ -176,7 +176,7 @@ func TestPostDispatchReport_AbortsOnClose(t *testing.T) { } } -func TestEnqueueDispatchReportsDroppedSnapshotWhenLaneFull(t *testing.T) { +func TestEnqueueDispatchDefersDroppedSnapshotReportWhenLaneFull(t *testing.T) { t.Parallel() snapshotCh := make(chan dispatchRequest, 1) snapshotCh <- dispatchRequest{msg: raftpb.Message{Type: messageTypePtr(raftpb.MsgSnap), To: uint64Ptr(2)}} @@ -195,10 +195,10 @@ func TestEnqueueDispatchReportsDroppedSnapshotWhenLaneFull(t *testing.T) { require.Equal(t, uint64(1), e.DispatchDropCount()) select { case got := <-e.dispatchReportCh: - require.Equal(t, dispatchReport{to: 2, msgType: raftpb.MsgSnap}, got) + t.Fatalf("dropped MsgSnap should defer without queueing: %+v", got) default: - t.Fatal("expected dropped MsgSnap to report SnapshotFailure input") } + require.Equal(t, []dispatchReport{{to: 2, msgType: raftpb.MsgSnap}}, e.deferredReadyDispatchReports) } func TestEnqueueDispatchReportsDroppedRegularMessageWhenLaneFull(t *testing.T) { diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index f33257500..b87a00d13 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -326,15 +326,16 @@ type Engine struct { nextRequestID atomic.Uint64 - proposeCh chan proposalRequest - readCh chan readRequest - adminCh chan adminRequest - stepCh chan raftpb.Message - priorityStepCh chan raftpb.Message - priorityStepBurst int - dispatchReportCh chan dispatchReport - peerDispatchers map[uint64]*peerQueues - perPeerQueueSize int + proposeCh chan proposalRequest + readCh chan readRequest + adminCh chan adminRequest + stepCh chan raftpb.Message + priorityStepCh chan raftpb.Message + priorityStepBurst int + dispatchReportCh chan dispatchReport + deferredReadyDispatchReports []dispatchReport + peerDispatchers map[uint64]*peerQueues + perPeerQueueSize int // dispatcherLanesEnabled toggles the opt-in multi-lane dispatcher layout. Captured // once at Open from ELASTICKV_RAFT_DISPATCHER_LANES so the run-time code // path is branch-free per message and does not need to re-read env vars. @@ -927,10 +928,10 @@ func waitForOpen(ctx context.Context, engine *Engine, waitForLeader bool) (*Engi if !waitForLeader { return engine, nil } - if err := waitForEngineSignal(ctx, engine, engine.startedCh); err != nil { + if err := waitForOpenSignal(ctx, engine, engine.startedCh); err != nil { return nil, err } - if err := waitForEngineSignal(ctx, engine, engine.leaderReady); err != nil { + if err := waitForOpenSignal(ctx, engine, engine.leaderReady); err != nil { return nil, err } return engine, nil @@ -940,13 +941,19 @@ func (e *Engine) WaitStarted(ctx context.Context) error { if e == nil { return errors.WithStack(errNilEngine) } - return waitForEngineSignal(ctx, e, e.startedCh) + return waitForEngineSignal(ctx, e, e.startedCh, false) } -func waitForEngineSignal(ctx context.Context, engine *Engine, ready <-chan struct{}) error { +func waitForOpenSignal(ctx context.Context, engine *Engine, ready <-chan struct{}) error { + return waitForEngineSignal(ctx, engine, ready, true) +} + +func waitForEngineSignal(ctx context.Context, engine *Engine, ready <-chan struct{}, closeOnCancel bool) error { select { case <-ctx.Done(): - _ = engine.Close() + if closeOnCancel { + _ = engine.Close() + } return errors.WithStack(ctx.Err()) case <-ready: return nil @@ -1720,6 +1727,10 @@ func (e *Engine) run() { e.fail(err) return } + if err := e.persistStartupAppliedIndex(); err != nil { + e.fail(err) + return + } e.markStarted() for { @@ -1868,7 +1879,7 @@ func (e *Engine) handleDispatchReport(report dispatchReport) { // If the channel is full (unlikely — the buffer is sized to MaxInflightMsg), // the report is dropped and logged; this is acceptable because raft will retry // on the next tick and we only need eventual consistency between transport -// state and Progress state. Successful MsgSnap reports are the exception; see +// state and Progress state. MsgSnap reports are the exception; see // postReliableDispatchReport. func (e *Engine) postDispatchReport(report dispatchReport) { select { @@ -1883,9 +1894,10 @@ func (e *Engine) postDispatchReport(report dispatchReport) { } // postReliableDispatchReport delivers a dispatch outcome that must not be -// dropped. SnapshotFinish is not eventually consistent with ordinary -// unreachable reports: if it is lost, raft can keep the follower's Progress in -// StateSnapshot after the follower accepted the snapshot. +// dropped. SnapshotFinish and SnapshotFailure are not eventually consistent +// with ordinary unreachable reports: if either is lost, raft can keep the +// follower's Progress in StateSnapshot after the follower accepted or missed +// the snapshot. func (e *Engine) postReliableDispatchReport(report dispatchReport) { if e.dispatchReportCh == nil { return @@ -2189,6 +2201,7 @@ func (e *Engine) drainReady() error { e.releaseProtectedReceivedFSMSnapshotsUpTo(e.appliedIndex.Load()) e.handleReadStates(rd.ReadStates) e.rawNode.Advance(rd) + e.flushDeferredDispatchReports() if err := e.maybePersistLocalSnapshot(); err != nil { return err } @@ -2229,6 +2242,9 @@ func (e *Engine) persistReadyWithSnapshotLocked(rd etcdraft.Ready) error { if err := persistReadyToWAL(e.persist, rd); err != nil { return err } + if err := e.persistReceivedSnapshotAppliedIndex(rd.Snapshot); err != nil { + return err + } e.releaseProtectedReceivedFSMSnapshotsUpToLocked(snapshotIndex(rd.Snapshot)) return nil } @@ -2547,15 +2563,6 @@ func (e *Engine) applyReadySnapshotLocked(snapshot *raftpb.Snapshot) error { if err != nil { return errors.Wrapf(err, "decode snapshot token index=%d", snapshot.GetMetadata().GetIndex()) } - // B3/follow-up: also call SetDurableAppliedIndex(tok.Index) here - // after Restore so peer-after-InstallSnapshot populates the meta - // key. The local-snapshot persist path already bumps the live - // store (engine.persistLocalSnapshotPayload), but the receiving - // node's restored store inherits the pre-bump value embedded in - // the snapshot artifact. Design Non-Goals § - // docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md#non-goals - // scopes this out of Branch 2; see PR #915 round-4/5 codex P2 on - // engine.go:4077 for the rationale. if err := openAndRestoreFSMSnapshot(e.fsm, fsmSnapPath(e.fsmSnapDir, tok.Index), tok.CRC32C); err != nil { return errors.Wrapf(err, "restore fsm snapshot file index=%d crc=%08x", tok.Index, tok.CRC32C) } @@ -2565,7 +2572,6 @@ func (e *Engine) applyReadySnapshotLocked(snapshot *raftpb.Snapshot) error { return errors.Wrapf(err, "restore fsm from legacy snapshot payload index=%d", snapshot.GetMetadata().GetIndex()) } } - if err := e.storage.ApplySnapshot(snapshot); err != nil { return errors.Wrapf(err, "apply snapshot to raft storage index=%d term=%d", snapshot.GetMetadata().GetIndex(), snapshot.GetMetadata().GetTerm()) @@ -3535,29 +3541,60 @@ func (e *Engine) createConfigSnapshot(index uint64, confState raftpb.ConfState, } } +// setDurableAppliedIndex pins the FSM's durable applied index to a +// known raft log index. FSMs that do not expose +// raftengine.AppliedIndexWriter silently no-op; the skip optimisation +// falls back to full restore for them (legacy test fakes, in-memory +// backends). +func (e *Engine) setDurableAppliedIndex(index uint64) error { + w, ok := e.fsm.(raftengine.AppliedIndexWriter) + if !ok { + return nil + } + return errors.WithStack(w.SetDurableAppliedIndex(index)) +} + // bumpDurableAppliedIndexBeforeSave pins the FSM's durable applied // index to `index` BEFORE the engine calls persist.SaveSnap, so a // successful snapshot persist always implies LastAppliedIndex >= // snap.Metadata.Index — closes the HLC-lease-only / encryption-only // fallback (PR #910 design §6). // -// FSMs that do not expose raftengine.AppliedIndexWriter silently -// no-op; the skip optimisation falls back to full restore for them -// (legacy test fakes, in-memory backends). pebble.Sync is forced on -// the writer side regardless of ELASTICKV_FSM_SYNC_MODE — once -// persist.SaveSnap returns, WAL compaction discards every log entry -// at or before snap.Metadata.Index, so there is no source to replay -// the meta key bump from. +// pebble.Sync is forced on the writer side regardless of +// ELASTICKV_FSM_SYNC_MODE — once persist.SaveSnap returns, WAL +// compaction discards every log entry at or before snap.Metadata.Index, +// so there is no source to replay the meta key bump from. // // Used by BOTH snapshot persist sites: persistCreatedSnapshot (this // file) and e.persistLocalSnapshotPayload (the steady-state // SnapshotCount-triggered hot path). func (e *Engine) bumpDurableAppliedIndexBeforeSave(index uint64) error { - w, ok := e.fsm.(raftengine.AppliedIndexWriter) - if !ok { + return e.setDurableAppliedIndex(index) +} + +func (e *Engine) persistStartupAppliedIndex() error { + if e.applied == 0 { return nil } - return errors.WithStack(w.SetDurableAppliedIndex(index)) + return e.setDurableAppliedIndex(e.applied) +} + +// persistReceivedSnapshotAppliedIndex runs only after persistReadyToWAL has +// saved the incoming raft snapshot. A crash before this point must fall back to +// a full FSM restore instead of letting the FSM meta index get ahead of the +// local durable raft snapshot. +func (e *Engine) persistReceivedSnapshotAppliedIndex(snapshot *raftpb.Snapshot) error { + if etcdraft.IsEmptySnap(snapshot) { + return nil + } + index := snapshot.GetMetadata().GetIndex() + if index == 0 { + return nil + } + if err := e.setDurableAppliedIndex(index); err != nil { + return errors.Wrapf(err, "persist durable applied index from snapshot index=%d", index) + } + return nil } func (e *Engine) persistCreatedSnapshot(snap raftpb.Snapshot) error { @@ -4535,7 +4572,7 @@ func (e *Engine) handleDispatchRequest(ctx context.Context, req dispatchRequest) // out of StateReplicate / StateSnapshot. Without this the leader keeps // Progress stuck and never retries sendAppend/sendSnap for the peer, // leaving the follower indefinitely stale even after heartbeats resume. - e.postDispatchReport(dispatchReport{to: req.msg.GetTo(), msgType: req.msg.GetType()}) + e.reportFailedDispatch(req.msg) } func (e *Engine) reportSuccessfulDispatch(msg raftpb.Message) { @@ -4942,10 +4979,39 @@ func (e *Engine) recordDroppedDispatch(msg raftpb.Message) { } func (e *Engine) reportDroppedDispatch(msg raftpb.Message) { + report := dispatchReport{to: msg.GetTo(), msgType: msg.GetType()} + if msg.GetType() == raftpb.MsgSnap { + e.deferReadyDispatchReport(report) + return + } if e.dispatchReportCh == nil { return } - e.postDispatchReport(dispatchReport{to: msg.GetTo(), msgType: msg.GetType()}) + e.postDispatchReport(report) +} + +func (e *Engine) reportFailedDispatch(msg raftpb.Message) { + report := dispatchReport{to: msg.GetTo(), msgType: msg.GetType()} + if msg.GetType() == raftpb.MsgSnap { + e.postReliableDispatchReport(report) + return + } + e.postDispatchReport(report) +} + +func (e *Engine) deferReadyDispatchReport(report dispatchReport) { + e.deferredReadyDispatchReports = append(e.deferredReadyDispatchReports, report) +} + +func (e *Engine) flushDeferredDispatchReports() { + if len(e.deferredReadyDispatchReports) == 0 { + return + } + reports := e.deferredReadyDispatchReports + e.deferredReadyDispatchReports = nil + for _, report := range reports { + e.handleDispatchReport(report) + } } // dispatchErrorCodeOf extracts the grpc status code name from err, or diff --git a/internal/raftengine/etcd/engine_applied_index_test.go b/internal/raftengine/etcd/engine_applied_index_test.go index d91ef433b..8ebb2ecfe 100644 --- a/internal/raftengine/etcd/engine_applied_index_test.go +++ b/internal/raftengine/etcd/engine_applied_index_test.go @@ -60,7 +60,9 @@ func (f *recordingAppliedIndexFSM) SetDurableAppliedIndex(idx uint64) error { f.failNext = false return f.failErr } - f.rec.record("bump", idx) + if f.rec != nil { + f.rec.record("bump", idx) + } return nil } @@ -338,6 +340,54 @@ func TestPersistCreatedSnapshot_BumpErrorAborts(t *testing.T) { "failed bump MUST NOT have recorded; SaveSnap MUST NOT have run") } +func TestPersistReadyWithSnapshot_BumpsAppliedIndexAfterWALSnapshot(t *testing.T) { + rec := &applyIndexOrderRecorder{} + fsm := &recordingAppliedIndexFSM{rec: rec} + persist := &recordingPersistStorage{rec: rec} + fsmSnapDir := t.TempDir() + const index uint64 = 77 + crc, _ := writeFSMFileForTest(t, fsmSnapDir, index, []byte("snapshot-payload")) + e := &Engine{ + storage: etcdraft.NewMemoryStorage(), + fsm: fsm, + persist: persist, + dataDir: t.TempDir(), + fsmSnapDir: fsmSnapDir, + } + + snap := appliedIndexTestSnapshot(index, encodeSnapshotToken(index, crc)) + require.NoError(t, e.persistReadyWithSnapshotLocked(etcdraft.Ready{Snapshot: &snap})) + + require.Equal(t, []orderEvent{ + {kind: "save", index: index}, + {kind: "bump", index: index}, + }, rec.snapshot(), + "received snapshot restore MUST publish its applied index only after the raft snapshot is durable") + require.Equal(t, index, e.applied) +} + +func TestPersistReadyWithSnapshot_BumpErrorAfterWALSnapshotSurfaces(t *testing.T) { + rec := &applyIndexOrderRecorder{} + fsm := &recordingAppliedIndexFSM{rec: rec, failNext: true, failErr: io.ErrShortBuffer} + persist := &recordingPersistStorage{rec: rec} + fsmSnapDir := t.TempDir() + const index uint64 = 78 + crc, _ := writeFSMFileForTest(t, fsmSnapDir, index, []byte("snapshot-payload")) + e := &Engine{ + storage: etcdraft.NewMemoryStorage(), + fsm: fsm, + persist: persist, + dataDir: t.TempDir(), + fsmSnapDir: fsmSnapDir, + } + + snap := appliedIndexTestSnapshot(index, encodeSnapshotToken(index, crc)) + err := e.persistReadyWithSnapshotLocked(etcdraft.Ready{Snapshot: &snap}) + require.Error(t, err, "received snapshot bump failure MUST be surfaced") + require.Equal(t, []orderEvent{{kind: "save", index: index}}, rec.snapshot(), + "received snapshot path must not bump before the WAL snapshot is durable") +} + // --- Site 2: persistLocalSnapshotPayload (steady-state hot path) --- // // These mirror the Site 1 tests above but exercise the engine's diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 87e8447df..ca92e1ff6 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -114,6 +114,18 @@ type blockingApplyStateMachine struct { startOnce sync.Once } +type recordingStartupApplyStateMachine struct { + *blockingApplyStateMachine + rec *applyIndexOrderRecorder +} + +func (s *recordingStartupApplyStateMachine) SetDurableAppliedIndex(idx uint64) error { + if s.rec != nil { + s.rec.record("bump", idx) + } + return nil +} + type blockingSnapshot struct { started chan struct{} release chan struct{} @@ -789,6 +801,58 @@ func TestSendMessagesDoesNotBlockWhenDispatchQueueIsFull(t *testing.T) { } } +func TestReportFailedDispatchReliablyQueuesMsgSnapFailure(t *testing.T) { + engine := &Engine{ + dispatchReportCh: make(chan dispatchReport, 1), + closeCh: make(chan struct{}), + } + engine.dispatchReportCh <- dispatchReport{to: 2, msgType: raftpb.MsgHeartbeat} + + done := make(chan struct{}) + go func() { + engine.reportFailedDispatch(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgSnap), + To: uint64Ptr(3), + }) + close(done) + }() + + select { + case <-done: + t.Fatal("MsgSnap failure report must wait instead of dropping when report channel is full") + case <-time.After(20 * time.Millisecond): + } + + <-engine.dispatchReportCh + requireSignal(t, done, time.Second, "MsgSnap failure report was not delivered after report channel drained") + report := <-engine.dispatchReportCh + require.Equal(t, uint64(3), report.to) + require.Equal(t, raftpb.MsgSnap, report.msgType) + require.False(t, report.snapshotFinish) +} + +func TestReportDroppedDispatchDefersMsgSnapUntilReadyAdvanced(t *testing.T) { + engine := &Engine{ + dispatchReportCh: make(chan dispatchReport, 1), + closeCh: make(chan struct{}), + } + queued := dispatchReport{to: 2, msgType: raftpb.MsgHeartbeat} + engine.dispatchReportCh <- queued + + engine.reportDroppedDispatch(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgSnap), + To: uint64Ptr(3), + }) + + require.Equal(t, queued, <-engine.dispatchReportCh) + select { + case report := <-engine.dispatchReportCh: + t.Fatalf("dropped MsgSnap report should not enqueue from the event loop: %+v", report) + default: + } + require.Equal(t, []dispatchReport{{to: 3, msgType: raftpb.MsgSnap}}, engine.deferredReadyDispatchReports) +} + func TestStopDispatchWorkersCancelsInflightDispatch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -1554,9 +1618,13 @@ func TestOpenMultiNodeWaitStartedWaitsForCommittedTailDrain(t *testing.T) { }}, })) - fsm := &blockingApplyStateMachine{ - started: make(chan struct{}), - release: make(chan struct{}), + rec := &applyIndexOrderRecorder{} + fsm := &recordingStartupApplyStateMachine{ + blockingApplyStateMachine: &blockingApplyStateMachine{ + started: make(chan struct{}), + release: make(chan struct{}), + }, + rec: rec, } done := make(chan openResult, 1) go func() { @@ -1591,6 +1659,8 @@ func TestOpenMultiNodeWaitStartedWaitsForCommittedTailDrain(t *testing.T) { requireStartupWaitResult(t, waitDone, time.Second, "WaitStarted did not return after committed tail drain completed") requireSignal(t, result.engine.startedCh, time.Second, "engine did not mark started after committed tail drain completed") + require.Equal(t, []orderEvent{{kind: "bump", index: 2}}, rec.snapshot(), + "startup must persist the committed tail applied index before marking the engine started") } type openResult struct { @@ -1638,6 +1708,24 @@ func requireStartupWaitResult(t *testing.T, ch <-chan error, timeout time.Durati } } +func TestWaitStartedTimeoutDoesNotCloseEngine(t *testing.T) { + engine := &Engine{ + startedCh: make(chan struct{}), + doneCh: make(chan struct{}), + closeCh: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := engine.WaitStarted(ctx) + require.ErrorIs(t, err, context.Canceled) + select { + case <-engine.closeCh: + t.Fatal("WaitStarted timeout must not close an already-returned engine") + default: + } +} + func TestOpenMultiNodeReplicatesOverGRPCTransport(t *testing.T) { nodes, peers := newTransportTestNodes(t, 3) startTransportTestServers(nodes, peers) diff --git a/internal/raftengine/statemachine.go b/internal/raftengine/statemachine.go index d834ad318..9c796f4f8 100644 --- a/internal/raftengine/statemachine.go +++ b/internal/raftengine/statemachine.go @@ -69,24 +69,25 @@ type AppliedIndexReader interface { } // AppliedIndexWriter is an OPTIONAL extension that lets the engine -// pin the FSM's durable applied-index to a known value at snapshot -// persist time. See docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md +// pin the FSM's durable applied-index to a known value at raft +// durability boundaries. See docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md // §6 "HLC lease entries — checkpoint at snapshot persist". // -// The engine calls SetDurableAppliedIndex(snap.Metadata.Index) -// before it calls persist.SaveSnap, so that on every successful -// snapshot persist the invariant `LastAppliedIndex >= -// snapshot.Metadata.Index` holds unconditionally — closing the -// HLC-lease-only / encryption-only fallback that would otherwise -// leave LastAppliedIndex stuck at the last data-Apply index. +// The engine calls SetDurableAppliedIndex at local snapshot persist, +// after received-snapshot WAL persistence, and at startup +// committed-tail drain boundaries, so the invariant +// `LastAppliedIndex >= the locally durable raft state` holds once the +// engine is store-ready. This closes the HLC-lease-only / +// encryption-only fallback that would otherwise leave LastAppliedIndex +// stuck at the last data-Apply index. // // Implementations MUST persist the value with pebble.Sync (or the // equivalent strong-durability flag for the backing store) // regardless of ELASTICKV_FSM_SYNC_MODE. The checkpoint is the only -// durable carrier of metaAppliedIndex at this point — once -// persist.SaveSnap returns, WAL compaction discards every log entry -// at or before snap.Metadata.Index, so there is no source to replay -// the meta key bump from. +// durable carrier of metaAppliedIndex at local snapshot persist time +// — once persist.SaveSnap returns, WAL compaction discards every log +// entry at or before snap.Metadata.Index, so there is no source to +// replay the meta key bump from. type AppliedIndexWriter interface { SetDurableAppliedIndex(idx uint64) error } From 3e988efca07dd77de287b5241b7a7457e3089e94 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 07:14:15 +0900 Subject: [PATCH 095/162] Enforce current route fences at apply time --- adapter/dynamodb_transact.go | 4 ++ adapter/retryable_write_fence_test.go | 4 ++ adapter/sqs_messages.go | 2 +- adapter/sqs_reaper.go | 4 +- adapter/sqs_redrive.go | 2 +- distribution/engine.go | 13 +++-- kv/fsm.go | 28 ++++++---- kv/fsm_migration_fence_test.go | 41 +++++---------- kv/sharded_coordinator.go | 54 +++++++++++--------- kv/sharded_coordinator_del_prefix_test.go | 62 +++++++++++++++++++++-- 10 files changed, 138 insertions(+), 76 deletions(-) diff --git a/adapter/dynamodb_transact.go b/adapter/dynamodb_transact.go index 9569b6053..e1f6ab221 100644 --- a/adapter/dynamodb_transact.go +++ b/adapter/dynamodb_transact.go @@ -1108,6 +1108,10 @@ func isRetryableTransactWriteError(err error) bool { return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || errors.Is(err, kv.ErrRouteWriteFenced) } +func isIgnorableTransactRaceError(err error) bool { + return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) +} + func shouldPreserveTransactWriteAttempt(err error) bool { return isRetryableTransactWriteError(err) && !errors.Is(err, kv.ErrRouteWriteFenced) } diff --git a/adapter/retryable_write_fence_test.go b/adapter/retryable_write_fence_test.go index e5b436c06..176ac68ca 100644 --- a/adapter/retryable_write_fence_test.go +++ b/adapter/retryable_write_fence_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -14,4 +15,7 @@ func TestWriteFenceErrorsAreAdapterRetryable(t *testing.T) { require.True(t, isRetryableS3MutationErr(kv.ErrRouteWriteFenced)) require.True(t, isRetryableTransactWriteError(kv.ErrRouteWriteFenced)) require.False(t, shouldPreserveTransactWriteAttempt(kv.ErrRouteWriteFenced)) + require.False(t, isIgnorableTransactRaceError(kv.ErrRouteWriteFenced)) + require.True(t, isIgnorableTransactRaceError(store.ErrWriteConflict)) + require.True(t, isIgnorableTransactRaceError(kv.ErrTxnLocked)) } diff --git a/adapter/sqs_messages.go b/adapter/sqs_messages.go index 516d53d87..83bfe6009 100644 --- a/adapter/sqs_messages.go +++ b/adapter/sqs_messages.go @@ -1292,7 +1292,7 @@ func (s *SQSServer) expireMessage(ctx context.Context, queueName string, meta *s Elems: elems, } if _, err := s.coordinator.Dispatch(ctx, req); err != nil { - if isRetryableTransactWriteError(err) { + if isIgnorableTransactRaceError(err) { return nil } return errors.WithStack(err) diff --git a/adapter/sqs_reaper.go b/adapter/sqs_reaper.go index ae7f6c20d..a25b83ca6 100644 --- a/adapter/sqs_reaper.go +++ b/adapter/sqs_reaper.go @@ -664,7 +664,7 @@ func (s *SQSServer) reapOneRecord(ctx context.Context, queueName string, meta *s return err } if _, err := s.coordinator.Dispatch(ctx, req); err != nil { - if isRetryableTransactWriteError(err) { + if isIgnorableTransactRaceError(err) { return nil } return errors.WithStack(err) @@ -873,7 +873,7 @@ func (s *SQSServer) dispatchDedupDelete(ctx context.Context, key []byte, readTS }, } if _, err := s.coordinator.Dispatch(ctx, req); err != nil { - if isRetryableTransactWriteError(err) { + if isIgnorableTransactRaceError(err) { return nil } return errors.WithStack(err) diff --git a/adapter/sqs_redrive.go b/adapter/sqs_redrive.go index 3fb13b8d3..cfc4437d0 100644 --- a/adapter/sqs_redrive.go +++ b/adapter/sqs_redrive.go @@ -237,7 +237,7 @@ func (s *SQSServer) redriveCandidateToDLQ( return false, err } if _, err := s.coordinator.Dispatch(ctx, req); err != nil { - if isRetryableTransactWriteError(err) { + if isIgnorableTransactRaceError(err) { return true, nil } return false, errors.WithStack(err) diff --git a/distribution/engine.go b/distribution/engine.go index 96d9a4c8e..16b4ad4b8 100644 --- a/distribution/engine.go +++ b/distribution/engine.go @@ -164,11 +164,16 @@ func (s RouteHistorySnapshot) Version() uint64 { return s.version } // scan from O(N) to "first non-covering gap" without changing the // resolution semantics). func (s RouteHistorySnapshot) OwnerOf(key []byte) (uint64, bool) { - r, ok := s.RouteOf(key) - if !ok { - return 0, false + for _, r := range s.routes { + if bytes.Compare(key, r.Start) < 0 { + break + } + if r.End != nil && bytes.Compare(key, r.End) >= 0 { + continue + } + return r.GroupID, true } - return r.GroupID, true + return 0, false } // RouteOf returns the route that covered key at this snapshot's version. diff --git a/kv/fsm.go b/kv/fsm.go index bc5da0b36..88c7b95cc 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -828,21 +828,27 @@ func (f *kvFSM) verifyWriteFence(r *pb.Request) error { return nil } observedVer := r.GetObservedRouteVersion() - if !f.writeFenceHistoryReady(observedVer) { + if !f.writeFenceHistoryReady() { return nil } - observedSnap, ok := f.routes.SnapshotAt(observedVer) + currentSnap, ok := f.routes.Current() if !ok { - return errors.WithStack(ErrComposed1VersionGCd) - } - if err := verifyWriteFenceFromSnapshot(r.GetMutations(), observedSnap, observedVer, "observed"); err != nil { - return err + return nil } - currentSnap, ok := f.routes.Current() - if !ok || currentSnap.Version() == observedSnap.Version() { - return nil + if observedVer != 0 { + observedSnap, ok := f.routes.SnapshotAt(observedVer) + if !ok { + return errors.WithStack(ErrComposed1VersionGCd) + } + if err := verifyWriteFenceFromSnapshot(r.GetMutations(), observedSnap, observedVer, "observed"); err != nil { + return err + } + if currentSnap.Version() == observedSnap.Version() { + return nil + } } + return verifyWriteFenceFromSnapshot(r.GetMutations(), currentSnap, currentSnap.Version(), "current") } @@ -859,8 +865,8 @@ func requestBypassesWriteFence(r *pb.Request) bool { return false } -func (f *kvFSM) writeFenceHistoryReady(observedVer uint64) bool { - return f.routes != nil && f.shardGroupID != 0 && observedVer != 0 +func (f *kvFSM) writeFenceHistoryReady() bool { + return f.routes != nil && f.shardGroupID != 0 } func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, snap RouteSnapshot, snapVer uint64, phase string) error { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 959358513..f1c027477 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -39,18 +39,14 @@ func newS3BucketAuxiliaryWriteFencedFSM(t *testing.T, bucket string) *kvFSM { return newComposed1FSM(t, engine, 1) } -func TestFSMAppliesRawPointWriteDespiteLocalWriteFencedRoute(t *testing.T) { +func TestFSMRejectsCurrentWriteFencedRawPointWrite(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, }, 10) - require.NoError(t, err) - - got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + require.ErrorIs(t, err, ErrRouteWriteFenced) } func TestFSMRejectsObservedWriteFencedRawPointWrite(t *testing.T) { @@ -84,7 +80,7 @@ func TestFSMRejectsCurrentWriteFenceAfterObservedActiveRawPointWrite(t *testing. require.ErrorIs(t, err, ErrRouteWriteFenced) } -func TestFSMAppliesS3BucketAuxiliaryPointWriteDespiteLocalWriteFencedRoute(t *testing.T) { +func TestFSMRejectsCurrentWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { t.Parallel() ctx := context.Background() @@ -98,11 +94,7 @@ func TestFSMAppliesS3BucketAuxiliaryPointWriteDespiteLocalWriteFencedRoute(t *te err := fsm.handleRawRequest(ctx, &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, }, 10) - require.NoError(t, err) - - got, getErr := fsm.store.GetAt(ctx, key, ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + require.ErrorIs(t, err, ErrRouteWriteFenced) } } @@ -121,7 +113,7 @@ func TestFSMRejectsObservedWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) require.ErrorIs(t, err, ErrRouteWriteFenced) } -func TestFSMAppliesDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { +func TestFSMRejectsCurrentWriteFencedDelPrefix(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -130,10 +122,7 @@ func TestFSMAppliesDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, }, 10) - require.NoError(t, err) - - _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) - require.Error(t, getErr) + require.ErrorIs(t, err, ErrRouteWriteFenced) } func TestFSMRejectsObservedWriteFencedDelPrefix(t *testing.T) { @@ -149,7 +138,7 @@ func TestFSMRejectsObservedWriteFencedDelPrefix(t *testing.T) { require.ErrorIs(t, err, ErrRouteWriteFenced) } -func TestFSMAppliesFullRangeDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { +func TestFSMRejectsCurrentWriteFencedFullRangeDelPrefix(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -158,13 +147,10 @@ func TestFSMAppliesFullRangeDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: nil}}, }, 10) - require.NoError(t, err) - - _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) - require.Error(t, getErr) + require.ErrorIs(t, err, ErrRouteWriteFenced) } -func TestFSMAppliesBroadInternalDelPrefixDespiteLocalWriteFencedRoute(t *testing.T) { +func TestFSMRejectsCurrentWriteFencedBroadInternalDelPrefix(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -174,13 +160,10 @@ func TestFSMAppliesBroadInternalDelPrefixDespiteLocalWriteFencedRoute(t *testing err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("!redis|")}}, }, 10) - require.NoError(t, err) - - _, getErr := fsm.store.GetAt(context.Background(), key, ^uint64(0)) - require.Error(t, getErr) + require.ErrorIs(t, err, ErrRouteWriteFenced) } -func TestFSMAppliesPrepareAndAbortDespiteLocalWriteFencedRoute(t *testing.T) { +func TestFSMRejectsCurrentWriteFencedPrepareButAllowsAbort(t *testing.T) { t.Parallel() ctx := context.Background() @@ -194,7 +177,7 @@ func TestFSMAppliesPrepareAndAbortDespiteLocalWriteFencedRoute(t *testing.T) { {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, }, } - require.NoError(t, fsm.handleTxnRequest(ctx, prepare, 10)) + require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) abort := &pb.Request{ IsTxn: true, diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index e8bfb5e80..147abc067 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -700,11 +700,6 @@ func (c *ShardedCoordinator) Dispatch(ctx context.Context, reqs *OperationGroup[ return nil, err } - c.maybeAutoPinRawObservedRouteVersion(reqs) - if resp, handled, err := c.dispatchBeforeShardRouting(ctx, reqs); handled { - return resp, err - } - // Capture whether the caller supplied a non-zero StartTS BEFORE // the coordinator-allocates-on-zero branch below mutates the // field. A caller-supplied StartTS names a specific snapshot @@ -736,10 +731,13 @@ func (c *ShardedCoordinator) Dispatch(ctx context.Context, reqs *OperationGroup[ } if reqs.IsTxn { + if resp, handled, err := c.dispatchBeforeShardRouting(ctx, reqs); handled { + return resp, err + } return c.dispatchTxnWithComposed1Retry(ctx, reqs, callerSuppliedStartTS) } - return c.dispatchNonTxn(ctx, reqs) + return c.dispatchRawWithComposed1Retry(ctx, reqs) } func (c *ShardedCoordinator) dispatchBeforeShardRouting(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, bool, error) { @@ -815,16 +813,17 @@ func (c *ShardedCoordinator) dispatchBeforeShardRouting(ctx context.Context, req // gate spuriously rejects resolver-routed commits even when // the resolver picked the correct gid. Skip the auto-pin // for any resolver-recognised key; the request flows with -// ObservedRouteVersion=0 and the M3 gate short-circuits — -// restoring the pre-auto-pin behaviour for resolver-routed -// txns. Resolver-aware M3 is M5+ work (codex P1 on -// 6a458a28, PR #900). +// ObservedRouteVersion=0 and the Composed-1 owner gate +// short-circuits — restoring the pre-auto-pin behaviour for +// resolver-routed txns. Resolver-aware M3 is M5+ work (PR #900). // // The non-auto-pin case (request flows with ObservedRouteVersion=0, -// M3 gate short-circuits) is the safe non-regressing posture for -// non-migrated callers — the gate cannot retroactively pin reads -// it was not present for. Adapters that want M3 protection must -// migrate to pin at BeginTxn per §4.1. +// Composed-1 owner gate short-circuits) is the safe non-regressing +// posture for non-migrated callers — the owner gate cannot +// retroactively pin reads it was not present for. The write-fence +// gate still checks the current route snapshot at apply time. +// Adapters that want full M3 owner protection must migrate to pin at +// BeginTxn per §4.1. // // Extracted from dispatchTxnWithComposed1Retry to keep its // cyclomatic complexity in the cyclop budget. @@ -838,16 +837,6 @@ func (c *ShardedCoordinator) maybeAutoPinObservedRouteVersion(reqs *OperationGro reqs.ObservedRouteVersion = c.engine.Version() } -func (c *ShardedCoordinator) maybeAutoPinRawObservedRouteVersion(reqs *OperationGroup[OP]) { - if c.engine == nil || reqs == nil || reqs.IsTxn || reqs.ObservedRouteVersion != 0 { - return - } - if c.anyResolverClaimedKey(reqs.Elems) { - return - } - reqs.ObservedRouteVersion = c.engine.Version() -} - // anyResolverClaimedKey reports whether any element's key is // claimed by the partition resolver. Returns false when the // router has no resolver installed (the most common case) so the @@ -1006,6 +995,23 @@ func (c *ShardedCoordinator) dispatchNonTxn(ctx context.Context, reqs *Operation return &CoordinateResponse{CommitIndex: r.CommitIndex}, nil } +func (c *ShardedCoordinator) dispatchRawWithComposed1Retry(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, error) { + for attempt := 0; attempt <= composed1RetryAttempts; attempt++ { + resp, handled, err := c.dispatchBeforeShardRouting(ctx, reqs) + if !handled { + resp, err = c.dispatchNonTxn(ctx, reqs) + } + if err == nil { + return resp, nil + } + if !errors.Is(err, ErrComposed1VersionGCd) || attempt == composed1RetryAttempts || c.engine == nil { + return resp, err + } + reqs.ObservedRouteVersion = c.engine.Version() + } + return nil, errors.WithStack(ErrInvalidRequest) +} + // hasDelPrefixElem returns true if any element is a DelPrefix operation. func hasDelPrefixElem(elems []*Elem[OP]) bool { for _, e := range elems { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 7b52fe105..d5f8632d3 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -122,7 +122,7 @@ func TestShardedCoordinator_DelPrefixBroadcastsToAllGroups(t *testing.T) { "same DEL_PREFIX element must use the same timestamp across shards") } -func TestShardedCoordinator_DelPrefixCarriesObservedRouteVersion(t *testing.T) { +func TestShardedCoordinator_DelPrefixDoesNotAutoPinObservedRouteVersion(t *testing.T) { t.Parallel() engine := distribution.NewEngine() @@ -142,10 +142,10 @@ func TestShardedCoordinator_DelPrefixCarriesObservedRouteVersion(t *testing.T) { }) require.NoError(t, err) require.Len(t, txn.requests, 1) - require.Equal(t, uint64(7), txn.requests[0].GetObservedRouteVersion()) + require.Zero(t, txn.requests[0].GetObservedRouteVersion()) } -func TestShardedCoordinator_RawWriteCarriesObservedRouteVersion(t *testing.T) { +func TestShardedCoordinator_RawWriteDoesNotAutoPinObservedRouteVersion(t *testing.T) { t.Parallel() engine := distribution.NewEngine() @@ -165,7 +165,61 @@ func TestShardedCoordinator_RawWriteCarriesObservedRouteVersion(t *testing.T) { }) require.NoError(t, err) require.Len(t, txn.requests, 1) - require.Equal(t, uint64(9), txn.requests[0].GetObservedRouteVersion()) + require.Zero(t, txn.requests[0].GetObservedRouteVersion()) +} + +func TestShardedCoordinator_RetriesRawWriteWhenObservedRouteVersionGCd(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 9, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: nil, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + txn := &recordingTransactional{ + errs: []error{ErrComposed1VersionGCd}, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + ObservedRouteVersion: 3, + Elems: []*Elem[OP]{{Op: Put, Key: []byte("k"), Value: []byte("v")}}, + }) + require.NoError(t, err) + require.Len(t, txn.requests, 2) + require.Equal(t, uint64(3), txn.requests[0].GetObservedRouteVersion()) + require.Equal(t, uint64(9), txn.requests[1].GetObservedRouteVersion()) +} + +func TestShardedCoordinator_RetriesDelPrefixWhenObservedRouteVersionGCd(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: nil, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + txn := &recordingTransactional{ + errs: []error{ErrComposed1VersionGCd}, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + ObservedRouteVersion: 2, + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("user:")}}, + }) + require.NoError(t, err) + require.Len(t, txn.requests, 2) + require.Equal(t, uint64(2), txn.requests[0].GetObservedRouteVersion()) + require.Equal(t, uint64(7), txn.requests[1].GetObservedRouteVersion()) } func TestShardedCoordinatorRejectsPointWriteOnWriteFencedRoute(t *testing.T) { From 04e8053bcb6649612ba8388f98c524c0268031a1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 07:52:00 +0900 Subject: [PATCH 096/162] Prioritize received raft snapshots --- internal/raftengine/etcd/engine.go | 40 +++++++++++++++++++++---- internal/raftengine/etcd/engine_test.go | 22 ++++++++++++++ 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index b87a00d13..5316967db 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -60,9 +60,10 @@ const ( // from shrinking the shared inbound queue enough to drop heartbeats. minInboundQueueCapacity = 128 // priorityStepQueueCapacity is the inbound control-plane queue size. - // Heartbeats, votes, read-index responses, and timeout-now messages are - // tiny but time-sensitive; keeping them off the bulk stepCh prevents a - // MsgApp burst from forcing followers into avoidable elections. + // Heartbeats, votes, read-index responses, timeout-now messages, and + // received snapshot tokens are tiny but time-sensitive; keeping them off the + // bulk stepCh prevents a MsgApp burst from forcing followers into avoidable + // elections or rejecting a catch-up snapshot after its payload was streamed. priorityStepQueueCapacity = 1024 // priorityStepBurstLimit bounds consecutive non-blocking priority drains // so a sustained control-message stream cannot starve Tick, proposals, or @@ -2493,6 +2494,14 @@ func isPriorityMsg(t raftpb.MessageType) bool { t == raftpb.MsgTimeoutNow } +func isInboundPriorityMsg(t raftpb.MessageType) bool { + return isPriorityMsg(t) || t == raftpb.MsgSnap +} + +func isBlockingInboundStepMsg(t raftpb.MessageType) bool { + return t == raftpb.MsgSnap +} + // selectDispatchLane picks the per-peer channel for msgType. In the legacy // layout it returns pd.heartbeatResp for MsgHeartbeatResp, pd.heartbeat for // other priority control traffic, and pd.normal for everything else. In the @@ -4346,9 +4355,10 @@ func maxAppliedIndex(snapshot raftpb.Snapshot) uint64 { } func (e *Engine) enqueueStep(ctx context.Context, msg raftpb.Message) error { - ch := e.stepCh - if isPriorityMsg(msg.GetType()) && e.priorityStepCh != nil { - ch = e.priorityStepCh + ch := e.stepChannelFor(msg.GetType()) + + if isBlockingInboundStepMsg(msg.GetType()) { + return e.enqueueBlockingStep(ctx, ch, msg) } select { @@ -4364,6 +4374,24 @@ func (e *Engine) enqueueStep(ctx context.Context, msg raftpb.Message) error { } } +func (e *Engine) stepChannelFor(msgType raftpb.MessageType) chan raftpb.Message { + if isInboundPriorityMsg(msgType) && e.priorityStepCh != nil { + return e.priorityStepCh + } + return e.stepCh +} + +func (e *Engine) enqueueBlockingStep(ctx context.Context, ch chan raftpb.Message, msg raftpb.Message) error { + select { + case <-ctx.Done(): + return errors.WithStack(ctx.Err()) + case <-e.doneCh: + return e.currentErrorOrClosed() + case ch <- msg: + return nil + } +} + func (e *Engine) handleTransportMessage(ctx context.Context, msg raftpb.Message) error { select { case <-ctx.Done(): diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index ca92e1ff6..6bc836704 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -647,6 +647,28 @@ func TestEnqueueStepPriorityBypassesFullBulkQueue(t *testing.T) { require.Equal(t, uint64(1), engine.StepQueueFullCount()) } +func TestEnqueueStepSnapshotBypassesFullBulkQueue(t *testing.T) { + engine := &Engine{ + doneCh: make(chan struct{}), + stepCh: make(chan raftpb.Message, 1), + priorityStepCh: make(chan raftpb.Message, 1), + } + engine.stepCh <- raftpb.Message{Type: messageTypePtr(raftpb.MsgApp)} + + err := engine.enqueueStep(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgSnap)}) + require.NoError(t, err) + require.Equal(t, uint64(0), engine.StepQueueFullCount()) + + select { + case msg := <-engine.priorityStepCh: + require.Equal(t, raftpb.MsgSnap, msg.GetType()) + default: + t.Fatal("snapshot step was not enqueued on the priority queue") + } + + require.Len(t, engine.stepCh, 1) +} + func TestHandleEventDrainsPriorityStepBeforeBulkStep(t *testing.T) { engine := &Engine{ closeCh: make(chan struct{}), From d818602593249f899bad1ce6d0b11da4f23508dc Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 08:23:05 +0900 Subject: [PATCH 097/162] Stabilize snapshot catch-up and fence checks --- adapter/dynamodb_transact.go | 2 + internal/raftengine/etcd/engine.go | 11 +++++ internal/raftengine/etcd/engine_test.go | 34 ++++++++++++++ internal/raftengine/etcd/fsm_snapshot_file.go | 31 ++++++++++++- .../raftengine/etcd/fsm_snapshot_file_test.go | 24 ++++++++++ kv/fsm.go | 3 -- kv/fsm_migration_fence_test.go | 46 +++++++++++++++++++ 7 files changed, 147 insertions(+), 4 deletions(-) diff --git a/adapter/dynamodb_transact.go b/adapter/dynamodb_transact.go index e1f6ab221..52fd98a42 100644 --- a/adapter/dynamodb_transact.go +++ b/adapter/dynamodb_transact.go @@ -1109,6 +1109,8 @@ func isRetryableTransactWriteError(err error) bool { } func isIgnorableTransactRaceError(err error) bool { + // Route fences reject before applying the write. They must be retried or + // propagated, not swallowed as "another worker already completed it". return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) } diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 5316967db..d36e7cf8a 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -927,6 +927,17 @@ func newRawNode(cfg OpenConfig, storage *etcdraft.MemoryStorage, applied uint64) func waitForOpen(ctx context.Context, engine *Engine, waitForLeader bool) (*Engine, error) { if !waitForLeader { + select { + case <-ctx.Done(): + _ = engine.Close() + return nil, errors.WithStack(ctx.Err()) + case <-engine.doneCh: + if err := engine.currentError(); err != nil { + return nil, err + } + return nil, errors.WithStack(errClosed) + default: + } return engine, nil } if err := waitForOpenSignal(ctx, engine, engine.startedCh); err != nil { diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 6bc836704..148f1d149 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1748,6 +1748,40 @@ func TestWaitStartedTimeoutDoesNotCloseEngine(t *testing.T) { } } +func TestWaitForOpenCanceledContextClosesMultiNodeEngine(t *testing.T) { + engine := &Engine{ + doneCh: make(chan struct{}), + closeCh: make(chan struct{}), + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + type result struct { + engine *Engine + err error + } + resultCh := make(chan result, 1) + go func() { + opened, err := waitForOpen(ctx, engine, false) + resultCh <- result{engine: opened, err: err} + }() + + select { + case <-engine.closeCh: + case <-time.After(time.Second): + t.Fatal("canceled multi-node Open must close the engine before returning") + } + close(engine.doneCh) + + select { + case got := <-resultCh: + require.Nil(t, got.engine) + require.ErrorIs(t, got.err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("waitForOpen did not return after Close completed") + } +} + func TestOpenMultiNodeReplicatesOverGRPCTransport(t *testing.T) { nodes, peers := newTransportTestNodes(t, 3) startTransportTestServers(nodes, peers) diff --git a/internal/raftengine/etcd/fsm_snapshot_file.go b/internal/raftengine/etcd/fsm_snapshot_file.go index 574724cae..78ca12ff4 100644 --- a/internal/raftengine/etcd/fsm_snapshot_file.go +++ b/internal/raftengine/etcd/fsm_snapshot_file.go @@ -831,7 +831,36 @@ func fsmSnapshotPairRestorable(snapDir, fsmSnapDir, snapName string, term, index if !ok { return false } - return verifyFSMSnapshotFileWithToken(fsmSnapPath(fsmSnapDir, index), tok.CRC32C, true) == nil + // Prewrite cleanup runs on the snapshot receive hot path before gRPC starts + // draining payload chunks. Only do a footer/token check here; full-payload + // CRC remains in the actual restore/open paths. + return fsmSnapshotFooterMatchesToken(fsmSnapPath(fsmSnapDir, index), tok.CRC32C) == nil +} + +func fsmSnapshotFooterMatchesToken(path string, tokenCRC uint32) error { + f, err := os.Open(path) + if err != nil { + return statFSMFileError(err) + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return errors.WithStack(err) + } + if info.Size() < fsmMinFileSize { + return errors.Wrapf(ErrFSMSnapshotTooSmall, + "file too small: %d bytes (minimum %d)", info.Size(), fsmMinFileSize) + } + footer, err := readFSMFooter(f, info.Size()) + if err != nil { + return err + } + if footer != tokenCRC { + return errors.Wrapf(ErrFSMSnapshotTokenCRC, + "path=%s footer=%08x token=%08x", path, footer, tokenCRC) + } + return nil } func snapshotTokenFromSnapFile(snapDir, snapName string, term, index uint64) (snapshotToken, bool) { diff --git a/internal/raftengine/etcd/fsm_snapshot_file_test.go b/internal/raftengine/etcd/fsm_snapshot_file_test.go index a7d509b44..f5b93f83a 100644 --- a/internal/raftengine/etcd/fsm_snapshot_file_test.go +++ b/internal/raftengine/etcd/fsm_snapshot_file_test.go @@ -446,6 +446,30 @@ func TestPrepareFSMSnapshotWriteKeepsTokenMatchingFallbackPair(t *testing.T) { require.FileExists(t, fsmSnapPath(fsmSnapDir, 100)) } +func TestPrewriteRestorableCheckUsesSnapshotFooterOnly(t *testing.T) { + snapDir := t.TempDir() + fsmSnapDir := t.TempDir() + payload := []byte("payload") + + crc, path := writeFSMFileForTest(t, fsmSnapDir, 100, payload) + createTokenSnapFileWithTerm(t, snapDir, 1, 100, crc) + + f, err := os.OpenFile(path, os.O_WRONLY, 0) + require.NoError(t, err) + _, err = f.WriteAt([]byte("X"), 0) + require.NoError(t, err) + require.NoError(t, f.Close()) + + require.True(t, fsmSnapshotPairRestorable( + snapDir, + fsmSnapDir, + "0000000000000001-0000000000000064.snap", + 1, + 100, + )) + require.ErrorIs(t, verifyFSMSnapshotFile(path, crc), ErrFSMSnapshotFileCRC) +} + func TestPrepareFSMSnapshotWriteKeepsWALValidAndRestorableFallbacksWhenWALValidCandidateIsBroken(t *testing.T) { snapDir := t.TempDir() fsmSnapDir := t.TempDir() diff --git a/kv/fsm.go b/kv/fsm.go index 88c7b95cc..1e1b3fb17 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -886,9 +886,6 @@ func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, snap RouteSnapshot, } continue } - if len(mut.Key) == 0 { - continue - } rKey := routeKey(mut.Key) if snap.WriteFencedForKey(rKey) { return errors.Wrapf(ErrRouteWriteFenced, diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index f1c027477..f668821c8 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -21,6 +21,17 @@ func newWriteFencedFSM(t *testing.T) *kvFSM { return newComposed1FSM(t, engine, 1) } +func newFirstRouteWriteFencedFSM(t *testing.T) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateWriteFenced}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }) + return newComposed1FSM(t, engine, 1) +} + func s3BucketAuxiliaryFenceRoutes(bucket string, rawGroupID, fencedGroupID uint64) []distribution.RouteDescriptor { start := s3keys.RoutePrefixForBucketAnyGeneration(bucket) end := prefixScanEnd(start) @@ -49,6 +60,16 @@ func TestFSMRejectsCurrentWriteFencedRawPointWrite(t *testing.T) { require.ErrorIs(t, err, ErrRouteWriteFenced) } +func TestFSMRejectsCurrentWriteFencedEmptyRawPointWrite(t *testing.T) { + t.Parallel() + + fsm := newFirstRouteWriteFencedFSM(t) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(""), Value: []byte("v")}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMRejectsObservedWriteFencedRawPointWrite(t *testing.T) { t.Parallel() @@ -80,6 +101,31 @@ func TestFSMRejectsCurrentWriteFenceAfterObservedActiveRawPointWrite(t *testing. require.ErrorIs(t, err, ErrRouteWriteFenced) } +func TestFSMRejectsCurrentWriteFencedUnpinnedPrepare(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, + }) + + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMRejectsCurrentWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { t.Parallel() From db9843abde7515833eb43025f93af213d4567aa6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 09:05:04 +0900 Subject: [PATCH 098/162] Stabilize startup reads and snapshot recovery --- adapter/distribution_server.go | 27 ++++++ adapter/distribution_server_test.go | 29 +++++++ adapter/grpc.go | 30 +++++++ adapter/grpc_test.go | 30 +++++++ internal/raftengine/etcd/engine.go | 57 ++++++++----- internal/raftengine/etcd/engine_test.go | 56 +++++++++++- internal/raftengine/etcd/fsm_snapshot_file.go | 78 +++++++++++++++-- .../raftengine/etcd/fsm_snapshot_file_test.go | 28 +++++- internal/raftengine/etcd/snapshot_spool.go | 14 +-- .../raftengine/etcd/snapshot_spool_test.go | 4 +- kv/fsm.go | 26 ++++++ kv/shard_key_test.go | 28 +++++- kv/sharded_coordinator.go | 31 +++++++ kv/sharded_coordinator_del_prefix_test.go | 85 ++++++++++++++++++- kv/sharded_coordinator_txn_test.go | 4 + main.go | 9 +- 16 files changed, 494 insertions(+), 42 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 3d3276f5c..f8b973f3d 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -23,6 +23,7 @@ type DistributionServer struct { catalog *distribution.CatalogStore coordinator kv.Coordinator readTracker *kv.ActiveTimestampTracker + readBlocked func() bool reloadRetry struct { attempts int interval time.Duration @@ -47,6 +48,12 @@ func WithDistributionActiveTimestampTracker(tracker *kv.ActiveTimestampTracker) } } +func WithDistributionReadGate(blocked func() bool) DistributionServerOption { + return func(s *DistributionServer) { + s.readBlocked = blocked + } +} + // WithCatalogReloadRetryPolicy configures the retry policy used after split // commit when waiting for the local catalog snapshot to become visible. func WithCatalogReloadRetryPolicy(attempts int, interval time.Duration) DistributionServerOption { @@ -96,6 +103,20 @@ func NewDistributionServer(e *distribution.Engine, catalog *distribution.Catalog return s } +func (s *DistributionServer) SetReadGate(blocked func() bool) { + if s != nil { + s.readBlocked = blocked + } +} + +func (s *DistributionServer) requireReadReady() error { + if s != nil && s.readBlocked != nil && s.readBlocked() { + //nolint:wrapcheck // Preserve the gRPC status code for startup readers. + return status.Error(codes.Unavailable, "distribution startup has not completed") + } + return nil +} + // UpdateRoute allows updating route information. func (s *DistributionServer) UpdateRoute(start, end []byte, group uint64) { s.engine.UpdateRoute(start, end, group) @@ -103,6 +124,9 @@ func (s *DistributionServer) UpdateRoute(start, end []byte, group uint64) { // GetRoute returns route for a key. func (s *DistributionServer) GetRoute(ctx context.Context, req *pb.GetRouteRequest) (*pb.GetRouteResponse, error) { + if err := s.requireReadReady(); err != nil { + return nil, err + } r, ok := s.engine.GetRoute(req.Key) if !ok { return &pb.GetRouteResponse{}, nil @@ -122,6 +146,9 @@ func (s *DistributionServer) GetTimestamp(ctx context.Context, req *pb.GetTimest // ListRoutes returns all durable routes from catalog storage. func (s *DistributionServer) ListRoutes(ctx context.Context, req *pb.ListRoutesRequest) (*pb.ListRoutesResponse, error) { + if err := s.requireReadReady(); err != nil { + return nil, err + } snapshot, err := s.loadCatalogSnapshot(ctx) if err != nil { return nil, err diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index f7cc59f17..876ef40a5 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -38,6 +38,35 @@ func TestDistributionServerGetRoute_HitAndMiss(t *testing.T) { require.Nil(t, miss.End) } +func TestDistributionServerRouteReadsHonorStartupGate(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + engine.UpdateRoute([]byte("a"), nil, 1) + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + _, err := catalog.Save(context.Background(), 0, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + + blocked := true + s := NewDistributionServer(engine, catalog, WithDistributionReadGate(func() bool { return blocked })) + + _, err = s.GetRoute(context.Background(), &pb.GetRouteRequest{Key: []byte("a")}) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + + _, err = s.ListRoutes(context.Background(), &pb.ListRoutesRequest{}) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + + blocked = false + _, err = s.GetRoute(context.Background(), &pb.GetRouteRequest{Key: []byte("a")}) + require.NoError(t, err) + _, err = s.ListRoutes(context.Background(), &pb.ListRoutesRequest{}) + require.NoError(t, err) +} + func TestDistributionServerGetTimestamp_IsMonotonic(t *testing.T) { t.Parallel() diff --git a/adapter/grpc.go b/adapter/grpc.go index 773dacb93..21f3e1761 100644 --- a/adapter/grpc.go +++ b/adapter/grpc.go @@ -25,6 +25,7 @@ type GRPCServer struct { grpcTranscoder *grpcTranscoder coordinator kv.Coordinator store store.MVCCStore + readBlocked func() bool closeStore bool closeOnce sync.Once @@ -66,6 +67,12 @@ func WithCloseStore() GRPCServerOption { } } +func WithGRPCReadGate(blocked func() bool) GRPCServerOption { + return func(s *GRPCServer) { + s.readBlocked = blocked + } +} + func NewGRPCServer(store store.MVCCStore, coordinate kv.Coordinator, opts ...GRPCServerOption) *GRPCServer { s := &GRPCServer{ log: slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ @@ -84,6 +91,14 @@ func NewGRPCServer(store store.MVCCStore, coordinate kv.Coordinator, opts ...GRP return s } +func (r *GRPCServer) requireReadReady() error { + if r != nil && r.readBlocked != nil && r.readBlocked() { + //nolint:wrapcheck // Preserve the gRPC status code for startup readers. + return status.Error(codes.Unavailable, "startup rotation has not completed") + } + return nil +} + func (r *GRPCServer) Close() error { if r == nil { return nil @@ -107,6 +122,9 @@ func (r *GRPCServer) clock() *kv.HLC { } func (r *GRPCServer) RawGet(ctx context.Context, req *pb.RawGetRequest) (*pb.RawGetResponse, error) { + if err := r.requireReadReady(); err != nil { + return nil, err + } readTS := req.GetTs() if readTS == 0 { readTS = globalSnapshotTS(ctx, r.clock(), r.store) @@ -143,6 +161,9 @@ func (r *GRPCServer) RawGet(ctx context.Context, req *pb.RawGetRequest) (*pb.Raw } func (r *GRPCServer) RawLatestCommitTS(ctx context.Context, req *pb.RawLatestCommitTSRequest) (*pb.RawLatestCommitTSResponse, error) { + if err := r.requireReadReady(); err != nil { + return nil, err + } key := req.GetKey() if len(key) == 0 { // No key: return the store's global last-committed watermark. @@ -173,6 +194,9 @@ func (r *GRPCServer) RawLatestCommitTS(ctx context.Context, req *pb.RawLatestCom } func (r *GRPCServer) RawScanAt(ctx context.Context, req *pb.RawScanAtRequest) (*pb.RawScanAtResponse, error) { + if err := r.requireReadReady(); err != nil { + return nil, err + } limit64 := req.GetLimit() limit, err := rawScanLimit(limit64) if err != nil { @@ -366,6 +390,9 @@ func (r *GRPCServer) Put(ctx context.Context, req *pb.PutRequest) (*pb.PutRespon } func (r *GRPCServer) Get(ctx context.Context, req *pb.GetRequest) (*pb.GetResponse, error) { + if err := r.requireReadReady(); err != nil { + return nil, err + } h := murmur3.New64() if _, err := h.Write(req.Key); err != nil { return nil, errors.WithStack(err) @@ -413,6 +440,9 @@ func (r *GRPCServer) Delete(ctx context.Context, req *pb.DeleteRequest) (*pb.Del } func (r *GRPCServer) Scan(ctx context.Context, req *pb.ScanRequest) (*pb.ScanResponse, error) { + if err := r.requireReadReady(); err != nil { + return nil, err + } limit, err := internal.Uint64ToInt(req.Limit) if err != nil { return &pb.ScanResponse{ diff --git a/adapter/grpc_test.go b/adapter/grpc_test.go index c70818adb..db5b8ce68 100644 --- a/adapter/grpc_test.go +++ b/adapter/grpc_test.go @@ -290,6 +290,36 @@ func TestGRPCServer_RawReadFenceHelpersStampCurrentRouteVersion(t *testing.T) { require.Equal(t, uint64(55), st.scanReadRouteVersion) } +func TestGRPCServer_ReadsHonorStartupGate(t *testing.T) { + t.Parallel() + + blocked := true + s := NewGRPCServer(store.NewMVCCStore(), nil, WithGRPCReadGate(func() bool { return blocked })) + ctx := context.Background() + + _, err := s.RawGet(ctx, &pb.RawGetRequest{Key: []byte("k"), Ts: 10}) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + _, err = s.RawLatestCommitTS(ctx, &pb.RawLatestCommitTSRequest{}) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + _, err = s.RawScanAt(ctx, &pb.RawScanAtRequest{Limit: 10, Ts: 10}) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + _, err = s.Get(ctx, &pb.GetRequest{Key: []byte("k")}) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + _, err = s.Scan(ctx, &pb.ScanRequest{Limit: 10}) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + + blocked = false + _, err = s.RawGet(ctx, &pb.RawGetRequest{Key: []byte("k"), Ts: 10}) + require.NoError(t, err) + _, err = s.Scan(ctx, &pb.ScanRequest{Limit: 10}) + require.NoError(t, err) +} + func TestGRPCServer_RawReadFenceHelpersKeepCallerRouteVersion(t *testing.T) { t.Parallel() diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index d36e7cf8a..65ef0903f 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -85,6 +85,12 @@ const ( // upside is that a ~5 s transient pause (election-timeout scale) // no longer drops heartbeats and forces the peers' lease to expire. defaultHeartbeatBufPerPeer = 512 + // defaultReadIndexRespBufPerPeer sizes the dedicated follower-to-leader + // ReadIndex heartbeat response lane. etcd/raft encodes ReadIndex + // completions as MsgHeartbeatResp messages with Context set; unlike plain + // heartbeat acks, each context is tied to a caller waiting in handleRead + // and must not be coalesced or dropped behind superseded empty acks. + defaultReadIndexRespBufPerPeer = 512 // defaultHeartbeatRespBufPerPeer sizes the dedicated follower-to-leader // heartbeat response lane. When a follower is receiving a large snapshot, // the leader may continue to send heartbeats while the follower's outbound @@ -577,21 +583,23 @@ type dispatchRequest struct { // peerQueues holds separate dispatch channels per peer so that heartbeats // are never blocked behind large log-entry RPCs. // -// Legacy 3-lane layout (default): heartbeat + heartbeatResp + normal. +// Legacy 4-lane layout (default): heartbeat + heartbeatResp + +// readIndexResp + normal. // -// 5-lane layout (opt-in via ELASTICKV_RAFT_DISPATCHER_LANES=1): heartbeat + -// heartbeatResp + replication (MsgApp/MsgAppResp) + snapshot (MsgSnap) + -// other. Each lane gets its own goroutine so a bulky MsgSnap transfer cannot -// stall MsgApp replication and vice versa. Per-peer ordering within a given -// message type is preserved because a single peer's MsgApp stream all share -// one lane and one worker. +// 6-lane layout (opt-in via ELASTICKV_RAFT_DISPATCHER_LANES=1): heartbeat + +// heartbeatResp + readIndexResp + replication (MsgApp/MsgAppResp) + snapshot +// (MsgSnap) + other. Each lane gets its own goroutine so a bulky MsgSnap +// transfer cannot stall MsgApp replication and vice versa. Per-peer ordering +// within a given message type is preserved because a single peer's MsgApp +// stream all share one lane and one worker. type peerQueues struct { normal chan dispatchRequest heartbeat chan dispatchRequest heartbeatResp chan dispatchRequest - replication chan dispatchRequest // 5-lane mode only; nil otherwise - snapshot chan dispatchRequest // 5-lane mode only; nil otherwise - other chan dispatchRequest // 5-lane mode only; nil otherwise + readIndexResp chan dispatchRequest + replication chan dispatchRequest // 6-lane mode only; nil otherwise + snapshot chan dispatchRequest // 6-lane mode only; nil otherwise + other chan dispatchRequest // 6-lane mode only; nil otherwise ctx context.Context cancel context.CancelFunc } @@ -2416,7 +2424,7 @@ func (e *Engine) enqueueDispatchMessage(msg raftpb.Message) error { e.recordDroppedDispatch(msg) return nil } - ch := e.selectDispatchLane(pd, msg.GetType()) + ch := e.selectDispatchLaneForMessage(pd, msg) // Avoid the expensive deep-clone in prepareDispatchRequest when the channel // is already full. The len/cap check is safe here because this function is // only ever called from the single engine event-loop goroutine. @@ -2519,6 +2527,14 @@ func isBlockingInboundStepMsg(t raftpb.MessageType) bool { // opt-in multi-lane layout it additionally partitions the non-heartbeat traffic // so that MsgApp/MsgAppResp and MsgSnap do not share a goroutine and cannot // block each other. +func (e *Engine) selectDispatchLaneForMessage(pd *peerQueues, msg raftpb.Message) chan dispatchRequest { + msgType := msg.GetType() + if msgType == raftpb.MsgHeartbeatResp && len(msg.GetContext()) > 0 && pd.readIndexResp != nil { + return pd.readIndexResp + } + return e.selectDispatchLane(pd, msgType) +} + func (e *Engine) selectDispatchLane(pd *peerQueues, msgType raftpb.MessageType) chan dispatchRequest { if msgType == raftpb.MsgHeartbeatResp && pd.heartbeatResp != nil { return pd.heartbeatResp @@ -4457,24 +4473,25 @@ func (e *Engine) startPeerDispatcher(nodeID uint64) { pd := &peerQueues{ heartbeat: make(chan dispatchRequest, defaultHeartbeatBufPerPeer), heartbeatResp: make(chan dispatchRequest, defaultHeartbeatRespBufPerPeer), + readIndexResp: make(chan dispatchRequest, defaultReadIndexRespBufPerPeer), ctx: ctx, cancel: cancel, } var workers []chan dispatchRequest if e.dispatcherLanesEnabled { - // 5-lane layout: split MsgHeartbeatResp, MsgApp/MsgAppResp - // (replication), MsgSnap (snapshot), and misc (other) onto independent - // goroutines so a bulky snapshot transfer cannot stall replication or - // follower heartbeat responses. Each channel still serves a single - // peer, so within-type ordering (the raft invariant we care about for - // MsgApp) is preserved. + // 6-lane layout: split MsgHeartbeatResp, ReadIndex heartbeat + // responses, MsgApp/MsgAppResp (replication), MsgSnap (snapshot), and + // misc (other) onto independent goroutines so a bulky snapshot transfer + // cannot stall replication or follower heartbeat responses. Each channel + // still serves a single peer, so within-type ordering (the raft invariant + // we care about for MsgApp) is preserved. pd.replication = make(chan dispatchRequest, size) pd.snapshot = make(chan dispatchRequest, defaultSnapshotLaneBufPerPeer) pd.other = make(chan dispatchRequest, defaultOtherLaneBufPerPeer) - workers = []chan dispatchRequest{pd.heartbeat, pd.heartbeatResp, pd.replication, pd.snapshot, pd.other} + workers = []chan dispatchRequest{pd.heartbeat, pd.heartbeatResp, pd.readIndexResp, pd.replication, pd.snapshot, pd.other} } else { pd.normal = make(chan dispatchRequest, size) - workers = []chan dispatchRequest{pd.normal, pd.heartbeat, pd.heartbeatResp} + workers = []chan dispatchRequest{pd.normal, pd.heartbeat, pd.heartbeatResp, pd.readIndexResp} } e.peerDispatchers[nodeID] = pd e.dispatchWG.Add(len(workers)) @@ -4549,7 +4566,7 @@ func dispatcherLanesEnabledFromEnv() bool { // drain loops in runDispatchWorker exit. It is safe to call with any dispatch // layout because unused lanes are nil. func closePeerLanes(pd *peerQueues) { - for _, ch := range []chan dispatchRequest{pd.heartbeat, pd.heartbeatResp, pd.normal, pd.replication, pd.snapshot, pd.other} { + for _, ch := range []chan dispatchRequest{pd.heartbeat, pd.heartbeatResp, pd.readIndexResp, pd.normal, pd.replication, pd.snapshot, pd.other} { if ch != nil { close(ch) } diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 148f1d149..ce81a539d 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -940,6 +940,7 @@ func TestUpsertPeerStartsDispatcherAndAcceptsMessages(t *testing.T) { require.True(t, ok, "dispatcher must be created on upsert") require.Equal(t, defaultHeartbeatBufPerPeer, cap(pd.heartbeat)) require.Equal(t, defaultHeartbeatRespBufPerPeer, cap(pd.heartbeatResp)) + require.Equal(t, defaultReadIndexRespBufPerPeer, cap(pd.readIndexResp)) require.Equal(t, 4, cap(pd.normal)) require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{Type: messageTypePtr(raftpb.MsgHeartbeat), To: uint64Ptr(2)})) @@ -962,6 +963,7 @@ func TestRemovePeerClosesDispatcherAndDropsSubsequentMessages(t *testing.T) { normal: make(chan dispatchRequest, 4), heartbeat: make(chan dispatchRequest, 4), heartbeatResp: make(chan dispatchRequest, 4), + readIndexResp: make(chan dispatchRequest, 4), ctx: ctx, cancel: cancel, } @@ -971,10 +973,11 @@ func TestRemovePeerClosesDispatcherAndDropsSubsequentMessages(t *testing.T) { peerDispatchers: map[uint64]*peerQueues{2: pd}, dispatchStopCh: stopCh, } - engine.dispatchWG.Add(3) + engine.dispatchWG.Add(4) go engine.runDispatchWorker(ctx, pd.normal) go engine.runDispatchWorker(ctx, pd.heartbeat) go engine.runDispatchWorker(ctx, pd.heartbeatResp) + go engine.runDispatchWorker(ctx, pd.readIndexResp) engine.removePeer(2) @@ -1583,6 +1586,44 @@ func TestEnqueueDispatchMessageCoalescesPlainHeartbeatBehindReadIndex(t *testing require.Equal(t, uint64(11), req.msg.GetIndex()) } +func TestEnqueueDispatchMessagePreservesIncomingReadIndexHeartbeatWhenRespLaneFull(t *testing.T) { + t.Parallel() + pd := &peerQueues{ + heartbeat: make(chan dispatchRequest, 1), + heartbeatResp: make(chan dispatchRequest, 2), + readIndexResp: make(chan dispatchRequest, 1), + } + engine := &Engine{ + nodeID: 1, + peerDispatchers: map[uint64]*peerQueues{ + 2: pd, + }, + } + pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Context: []byte("read-index-a"), + }) + pd.heartbeatResp <- prepareDispatchRequest(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Context: []byte("read-index-b"), + }) + + require.NoError(t, engine.enqueueDispatchMessage(raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + To: uint64Ptr(2), + Context: []byte("read-index-c"), + })) + + require.Zero(t, engine.DispatchDropCount()) + require.Len(t, pd.heartbeatResp, 2) + require.Len(t, pd.readIndexResp, 1) + req := <-pd.readIndexResp + require.Equal(t, raftpb.MsgHeartbeatResp, req.msg.GetType()) + require.Equal(t, []byte("read-index-c"), req.msg.Context) +} + func TestMaxAppliedIndexStartsFromSnapshotIndex(t *testing.T) { storage := etcdraft.NewMemoryStorage() snap := raftTestSnapshot(5, 2, []uint64{1}, nil) @@ -2335,6 +2376,7 @@ func TestSelectDispatchLane_LegacyThreeLane(t *testing.T) { normal: make(chan dispatchRequest, 1), heartbeat: make(chan dispatchRequest, 1), heartbeatResp: make(chan dispatchRequest, 1), + readIndexResp: make(chan dispatchRequest, 1), } cases := map[raftpb.MessageType]chan dispatchRequest{ @@ -2355,6 +2397,11 @@ func TestSelectDispatchLane_LegacyThreeLane(t *testing.T) { got := engine.selectDispatchLane(pd, mt) require.Equalf(t, want, got, "legacy mode routing for %s", mt) } + got := engine.selectDispatchLaneForMessage(pd, raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + Context: []byte("read-index"), + }) + require.Equal(t, pd.readIndexResp, got) } // TestSelectDispatchLane_FiveLane verifies that, when ELASTICKV_RAFT_DISPATCHER_LANES @@ -2367,6 +2414,7 @@ func TestSelectDispatchLane_FiveLane(t *testing.T) { pd := &peerQueues{ heartbeat: make(chan dispatchRequest, 1), heartbeatResp: make(chan dispatchRequest, 1), + readIndexResp: make(chan dispatchRequest, 1), replication: make(chan dispatchRequest, 1), snapshot: make(chan dispatchRequest, 1), other: make(chan dispatchRequest, 1), @@ -2390,6 +2438,11 @@ func TestSelectDispatchLane_FiveLane(t *testing.T) { got := engine.selectDispatchLane(pd, mt) require.Equalf(t, want, got, "multi-lane mode routing for %s", mt) } + got := engine.selectDispatchLaneForMessage(pd, raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeatResp), + Context: []byte("read-index"), + }) + require.Equal(t, pd.readIndexResp, got) } // TestSelectDispatchLane_MsgPropReachesDefaultFallback verifies that MsgProp, @@ -2403,6 +2456,7 @@ func TestSelectDispatchLane_MsgPropReachesDefaultFallback(t *testing.T) { pd := &peerQueues{ heartbeat: make(chan dispatchRequest, 1), heartbeatResp: make(chan dispatchRequest, 1), + readIndexResp: make(chan dispatchRequest, 1), replication: make(chan dispatchRequest, 1), snapshot: make(chan dispatchRequest, 1), other: make(chan dispatchRequest, 1), diff --git a/internal/raftengine/etcd/fsm_snapshot_file.go b/internal/raftengine/etcd/fsm_snapshot_file.go index 78ca12ff4..8e31a05d3 100644 --- a/internal/raftengine/etcd/fsm_snapshot_file.go +++ b/internal/raftengine/etcd/fsm_snapshot_file.go @@ -619,6 +619,7 @@ type snapFileCandidate struct { index uint64 restorable bool walValid bool + tokenCRC uint32 } type prewriteSnapshotRetention struct { @@ -655,7 +656,7 @@ func purgeOlderSnapshotPairsBeforeWrite( return candidates[i].index < candidates[j].index }) - retention := keepRestorablePrewriteSnapshots(candidates) + retention := keepVerifiedPrewriteSnapshots(fsmSnapDir, candidates) var combined error combined = errors.CombineErrors(combined, purgeUnretainedPrewriteSnapshots(snapDir, fsmSnapDir, candidates, retention)) combined = errors.CombineErrors(combined, removePrewriteFSMOrphansBeforeIndex( @@ -746,6 +747,69 @@ func keepRestorablePrewriteSnapshots(candidates []snapFileCandidate) prewriteSna return retention } +func keepVerifiedPrewriteSnapshots(fsmSnapDir string, candidates []snapFileCandidate) prewriteSnapshotRetention { + retention := keepRestorablePrewriteSnapshots(candidates) + if fsmSnapDir == "" || len(candidates) <= prewriteSnapKeep { + return retention + } + for { + if !invalidateUnverifiedRetainedPrewriteSnapshots(fsmSnapDir, candidates, retention) { + return retention + } + retention = keepRestorablePrewriteSnapshots(candidates) + if !retentionHasRestorable(candidates, retention) { + return keepAllPrewriteSnapshots(candidates) + } + } +} + +func invalidateUnverifiedRetainedPrewriteSnapshots( + fsmSnapDir string, + candidates []snapFileCandidate, + retention prewriteSnapshotRetention, +) bool { + invalidated := false + for i := range candidates { + candidate := &candidates[i] + if !candidate.restorable || !retention.keep[candidate.name] || !retainedPrewriteSnapshotPrunesOlder(candidates, retention, *candidate) { + continue + } + if verifyFSMSnapshotFileWithToken(fsmSnapPath(fsmSnapDir, candidate.index), candidate.tokenCRC, true) == nil { + continue + } + candidate.restorable = false + invalidated = true + } + return invalidated +} + +func retainedPrewriteSnapshotPrunesOlder(candidates []snapFileCandidate, retention prewriteSnapshotRetention, retained snapFileCandidate) bool { + for _, candidate := range candidates { + if candidate.index >= retained.index || retention.keep[candidate.name] { + continue + } + return true + } + return false +} + +func retentionHasRestorable(candidates []snapFileCandidate, retention prewriteSnapshotRetention) bool { + for _, candidate := range candidates { + if candidate.restorable && retention.keep[candidate.name] { + return true + } + } + return false +} + +func keepAllPrewriteSnapshots(candidates []snapFileCandidate) prewriteSnapshotRetention { + retention := prewriteSnapshotRetention{keep: make(map[string]bool, len(candidates))} + for _, candidate := range candidates { + retention.keep[candidate.name] = true + } + return retention +} + func keepNewestMatchingPrewriteSnapshots( candidates []snapFileCandidate, retention *prewriteSnapshotRetention, @@ -813,28 +877,30 @@ func collectPrewriteSnapCandidates( if index == 0 || index >= nextIndex { continue } + restorable, tokenCRC := fsmSnapshotPairRestorable(snapDir, fsmSnapDir, e.Name(), term, index) candidates = append(candidates, snapFileCandidate{ name: e.Name(), index: index, - restorable: fsmSnapshotPairRestorable(snapDir, fsmSnapDir, e.Name(), term, index), + restorable: restorable, walValid: walValidIndexes == nil || walValidIndexes[walSnapshotKey{term: term, index: index}], + tokenCRC: tokenCRC, }) } return candidates } -func fsmSnapshotPairRestorable(snapDir, fsmSnapDir, snapName string, term, index uint64) bool { +func fsmSnapshotPairRestorable(snapDir, fsmSnapDir, snapName string, term, index uint64) (bool, uint32) { if fsmSnapDir == "" { - return false + return false, 0 } tok, ok := snapshotTokenFromSnapFile(snapDir, snapName, term, index) if !ok { - return false + return false, 0 } // Prewrite cleanup runs on the snapshot receive hot path before gRPC starts // draining payload chunks. Only do a footer/token check here; full-payload // CRC remains in the actual restore/open paths. - return fsmSnapshotFooterMatchesToken(fsmSnapPath(fsmSnapDir, index), tok.CRC32C) == nil + return fsmSnapshotFooterMatchesToken(fsmSnapPath(fsmSnapDir, index), tok.CRC32C) == nil, tok.CRC32C } func fsmSnapshotFooterMatchesToken(path string, tokenCRC uint32) error { diff --git a/internal/raftengine/etcd/fsm_snapshot_file_test.go b/internal/raftengine/etcd/fsm_snapshot_file_test.go index f5b93f83a..60cc33344 100644 --- a/internal/raftengine/etcd/fsm_snapshot_file_test.go +++ b/internal/raftengine/etcd/fsm_snapshot_file_test.go @@ -460,16 +460,40 @@ func TestPrewriteRestorableCheckUsesSnapshotFooterOnly(t *testing.T) { require.NoError(t, err) require.NoError(t, f.Close()) - require.True(t, fsmSnapshotPairRestorable( + restorable, _ := fsmSnapshotPairRestorable( snapDir, fsmSnapDir, "0000000000000001-0000000000000064.snap", 1, 100, - )) + ) + require.True(t, restorable) require.ErrorIs(t, verifyFSMSnapshotFile(path, crc), ErrFSMSnapshotFileCRC) } +func TestPrepareFSMSnapshotWriteVerifiesRetainedFooterOnlyFallbackBeforePruning(t *testing.T) { + snapDir := t.TempDir() + fsmSnapDir := t.TempDir() + + crc100, _ := writeFSMFileForTest(t, fsmSnapDir, 100, []byte("valid fallback")) + createTokenSnapFileWithTerm(t, snapDir, 1, 100, crc100) + crc200, path200 := writeFSMFileForTest(t, fsmSnapDir, 200, []byte("corrupt newest fallback")) + createTokenSnapFileWithTerm(t, snapDir, 1, 200, crc200) + + f, err := os.OpenFile(path200, os.O_WRONLY, 0) + require.NoError(t, err) + _, err = f.WriteAt([]byte("X"), 0) + require.NoError(t, err) + require.NoError(t, f.Close()) + + require.NoError(t, prepareFSMSnapshotWrite(snapDir, fsmSnapDir, 300)) + + require.FileExists(t, filepath.Join(snapDir, "0000000000000001-0000000000000064.snap")) + require.FileExists(t, fsmSnapPath(fsmSnapDir, 100)) + require.NoFileExists(t, filepath.Join(snapDir, "0000000000000001-00000000000000c8.snap")) + require.NoFileExists(t, fsmSnapPath(fsmSnapDir, 200)) +} + func TestPrepareFSMSnapshotWriteKeepsWALValidAndRestorableFallbacksWhenWALValidCandidateIsBroken(t *testing.T) { snapDir := t.TempDir() fsmSnapDir := t.TempDir() diff --git a/internal/raftengine/etcd/snapshot_spool.go b/internal/raftengine/etcd/snapshot_spool.go index cb1b3bc8e..678d7857a 100644 --- a/internal/raftengine/etcd/snapshot_spool.go +++ b/internal/raftengine/etcd/snapshot_spool.go @@ -33,6 +33,8 @@ const maxSnapshotPayloadBytesEnvVar = "ELASTICKV_RAFT_MAX_SNAPSHOT_PAYLOAD_BYTES const snapshotSpoolMinFreeBytesEnvVar = "ELASTICKV_RAFT_SNAPSHOT_SPOOL_MIN_FREE_BYTES" +const defaultReceiveSnapshotSpoolMinFreeBytes int64 = 1 << 30 // 1 GiB + // resolveMaxSnapshotPayloadBytes evaluates the env override once per spool // creation. Snapshots are infrequent enough that one Getenv + ParseInt per // spool is invisible in profiles, and resolving at construction means tests @@ -94,11 +96,13 @@ func newSnapshotSpool(dir string) (*snapshotSpool, error) { func newReceiveSnapshotSpool(dir string) (*snapshotSpool, error) { maxSize := resolveMaxSnapshotPayloadBytes() - // Keep one full max-size snapshot worth of headroom after receive-side - // spooling. Applying a token snapshot restores the FSM from the completed - // .fsm into a new local store directory, so a node can transiently need old - // snapshot + incoming .fsm + restored store bytes. - return newSnapshotSpoolWithLimits(dir, maxSize, resolveSnapshotSpoolMinFreeBytes(maxSize)) + // Keep a fixed emergency reserve after receive-side spooling. Tying this + // reserve to the configured maximum snapshot size made the default 16 GiB + // cap also require 16 GiB of free space after every chunk; production nodes + // with enough space for a real 13 GiB FSM snapshot were rejecting the stream + // around 4 GiB and retrying forever. Operators that restore into a layout + // needing a larger reserve can still raise it with the env knob. + return newSnapshotSpoolWithLimits(dir, maxSize, resolveSnapshotSpoolMinFreeBytes(defaultReceiveSnapshotSpoolMinFreeBytes)) } func newSnapshotSpoolWithLimits(dir string, maxSize, minFreeBytes int64) (*snapshotSpool, error) { diff --git a/internal/raftengine/etcd/snapshot_spool_test.go b/internal/raftengine/etcd/snapshot_spool_test.go index bed5fd223..b0facd96d 100644 --- a/internal/raftengine/etcd/snapshot_spool_test.go +++ b/internal/raftengine/etcd/snapshot_spool_test.go @@ -97,7 +97,7 @@ func TestSnapshotSpool_OverrideInvalidFallsBack(t *testing.T) { require.Equal(t, defaultMaxSnapshotPayloadBytes, spool.maxSize) } -func TestSnapshotSpoolDefaultReserveTracksPayloadCap(t *testing.T) { +func TestSnapshotSpoolDefaultReserveUsesFixedEmergencyHeadroom(t *testing.T) { const spoolCap = int64(4096) t.Setenv(maxSnapshotPayloadBytesEnvVar, strconv.FormatInt(spoolCap, 10)) @@ -106,7 +106,7 @@ func TestSnapshotSpoolDefaultReserveTracksPayloadCap(t *testing.T) { t.Cleanup(func() { _ = spool.Close() }) require.Equal(t, spoolCap, spool.maxSize) - require.Equal(t, spoolCap, spool.minFreeBytes) + require.Equal(t, defaultReceiveSnapshotSpoolMinFreeBytes, spool.minFreeBytes) } func TestSnapshotSpoolMaterializeDoesNotReserveDiskHeadroom(t *testing.T) { diff --git a/kv/fsm.go b/kv/fsm.go index 1e1b3fb17..b2e74b1db 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -546,6 +546,9 @@ func routePrefixRange(prefix []byte) ([]byte, []byte) { if start, ok := s3keys.BucketGenerationRoutePrefixForCleanupPrefix(prefix); ok { return start, prefixScanEnd(start) } + if start, ok := dynamoExactCleanupRouteKey(prefix); ok { + return start, routePointRangeEnd(start) + } if routeKeyspaceWideRawPrefix(prefix) { return []byte(""), nil } @@ -553,6 +556,29 @@ func routePrefixRange(prefix []byte) ([]byte, []byte) { return start, prefixScanEnd(start) } +func dynamoExactCleanupRouteKey(prefix []byte) ([]byte, bool) { + switch { + case bytes.HasPrefix(prefix, dynamoTableMetaPrefixBytes), + bytes.HasPrefix(prefix, dynamoTableGenerationPrefixBytes), + bytes.HasPrefix(prefix, dynamoItemPrefixBytes), + bytes.HasPrefix(prefix, dynamoGSIPrefixBytes): + default: + return nil, false + } + start := routeKey(prefix) + if len(start) == 0 || bytes.Equal(start, prefix) || !bytes.HasPrefix(start, dynamoRoutePrefixBytes) { + return nil, false + } + return start, true +} + +func routePointRangeEnd(start []byte) []byte { + end := make([]byte, 0, len(start)+1) + end = append(end, start...) + end = append(end, 0) + return end +} + func routeKeyspaceWideRawPrefix(prefix []byte) bool { if !rawPrefixMayContainRouteMappedKeys(prefix) { return false diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 817a1d809..6f6f2a653 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -306,7 +306,7 @@ func TestRoutePrefixRangeTreatsBroadMappedPrefixesAsFullKeyspace(t *testing.T) { name: "dynamo table cleanup prefix", prefix: []byte(DynamoItemPrefix + tableSegment + "|7|"), wantStart: dynamoTableRoute, - wantEnd: prefixScanEnd(dynamoTableRoute), + wantEnd: routePointRangeEnd(dynamoTableRoute), }, { name: "broad redis namespace", @@ -343,6 +343,32 @@ func TestRoutePrefixRangeTreatsBroadMappedPrefixesAsFullKeyspace(t *testing.T) { } } +func TestRoutePrefixRangeTreatsDynamoCleanupAsExactRouteKey(t *testing.T) { + t.Parallel() + + fooSegment := base64.RawURLEncoding.EncodeToString([]byte("foo")) + foobarSegment := base64.RawURLEncoding.EncodeToString([]byte("foobar")) + fooRoute := dynamoRouteTableKey([]byte(fooSegment)) + foobarRoute := dynamoRouteTableKey([]byte(foobarSegment)) + + start, end := routePrefixRange([]byte(DynamoItemPrefix + fooSegment + "|7|")) + + require.Equal(t, fooRoute, start) + require.Equal(t, routePointRangeEnd(fooRoute), end) + require.False(t, rangesIntersectForTest(start, end, foobarRoute, prefixScanEnd(foobarRoute)), + "cleanup for table foo must not intersect table foobar's route") +} + +func rangesIntersectForTest(aStart, aEnd, bStart, bEnd []byte) bool { + if aEnd != nil && bytes.Compare(aEnd, bStart) <= 0 { + return false + } + if bEnd != nil && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true +} + func TestRouteKeyFilterTreatsNilAndEmptyEndAsInfinity(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 147abc067..73420bb86 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1007,11 +1007,42 @@ func (c *ShardedCoordinator) dispatchRawWithComposed1Retry(ctx context.Context, if !errors.Is(err, ErrComposed1VersionGCd) || attempt == composed1RetryAttempts || c.engine == nil { return resp, err } + if !c.canRetryRawVersionGC(reqs) { + return resp, err + } reqs.ObservedRouteVersion = c.engine.Version() } return nil, errors.WithStack(ErrInvalidRequest) } +func (c *ShardedCoordinator) canRetryRawVersionGC(reqs *OperationGroup[OP]) bool { + if c == nil || c.router == nil || reqs == nil || hasDelPrefixElem(reqs.Elems) { + return false + } + var ( + firstGID uint64 + seen bool + ) + for _, elem := range reqs.Elems { + if elem == nil { + return false + } + gid, ok := c.router.ResolveGroup(elem.Key) + if !ok { + return false + } + if !seen { + firstGID = gid + seen = true + continue + } + if gid != firstGID { + return false + } + } + return seen +} + // hasDelPrefixElem returns true if any element is a DelPrefix operation. func hasDelPrefixElem(elems []*Elem[OP]) bool { for _, e := range elems { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index d5f8632d3..cd314eef6 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -195,7 +195,7 @@ func TestShardedCoordinator_RetriesRawWriteWhenObservedRouteVersionGCd(t *testin require.Equal(t, uint64(9), txn.requests[1].GetObservedRouteVersion()) } -func TestShardedCoordinator_RetriesDelPrefixWhenObservedRouteVersionGCd(t *testing.T) { +func TestShardedCoordinator_DoesNotRetryDelPrefixWhenObservedRouteVersionGCd(t *testing.T) { t.Parallel() engine := distribution.NewEngine() @@ -216,10 +216,87 @@ func TestShardedCoordinator_RetriesDelPrefixWhenObservedRouteVersionGCd(t *testi ObservedRouteVersion: 2, Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("user:")}}, }) - require.NoError(t, err) - require.Len(t, txn.requests, 2) + require.ErrorIs(t, err, ErrComposed1VersionGCd) + require.Len(t, txn.requests, 1) require.Equal(t, uint64(2), txn.requests[0].GetObservedRouteVersion()) - require.Equal(t, uint64(7), txn.requests[1].GetObservedRouteVersion()) +} + +func TestShardedCoordinator_DoesNotRetryMultiShardRawWriteWhenObservedRouteVersionGCd(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 11, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{errs: []error{ErrComposed1VersionGCd}} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + ObservedRouteVersion: 3, + Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("a"), Value: []byte("v1")}, + {Op: Put, Key: []byte("z"), Value: []byte("v2")}, + }, + }) + + require.ErrorIs(t, err, ErrComposed1VersionGCd) + require.Len(t, g1Txn.requests, 1) + require.Len(t, g2Txn.requests, 1) + require.Equal(t, uint64(3), g1Txn.requests[0].GetObservedRouteVersion()) + require.Equal(t, uint64(3), g2Txn.requests[0].GetObservedRouteVersion()) +} + +func TestShardedCoordinator_DoesNotRetryRawWriteWhenRetryRouteBecomesMultiShard(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 11, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + g1Txn := &recordingTransactional{ + errs: []error{ErrComposed1VersionGCd}, + onCommit: func(call int, _ *pb.Request) { + if call != 0 { + return + } + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 12, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + }, + } + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + ObservedRouteVersion: 3, + Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("a"), Value: []byte("v1")}, + {Op: Put, Key: []byte("z"), Value: []byte("v2")}, + }, + }) + + require.ErrorIs(t, err, ErrComposed1VersionGCd) + require.Len(t, g1Txn.requests, 1) + require.Len(t, g2Txn.requests, 0) + require.Equal(t, uint64(3), g1Txn.requests[0].GetObservedRouteVersion()) } func TestShardedCoordinatorRejectsPointWriteOnWriteFencedRoute(t *testing.T) { diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 964f2713f..f3e29b377 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -19,6 +19,7 @@ type recordingTransactional struct { requests []*pb.Request responses []*TransactionResponse errs []error + onCommit func(call int, req *pb.Request) } func (s *recordingTransactional) Commit(_ context.Context, reqs []*pb.Request) (*TransactionResponse, error) { @@ -30,6 +31,9 @@ func (s *recordingTransactional) Commit(_ context.Context, reqs []*pb.Request) ( } s.requests = append(s.requests, cloneTxnRequest(reqs[0])) call := len(s.requests) - 1 + if s.onCommit != nil { + s.onCommit(call, s.requests[call]) + } if call < len(s.errs) && s.errs[call] != nil { return nil, s.errs[call] } diff --git a/main.go b/main.go index a630690d0..734436134 100644 --- a/main.go +++ b/main.go @@ -1564,6 +1564,9 @@ func prepareRuntimeServerRunner(waitRotateOnStartup startupRotationWaiter, in se }) } publicKVGate := &startupPublicKVGate{} + if in.distServer != nil { + in.distServer.SetReadGate(publicKVGate.blocked) + } installHLCLeaseRenewalBlocker(in.coordinate, waitRotateOnStartup.BlockMutators) adapterCoordinate := startupGatedCoordinator{ inner: in.coordinate, @@ -2185,6 +2188,7 @@ func startRaftServers( shardGroups map[uint64]*kv.ShardGroup, shardStore *kv.ShardStore, coordinate kv.Coordinator, + readGate func() bool, distServer *adapter.DistributionServer, relay *adapter.RedisPubSubRelay, proposalObserverForGroup func(uint64) kv.ProposalObserver, @@ -2218,7 +2222,7 @@ func startRaftServers( } gs := grpc.NewServer(opts...) trx := kv.NewTransactionWithProposer(proposerForGroup(rt, shardGroups), kv.WithProposalObserver(observerForGroup(proposalObserverForGroup, rt.spec.id))) - grpcSvc := adapter.NewGRPCServer(shardStore, coordinate) + grpcSvc := adapter.NewGRPCServer(shardStore, coordinate, adapter.WithGRPCReadGate(readGate)) pb.RegisterRawKVServer(gs, grpcSvc) pb.RegisterTransactionalKVServer(gs, grpcSvc) pb.RegisterInternalServer(gs, adapter.NewInternalWithEngine( @@ -2658,8 +2662,10 @@ func (r *runtimeServerRunner) startRaftTransport() error { return r.startupFailure(err) } adminGRPCOpts := r.adminGRPCOpts + var readGate func() bool if r.publicKVGate != nil { adminGRPCOpts.unary = append(adminGRPCOpts.unary, r.publicKVGate.unaryInterceptor) + readGate = r.publicKVGate.blocked } forwardDeps := adminForwardServerDeps{ tables: newDynamoTablesSource(r.dynamoServer), @@ -2674,6 +2680,7 @@ func (r *runtimeServerRunner) startRaftTransport() error { r.shardGroups, r.shardStore, r.coordinate, + readGate, r.distServer, r.pubsubRelay, func(groupID uint64) kv.ProposalObserver { From a6d7d7609c8cb502773ca293ea4964f6640f373b Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 09:26:39 +0900 Subject: [PATCH 099/162] Reset stale raft peer connections --- internal/raftengine/etcd/grpc_transport.go | 22 ++++++++++++-- .../raftengine/etcd/grpc_transport_test.go | 29 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/internal/raftengine/etcd/grpc_transport.go b/internal/raftengine/etcd/grpc_transport.go index 32d82e83e..50ea58460 100644 --- a/internal/raftengine/etcd/grpc_transport.go +++ b/internal/raftengine/etcd/grpc_transport.go @@ -533,7 +533,9 @@ func (t *GRPCTransport) dispatchRegular(ctx context.Context, msg raftpb.Message) } req := &pb.EtcdRaftMessage{Message: raw} if isPriorityMsg(msg.GetType()) || !t.sendStreamEnabledNow() || !t.allowPeerStreamProbe(peer.Address, time.Now()) { - return t.dispatchRegularUnary(ctx, client, req) + err := t.dispatchRegularUnary(ctx, client, req) + t.closePeerConnOnRetryableDialError(peer.Address, err) + return err } err = t.dispatchRegularStream(ctx, peer.Address, client, req) if err == nil { @@ -541,11 +543,16 @@ func (t *GRPCTransport) dispatchRegular(ctx context.Context, msg raftpb.Message) } if grpcStatusCode(err) == codes.Unimplemented { t.markPeerStreamUnsupported(peer.Address) - return t.dispatchRegularUnary(ctx, client, req) + err := t.dispatchRegularUnary(ctx, client, req) + t.closePeerConnOnRetryableDialError(peer.Address, err) + return err } if isSendStreamDisabled(err) { - return t.dispatchRegularUnary(ctx, client, req) + err := t.dispatchRegularUnary(ctx, client, req) + t.closePeerConnOnRetryableDialError(peer.Address, err) + return err } + t.closePeerConnOnRetryableDialError(peer.Address, err) return errors.WithStack(err) } @@ -554,6 +561,15 @@ func (t *GRPCTransport) dispatchRegularUnary(ctx context.Context, client pb.Etcd return errors.WithStack(err) } +func (t *GRPCTransport) closePeerConnOnRetryableDialError(address string, err error) { + if grpcStatusCode(err) != codes.Unavailable { + return + } + t.mu.Lock() + defer t.mu.Unlock() + t.closePeerConnLocked(address) +} + func (t *GRPCTransport) dispatchRegularStream(ctx context.Context, address string, client pb.EtcdRaftClient, req *pb.EtcdRaftMessage) error { stream, err := t.streamFor(ctx, address, client) if err != nil { diff --git a/internal/raftengine/etcd/grpc_transport_test.go b/internal/raftengine/etcd/grpc_transport_test.go index 02ffcb97f..fac1feb9b 100644 --- a/internal/raftengine/etcd/grpc_transport_test.go +++ b/internal/raftengine/etcd/grpc_transport_test.go @@ -19,6 +19,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" ) @@ -908,6 +909,30 @@ func TestDispatchRegularUsesUnaryForPriorityMessages(t *testing.T) { require.Zero(t, client.sendStreamCalls.Load()) } +func TestDispatchRegularDropsCachedPeerConnAfterUnaryUnavailable(t *testing.T) { + const addr = "host:2" + transport := NewGRPCTransport([]Peer{{NodeID: 2, Address: addr}}) + t.Cleanup(func() { require.NoError(t, transport.Close()) }) + client := &testEtcdRaftClient{sendErr: status.Error(codes.Unavailable, "connection refused")} + injectClient(t, transport, addr, client) + + err := transport.dispatchRegular(context.Background(), raftpb.Message{ + Type: messageTypePtr(raftpb.MsgHeartbeat), + From: uint64Ptr(1), + To: uint64Ptr(2), + Term: uint64Ptr(4), + Commit: uint64Ptr(22), + }) + require.Error(t, err) + require.Equal(t, codes.Unavailable, grpcStatusCode(err)) + + transport.mu.RLock() + _, cached := transport.clients[addr] + transport.mu.RUnlock() + require.False(t, cached) + require.Equal(t, int32(1), client.sendCalls.Load()) +} + func TestDispatchRegularUsesUnaryWhenSendStreamDisabled(t *testing.T) { t.Setenv(sendStreamEnabledEnvVar, "false") const addr = "host:2" @@ -1522,12 +1547,16 @@ type testEtcdRaftClient struct { blockSendStreamUntilContext bool sendStreamStarted chan struct{} releaseSendStream chan struct{} + sendErr error sendCalls atomic.Int32 sendStreamCalls atomic.Int32 } func (c *testEtcdRaftClient) Send(_ context.Context, _ *pb.EtcdRaftMessage, _ ...grpc.CallOption) (*pb.EtcdRaftAck, error) { c.sendCalls.Add(1) + if c.sendErr != nil { + return nil, c.sendErr + } return &pb.EtcdRaftAck{}, nil } From c8f865ea84ca03de6d96a604494b5902e974af19 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 10:04:49 +0900 Subject: [PATCH 100/162] Stabilize snapshot catch-up and fence routing --- internal/raftengine/etcd/engine.go | 3 + .../etcd/engine_applied_index_test.go | 12 ++ internal/raftengine/etcd/grpc_transport.go | 121 ++++++++++++++++-- .../raftengine/etcd/grpc_transport_test.go | 52 ++++++++ kv/coordinator.go | 7 +- kv/fsm.go | 24 +++- kv/fsm_migration_fence_test.go | 28 ++++ kv/sharded_coordinator.go | 52 +++++++- kv/sharded_coordinator_partition_test.go | 43 +++++++ proto/internal.pb.go | 17 ++- proto/internal.proto | 5 + 11 files changed, 341 insertions(+), 23 deletions(-) diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 65ef0903f..a273f5e8d 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -3386,6 +3386,9 @@ func (e *Engine) protectReceivedFSMSnapshot(index uint64) bool { if e.protectedReceivedFSMSnaps == nil { e.protectedReceivedFSMSnaps = make(map[uint64]int, 1) } + if e.protectedReceivedFSMSnaps[index] > 0 { + return false + } e.protectedReceivedFSMSnaps[index]++ return true } diff --git a/internal/raftengine/etcd/engine_applied_index_test.go b/internal/raftengine/etcd/engine_applied_index_test.go index 8ebb2ecfe..cf4596306 100644 --- a/internal/raftengine/etcd/engine_applied_index_test.go +++ b/internal/raftengine/etcd/engine_applied_index_test.go @@ -208,6 +208,18 @@ func TestProtectReceivedFSMSnapshotWaitsOnSnapshotMuForAlreadyAppliedIndex(t *te require.Empty(t, e.protectedReceivedFSMSnaps) } +func TestProtectReceivedFSMSnapshotRejectsDuplicateInFlightIndex(t *testing.T) { + e := &Engine{} + + require.True(t, e.protectReceivedFSMSnapshot(9)) + require.False(t, e.protectReceivedFSMSnapshot(9)) + require.Equal(t, map[uint64]int{9: 1}, e.protectedReceivedFSMSnaps) + + e.unprotectReceivedFSMSnapshot(9) + require.Empty(t, e.protectedReceivedFSMSnaps) + require.True(t, e.protectReceivedFSMSnapshot(9)) +} + func TestUnprotectReceivedFSMSnapshotTokenIfApplied(t *testing.T) { e := &Engine{ protectedReceivedFSMSnaps: map[uint64]int{9: 1}, diff --git a/internal/raftengine/etcd/grpc_transport.go b/internal/raftengine/etcd/grpc_transport.go index 50ea58460..29f67129f 100644 --- a/internal/raftengine/etcd/grpc_transport.go +++ b/internal/raftengine/etcd/grpc_transport.go @@ -41,6 +41,7 @@ var ( errSnapshotMetadataDuplicate = errors.New("etcd raft snapshot metadata was sent more than once") errSnapshotMessageNil = errors.New("etcd raft snapshot message is required") errSnapshotStreamShort = errors.New("etcd raft snapshot stream closed before final chunk") + errSnapshotDispatchBusy = errors.New("etcd raft snapshot dispatch already in progress") errReceivedFSMSnapshotStale = errors.New("etcd raft received fsm snapshot is stale") errPeerStreamClosed = errors.New("etcd raft SendStream closed") errSendStreamDisabled = errors.New("etcd raft SendStream is disabled") @@ -132,7 +133,8 @@ type GRPCTransport struct { // bridgeSem limits concurrent bridge-mode snapshot materializations so // that aggregate in-memory allocation stays bounded even when multiple // dispatch workers run simultaneously. - bridgeSem chan struct{} + bridgeSem chan struct{} + snapshotSendSem chan struct{} } func NewGRPCTransport(peers []Peer) *GRPCTransport { @@ -167,6 +169,7 @@ func NewGRPCTransport(peers []Peer) *GRPCTransport { sendStreamCancel: sendStreamCancel, snapshotChunkSize: defaultSnapshotChunkSize, bridgeSem: make(chan struct{}, defaultBridgeMaterializeLimit), + snapshotSendSem: make(chan struct{}, 1), } } @@ -389,6 +392,10 @@ func isSnapshotMsg(msg raftpb.Message) bool { func (t *GRPCTransport) dispatchSnapshot(ctx context.Context, msg raftpb.Message) error { ctx, cancel := transportContext(ctx, defaultSnapshotDispatchTimeout) defer cancel() + if !t.tryAcquireSnapshotSend() { + return errors.WithStack(errors.Mark(status.Error(codes.ResourceExhausted, errSnapshotDispatchBusy.Error()), errSnapshotDispatchBusy)) + } + defer t.releaseSnapshotSend() // Prefer streaming when the snapshot holds a token and an opener is wired. // This avoids materialising the full FSM payload in memory on the sender. @@ -413,6 +420,28 @@ func (t *GRPCTransport) dispatchSnapshot(ctx context.Context, msg raftpb.Message return t.sendSnapshot(ctx, patched) } +func (t *GRPCTransport) tryAcquireSnapshotSend() bool { + if t == nil || t.snapshotSendSem == nil { + return true + } + select { + case t.snapshotSendSem <- struct{}{}: + return true + default: + return false + } +} + +func (t *GRPCTransport) releaseSnapshotSend() { + if t == nil || t.snapshotSendSem == nil { + return + } + select { + case <-t.snapshotSendSem: + default: + } +} + // streamFSMSnapshot streams the .fsm payload file directly to the peer using // chunked gRPC without loading the full content into memory. func (t *GRPCTransport) streamFSMSnapshot(ctx context.Context, msg raftpb.Message, index uint64, openFn func(uint64) (io.ReadCloser, error)) error { @@ -1247,6 +1276,11 @@ func (t *GRPCTransport) receiveSnapshotStream(stream pb.EtcdRaft_SendSnapshotSer if err != nil { return raftpb.Message{}, err } + protection, err := protectReceivedSnapshotMetadata(metadata, fsmSnapDir, protectFn, unprotectFn) + if err != nil { + return raftpb.Message{}, err + } + defer protection.releaseUnlessHandedOff() spool, err := newReceiveSnapshotSpool(spoolPlacement) if err != nil { return raftpb.Message{}, err @@ -1275,10 +1309,12 @@ func (t *GRPCTransport) receiveSnapshotStream(stream pb.EtcdRaft_SendSnapshotSer metadata, firstPayloadChunk, preparedFSMWrite, + protection.protected, ) if err != nil { return raftpb.Message{}, err } + protection.handoff() index := uint64(0) if msg.Snapshot != nil { index = msg.Snapshot.GetMetadata().GetIndex() @@ -1292,6 +1328,50 @@ func (t *GRPCTransport) receiveSnapshotStream(stream pb.EtcdRaft_SendSnapshotSer return msg, nil } +type receivedSnapshotProtection struct { + index uint64 + protected bool + handedOff bool + unprotectFn func(uint64) +} + +func protectReceivedSnapshotMetadata( + metadata raftpb.Message, + fsmSnapDir string, + protectFn func(uint64) bool, + unprotectFn func(uint64), +) (receivedSnapshotProtection, error) { + if fsmSnapDir == "" || metadata.Snapshot == nil { + return receivedSnapshotProtection{}, nil + } + index := metadata.Snapshot.GetMetadata().GetIndex() + if index == 0 || protectFn == nil { + return receivedSnapshotProtection{}, nil + } + if !protectFn(index) { + return receivedSnapshotProtection{}, errors.WithStack(errReceivedFSMSnapshotStale) + } + return receivedSnapshotProtection{ + index: index, + protected: true, + unprotectFn: unprotectFn, + }, nil +} + +func (p *receivedSnapshotProtection) handoff() { + if p == nil { + return + } + p.handedOff = true +} + +func (p *receivedSnapshotProtection) releaseUnlessHandedOff() { + if p == nil || !p.protected || p.handedOff || p.unprotectFn == nil { + return + } + p.unprotectFn(p.index) +} + // drainSnapshotChunks consumes the SendSnapshot stream into spool, computes // CRC32C over the payload bytes as they hit disk, and on the final chunk // hands off to finalizeReceivedSnapshot — which decides between the @@ -1308,7 +1388,7 @@ func drainSnapshotChunks( unprotectFn func(uint64), ) (raftpb.Message, int64, error) { var metadata raftpb.Message - return drainSnapshotChunksFrom(stream, spool, fsmSnapDir, prepareFn, protectFn, unprotectFn, metadata, nil, false) + return drainSnapshotChunksFrom(stream, spool, fsmSnapDir, prepareFn, protectFn, unprotectFn, metadata, nil, false, false) } func drainSnapshotChunksFrom( @@ -1321,6 +1401,7 @@ func drainSnapshotChunksFrom( metadata raftpb.Message, firstPayloadChunk *pb.EtcdRaftSnapshotChunk, preparedFSMWrite bool, + preprotected bool, ) (raftpb.Message, int64, error) { seenMetadata := metadata.Snapshot != nil // Wrap spool with crc32CWriter so the CRC accumulates as bytes hit @@ -1353,7 +1434,7 @@ func drainSnapshotChunksFrom( } payloadBytes += int64(len(chunk.Chunk)) if chunk.Final { - msg, err := finalizeReceivedSnapshot(metadata, spool, crcWriter.Sum32(), fsmSnapDir, protectFn, unprotectFn, seenMetadata) + msg, err := finalizeReceivedSnapshot(metadata, spool, crcWriter.Sum32(), fsmSnapDir, protectFn, unprotectFn, seenMetadata, preprotected) if err != nil { return raftpb.Message{}, 0, err } @@ -1436,6 +1517,7 @@ func finalizeReceivedSnapshot( protectFn func(uint64) bool, unprotectFn func(uint64), seenMetadata bool, + preprotected bool, ) (raftpb.Message, error) { if !seenMetadata || metadata.Snapshot == nil { return raftpb.Message{}, errors.WithStack(errSnapshotMetadataNil) @@ -1447,23 +1529,38 @@ func finalizeReceivedSnapshot( // rename to). return buildSnapshotMessage(metadata, spool, seenMetadata) } - protected := false - if protectFn != nil { - if !protectFn(index) { - return raftpb.Message{}, errors.WithStack(errReceivedFSMSnapshotStale) - } - protected = true + protected, err := protectReceivedSnapshotForFinalize(index, preprotected, protectFn) + if err != nil { + return raftpb.Message{}, err } if err := spool.FinalizeAsFSMFile(fsmSnapDir, index, crc32c); err != nil { - if protected && unprotectFn != nil { - unprotectFn(index) - } + releaseFinalizeSnapshotProtection(index, protected, preprotected, unprotectFn) return raftpb.Message{}, err } metadata.Snapshot.Data = encodeSnapshotToken(index, crc32c) return metadata, nil } +func protectReceivedSnapshotForFinalize(index uint64, preprotected bool, protectFn func(uint64) bool) (bool, error) { + if preprotected { + return true, nil + } + if protectFn == nil { + return false, nil + } + if !protectFn(index) { + return false, errors.WithStack(errReceivedFSMSnapshotStale) + } + return true, nil +} + +func releaseFinalizeSnapshotProtection(index uint64, protected bool, preprotected bool, unprotectFn func(uint64)) { + if !protected || preprotected || unprotectFn == nil { + return + } + unprotectFn(index) +} + func maybePrepareReceivedFSMSnapshotWrite( metadata raftpb.Message, fsmSnapDir string, diff --git a/internal/raftengine/etcd/grpc_transport_test.go b/internal/raftengine/etcd/grpc_transport_test.go index fac1feb9b..b7e43c703 100644 --- a/internal/raftengine/etcd/grpc_transport_test.go +++ b/internal/raftengine/etcd/grpc_transport_test.go @@ -125,6 +125,58 @@ func TestReceiveSnapshotStreamRejectsDuplicateMetadata(t *testing.T) { require.True(t, errors.Is(err, errSnapshotMetadataDuplicate)) } +func TestSnapshotSendGateAllowsOnlyOneInFlightSnapshot(t *testing.T) { + transport := NewGRPCTransport(nil) + + require.True(t, transport.tryAcquireSnapshotSend()) + require.False(t, transport.tryAcquireSnapshotSend()) + + transport.releaseSnapshotSend() + require.True(t, transport.tryAcquireSnapshotSend()) + transport.releaseSnapshotSend() +} + +func TestReceiveSnapshotStreamRejectsProtectedIndexBeforePayload(t *testing.T) { + const index = uint64(127) + metadata := raftpb.Message{ + Type: messageTypePtr(raftpb.MsgSnap), + From: uint64Ptr(1), + To: uint64Ptr(2), + Snapshot: &raftpb.Snapshot{ + Metadata: testSnapshotMetadata(index, 1, nil), + }, + } + raw, err := proto.Marshal(&metadata) + require.NoError(t, err) + + transport := NewGRPCTransport(nil) + fsmSnapDir := t.TempDir() + transport.SetFSMSnapDir(fsmSnapDir) + var protected []uint64 + transport.SetFSMSnapshotProtection( + func(got uint64) bool { + protected = append(protected, got) + return false + }, + func(uint64) { + t.Fatal("stale snapshot rejection must not unprotect an index it did not protect") + }, + ) + stream := &testSendSnapshotServer{ + chunks: []*pb.EtcdRaftSnapshotChunk{ + {Metadata: raw}, + {Chunk: []byte("payload that must not be read"), Final: true}, + }, + } + + _, err = transport.receiveSnapshotStream(stream) + require.Error(t, err) + require.True(t, errors.Is(err, errReceivedFSMSnapshotStale)) + require.Equal(t, []uint64{index}, protected) + require.Equal(t, 1, stream.index, "receiver must reject after metadata before reading payload chunks") + require.NoFileExists(t, fsmSnapPath(fsmSnapDir, index)) +} + // TestReceiveSnapshotStream_StreamingTokenWhenFSMSnapDirSet pins the // memory-safety win: when the receive transport has fsmSnapDir wired, // the spool file is renamed to fsmSnapPath(...) and Snapshot.Data is diff --git a/kv/coordinator.go b/kv/coordinator.go index 174c3b636..0c1567a26 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -1088,7 +1088,7 @@ func (c *Coordinate) dispatchTxn(ctx context.Context, reqs []*Elem[OP], readKeys // carries the option-2 one-phase dedup probe key for a retry that reuses // a failed attempt's write set. r, err := c.transactionManager.Commit(ctx, []*pb.Request{ - onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS, primary, reqs, readKeys, observedRouteVersion), + onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS, primary, reqs, readKeys, observedRouteVersion, nil), }) if err != nil { return nil, errors.WithStack(err) @@ -1265,7 +1265,7 @@ func (c *Coordinate) buildRedirectRequests(reqs *OperationGroup[OP]) ([]*pb.Requ commitTS = 0 } return []*pb.Request{ - onePhaseTxnRequestWithPrevCommit(reqs.StartTS, commitTS, reqs.PrevCommitTS, primary, reqs.Elems, reqs.ReadKeys, reqs.ObservedRouteVersion), + onePhaseTxnRequestWithPrevCommit(reqs.StartTS, commitTS, reqs.PrevCommitTS, primary, reqs.Elems, reqs.ReadKeys, reqs.ObservedRouteVersion, nil), }, nil } @@ -1319,7 +1319,7 @@ func elemToMutation(req *Elem[OP]) *pb.Mutation { // route catalog snapshot at txn-begin (M1 plumbing, see // docs/design/2026_05_29_implemented_composed1_cross_group_commit_guard.md). // Zero is the legacy "unpinned" sentinel. -func onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS uint64, primaryKey []byte, reqs []*Elem[OP], readKeys [][]byte, observedRouteVersion uint64) *pb.Request { +func onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS uint64, primaryKey []byte, reqs []*Elem[OP], readKeys [][]byte, observedRouteVersion uint64, writeFenceBypassKeys [][]byte) *pb.Request { muts := make([]*pb.Mutation, 0, len(reqs)+1) muts = append(muts, &pb.Mutation{ Op: pb.Op_PUT, @@ -1336,6 +1336,7 @@ func onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS uint64, pr Mutations: muts, ReadKeys: readKeys, ObservedRouteVersion: observedRouteVersion, + WriteFenceBypassKeys: writeFenceBypassKeys, } } diff --git a/kv/fsm.go b/kv/fsm.go index b2e74b1db..dedc7829a 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -867,7 +867,7 @@ func (f *kvFSM) verifyWriteFence(r *pb.Request) error { if !ok { return errors.WithStack(ErrComposed1VersionGCd) } - if err := verifyWriteFenceFromSnapshot(r.GetMutations(), observedSnap, observedVer, "observed"); err != nil { + if err := verifyWriteFenceFromSnapshot(r.GetMutations(), r.GetWriteFenceBypassKeys(), observedSnap, observedVer, "observed"); err != nil { return err } if currentSnap.Version() == observedSnap.Version() { @@ -875,7 +875,7 @@ func (f *kvFSM) verifyWriteFence(r *pb.Request) error { } } - return verifyWriteFenceFromSnapshot(r.GetMutations(), currentSnap, currentSnap.Version(), "current") + return verifyWriteFenceFromSnapshot(r.GetMutations(), r.GetWriteFenceBypassKeys(), currentSnap, currentSnap.Version(), "current") } func requestBypassesWriteFence(r *pb.Request) bool { @@ -895,7 +895,8 @@ func (f *kvFSM) writeFenceHistoryReady() bool { return f.routes != nil && f.shardGroupID != 0 } -func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, snap RouteSnapshot, snapVer uint64, phase string) error { +func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, writeFenceBypassKeys [][]byte, snap RouteSnapshot, snapVer uint64, phase string) error { + bypassKeys := writeFenceBypassKeySet(writeFenceBypassKeys) for _, mut := range mutations { if mut == nil { continue @@ -912,6 +913,9 @@ func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, snap RouteSnapshot, } continue } + if _, ok := bypassKeys[string(mut.Key)]; ok { + continue + } rKey := routeKey(mut.Key) if snap.WriteFencedForKey(rKey) { return errors.Wrapf(ErrRouteWriteFenced, @@ -928,6 +932,20 @@ func verifyWriteFenceFromSnapshot(mutations []*pb.Mutation, snap RouteSnapshot, return nil } +func writeFenceBypassKeySet(keys [][]byte) map[string]struct{} { + if len(keys) == 0 { + return nil + } + out := make(map[string]struct{}, len(keys)) + for _, key := range keys { + if len(key) == 0 { + continue + } + out[string(key)] = struct{}{} + } + return out +} + // verifyOwnerFromSnapshot is the shared per-mutation owner-check // loop used by verifyComposed1's observed-version and current- // version passes. `phase` is the diagnostic label ("observed" / diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index f668821c8..ecb771b38 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -81,6 +81,34 @@ func TestFSMRejectsObservedWriteFencedRawPointWrite(t *testing.T) { require.ErrorIs(t, err, ErrRouteWriteFenced) } +func TestFSMWriteFenceBypassAllowsMarkedRawPointWrite(t *testing.T) { + t.Parallel() + + fsm := newFirstRouteWriteFencedFSM(t) + key := []byte("!sqs|msg|data|p|partitioned-key") + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 10) + require.NoError(t, err) + + got, err := fsm.store.GetAt(context.Background(), key, 10) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) +} + +func TestFSMWriteFenceBypassDoesNotAllowDelPrefix(t *testing.T) { + t.Parallel() + + fsm := newFirstRouteWriteFencedFSM(t) + prefix := []byte("!sqs|msg|data|p|") + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + WriteFenceBypassKeys: [][]byte{prefix}, + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: prefix}}, + }, 10) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMRejectsCurrentWriteFenceAfterObservedActiveRawPointWrite(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 73420bb86..ec7c43843 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1143,6 +1143,42 @@ func (c *ShardedCoordinator) partitionResolverRecognisesPointKey(key []byte) boo return c.router.partitionResolver.RecognisesPartitionedKey(key) } +func (c *ShardedCoordinator) resolverClaimedWriteFenceBypassKeysForElems(elems []*Elem[OP]) [][]byte { + if len(elems) == 0 { + return nil + } + out := make([][]byte, 0, len(elems)) + for _, elem := range elems { + if elem == nil || elem.Op == DelPrefix || !c.partitionResolverClaimsPointKey(elem.Key) { + continue + } + out = append(out, bytes.Clone(elem.Key)) + } + return out +} + +func (c *ShardedCoordinator) resolverClaimedWriteFenceBypassKeys(muts []*pb.Mutation) [][]byte { + if len(muts) == 0 { + return nil + } + out := make([][]byte, 0, len(muts)) + for _, mut := range muts { + if mut == nil || mut.GetOp() == pb.Op_DEL_PREFIX || !c.partitionResolverClaimsPointKey(mut.Key) { + continue + } + out = append(out, bytes.Clone(mut.Key)) + } + return out +} + +func (c *ShardedCoordinator) partitionResolverClaimsPointKey(key []byte) bool { + if c == nil || c.router == nil || c.router.partitionResolver == nil || len(key) == 0 { + return false + } + _, ok := c.router.partitionResolver.ResolveGroup(key) + return ok +} + func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) error { if c == nil || c.engine == nil { return nil @@ -1337,7 +1373,7 @@ func (c *ShardedCoordinator) dispatchSingleShardTxn(ctx context.Context, startTS // carries the one-phase dedup probe key for a retry that reuses a failed // attempt's write set. resp, err := g.Txn.Commit(ctx, []*pb.Request{ - onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS, primaryKey, elems, readKeys, observedRouteVersion), + onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS, primaryKey, elems, readKeys, observedRouteVersion, c.resolverClaimedWriteFenceBypassKeysForElems(elems)), }) if err != nil { return nil, errors.WithStack(err) @@ -1369,6 +1405,7 @@ func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS Mutations: append([]*pb.Mutation{prepareMeta}, grouped[gid]...), ReadKeys: groupedReadKeys[gid], ObservedRouteVersion: observedRouteVersion, + WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(grouped[gid]), } if _, err := g.Txn.Commit(ctx, []*pb.Request{req}); err != nil { // Same WithoutCancel pattern as dispatchTxn's @@ -1421,6 +1458,7 @@ func (c *ShardedCoordinator) commitPrimaryTxn(ctx context.Context, startTS uint6 Ts: startTS, Mutations: append([]*pb.Mutation{meta}, keys...), ObservedRouteVersion: observedRouteVersion, + WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(keys), } r, err := g.Txn.Commit(ctx, []*pb.Request{req}) @@ -1470,6 +1508,7 @@ func (c *ShardedCoordinator) commitSecondaryTxns(ctx context.Context, startTS ui Ts: startTS, Mutations: append([]*pb.Mutation{meta}, keyMutations(grouped[gid])...), ObservedRouteVersion: observedRouteVersion, + WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(grouped[gid]), } r, err := commitSecondaryWithRetry(ctx, g, req) if err != nil { @@ -2067,6 +2106,7 @@ func (c *ShardedCoordinator) rawLogs(ctx context.Context, reqs *OperationGroup[O Ts: ts, Mutations: grouped[gid], ObservedRouteVersion: reqs.ObservedRouteVersion, + WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(grouped[gid]), }) } return logs, nil @@ -2107,7 +2147,11 @@ func (c *ShardedCoordinator) txnLogs(ctx context.Context, reqs *OperationGroup[O if err != nil { return nil, err } - return buildTxnLogs(reqs.StartTS, commitTS, grouped, gids, reqs.ObservedRouteVersion) + bypassKeysByGroup := make(map[uint64][][]byte, len(grouped)) + for gid, muts := range grouped { + bypassKeysByGroup[gid] = c.resolverClaimedWriteFenceBypassKeys(muts) + } + return buildTxnLogs(reqs.StartTS, commitTS, grouped, gids, reqs.ObservedRouteVersion, bypassKeysByGroup) } // observeMutation: counted pre-commit, so a mutation that subsequently @@ -2183,7 +2227,7 @@ func (c *ShardedCoordinator) groupMutations(reqs []*Elem[OP], label keyviz.Label return grouped, gids, nil } -func buildTxnLogs(startTS uint64, commitTS uint64, grouped map[uint64][]*pb.Mutation, gids []uint64, observedRouteVersion uint64) ([]*pb.Request, error) { +func buildTxnLogs(startTS uint64, commitTS uint64, grouped map[uint64][]*pb.Mutation, gids []uint64, observedRouteVersion uint64, writeFenceBypassKeysByGroup map[uint64][][]byte) ([]*pb.Request, error) { logs := make([]*pb.Request, 0, len(gids)*txnPhaseCount) for _, gid := range gids { muts := grouped[gid] @@ -2200,6 +2244,7 @@ func buildTxnLogs(startTS uint64, commitTS uint64, grouped map[uint64][]*pb.Muta {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms, CommitTS: 0})}, }, muts...), ObservedRouteVersion: observedRouteVersion, + WriteFenceBypassKeys: writeFenceBypassKeysByGroup[gid], }, &pb.Request{ IsTxn: true, @@ -2209,6 +2254,7 @@ func buildTxnLogs(startTS uint64, commitTS uint64, grouped map[uint64][]*pb.Muta {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, LockTTLms: 0, CommitTS: commitTS})}, }, keys...), ObservedRouteVersion: observedRouteVersion, + WriteFenceBypassKeys: writeFenceBypassKeysByGroup[gid], }, ) } diff --git a/kv/sharded_coordinator_partition_test.go b/kv/sharded_coordinator_partition_test.go index e9b5c16bb..7b224db8a 100644 --- a/kv/sharded_coordinator_partition_test.go +++ b/kv/sharded_coordinator_partition_test.go @@ -151,6 +151,8 @@ func TestShardedCoordinatorWriteFencePrecheckHonoursPartitionResolver(t *testing require.Equal(t, uint64(42), resp.CommitIndex) require.Empty(t, g1.requests, "engine route fence must not preempt resolver-owned keys") require.Len(t, g42.requests, 1) + require.Equal(t, [][]byte{key}, g42.requests[0].GetWriteFenceBypassKeys(), + "resolver-owned point writes must carry the FSM write-fence bypass marker") } func TestShardedCoordinatorWriteFencePrecheckSkipsUnresolvedPartitionKey(t *testing.T) { @@ -185,6 +187,47 @@ func TestShardedCoordinatorWriteFencePrecheckSkipsUnresolvedPartitionKey(t *test require.Empty(t, g1.requests) } +func TestShardedCoordinatorTxnCarriesResolverWriteFenceBypassKeys(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, + }, + })) + + g1 := &recordingTransactional{ + responses: []*TransactionResponse{{CommitIndex: 1}}, + } + g42 := &recordingTransactional{ + responses: []*TransactionResponse{{CommitIndex: 42}}, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1, Store: store.NewMVCCStore()}, + 42: {Txn: g42, Store: store.NewMVCCStore()}, + }, 1, NewHLC(), nil) + + key := []byte("!sqs|msg|data|p|txn-partitioned-key") + coord.WithPartitionResolver(&stubResolver{claim: map[string]uint64{ + string(key): 42, + }}) + + resp, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + IsTxn: true, + StartTS: 100, + CommitTS: 200, + Elems: []*Elem[OP]{{Op: Put, Key: key, Value: []byte("v")}}, + }) + require.NoError(t, err) + require.NotNil(t, resp) + require.Empty(t, g1.requests) + require.Len(t, g42.requests, 1) + require.Equal(t, [][]byte{key}, g42.requests[0].GetWriteFenceBypassKeys(), + "single-shard txn prepare/one-phase request must preserve the resolver-owned point key for the FSM") +} + // TestShardedCoordinator_DispatchSplitsMutationsByResolverGroup is // the genuine regression for the Gemini-HIGH groupMutations // bypass: a Dispatch with mutations belonging to TWO different diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 0e2bd4664..7490809ce 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -208,6 +208,11 @@ type Request struct { // is plumbing only — the FSM ignores the value, so all existing // callers see no behaviour change. ObservedRouteVersion uint64 `protobuf:"varint,6,opt,name=observed_route_version,json=observedRouteVersion,proto3" json:"observed_route_version,omitempty"` + // write_fence_bypass_keys carries point keys whose owner was resolved by a + // higher-level partition resolver before the request was appended to Raft. + // The FSM uses it only to skip the byte-route write fence for those exact + // point mutations; prefix deletes and unmarked keys still fail closed. + WriteFenceBypassKeys [][]byte `protobuf:"bytes,7,rep,name=write_fence_bypass_keys,json=writeFenceBypassKeys,proto3" json:"write_fence_bypass_keys,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -284,6 +289,13 @@ func (x *Request) GetObservedRouteVersion() uint64 { return 0 } +func (x *Request) GetWriteFenceBypassKeys() [][]byte { + if x != nil { + return x.WriteFenceBypassKeys + } + return nil +} + type RaftCommand struct { state protoimpl.MessageState `protogen:"open.v1"` Requests []*Request `protobuf:"bytes,1,rep,name=requests,proto3" json:"requests,omitempty"` @@ -909,14 +921,15 @@ const file_internal_proto_rawDesc = "" + "\bMutation\x12\x13\n" + "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + - "\x05value\x18\x03 \x01(\fR\x05value\"\xca\x01\n" + + "\x05value\x18\x03 \x01(\fR\x05value\"\x81\x02\n" + "\aRequest\x12\x15\n" + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + "\x02ts\x18\x03 \x01(\x04R\x02ts\x12'\n" + "\tmutations\x18\x04 \x03(\v2\t.MutationR\tmutations\x12\x1b\n" + "\tread_keys\x18\x05 \x03(\fR\breadKeys\x124\n" + - "\x16observed_route_version\x18\x06 \x01(\x04R\x14observedRouteVersion\"3\n" + + "\x16observed_route_version\x18\x06 \x01(\x04R\x14observedRouteVersion\x125\n" + + "\x17write_fence_bypass_keys\x18\a \x03(\fR\x14writeFenceBypassKeys\"3\n" + "\vRaftCommand\x12$\n" + "\brequests\x18\x01 \x03(\v2\b.RequestR\brequests\"M\n" + "\x0eForwardRequest\x12\x15\n" + diff --git a/proto/internal.proto b/proto/internal.proto index 1f7c77712..f58257cec 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -55,6 +55,11 @@ message Request { // is plumbing only — the FSM ignores the value, so all existing // callers see no behaviour change. uint64 observed_route_version = 6; + // write_fence_bypass_keys carries point keys whose owner was resolved by a + // higher-level partition resolver before the request was appended to Raft. + // The FSM uses it only to skip the byte-route write fence for those exact + // point mutations; prefix deletes and unmarked keys still fail closed. + repeated bytes write_fence_bypass_keys = 7; } message RaftCommand { From b553090f299c5243ed2307b67ece1bf1f90d9930 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 10:40:25 +0900 Subject: [PATCH 101/162] Keep SQS receive route fences visible --- adapter/sqs_messages.go | 7 +- adapter/sqs_receive_route_fence_test.go | 86 +++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 adapter/sqs_receive_route_fence_test.go diff --git a/adapter/sqs_messages.go b/adapter/sqs_messages.go index 83bfe6009..a75e0c96d 100644 --- a/adapter/sqs_messages.go +++ b/adapter/sqs_messages.go @@ -1340,8 +1340,9 @@ func (s *SQSServer) rotateMessagesForDelivery( // - (msg, false, nil) → delivered, caller appends. // - (nil, true, nil) → expected race; skip this candidate only. // Covers ErrKeyNotFound (someone deleted the record between the -// vis-index scan and our GetAt) and ErrWriteConflict on dispatch -// (another receive rotated the same record). +// vis-index scan and our GetAt), plus non-fence dispatch races +// like ErrWriteConflict / ErrTxnLocked (another receive rotated +// the same record). // - (nil, false, err) → non-retryable failure; propagate up the // stack so ReceiveMessage returns an actionable 5xx instead of // a false-empty 200. @@ -1441,7 +1442,7 @@ func (s *SQSServer) commitReceiveRotation(ctx context.Context, queueName string, return nil, false, err } if _, err := s.coordinator.Dispatch(ctx, req); err != nil { - if isRetryableTransactWriteError(err) { + if isIgnorableTransactRaceError(err) { return nil, true, nil } return nil, false, errors.WithStack(err) diff --git a/adapter/sqs_receive_route_fence_test.go b/adapter/sqs_receive_route_fence_test.go new file mode 100644 index 000000000..9c584f791 --- /dev/null +++ b/adapter/sqs_receive_route_fence_test.go @@ -0,0 +1,86 @@ +package adapter + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/kv" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +type receiveRotationErrorCoordinator struct { + stubAdapterCoordinator + err error +} + +func (c *receiveRotationErrorCoordinator) Dispatch(context.Context, *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { + return nil, c.err +} + +func TestSQSReceiveRotationClassifiesDispatchErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + wantSkip bool + wantErr error + }{ + {name: "write conflict", err: store.ErrWriteConflict, wantSkip: true}, + {name: "txn locked", err: kv.ErrTxnLocked, wantSkip: true}, + {name: "route write fenced", err: kv.ErrRouteWriteFenced, wantErr: kv.ErrRouteWriteFenced}, + } + + for _, tt := range tests { + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + const queueName = "receive-route-fence" + const messageID = "msg-1" + const gen = uint64(1) + const visibleAt = int64(1000) + + meta := &sqsQueueMeta{Name: queueName, Generation: gen, PartitionCount: 1} + rec := &sqsMessageRecord{ + MessageID: messageID, + Body: []byte("body"), + MD5OfBody: sqsMD5Hex([]byte("body")), + SendTimestampMillis: visibleAt, + VisibleAtMillis: visibleAt, + QueueGeneration: gen, + } + cand := sqsMsgCandidate{ + visKey: sqsMsgVisKeyDispatch(meta, queueName, 0, gen, visibleAt, messageID), + messageID: messageID, + partition: 0, + } + dataKey := sqsMsgDataKeyDispatch(meta, queueName, 0, gen, messageID) + srv := &SQSServer{ + coordinator: &receiveRotationErrorCoordinator{err: tt.err}, + } + + msg, skip, err := srv.commitReceiveRotation( + context.Background(), + queueName, + meta, + cand, + dataKey, + rec, + uint64(visibleAt), + sqsReceiveOptions{VisibilityTimeout: 30}, + nil, + fifoLockAcquire, + ) + + require.Nil(t, msg) + require.Equal(t, tt.wantSkip, skip) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + }) + } +} From d2b22acc76035ec00d4f79c6d83bed7b232e16b7 Mon Sep 17 00:00:00 2001 From: bootjp Date: Wed, 15 Jul 2026 19:45:09 +0900 Subject: [PATCH 102/162] Stabilize redis proxy and raft recovery --- adapter/redis_txn.go | 96 ++++++++-- adapter/redis_txn_test.go | 86 +++++++++ cmd/redis-proxy/main.go | 122 ++++++++----- ..._implemented_etcd_snapshot_disk_offload.md | 4 +- ...29_implemented_snapshot_logical_decoder.md | 4 +- .../2026_04_29_proposed_logical_backup.md | 12 +- .../2026_05_28_implemented_tla_safety_spec.md | 8 +- ...implemented_idempotent_snapshot_restore.md | 166 +++++++----------- .../2026_06_12_proposed_scaling_roadmap.md | 2 +- internal/raftengine/etcd/engine.go | 47 +++-- .../etcd/engine_applied_index_test.go | 22 +++ internal/raftengine/etcd/engine_test.go | 62 ++++++- internal/raftengine/etcd/fsm_snapshot_file.go | 70 +++++++- internal/raftengine/etcd/grpc_transport.go | 18 +- .../raftengine/etcd/grpc_transport_test.go | 54 ++++-- .../raftengine/etcd/snapshot_spool_test.go | 42 +++++ internal/raftengine/etcd/wal_store.go | 162 ++++++----------- .../etcd/wal_store_skip_gate_test.go | 98 +++++------ internal/raftengine/statemachine.go | 26 ++- kv/coordinator.go | 22 ++- kv/fsm.go | 31 ++-- kv/lease_read_test.go | 8 +- kv/lease_warmup_test.go | 64 +++++++ kv/sharded_coordinator.go | 2 +- .../dashboards/elastickv-redis-summary.json | 2 +- monitoring/hotpath.go | 2 +- monitoring/hotpath_test.go | 2 +- proxy/config.go | 3 + proxy/dualwrite.go | 122 ++++++++++--- proxy/metrics.go | 11 +- proxy/noop_backend.go | 38 ++++ proxy/proxy.go | 15 +- proxy/proxy_test.go | 105 +++++++++-- proxy/pubsub.go | 10 +- proxy/raw_redis_proxy.go | 156 ++++++++++++++++ proxy/raw_redis_proxy_test.go | 77 ++++++++ 36 files changed, 1282 insertions(+), 489 deletions(-) create mode 100644 proxy/noop_backend.go create mode 100644 proxy/raw_redis_proxy.go create mode 100644 proxy/raw_redis_proxy_test.go diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index 53b937ed2..dcbcc3b48 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -98,9 +98,11 @@ type txnValue struct { } type stringReplacement struct { - key []byte - value []byte - ttl *time.Time + key []byte + value []byte + ttl *time.Time + rawTyp redisValueType + rawTypKnown bool } type txnContext struct { @@ -617,18 +619,48 @@ func (t *txnContext) loadTTLState(key []byte) (*ttlTxnState, error) { } func (t *txnContext) stagedKeyType(key []byte) (redisValueType, error) { + view, err := t.stagedKeyTypeView(key) + if err != nil { + return redisTypeNone, err + } + return view.typ, nil +} + +type txnKeyTypeView struct { + typ redisValueType + rawTyp redisValueType + rawTypKnown bool +} + +func (t *txnContext) stagedKeyTypeView(key []byte) (txnKeyTypeView, error) { k := string(key) - if _, ok := t.replacers[k]; ok { - return redisTypeString, nil + if repl, ok := t.replacers[k]; ok { + return txnKeyTypeView{ + typ: redisTypeString, + rawTyp: repl.rawTyp, + rawTypKnown: repl.rawTypKnown, + }, nil } if typ, ok := t.stagedPositiveKeyType(k); ok { - return typ, nil + return txnKeyTypeView{typ: typ}, nil } if t.hasStagedTypeDeletion(k) { - return redisTypeNone, nil + return txnKeyTypeView{typ: redisTypeNone}, nil } t.trackTypeReadKeys(key) - return t.server.keyTypeAt(t.ctxOrBackground(), key, t.startTS) + rawTyp, err := t.server.rawKeyTypeAt(t.ctxOrBackground(), key, t.startTS) + if err != nil { + return txnKeyTypeView{}, err + } + typ, err := t.server.applyTTLFilter(t.ctxOrBackground(), key, t.startTS, rawTyp) + if err != nil { + return txnKeyTypeView{}, err + } + return txnKeyTypeView{ + typ: typ, + rawTyp: rawTyp, + rawTypKnown: true, + }, nil } func (t *txnContext) stagedPositiveKeyType(key string) (redisValueType, bool) { @@ -721,10 +753,11 @@ func (t *txnContext) applySet(cmd redcon.Command) (redisResult, error) { if err != nil { return redisResult{}, err } - typ, err := t.stagedKeyType(cmd.Args[1]) + typeView, err := t.stagedKeyTypeView(cmd.Args[1]) if err != nil { return redisResult{}, err } + typ := typeView.typ // NX/XX: skip the write if the key-existence condition is not met. exists := typ != redisTypeNone @@ -739,7 +772,8 @@ func (t *txnContext) applySet(cmd redcon.Command) (redisResult, error) { if err != nil { return redisResult{}, err } - t.stageStringReplacement(cmd.Args[1], cmd.Args[2], opts.ttl) + t.trackWideCollectionFenceReads(cmd.Args[1]) + t.stageStringReplacementWithRawType(cmd.Args[1], cmd.Args[2], opts.ttl, typeView.rawTyp, typeView.rawTypKnown) return applySetResult(opts, oldValue), nil } @@ -777,15 +811,32 @@ func cloneTimePtr(in *time.Time) *time.Time { } func (t *txnContext) stageStringReplacement(key, value []byte, ttl *time.Time) { + t.stageStringReplacementWithRawType(key, value, ttl, redisTypeNone, false) +} + +func (t *txnContext) stageStringReplacementWithRawType(key, value []byte, ttl *time.Time, rawTyp redisValueType, rawTypKnown bool) { if t.replacers == nil { t.replacers = map[string]*stringReplacement{} } k := string(key) - t.replacers[k] = &stringReplacement{ - key: bytes.Clone(key), - value: bytes.Clone(value), - ttl: cloneTimePtr(ttl), + if repl, ok := t.replacers[k]; ok { + repl.value = bytes.Clone(value) + repl.ttl = cloneTimePtr(ttl) + if rawTypKnown { + repl.rawTyp = rawTyp + repl.rawTypKnown = true + } + delete(t.deletedKeys, k) + return + } + repl := &stringReplacement{ + key: bytes.Clone(key), + value: bytes.Clone(value), + ttl: cloneTimePtr(ttl), + rawTyp: rawTyp, + rawTypKnown: rawTypKnown, } + t.replacers[k] = repl delete(t.deletedKeys, k) } @@ -1576,11 +1627,13 @@ func (t *txnContext) buildReplacementElems(ctx context.Context) ([]*kv.Elem[kv.O for _, k := range keys { repl := t.replacers[k] t.trackWideCollectionFenceReads(repl.key) - deleteElems, _, err := t.server.deleteLogicalKeyElems(ctx, repl.key, t.startTS) - if err != nil { - return nil, err + if repl.needsFullLogicalDelete() { + deleteElems, _, err := t.server.deleteLogicalKeyElems(ctx, repl.key, t.startTS) + if err != nil { + return nil, err + } + elems = append(elems, deleteElems...) } - elems = append(elems, deleteElems...) elems = append(elems, redisTxnWideCollectionFenceElems(repl.key)...) elems = append(elems, &kv.Elem[kv.OP]{ Op: kv.Put, @@ -1596,6 +1649,13 @@ func (t *txnContext) buildReplacementElems(ctx context.Context) ([]*kv.Elem[kv.O return elems, nil } +func (r *stringReplacement) needsFullLogicalDelete() bool { + if !r.rawTypKnown { + return true + } + return isNonStringCollectionType(r.rawTyp) +} + func (t *txnContext) buildLogicalDeletionElems(ctx context.Context) ([]*kv.Elem[kv.OP], error) { if len(t.logicalDeletes) == 0 { return nil, nil diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index 3349c4b8b..c8e49c1d9 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -684,6 +684,92 @@ func TestRedisTxnSetReplacementConflictsWithConcurrentWideHashWrite(t *testing.T "SET replacement in MULTI must conflict with concurrent HSET of a new field") } +func TestRedisTxnSetReplacementTracksWideFencesBeforeBuild(t *testing.T) { + t.Parallel() + + ctx := context.Background() + server, st := newRedisStorageMigrationTestServer(t) + key := []byte("set-replace:fence-read") + + txn := newRedisTxnTestContext(server) + res, err := txn.applySet(redcon.Command{Args: [][]byte{[]byte(cmdSet), key, []byte("string")}}) + require.NoError(t, err) + require.Equal(t, "OK", res.str) + for _, fenceKey := range redisTxnWideCollectionFenceKeys(key) { + require.Contains(t, txn.readKeys, string(fenceKey)) + } + + require.NoError(t, st.PutAt(ctx, redisTxnWideHashFenceKey(key), []byte{}, redisTxnTestStartTS+1, 0)) + require.ErrorIs(t, txn.validateReadSet(ctx), store.ErrWriteConflict) +} + +func TestRedisTxnSetReplacementSkipsWideCleanupForRawStringOrMissing(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cases := []struct { + name string + seed func(store.MVCCStore, []byte) + }{ + {name: "missing"}, + { + name: "string", + seed: func(st store.MVCCStore, key []byte) { + require.NoError(t, st.PutAt(ctx, redisStrKey(key), encodeRedisStr([]byte("old"), nil), redisTxnTestStartTS, 0)) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + server, st := newRedisStorageMigrationTestServer(t) + key := []byte("set-replace:no-wide-cleanup:" + tc.name) + if tc.seed != nil { + tc.seed(st, key) + } + + txn := newRedisTxnTestContext(server) + res, err := txn.applySet(redcon.Command{Args: [][]byte{[]byte(cmdSet), key, []byte("string")}}) + require.NoError(t, err) + require.Equal(t, "OK", res.str) + + elems, err := txn.buildReplacementElems(ctx) + require.NoError(t, err) + require.False(t, elemKeysContain(elems, store.HashMetaKey(key))) + require.False(t, elemKeysContain(elems, store.SetMetaKey(key))) + require.False(t, elemKeysContain(elems, store.ZSetMetaKey(key))) + require.False(t, elemKeysContain(elems, store.ListMetaKey(key))) + require.False(t, elemKeysContain(elems, store.StreamMetaKey(key))) + require.True(t, elemKeysContain(elems, redisStrKey(key))) + }) + } +} + +func TestRedisTxnSetReplacementDeletesExpiredRawHash(t *testing.T) { + t.Parallel() + + ctx := context.Background() + server, st := newRedisStorageMigrationTestServer(t) + key := []byte("set-replace:expired-hash") + expired := time.Now().Add(-time.Hour) + require.NoError(t, st.PutAt(ctx, store.HashFieldKey(key, []byte("old")), []byte("v"), redisTxnTestStartTS, 0)) + require.NoError(t, st.PutAt(ctx, store.HashMetaKey(key), store.MarshalHashMeta(store.HashMeta{Len: 1}), redisTxnTestStartTS, 0)) + require.NoError(t, st.PutAt(ctx, redisTTLKey(key), encodeRedisTTL(expired), redisTxnTestStartTS, 0)) + + txn := newRedisTxnTestContext(server) + res, err := txn.applySet(redcon.Command{Args: [][]byte{[]byte(cmdSet), key, []byte("string")}}) + require.NoError(t, err) + require.Equal(t, "OK", res.str) + + elems, err := txn.buildReplacementElems(ctx) + require.NoError(t, err) + require.True(t, elemKeysContain(elems, store.HashFieldKey(key, []byte("old")))) + require.True(t, elemKeysContain(elems, store.HashMetaKey(key))) + require.True(t, elemKeysContain(elems, redisStrKey(key))) +} + func TestRedisTxnSetReplacementConflictsWithConcurrentListPush(t *testing.T) { t.Parallel() diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index ec2622eb3..4ad067987 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -7,6 +7,7 @@ import ( "log/slog" "net" "net/http" + "net/http/pprof" "os" "os/signal" "strings" @@ -57,6 +58,8 @@ func run() error { flag.StringVar(&cfg.SentryEnv, "sentry-env", cfg.SentryEnv, "Sentry environment") flag.Float64Var(&cfg.SentrySampleRate, "sentry-sample", cfg.SentrySampleRate, "Sentry sample rate") flag.StringVar(&cfg.MetricsAddr, "metrics", cfg.MetricsAddr, "Prometheus metrics address") + flag.StringVar(&cfg.PProfAddr, "pprof", cfg.PProfAddr, "pprof listen address (empty = disabled)") + flag.BoolVar(&cfg.RedisOnlyRaw, "redis-only-raw", cfg.RedisOnlyRaw, "Use raw TCP bridging in redis-only mode") flag.Parse() mode, err := parseRuntimeOptions(modeStr, primaryPoolSize, elasticKVPoolSize, secondaryWriteConcurrency, secondaryScriptConcurrency) @@ -78,11 +81,38 @@ func run() error { sentryReporter := proxy.NewSentryReporter(cfg.SentryDSN, cfg.SentryEnv, cfg.SentrySampleRate, logger) defer sentryReporter.Flush(sentryFlushTimeout) - // Prometheus reg := prometheus.NewRegistry() metrics := proxy.NewProxyMetrics(reg) - // Backends + primary, secondary, err := newBackends(cfg, primaryPoolSize, elasticKVPoolSize, logger) + if err != nil { + return err + } + defer primary.Close() + defer secondary.Close() + + dual := proxy.NewDualWriter(primary, secondary, cfg, metrics, sentryReporter, logger) + defer dual.Close() // wait for in-flight async goroutines + srv := proxy.NewProxyServer(cfg, dual, metrics, sentryReporter, logger) + + // Context for graceful shutdown + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + go serveMetrics(ctx, cfg.MetricsAddr, reg, logger) + + if cfg.PProfAddr != "" { + go servePProf(ctx, cfg.PProfAddr, logger) + } + + // Start proxy + if err := srv.ListenAndServe(ctx); err != nil { + return fmt.Errorf("proxy server: %w", err) + } + return nil +} + +func newBackends(cfg proxy.ProxyConfig, primaryPoolSize, elasticKVPoolSize int, logger *slog.Logger) (proxy.Backend, proxy.Backend, error) { primaryOpts := proxy.DefaultBackendOptions() primaryOpts.DB = cfg.PrimaryDB primaryOpts.Password = cfg.PrimaryPassword @@ -94,59 +124,63 @@ func run() error { secondarySeeds := parseAddrList(cfg.SecondaryAddr) if len(secondarySeeds) == 0 { - return fmt.Errorf("at least one secondary address is required") + return nil, nil, fmt.Errorf("at least one secondary address is required") } - var primary, secondary proxy.Backend switch cfg.Mode { - case proxy.ModeElasticKVPrimary, proxy.ModeElasticKVOnly: - primary = proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger) - secondary = proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts) - case proxy.ModeRedisOnly, proxy.ModeDualWrite, proxy.ModeDualWriteShadow: - primary = proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts) - secondary = proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger) + case proxy.ModeElasticKVPrimary: + return proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger), + proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts), nil + case proxy.ModeElasticKVOnly: + return proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger), + proxy.NewNoopBackend("redis"), nil + case proxy.ModeRedisOnly: + return proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts), + proxy.NewNoopBackend("elastickv"), nil + case proxy.ModeDualWrite, proxy.ModeDualWriteShadow: + return proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts), + proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger), nil + default: + return nil, nil, fmt.Errorf("unsupported mode: %s", cfg.Mode.String()) } - defer primary.Close() - defer secondary.Close() +} - dual := proxy.NewDualWriter(primary, secondary, cfg, metrics, sentryReporter, logger) - defer dual.Close() // wait for in-flight async goroutines - srv := proxy.NewProxyServer(cfg, dual, metrics, sentryReporter, logger) +func serveMetrics(ctx context.Context, addr string, reg *prometheus.Registry, logger *slog.Logger) { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) + serveHTTP(ctx, addr, mux, "metrics", logger) +} - // Context for graceful shutdown - ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer cancel() +func servePProf(ctx context.Context, addr string, logger *slog.Logger) { + mux := http.NewServeMux() + mux.HandleFunc("/debug/pprof/", pprof.Index) + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + serveHTTP(ctx, addr, mux, "pprof", logger) +} - // Start metrics server +func serveHTTP(ctx context.Context, addr string, handler http.Handler, name string, logger *slog.Logger) { + var lc net.ListenConfig + ln, err := lc.Listen(ctx, "tcp", addr) + if err != nil { + logger.Error(name+" listen failed", "addr", addr, "err", err) + return + } + srv := &http.Server{Handler: handler, ReadHeaderTimeout: time.Second} go func() { - mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) - var lc net.ListenConfig - ln, err := lc.Listen(ctx, "tcp", cfg.MetricsAddr) - if err != nil { - logger.Error("metrics listen failed", "addr", cfg.MetricsAddr, "err", err) - return - } - metricsSrv := &http.Server{Handler: mux, ReadHeaderTimeout: time.Second} - go func() { - <-ctx.Done() - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), metricsShutdownTimeout) - defer shutdownCancel() - if err := metricsSrv.Shutdown(shutdownCtx); err != nil { - logger.Warn("metrics server shutdown error", "err", err) - } - }() - logger.Info("metrics server starting", "addr", cfg.MetricsAddr) - if err := metricsSrv.Serve(ln); err != nil && err != http.ErrServerClosed { - logger.Error("metrics server error", "err", err) + <-ctx.Done() + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), metricsShutdownTimeout) + defer shutdownCancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + logger.Warn(name+" server shutdown error", "err", err) } }() - - // Start proxy - if err := srv.ListenAndServe(ctx); err != nil { - return fmt.Errorf("proxy server: %w", err) + logger.Info(name+" server starting", "addr", addr) + if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed { + logger.Error(name+" server error", "err", err) } - return nil } func parseRuntimeOptions(modeStr string, primaryPoolSize, elasticKVPoolSize, secondaryWriteConcurrency, secondaryScriptConcurrency int) (proxy.ProxyMode, error) { diff --git a/docs/design/2026_04_14_implemented_etcd_snapshot_disk_offload.md b/docs/design/2026_04_14_implemented_etcd_snapshot_disk_offload.md index 5312c3784..b800f2906 100644 --- a/docs/design/2026_04_14_implemented_etcd_snapshot_disk_offload.md +++ b/docs/design/2026_04_14_implemented_etcd_snapshot_disk_offload.md @@ -5,7 +5,7 @@ ### Observed Symptoms Clusters using the etcd engine exhibit large memory spikes across all nodes at snapshot -creation intervals (`defaultSnapshotEvery = 10,000` entries). The spike magnitude scales +creation intervals (`defaultSnapshotEvery = 100,000` entries). The spike magnitude scales with FSM data size, and simultaneous spikes across multiple nodes compound the pressure. ### Root Cause @@ -34,7 +34,7 @@ storage.CreateSnapshot(applied, &confState, payload) ``` `MemoryStorage` holds `raftpb.Snapshot.Data = payload` until the next snapshot is created -(i.e., until another 10,000 entries are processed), keeping the full FSM export resident +(i.e., until another 100,000 entries are processed), keeping the full FSM export resident in memory the entire time. #### Problem 3: Re-allocation when sending to followers diff --git a/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md b/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md index 07205fc7c..f4f721ba4 100644 --- a/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md +++ b/docs/design/2026_04_29_implemented_snapshot_logical_decoder.md @@ -37,7 +37,7 @@ which is a different framing the decoder/encoder do not touch. See `2026_05_25_implemented_snapshot_logical_encoder.md` §"Why a separate design doc" item 3. -Snapshots are taken automatically every `defaultSnapshotEvery = 10000` +Snapshots are taken automatically every `defaultSnapshotEvery = 100000` log entries (`internal/raftengine/etcd/engine.go:92`) and stored under `{dataDir}/fsm-snap/.fsm`. They are crash-consistent by construction — the writer takes a Pebble snapshot at the FSM's @@ -608,7 +608,7 @@ bespoke parser, the format has failed its goal. - **Staleness.** Whatever was written between the snapshot's `applied_index` and "now" is not in the snapshot, so it is not in the decoded output. The gap is bounded by `SnapshotEvery × write_rate` - (default 10000 entries; for a write-heavy cluster, seconds; for a + (default 100000 entries; for a write-heavy cluster, minutes; for a quiet one, hours). - **Cadence is not user-controlled.** "Snapshot now" requires a Raft trigger; the decoder cannot create a fresh snapshot, only consume diff --git a/docs/design/2026_04_29_proposed_logical_backup.md b/docs/design/2026_04_29_proposed_logical_backup.md index 39350d188..1e307c832 100644 --- a/docs/design/2026_04_29_proposed_logical_backup.md +++ b/docs/design/2026_04_29_proposed_logical_backup.md @@ -884,7 +884,7 @@ refuse if: remaining_headroom < --snapshot-headroom-entries ``` `SnapshotEvery` is the per-engine snapshot trigger (default -`defaultSnapshotEvery = 10000` from `internal/raftengine/etcd/engine.go:92`, +`defaultSnapshotEvery = 100000` from `internal/raftengine/etcd/engine.go:92`, overridden via the `ELASTICKV_RAFT_SNAPSHOT_COUNT` env var — there is no `--raftSnapshotEvery` CLI flag). The value is currently a private field on the etcd `Engine` struct (`internal/raftengine/etcd/engine.go:224`) @@ -906,9 +906,9 @@ implemented on the etcd backend by returning the place, `BeginBackup` reads each group's `SnapshotEvery` rather than hardcoding `defaultSnapshotEvery`, so an operator who tuned `ELASTICKV_RAFT_SNAPSHOT_COUNT` sees consistent behavior. With -`SnapshotEvery = 10000` and -`--snapshot-headroom-entries = 1000` (default; one-tenth of -SnapshotEvery), the check refuses backups when fewer than 1000 entries +`SnapshotEvery = 100000` and +`--snapshot-headroom-entries = 10000` (default; one-tenth of +SnapshotEvery), the check refuses backups when fewer than 10000 entries remain before the next snapshot fires — i.e. when an in-flight backup is at risk of triggering the snapshot-installation corner case. A freshly-snapshotted cluster has the *largest* remaining headroom and @@ -1341,7 +1341,7 @@ written. `s.groups[id]` map without a typed assertion fallback. Tests that mock `AdminGroup` (e.g. `adapter/admin_grpc_test.go`) gain one extra method to implement; they can return - `defaultSnapshotEvery = 10000` for parity with production. + `defaultSnapshotEvery = 100000` for parity with production. - Extend `kv/active_timestamp_tracker.go` with `PinWithDeadline`, `Extend`, and the per-second sweeper goroutine that reaps expired pins and emits the `backup_pin_expired` structured warning. @@ -1601,7 +1601,7 @@ Scope: out of this proposal; mentioned only to draw the boundary. | `TestAdminGRPCConnCacheReuse` | Two consecutive `BeginBackup` calls dialing the same peer share one underlying `*grpc.ClientConn` (verified via the cache size); shutdown of a peer evicts only that entry, not the whole cache; admin cache is independent of `ShardStore.connCache` (no cross-cache eviction) | | `TestBeginBackupPropagatesAdminAuthToken` | A cluster booted with `--adminToken` accepts a `BeginBackup` call carrying `authorization: Bearer `; the handler propagates the same metadata via `metadata.NewOutgoingContext` to every `GetNodeVersion` fan-out dial, so peers return their version (not `Unauthenticated`). A `BeginBackup` call without the token is itself rejected before any fan-out happens | | `TestVersionCacheRaceUnderLoad` | `go test -race` with 50 concurrent `GetRaftGroups` callers and async `GetNodeVersion` probe goroutines writing the cache simultaneously emits no data-race report; the `sync.Map` choice is enforced by the lack of a separate `versionCacheMu` field on `AdminServer` | -| `TestSnapshotEveryReadsFromEngine` | A node started with `ELASTICKV_RAFT_SNAPSHOT_COUNT=5000` reports `Engine.SnapshotEvery() == 5000`; `BeginBackup` uses 5000 (not the default 10000) when computing remaining headroom | +| `TestSnapshotEveryReadsFromEngine` | A node started with `ELASTICKV_RAFT_SNAPSHOT_COUNT=5000` reports `Engine.SnapshotEvery() == 5000`; `BeginBackup` uses 5000 (not the default 100000) when computing remaining headroom | | `TestRenewBackupRetriesLeaderElection` | Force a leader election mid-`RenewBackup`; the admin server retries `BackupExtend` up to 3 times with 500ms backoff and succeeds once the new leader is established, without aborting the dump | | `TestPinWithDeadlineExpiry` | `PinWithDeadline(ts, now+100ms)` is auto-released by the sweeper after the deadline; compactor unblocked; `backup_pin_expired` log emitted | | `TestBeginBackupWaitsForLaggingShard` | Force shard B's `applied_index` to lag; `BeginBackup` polls until it catches up or times out with `FailedPrecondition`; no scan starts in the timeout case | diff --git a/docs/design/2026_05_28_implemented_tla_safety_spec.md b/docs/design/2026_05_28_implemented_tla_safety_spec.md index cd94bb5d0..463fa16ba 100644 --- a/docs/design/2026_05_28_implemented_tla_safety_spec.md +++ b/docs/design/2026_05_28_implemented_tla_safety_spec.md @@ -199,8 +199,8 @@ cannot check them as state invariants. independently reach `ceiling + 1` before a fresh ceiling is renewed, the new leader's first `Next()` can tie or undercut the old leader's last commit. Bounding inter-node skew to less than - one ceiling window (< 3s with the current `hlcPhysicalWindowMs = - 3000ms`) keeps the window wide enough that the new leader cannot + one ceiling window (< 15s with the current `hlcPhysicalWindowMs = + 15000ms`) keeps the window wide enough that the new leader cannot independently reach the overflow value before a renewal applies. Should be surfaced in operator docs as a cluster prerequisite. - **(ii) Logical-counter handoff.** The 16-bit logical half of the HLC @@ -270,7 +270,7 @@ cannot check them as state invariants. `hlcPhysicalWindowMs` cannot serve any persistence timestamp — every client commit is rejected until renewal succeeds. This is a CP, not AP, trade-off and operators must size - `hlcPhysicalWindowMs` (currently 3s) relative to expected + `hlcPhysicalWindowMs` (currently 15s) relative to expected partition duration; see §9 risk 7. ### 5.2 OCC @@ -676,7 +676,7 @@ does not keep this document in `partial`. 7. **Fail-closed availability under partition.** HLC-4 precondition (iii) makes the ceiling-fence behaviour normative: a leader partitioned from the default group's quorum for longer than - `hlcPhysicalWindowMs` (currently 3s) cannot serve any persistence + `hlcPhysicalWindowMs` (currently 15s) cannot serve any persistence timestamp, so client commits are rejected until renewal succeeds. This is a CP, not AP, trade-off and is a stricter regime than the current implementation (which silently keeps issuing). Mitigation: diff --git a/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md b/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md index 2317fb9db..a3db4b2f2 100644 --- a/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md +++ b/docs/design/2026_06_02_implemented_idempotent_snapshot_restore.md @@ -297,9 +297,9 @@ func restoreSnapshotState(fsm StateMachine, snapshot raftpb.Snapshot, fsmSnapDir // The body restore is skipped, but we MUST still consume // the v1/v2 snapshot header so the FSM picks up the HLC // ceiling AND the Stage 8a cutover (see §5). Thread - // tok.CRC32C through so the skip path verifies the file - // before mutating FSM state, matching the existing - // openAndRestoreFSMSnapshot safety contract. + // tok.CRC32C through so the skip path verifies the cheap + // snapshot envelope before mutating FSM state. Full-body CRC + // remains on the restore path where body bytes are consumed. return applyHeaderStateOnSkip(fsm, fsmSnapPath(fsmSnapDir, tok.Index), tok.CRC32C) } return openAndRestoreFSMSnapshot(fsm, fsmSnapPath(fsmSnapDir, tok.Index), tok.CRC32C) @@ -423,42 +423,35 @@ or `internal/raftengine/etcd → kv` edges (test-only on both sides), and adding either to satisfy the round-6 design either duplicates the CRC verifier in `kv` or breaks the layering. -Round-7 keeps the CRC verifier in its existing package and splits the -seam into two phases — a **parse** phase that reads the header from a -caller-supplied reader (and drains the rest, for CRC coverage), and an -**apply** phase that is pure assignment. The engine orchestrates -size + footer + tee'd CRC computation around the parse phase, then -calls apply only after all three pass: +Round-7 keeps the envelope checks in the engine package and splits the +seam into two phases — a **parse** phase that reads only the header from +a caller-supplied reader, and an **apply** phase that is pure assignment. +The engine validates size + footer-vs-tokenCRC before parsing and calls +apply only after those checks and the header parse pass: ```go // internal/raftengine/statemachine.go (new, sibling to ApplyIndexAware) type SnapshotHeaderApplier interface { - // ParseSnapshotHeader reads the v1/v2 header from r, drains the - // remaining bytes (so a wrapping crc32 TeeReader covers the full - // payload), and returns the parsed (ceiling, cutover) pair WITHOUT - // mutating FSM state. Implementations MUST NOT touch any FSM - // fields here; the engine calls ApplySnapshotHeader separately - // only after the wrapping CRC verification passes. + // ParseSnapshotHeader reads the v1/v2 header from r and returns the + // parsed (ceiling, cutover) pair WITHOUT mutating FSM state. + // Implementations MUST NOT touch any FSM fields here; the engine + // calls ApplySnapshotHeader separately only after the snapshot file + // footer matches the raft token. // // Errors propagate from the underlying header parser - // (ErrSnapshotHeaderUnknownMagic / InvalidLength) or from the - // drain pass (I/O errors). FSM state stays untouched on error. + // (ErrSnapshotHeaderUnknownMagic / InvalidLength). FSM state stays + // untouched on error. ParseSnapshotHeader(r io.Reader) (ceiling, cutover uint64, err error) // ApplySnapshotHeader is pure assignment of the verified header // state. The engine calls this only after ParseSnapshotHeader - // returned and the wrapping crc32 hash matched the file footer. + // returned and the snapshot file's footer matched the raft token. ApplySnapshotHeader(ceiling, cutover uint64) } // internal/raftengine/etcd/wal_store.go -- never imports kv; -// CRC verification stays here where the helpers live. +// cheap snapshot envelope checks stay here where the helpers live. func applyHeaderStateOnSkip(fsm StateMachine, snapPath string, tokenCRC uint32) error { - setter, ok := fsm.(SnapshotHeaderApplier) - if !ok { - return nil // FSM has no header state; skip is harmless. - } - file, err := os.Open(snapPath) if err != nil { return statFSMFileError(err) } defer file.Close() @@ -480,29 +473,24 @@ func applyHeaderStateOnSkip(fsm StateMachine, snapPath string, tokenCRC uint32) "path=%s footer=%08x token=%08x", snapPath, footer, tokenCRC) } - // Step 3: full-body CRC. Wrap the payload in a crc32 TeeReader and - // hand it to the FSM's ParseSnapshotHeader for header parse + drain. - // The header bytes are included in the computed CRC because the - // FSM reads them from the tee'd reader. + setter, ok := fsm.(SnapshotHeaderApplier) + if !ok { + return nil // FSM has no header state; skip is harmless. + } + if _, err := file.Seek(0, io.SeekStart); err != nil { return errors.WithStack(err) } payloadSize := info.Size() - fsmFooterSize - h := crc32.New(crc32cTable) - tee := io.TeeReader(io.LimitReader(file, payloadSize), h) - ceiling, cutover, perr := setter.ParseSnapshotHeader(tee) + ceiling, cutover, perr := setter.ParseSnapshotHeader(io.LimitReader(file, payloadSize)) if perr != nil { - // ErrSnapshotHeaderUnknownMagic / InvalidLength / I/O error - // surfaced from the FSM's parse pass. State unchanged. + // ErrSnapshotHeaderUnknownMagic / InvalidLength surfaced from + // the FSM's parse pass. State unchanged. return errors.WithStack(perr) } - if h.Sum32() != footer { - return errors.Wrapf(ErrFSMSnapshotFileCRC, - "path=%s footer=%08x computed=%08x", snapPath, footer, h.Sum32()) - } - // All three checks passed; apply side-effects. + // Envelope checks and header parse passed; apply side-effects. setter.ApplySnapshotHeader(ceiling, cutover) return nil } @@ -510,16 +498,10 @@ func applyHeaderStateOnSkip(fsm StateMachine, snapPath string, tokenCRC uint32) // kv/fsm.go (new methods on kvFSM) -- kv.ReadSnapshotHeader stays inside kv; // no imports of internal/raftengine/etcd or its private helpers. func (f *kvFSM) ParseSnapshotHeader(r io.Reader) (uint64, uint64, error) { - // The engine has already wrapped r in a crc32 TeeReader sized at - // the body payload (file size minus 4-byte footer). We read the - // header, then drain the rest of the body so the engine's CRC - // covers every byte (matching restoreAndComputeCRC's behaviour). - br := bufio.NewReaderSize(r, 1<<20) //nolint:mnd // 1 MiB, local to kv - ceiling, cutover, err := ReadSnapshotHeader(br) + // The skip path already has the FSM body locally, so read only + // the snapshot header needed for HLC/cutover state. + ceiling, cutover, err := ReadSnapshotHeader(bufio.NewReaderSize(r, 4<<10)) if err != nil { return 0, 0, errors.WithStack(err) } - if _, err := io.Copy(io.Discard, br); err != nil { - return 0, 0, errors.WithStack(err) - } return ceiling, cutover, nil } @@ -531,27 +513,16 @@ func (f *kvFSM) ApplySnapshotHeader(ceiling, cutover uint64) { } ``` -**Cost note**. Step 3 reads the full snapshot file once (through the -crc32 TeeReader). For multi-GiB FSMs this is a non-trivial I/O cost -— but it is **strictly cheaper** than the restore path it replaces -(which also reads the file once via `restoreAndComputeCRC` AND -additionally writes a temp Pebble database with sstable / WAL output). -Observed restore wall-clock is dominated by Pebble writes, not reads; -eliding the writes preserves the bulk of the win. A future -optimisation could persist the HLC ceiling + cutover durably -(analogous to `metaAppliedIndex`) and elide the file read entirely — -out of scope here, flagged under Open Questions. - -**Why this seam shape**. The two-phase split lets the CRC verifier -stay co-located with its private helpers in -`internal/raftengine/etcd/fsm_snapshot_file.go`'s package, **and** -keeps the v1/v2 header parser inside `kv` where it already lives. -Neither package imports the other in production. The "do CRC on -engine side, side-effects on FSM side after verify" contract is -exactly the inversion of `openAndRestoreFSMSnapshot` (which inlines -`fsm.Restore` inside the CRC tee for performance reasons): for the -skip path we don't need a single-pass restore, so splitting the -phases costs nothing and buys layer hygiene. +**Cost note**. The skip path does not read the multi-GiB body. Startup +cost is bounded by opening the snapshot file, reading the footer, and +parsing the fixed-size header. Full-body CRC verification remains on +the execute/full-restore path, where the body bytes are actually used. + +**Why this seam shape**. The two-phase split keeps the snapshot +envelope checks in `internal/raftengine/etcd`, and keeps the v1/v2 +header parser inside `kv` where it already lives. Neither package +imports the other in production. The skip path deliberately avoids the +restore path's body CRC work because it does not consume body bytes. ### 6. Crash-safety argument @@ -617,7 +588,7 @@ window. By forcing `pebble.Sync` on `SetDurableAppliedIndex` we make the checkpoint at least as durable as the snapshot pointer that follows. Cost: +1 extra fsync per snapshot persist (rare; default -`SnapshotCount=10000`). Negligible vs. the savings, and the only +`SnapshotCount=100000`). Negligible vs. the savings, and the only way to keep the round-4 ordering proof intact under nosync mode. #### Encryption opcodes (`OpRegistration`/`OpBootstrap`/`OpRotation`) @@ -768,7 +739,7 @@ func (s *pebbleStore) SetDurableAppliedIndex(idx uint64) error { // because WAL compaction starts at the snapshot index, no future // replay can re-bump metaAppliedIndex from the lost lease/data // applies, and the skip permanently falls back. The +1 extra fsync - // per snapshot persist (rare; default SnapshotCount=10000) is the + // per snapshot persist (rare; default SnapshotCount=100000) is the // right price. return errors.WithStack(b.Commit(pebble.Sync)) } @@ -801,12 +772,12 @@ permanent fallback case is closed. **Cost**: one extra pebble `Batch.Commit` (Sync per `ELASTICKV_FSM_SYNC_MODE`) per snapshot persist. Snapshots fire on the -etcd raft `SnapshotCount` cadence (default 10000 entries), so this is -~one extra fsync per ~10000 entries — negligible. +etcd raft `SnapshotCount` cadence (default 100000 entries), so this is +~one extra fsync per ~100000 entries — negligible. **Why not bump on every HLC lease apply**. Option A (1 pebble batch per lease tick) costs ~1 fsync/sec/group continuously. Option B (the -snapshot-persist hook) costs ~1 fsync per 10000 entries. Both close +snapshot-persist hook) costs ~1 fsync per 100000 entries. Both close the skip gap; B costs ~10⁴× less and aligns with the natural durability boundary the engine already maintains. @@ -903,7 +874,7 @@ restoreSnapshotState skipped (FSM at index %d, snapshot at %d, ceiling=%d, cutov |---|---|---| | **B1** (this PR) | Design doc | None | | **B2** | `ApplyMutationsRaftAt` / `DeletePrefixAtRaftAt` overloads + meta-key bundling in both leaves + `pebbleStore.LastAppliedIndex()` (under `dbMu.RLock()`) + `pebbleStore.SetDurableAppliedIndex()` (under `dbMu.RLock()` + `applyMu.Lock()` RMW monotonic guard, **`pebble.Sync` unconditionally**) + `kvFSM.LastAppliedIndex()` directly satisfies `raftengine.AppliedIndexReader` (compile-time guard in `kv/fsm_applied_index_iface_check.go`) + `kvFSM.SetDurableAppliedIndex` forwarding + thread `f.pendingApplyIdx` into the data-Apply leaves + BOTH `persistCreatedSnapshot` (`engine.go:2679`) AND `e.persistLocalSnapshotPayload` (`engine.go:4032`, the SnapshotCount-triggered hot path) call `SetDurableAppliedIndex` BEFORE the corresponding `persist.SaveSnap` | Meta key starts being written on every data Apply AND at every snapshot persist (both config-snapshot and steady-state local-snapshot paths). Skip is still disabled. Soak in production for one release. | -| **B3** | `restoreSnapshotState` skip gate + `applyHeaderStateOnSkip(snapPath, tok.CRC32C)` orchestrating size + footer-vs-tokenCRC + full-body-CRC verification using `internal/raftengine/etcd`'s existing helpers (matching `openAndRestoreFSMSnapshot`'s safety contract) + two-phase `SnapshotHeaderApplier` seam on `kvFSM` (`ParseSnapshotHeader(r io.Reader) (ceiling, cutover, err)` + pure `ApplySnapshotHeader(ceiling, cutover)`) + metrics + INFO log | **User-visible cold-start win.** | +| **B3** | `restoreSnapshotState` skip gate + `applyHeaderStateOnSkip(snapPath, tok.CRC32C)` orchestrating size + footer-vs-tokenCRC checks using `internal/raftengine/etcd`'s existing helpers + two-phase `SnapshotHeaderApplier` seam on `kvFSM` (`ParseSnapshotHeader(r io.Reader) (ceiling, cutover, err)` + pure `ApplySnapshotHeader(ceiling, cutover)`) + metrics + INFO log. Full-body CRC remains on the execute/full-restore path. | **User-visible cold-start win.** | | **B4** | Lower `HEALTH_TIMEOUT_SECONDS` default once production data shows steady-state skip rate ≥ 90 % | Tighter ceiling; the env override remains honoured. | Each of B2–B3 ships behind tests: @@ -927,8 +898,8 @@ Each of B2–B3 ships behind tests: test asserts that `applyHeaderStateOnSkip` sets `f.hlc.PhysicalCeiling()` **and** `f.restoredCutover` for both v1 and v2 snapshot headers — the ceiling+cutover are invariant under - the optimisation. Three additional CRC-corruption tests (round-6, - one per failure mode) inject the corruption and drive the skip path + the optimisation. Additional envelope/header tests inject corruption + and drive the skip path through `applyHeaderStateOnSkip` (either directly or via `restoreSnapshotState` with `fsmAlreadyAtIndex` returning true), asserting the specific typed error surfaces and that the FSM did @@ -937,16 +908,15 @@ Each of B2–B3 ships behind tests: `ErrFSMSnapshotTooSmall`. - Pair the file with a wrong-token CRC → `ErrFSMSnapshotTokenCRC`. - - Flip one body byte (post-header) → - `ErrFSMSnapshotFileCRC` (round-6's full-body CRC pass catches - this; without round-6 the skip would silently install state - from a corrupt file). + - Flip one body byte (post-header) and assert the skip path still + succeeds without scanning the body. Body corruption is caught by + the full restore path if that path is needed. An idle-cluster integration test runs a 3-node cluster with `ELASTICKV_RAFT_SNAPSHOT_COUNT=10` (overriding the default - `defaultSnapshotEvery = 10000` at `engine.go:93` so the scenario is + `defaultSnapshotEvery = 100000` at `engine.go:93` so the scenario is tractable — at default + `hlcRenewalInterval = 1 s` an idle period - would need ≥ 20 000 s). With the override, the test issues no data + would need ≥ 200 000 s). With the override, the test issues no data writes for `2 × 10 × hlcRenewalInterval = 20 s`, takes a snapshot, restarts a node, and asserts the skip fires — proving the codex round-3 P2 scenario is closed end-to-end through the @@ -1061,7 +1031,7 @@ subsection + B2 row + B2 test list) closes the gap by bumping successful snapshot persist, `LastAppliedIndex >= snapshot.Index` holds unconditionally, so the skip fires reliably on the next restart. Cost: one extra pebble `Batch.Commit` per snapshot persist -(~one extra fsync per `SnapshotCount` entries, default 10000) versus +(~one extra fsync per `SnapshotCount` entries, default 100000) versus Option A's continuous ~1 fsync/sec/group. Lesson: "rare" should be a quantitative claim against the actual @@ -1138,16 +1108,13 @@ and silently apply `ceiling=0, cutover=0` for a too-short file. Round 6 threads `tok.CRC32C` through `applyHeaderStateOnSkip` and `SnapshotHeaderApplier.ApplySnapshotHeaderFromFile`, and the §5 -pseudocode runs the same three-step verification before applying any -side-effect. Cost: one extra full-file read on the skip path — still -strictly cheaper than the restore path it replaces (the same read -happens during `restoreAndComputeCRC`, plus restore additionally -writes a temp Pebble database via `restoreBatchLoopInto`). - -A follow-up optimisation that persists the HLC ceiling + cutover as -durable meta keys (analogous to `metaAppliedIndex`) would let the -skip path elide the file read entirely; flagged under Open Questions -but not part of this proposal. +pseudocode originally ran the same three-step verification before +applying any side-effect. Production profiling later showed that the +extra full-file read is too expensive for multi-GiB snapshots during +restart. The final implementation keeps the cheap fail-closed checks +that matter for the skipped header side-effect (size, footer-vs-token, +header parse), and leaves full-body CRC on the full restore path where +body bytes are consumed. Lesson: when a §X seam takes over a side-effect previously gated by existing fail-closed checks, **inventory those checks first and @@ -1176,12 +1143,11 @@ or force a layering change. Round 7 splits `SnapshotHeaderApplier` into two methods: `ParseSnapshotHeader(r io.Reader) (ceiling, cutover, err)` and the -pure-assignment `ApplySnapshotHeader(ceiling, cutover)`. The CRC -verification orchestration stays in `internal/raftengine/etcd/wal_store.go` -where the helpers already live; the engine wraps the file in a crc32 -TeeReader and hands the reader to `setter.ParseSnapshotHeader`, which -calls the still-in-`kv`-package `ReadSnapshotHeader(*bufio.Reader)` -and drains. No package imports change; no helpers need to be +pure-assignment `ApplySnapshotHeader(ceiling, cutover)`. The envelope +checks stay in `internal/raftengine/etcd/wal_store.go` where the helpers +already live; the engine hands a payload-limited reader to +`setter.ParseSnapshotHeader`, which calls the still-in-`kv`-package +`ReadSnapshotHeader`. No package imports change; no helpers need to be exported. **P2 line 660 — `SetDurableAppliedIndex` honoured nosync mode.** @@ -1201,7 +1167,7 @@ Round 7 pins `pebbleStore.SetDurableAppliedIndex` to `pebble.Sync` unconditionally. The checkpoint must be at least as durable as the snapshot pointer that immediately follows; there is no raft log entry to replay it from after WAL compaction. Cost: +1 extra fsync per -snapshot persist (rare, default `SnapshotCount=10000`). +snapshot persist (rare, default `SnapshotCount=100000`). Lesson: - (P2 line 410) **Before pseudocoding cross-package helper use, diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 6da71b736..4002f2209 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -124,7 +124,7 @@ Storage breakage at 1–10 TB/shard: - Snapshot transfer is a single-stream full-iter scan; at 1 TB it is hours, pins SSTs (blocks compaction reclamation), and any raft snapshot transfer serializes through it. -- WAL replay between `defaultSnapshotEvery = 10 000` raft entries +- WAL replay between `defaultSnapshotEvery = 100 000` raft entries can be multi-GiB; with `pebble.Sync` durability path the replay is tens of minutes. - Pebble L0CompactionThreshold / LBaseMaxBytes / compaction diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index a273f5e8d..b508b2f00 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -116,10 +116,10 @@ const ( // defaultSnapshotEvery is the fallback trigger threshold: take an FSM // snapshot once the applied index has advanced this many entries past // the last snapshot's index. etcd/raft itself uses 10_000 as a default, - // but with fat proposal payloads (e.g. Lua scripts) this can produce a - // multi-GiB WAL between snapshots. Operators can lower via + // but multi-GiB FSM snapshots can take tens of seconds to persist and + // contend with the Raft hot path. Operators can lower or raise via // ELASTICKV_RAFT_SNAPSHOT_COUNT without a rebuild. - defaultSnapshotEvery = 10_000 + defaultSnapshotEvery = 100_000 snapshotEveryEnvVar = "ELASTICKV_RAFT_SNAPSHOT_COUNT" defaultSnapshotQueueSize = 1 defaultAdminPollInterval = 10 * time.Millisecond @@ -1250,12 +1250,11 @@ func (e *Engine) recordDispatchErrorCode(code string) uint64 { } // StepQueueFullCount returns the total number of inbound raft messages -// that could not be enqueued into the selected inbound step queue -// because the channel was at capacity. This is the "etcd raft inbound -// step queue is full" signal from the task description: a spike -// indicates the local raft loop is starved, usually by something -// blocking the apply path such as -// the pre-#560 rawKeyTypeAt seek storm. +// that found the selected inbound step queue at capacity. Blocking inbound +// message classes wait for space after incrementing this counter; best-effort +// classes still return errStepQueueFull. A spike indicates the local raft loop +// is starved, usually by something blocking the apply path such as the +// pre-#560 rawKeyTypeAt seek storm. func (e *Engine) StepQueueFullCount() uint64 { if e == nil { return 0 @@ -2298,6 +2297,7 @@ func (e *Engine) handleStep(msg raftpb.Message) { commitBeforeStep := e.rawNode.Status().GetCommit() if err := e.rawNode.Step(&msg); err != nil { if errors.Is(err, etcdraft.ErrStepPeerNotFound) { + e.removeReceivedFSMSnapshotToken(msg) e.unprotectReceivedFSMSnapshotToken(msg) return } @@ -2305,9 +2305,11 @@ func (e *Engine) handleStep(msg raftpb.Message) { return } if e.unprotectReceivedFSMSnapshotTokenIfCommitted(msg, commitBeforeStep) { + e.removeReceivedFSMSnapshotToken(msg) return } if !e.rawNode.HasReady() { + e.removeReceivedFSMSnapshotToken(msg) e.unprotectReceivedFSMSnapshotToken(msg) return } @@ -2518,7 +2520,7 @@ func isInboundPriorityMsg(t raftpb.MessageType) bool { } func isBlockingInboundStepMsg(t raftpb.MessageType) bool { - return t == raftpb.MsgSnap + return t == raftpb.MsgSnap || t == raftpb.MsgApp || t == raftpb.MsgAppResp } // selectDispatchLane picks the per-peer channel for msgType. In the legacy @@ -3475,6 +3477,7 @@ func (e *Engine) releaseIgnoredReceivedFSMSnapshotSteps(rd etcdraft.Ready) { if index == readySnapshotIndex { continue } + e.removeReceivedFSMSnapshotIndex(index) for i := 0; i < count; i++ { e.unprotectReceivedFSMSnapshot(index) } @@ -3489,6 +3492,23 @@ func (e *Engine) unprotectReceivedFSMSnapshotToken(msg raftpb.Message) { e.unprotectReceivedFSMSnapshot(index) } +func (e *Engine) removeReceivedFSMSnapshotToken(msg raftpb.Message) { + index, ok := receivedFSMSnapshotTokenIndex(msg) + if !ok { + return + } + e.removeReceivedFSMSnapshotIndex(index) +} + +func (e *Engine) removeReceivedFSMSnapshotIndex(index uint64) { + if e == nil || e.fsmSnapDir == "" || index == 0 { + return + } + e.snapshotMu.Lock() + defer e.snapshotMu.Unlock() + removeWithWarn(fsmSnapPath(e.fsmSnapDir, index), "ignored received fsm snapshot") +} + func receivedFSMSnapshotTokenIndex(msg raftpb.Message) (uint64, bool) { if msg.GetType() != raftpb.MsgSnap || msg.GetSnapshot() == nil || !isSnapshotToken(msg.GetSnapshot().GetData()) { return 0, false @@ -4412,6 +4432,13 @@ func (e *Engine) stepChannelFor(msgType raftpb.MessageType) chan raftpb.Message } func (e *Engine) enqueueBlockingStep(ctx context.Context, ch chan raftpb.Message, msg raftpb.Message) error { + select { + case ch <- msg: + return nil + default: + e.stepQueueFullCount.Add(1) + } + select { case <-ctx.Done(): return errors.WithStack(ctx.Err()) diff --git a/internal/raftengine/etcd/engine_applied_index_test.go b/internal/raftengine/etcd/engine_applied_index_test.go index cf4596306..6d8d3e48f 100644 --- a/internal/raftengine/etcd/engine_applied_index_test.go +++ b/internal/raftengine/etcd/engine_applied_index_test.go @@ -268,8 +268,29 @@ func TestReleaseIgnoredReceivedFSMSnapshotStepsUnprotectsNonSnapshotReady(t *tes require.Empty(t, e.pendingReceivedFSMSnapshotStep) } +func TestReleaseIgnoredReceivedFSMSnapshotStepsRemovesIgnoredFSMFile(t *testing.T) { + fsmSnapDir := t.TempDir() + writeFSMFileForTest(t, fsmSnapDir, 10, []byte("ignored snapshot")) + e := &Engine{ + fsmSnapDir: fsmSnapDir, + protectedReceivedFSMSnaps: map[uint64]int{10: 1}, + pendingReceivedFSMSnapshotStep: map[uint64]int{ + 10: 1, + }, + } + + e.releaseIgnoredReceivedFSMSnapshotSteps(etcdraft.Ready{}) + + require.NoFileExists(t, fsmSnapPath(fsmSnapDir, 10)) + require.Empty(t, e.protectedReceivedFSMSnaps) + require.Empty(t, e.pendingReceivedFSMSnapshotStep) +} + func TestReleaseIgnoredReceivedFSMSnapshotStepsKeepsSnapshotReadyProtected(t *testing.T) { + fsmSnapDir := t.TempDir() + writeFSMFileForTest(t, fsmSnapDir, 10, []byte("accepted snapshot")) e := &Engine{ + fsmSnapDir: fsmSnapDir, protectedReceivedFSMSnaps: map[uint64]int{10: 1}, pendingReceivedFSMSnapshotStep: map[uint64]int{ 10: 1, @@ -283,6 +304,7 @@ func TestReleaseIgnoredReceivedFSMSnapshotStepsKeepsSnapshotReadyProtected(t *te require.Equal(t, map[uint64]int{10: 1}, e.protectedReceivedFSMSnaps) require.Empty(t, e.pendingReceivedFSMSnapshotStep) + require.FileExists(t, fsmSnapPath(fsmSnapDir, 10)) } // TestRecordingFSM_SatisfiesAppliedIndexWriter is a compile-time- diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index ce81a539d..713862a9c 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -599,7 +599,7 @@ func TestHandleTransportMessageWaitsForStartup(t *testing.T) { require.NoError(t, <-errCh) } -func TestEnqueueStepReturnsQueueFull(t *testing.T) { +func TestEnqueueStepBestEffortReturnsQueueFull(t *testing.T) { engine := &Engine{ doneCh: make(chan struct{}), stepCh: make(chan raftpb.Message, 1), @@ -608,20 +608,66 @@ func TestEnqueueStepReturnsQueueFull(t *testing.T) { require.Equal(t, uint64(0), engine.StepQueueFullCount()) - err := engine.enqueueStep(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgApp)}) + err := engine.enqueueStep(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgStorageAppend)}) require.Error(t, err) require.True(t, errors.Is(err, errStepQueueFull)) // The Prometheus hot-path dashboard relies on StepQueueFullCount - // advancing exactly once per rejected enqueue so the scraped rate - // equals the true drop rate, not a multiple of it. + // advancing exactly once per full enqueue attempt so the scraped + // rate equals the true congestion rate, not a multiple of it. require.Equal(t, uint64(1), engine.StepQueueFullCount()) - err = engine.enqueueStep(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgApp)}) + err = engine.enqueueStep(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgStorageAppend)}) require.Error(t, err) require.Equal(t, uint64(2), engine.StepQueueFullCount()) } +func TestEnqueueStepMsgAppWaitsForQueueSlot(t *testing.T) { + engine := &Engine{ + doneCh: make(chan struct{}), + stepCh: make(chan raftpb.Message, 1), + } + engine.stepCh <- raftpb.Message{Type: messageTypePtr(raftpb.MsgHeartbeat)} + + errCh := make(chan error, 1) + go func() { + errCh <- engine.enqueueStep(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgApp)}) + }() + + select { + case err := <-errCh: + t.Fatalf("MsgApp enqueue returned before the queue had room: %v", err) + case <-time.After(20 * time.Millisecond): + } + require.Equal(t, uint64(1), engine.StepQueueFullCount()) + + firstMsg := <-engine.stepCh + require.Equal(t, raftpb.MsgHeartbeat, firstMsg.GetType()) + select { + case err := <-errCh: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("MsgApp enqueue did not resume after the queue had room") + } + nextMsg := <-engine.stepCh + require.Equal(t, raftpb.MsgApp, nextMsg.GetType()) +} + +func TestEnqueueStepMsgAppReturnsContextDeadlineWhenQueueStaysFull(t *testing.T) { + engine := &Engine{ + doneCh: make(chan struct{}), + stepCh: make(chan raftpb.Message, 1), + } + engine.stepCh <- raftpb.Message{Type: messageTypePtr(raftpb.MsgHeartbeat)} + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + err := engine.enqueueStep(ctx, raftpb.Message{Type: messageTypePtr(raftpb.MsgApp)}) + require.Error(t, err) + require.True(t, errors.Is(err, context.DeadlineExceeded)) + require.Equal(t, uint64(1), engine.StepQueueFullCount()) +} + func TestEnqueueStepPriorityBypassesFullBulkQueue(t *testing.T) { engine := &Engine{ doneCh: make(chan struct{}), @@ -641,9 +687,11 @@ func TestEnqueueStepPriorityBypassesFullBulkQueue(t *testing.T) { t.Fatal("priority heartbeat was not enqueued") } - err = engine.enqueueStep(context.Background(), raftpb.Message{Type: messageTypePtr(raftpb.MsgApp)}) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + err = engine.enqueueStep(ctx, raftpb.Message{Type: messageTypePtr(raftpb.MsgApp)}) require.Error(t, err) - require.True(t, errors.Is(err, errStepQueueFull)) + require.True(t, errors.Is(err, context.DeadlineExceeded)) require.Equal(t, uint64(1), engine.StepQueueFullCount()) } diff --git a/internal/raftengine/etcd/fsm_snapshot_file.go b/internal/raftengine/etcd/fsm_snapshot_file.go index 8e31a05d3..e27bf0b0b 100644 --- a/internal/raftengine/etcd/fsm_snapshot_file.go +++ b/internal/raftengine/etcd/fsm_snapshot_file.go @@ -46,6 +46,12 @@ const ( // fsmWriteBufSize is the bufio.Writer buffer size used when writing .fsm files. fsmWriteBufSize = 1 << 20 // 1 MiB + // defaultMaxRetainedFSMSnapshotBytes bounds retained .fsm payload bytes + // after successful snapshot publication. Large production FSM snapshots can + // be tens of GiB; keeping defaultMaxSnapFiles full copies leaves too little + // headroom for the next receive-side spool. + defaultMaxRetainedFSMSnapshotBytes = int64(16 << 30) // 16 GiB + // fsmMaxInMemPayload is the maximum payload size that readFSMSnapshotPayload // will materialise into memory. Larger snapshots must use the streaming path // (openFSMSnapshotPayloadReader) to avoid OOM. 1 GiB is chosen as a generous @@ -53,6 +59,8 @@ const ( fsmMaxInMemPayload = int64(1 << 30) // 1 GiB ) +const maxRetainedFSMSnapshotBytesEnvVar = "ELASTICKV_RAFT_MAX_RETAINED_FSM_SNAPSHOT_BYTES" + var ( snapshotTokenMagic = [snapshotTokenMagicLen]byte{'E', 'K', 'V', 'T'} crc32cTable = crc32.MakeTable(crc32.Castagnoli) @@ -1081,7 +1089,7 @@ func purgeOldSnapshotFiles(snapDir, fsmSnapDir string) error { } snaps := collectSnapNames(entries) - if len(snaps) <= defaultMaxSnapFiles { + if len(snaps) == 0 { return nil } // Sort explicitly: os.ReadDir returns lexicographic order on most systems, @@ -1089,8 +1097,12 @@ func purgeOldSnapshotFiles(snapDir, fsmSnapDir string) error { // hex, so lexicographic == chronological order (oldest first). sort.Strings(snaps) + maxKeep := retainedSnapshotFileLimit(snaps, fsmSnapDir) + if len(snaps) <= maxKeep { + return nil + } var combined error - for _, name := range snaps[:len(snaps)-defaultMaxSnapFiles] { + for _, name := range snaps[:len(snaps)-maxKeep] { if err := purgeSnapPair(snapDir, fsmSnapDir, name); err != nil { combined = errors.CombineErrors(combined, err) } @@ -1101,6 +1113,60 @@ func purgeOldSnapshotFiles(snapDir, fsmSnapDir string) error { return errors.WithStack(combined) } +func retainedSnapshotFileLimit(snaps []string, fsmSnapDir string) int { + maxKeep := defaultMaxSnapFiles + if maxKeep < 1 { + maxKeep = 1 + } + if maxKeep > len(snaps) { + maxKeep = len(snaps) + } + if fsmSnapDir == "" { + return maxKeep + } + budget := maxRetainedFSMSnapshotBytes() + if budget <= 0 { + return maxKeep + } + for maxKeep > 1 && retainedFSMSnapshotBytes(snaps[len(snaps)-maxKeep:], fsmSnapDir) > budget { + maxKeep-- + } + return maxKeep +} + +func maxRetainedFSMSnapshotBytes() int64 { + raw := strings.TrimSpace(os.Getenv(maxRetainedFSMSnapshotBytesEnvVar)) + if raw == "" { + return defaultMaxRetainedFSMSnapshotBytes + } + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + slog.Warn("invalid max retained FSM snapshot bytes; using default", + "env", maxRetainedFSMSnapshotBytesEnvVar, + "value", raw, + "error", err, + ) + return defaultMaxRetainedFSMSnapshotBytes + } + return n +} + +func retainedFSMSnapshotBytes(snaps []string, fsmSnapDir string) int64 { + var total int64 + for _, name := range snaps { + idx := parseSnapFileIndex(name) + if idx == 0 { + continue + } + info, err := os.Stat(fsmSnapPath(fsmSnapDir, idx)) + if err != nil { + continue + } + total += info.Size() + } + return total +} + func collectSnapNames(entries []os.DirEntry) []string { var snaps []string for _, e := range entries { diff --git a/internal/raftengine/etcd/grpc_transport.go b/internal/raftengine/etcd/grpc_transport.go index 29f67129f..ab7fdf335 100644 --- a/internal/raftengine/etcd/grpc_transport.go +++ b/internal/raftengine/etcd/grpc_transport.go @@ -1087,8 +1087,11 @@ func snapshotMessageHeader(msg raftpb.Message) ([]byte, error) { } func sendSnapshotChunks(stream pb.EtcdRaft_SendSnapshotClient, header []byte, payload []byte, chunkSize int) error { + if err := sendSnapshotChunk(stream, &pb.EtcdRaftSnapshotChunk{Metadata: header}); err != nil { + return err + } if len(payload) == 0 { - return sendSnapshotChunk(stream, &pb.EtcdRaftSnapshotChunk{Metadata: header, Final: true}) + return sendSnapshotChunk(stream, &pb.EtcdRaftSnapshotChunk{Final: true}) } for offset := 0; offset < len(payload); offset += chunkSize { end := offset + chunkSize @@ -1099,9 +1102,6 @@ func sendSnapshotChunks(stream pb.EtcdRaft_SendSnapshotClient, header []byte, pa Chunk: payload[offset:end], Final: end == len(payload), } - if offset == 0 { - chunk.Metadata = header - } if err := sendSnapshotChunk(stream, chunk); err != nil { return err } @@ -1120,6 +1120,9 @@ func sendSnapshotReaderChunks(stream pb.EtcdRaft_SendSnapshotClient, header []by if chunkSize <= 0 { chunkSize = defaultSnapshotChunkSize } + if err := sendSnapshotChunk(stream, &pb.EtcdRaftSnapshotChunk{Metadata: header}); err != nil { + return err + } buffered := bufio.NewReaderSize(reader, chunkSize) current, err := readSnapshotChunk(buffered, chunkSize) if err != nil { @@ -1129,14 +1132,13 @@ func sendSnapshotReaderChunks(stream pb.EtcdRaft_SendSnapshotClient, header []by // a small snapshot. Include it so the receiver does not get an // empty snapshot.Data. return sendSnapshotChunk(stream, &pb.EtcdRaftSnapshotChunk{ - Metadata: header, - Chunk: current, - Final: true, + Chunk: current, + Final: true, }) } return errors.WithStack(err) } - return streamReaderChunks(stream, header, buffered, current, chunkSize) + return streamReaderChunks(stream, nil, buffered, current, chunkSize) } // streamReaderChunks drains buffered starting from `current` (the first full diff --git a/internal/raftengine/etcd/grpc_transport_test.go b/internal/raftengine/etcd/grpc_transport_test.go index b7e43c703..a57fc43bb 100644 --- a/internal/raftengine/etcd/grpc_transport_test.go +++ b/internal/raftengine/etcd/grpc_transport_test.go @@ -1549,10 +1549,13 @@ func TestSendSnapshotReaderChunksSmallPayloadPreservesData(t *testing.T) { err := sendSnapshotReaderChunks(client, header, bytes.NewReader(payload), defaultSnapshotChunkSize) require.NoError(t, err) - require.Len(t, client.chunks, 1) + require.Len(t, client.chunks, 2) require.Equal(t, header, client.chunks[0].Metadata) - require.Equal(t, payload, client.chunks[0].Chunk) - require.True(t, client.chunks[0].Final) + require.Empty(t, client.chunks[0].Chunk) + require.False(t, client.chunks[0].Final) + require.Empty(t, client.chunks[1].Metadata) + require.Equal(t, payload, client.chunks[1].Chunk) + require.True(t, client.chunks[1].Final) } func TestSendSnapshotReaderChunksEmptyPayloadSendsHeaderOnly(t *testing.T) { @@ -1562,10 +1565,13 @@ func TestSendSnapshotReaderChunksEmptyPayloadSendsHeaderOnly(t *testing.T) { err := sendSnapshotReaderChunks(client, header, bytes.NewReader(nil), defaultSnapshotChunkSize) require.NoError(t, err) - require.Len(t, client.chunks, 1) + require.Len(t, client.chunks, 2) require.Equal(t, header, client.chunks[0].Metadata) require.Empty(t, client.chunks[0].Chunk) - require.True(t, client.chunks[0].Final) + require.False(t, client.chunks[0].Final) + require.Empty(t, client.chunks[1].Metadata) + require.Empty(t, client.chunks[1].Chunk) + require.True(t, client.chunks[1].Final) } // testSnapshotSendClient captures chunks sent via sendSnapshotReaderChunks / sendSnapshotChunks. @@ -1718,19 +1724,23 @@ func TestSendSnapshotReaderChunksMultiChunk(t *testing.T) { err := sendSnapshotReaderChunks(client, header, bytes.NewReader(payload), chunkSize) require.NoError(t, err) - require.Len(t, client.chunks, 3) + require.Len(t, client.chunks, 4) require.Equal(t, header, client.chunks[0].Metadata) - require.Equal(t, []byte("1234"), client.chunks[0].Chunk) + require.Empty(t, client.chunks[0].Chunk) require.False(t, client.chunks[0].Final) require.Empty(t, client.chunks[1].Metadata) - require.Equal(t, []byte("abcd"), client.chunks[1].Chunk) + require.Equal(t, []byte("1234"), client.chunks[1].Chunk) require.False(t, client.chunks[1].Final) require.Empty(t, client.chunks[2].Metadata) - require.Equal(t, []byte("5678"), client.chunks[2].Chunk) - require.True(t, client.chunks[2].Final) + require.Equal(t, []byte("abcd"), client.chunks[2].Chunk) + require.False(t, client.chunks[2].Final) + + require.Empty(t, client.chunks[3].Metadata) + require.Equal(t, []byte("5678"), client.chunks[3].Chunk) + require.True(t, client.chunks[3].Final) } func TestSendSnapshotReaderChunksExactBoundary(t *testing.T) { @@ -1743,13 +1753,16 @@ func TestSendSnapshotReaderChunksExactBoundary(t *testing.T) { err := sendSnapshotReaderChunks(client, header, bytes.NewReader(payload), chunkSize) require.NoError(t, err) - require.Len(t, client.chunks, 2) + require.Len(t, client.chunks, 3) require.Equal(t, header, client.chunks[0].Metadata) - require.Equal(t, []byte("1234"), client.chunks[0].Chunk) + require.Empty(t, client.chunks[0].Chunk) require.False(t, client.chunks[0].Final) - require.Equal(t, []byte("5678"), client.chunks[1].Chunk) - require.True(t, client.chunks[1].Final) + require.Equal(t, []byte("1234"), client.chunks[1].Chunk) + require.False(t, client.chunks[1].Final) + + require.Equal(t, []byte("5678"), client.chunks[2].Chunk) + require.True(t, client.chunks[2].Final) } // TestSendSnapshotReaderChunksTrailingPartialChunk regressions a production @@ -1772,17 +1785,20 @@ func TestSendSnapshotReaderChunksTrailingPartialChunk(t *testing.T) { err := sendSnapshotReaderChunks(client, header, bytes.NewReader(payload), chunkSize) require.NoError(t, err) - require.Len(t, client.chunks, 3, "expected two full chunks plus a trailing partial") + require.Len(t, client.chunks, 4, "expected metadata, two full chunks, and a trailing partial") require.Equal(t, header, client.chunks[0].Metadata) - require.Equal(t, []byte("1234"), client.chunks[0].Chunk) + require.Empty(t, client.chunks[0].Chunk) require.False(t, client.chunks[0].Final) - require.Equal(t, []byte("5678"), client.chunks[1].Chunk) + require.Equal(t, []byte("1234"), client.chunks[1].Chunk) require.False(t, client.chunks[1].Final) - require.Equal(t, []byte("X"), client.chunks[2].Chunk) - require.True(t, client.chunks[2].Final) + require.Equal(t, []byte("5678"), client.chunks[2].Chunk) + require.False(t, client.chunks[2].Final) + + require.Equal(t, []byte("X"), client.chunks[3].Chunk) + require.True(t, client.chunks[3].Final) var delivered []byte for _, c := range client.chunks { diff --git a/internal/raftengine/etcd/snapshot_spool_test.go b/internal/raftengine/etcd/snapshot_spool_test.go index b0facd96d..211077312 100644 --- a/internal/raftengine/etcd/snapshot_spool_test.go +++ b/internal/raftengine/etcd/snapshot_spool_test.go @@ -374,6 +374,48 @@ func TestPurgeOldSnapFiles(t *testing.T) { require.NoError(t, err) } +func TestPurgeOldSnapFilesHonorsFSMByteBudget(t *testing.T) { + t.Setenv(maxRetainedFSMSnapshotBytesEnvVar, "100") + + snapDir := t.TempDir() + fsmSnapDir := t.TempDir() + + for i := uint64(1); i <= 3; i++ { + index := i * 10000 + createSnapFile(t, snapDir, index) + require.NoError(t, os.WriteFile(fsmSnapPath(fsmSnapDir, index), bytes.Repeat([]byte("x"), 60), 0o600)) + } + + require.NoError(t, purgeOldSnapshotFiles(snapDir, fsmSnapDir)) + + require.NoFileExists(t, filepath.Join(snapDir, fmt.Sprintf("%016x-%016x.snap", 1, uint64(10000)))) + require.NoFileExists(t, filepath.Join(snapDir, fmt.Sprintf("%016x-%016x.snap", 1, uint64(20000)))) + require.FileExists(t, filepath.Join(snapDir, fmt.Sprintf("%016x-%016x.snap", 1, uint64(30000)))) + require.NoFileExists(t, fsmSnapPath(fsmSnapDir, 10000)) + require.NoFileExists(t, fsmSnapPath(fsmSnapDir, 20000)) + require.FileExists(t, fsmSnapPath(fsmSnapDir, 30000)) +} + +func TestPurgeOldSnapFilesBudgetCanBeDisabled(t *testing.T) { + t.Setenv(maxRetainedFSMSnapshotBytesEnvVar, "0") + + snapDir := t.TempDir() + fsmSnapDir := t.TempDir() + + for i := uint64(1); i <= 4; i++ { + index := i * 10000 + createSnapFile(t, snapDir, index) + require.NoError(t, os.WriteFile(fsmSnapPath(fsmSnapDir, index), bytes.Repeat([]byte("x"), 60), 0o600)) + } + + require.NoError(t, purgeOldSnapshotFiles(snapDir, fsmSnapDir)) + + require.NoFileExists(t, filepath.Join(snapDir, fmt.Sprintf("%016x-%016x.snap", 1, uint64(10000)))) + require.FileExists(t, filepath.Join(snapDir, fmt.Sprintf("%016x-%016x.snap", 1, uint64(20000)))) + require.FileExists(t, filepath.Join(snapDir, fmt.Sprintf("%016x-%016x.snap", 1, uint64(30000)))) + require.FileExists(t, filepath.Join(snapDir, fmt.Sprintf("%016x-%016x.snap", 1, uint64(40000)))) +} + func TestPurgeOldSnapFilesUnderLimit(t *testing.T) { snapDir := t.TempDir() fsmSnapDir := t.TempDir() diff --git a/internal/raftengine/etcd/wal_store.go b/internal/raftengine/etcd/wal_store.go index f6fcc8a44..2a38ac3d0 100644 --- a/internal/raftengine/etcd/wal_store.go +++ b/internal/raftengine/etcd/wal_store.go @@ -2,7 +2,6 @@ package etcd import ( "bytes" - "hash/crc32" "io" "os" "path/filepath" @@ -142,22 +141,20 @@ func loadWalState(logger *zap.Logger, walDir, snapDir, fsmSnapDir string, fsm St return nil, err } - // Codex P1 #934: open the WAL BEFORE the skip-gate decision so we - // know the post-snapshot entry tail. The skip path is only safe - // when the FSM is at least as advanced as the last WAL entry; if - // the FSM is past `tok.Index` but the WAL still carries entries - // `tok.Index+1 .. have` (the normal interval between snapshots, - // since metaAppliedIndex advances on each Apply), those entries - // would re-apply onto a Pebble store that already contains them, - // hitting OCC conflicts and leaving the HLC below timestamps - // already on disk. Compute the WAL tail's last index and gate - // the skip on `have >= lastWalIndex`. + // Open the WAL before the skip-gate decision so we know the committed + // replay target for metrics and raw-node initialization. The skip + // gate itself only requires the FSM to be at least as fresh as the + // persisted snapshot index: Engine.Open seeds e.applied with the + // FSM's durable applied index, rawNodeAppliedForOpen trims the RawNode + // applied pointer when volatile/conf-change entries must replay, and + // applyNormalCommitted drops data-mutating duplicates while applying + // the remaining committed tail. w, hardState, entries, err := openAndReadWALWithRepair(logger, walDir, walSnapshotFor(snapshot)) if err != nil { return nil, err } - lastCommittedIndex := coldStartSkipThreshold(snapshot, hardState) - effectiveApplied, err := restoreSnapshotState(fsm, snapshot, lastCommittedIndex, fsmSnapDir, obs, logger) + committedReplayTarget := coldStartReplayTarget(snapshot, hardState) + effectiveApplied, err := restoreSnapshotState(fsm, snapshot, committedReplayTarget, fsmSnapDir, obs, logger) if err != nil { if closeErr := w.Close(); closeErr != nil { logger.Warn("WAL close failed after restoreSnapshotState error", @@ -221,25 +218,26 @@ func reportColdStartExecute(obs raftengine.ColdStartObserver, logger *zap.Logger ) } -// coldStartSkipThreshold returns the maximum log index the cold- -// start replay can deliver via Ready.CommittedEntries on this -// node: max(snapshot.Metadata.Index, hardState.Commit). The skip -// gate compares the FSM's durable applied index against this -// value; skip is only safe when the FSM is at least this fresh. +// coldStartReplayTarget returns the maximum log index cold-start replay +// can deliver via Ready.CommittedEntries on this node: +// max(snapshot.Metadata.Index, hardState.Commit). The skip gate uses the +// snapshot index for safety; this target is still reported in metrics and +// used by RawNode initialization to replay only the entries still needed +// after e.applied is seeded from the FSM's durable applied index. // // Followers can carry an UNCOMMITTED WAL suffix // (entries[n-1].Index > hardState.Commit). Raft does NOT surface // those entries in CommittedEntries until the leader confirms -// them. The previous gate used the WAL tail (entries[n-1].Index) +// them. The previous target used the WAL tail (entries[n-1].Index) // which forced a multi-GiB restore on every restart of any // follower with an uncommitted suffix, defeating the cold-start -// optimization. Codex P2 #934 round 3. +// optimization. // // The lower bound stays at the snapshot pointer because an empty // WAL still requires the FSM to be at least at the snapshot // index. Raft's invariant guarantees hardState.Commit >= 0; we do // not need to bound from below explicitly beyond snap.Index. -func coldStartSkipThreshold(snapshot raftpb.Snapshot, hardState raftpb.HardState) uint64 { +func coldStartReplayTarget(snapshot raftpb.Snapshot, hardState raftpb.HardState) uint64 { threshold := snapshot.GetMetadata().GetIndex() if hardState.GetCommit() > threshold { threshold = hardState.GetCommit() @@ -356,12 +354,12 @@ func loadPersistedSnapshot(logger *zap.Logger, walDir string, snapshotter *snap. // <= have or the Pebble store would observe them twice (OCC // conflicts; HLC ceiling inversion). The execute path returns // snapshot.Metadata.Index to leave engine behaviour unchanged. -func restoreSnapshotState(fsm StateMachine, snapshot raftpb.Snapshot, lastWalIndex uint64, fsmSnapDir string, obs raftengine.ColdStartObserver, logger *zap.Logger) (uint64, error) { +func restoreSnapshotState(fsm StateMachine, snapshot raftpb.Snapshot, replayTarget uint64, fsmSnapDir string, obs raftengine.ColdStartObserver, logger *zap.Logger) (uint64, error) { if etcdraft.IsEmptySnap(&snapshot) || len(snapshot.Data) == 0 || fsm == nil { return 0, nil } if isSnapshotToken(snapshot.Data) { - return restoreSnapshotStateFromToken(fsm, snapshot, lastWalIndex, fsmSnapDir, obs, logger) + return restoreSnapshotStateFromToken(fsm, snapshot, replayTarget, fsmSnapDir, obs, logger) } // Legacy format: full FSM payload embedded in snapshot.Data. if err := fsm.Restore(bytes.NewReader(snapshot.Data)); err != nil { @@ -376,32 +374,33 @@ func restoreSnapshotState(fsm StateMachine, snapshot raftpb.Snapshot, lastWalInd // - skip path: `have` (FSM is already past snapshot.Metadata.Index) // - execute path: snapshot.Metadata.Index (restored from snapshot) // -// The skip threshold is lastWalIndex (NOT tok.Index): the FSM must be -// at least as fresh as the last WAL entry the cold-start replay would -// deliver, otherwise entries between tok.Index and have would re-apply -// onto a Pebble store that already contains them. Codex P1 #934. +// The skip threshold is tok.Index: if the FSM already contains the +// snapshot state, the engine can seed e.applied with the FSM's durable +// applied index and let normal replay apply any committed tail after +// that point. Entries at or below e.applied are handled by the existing +// duplicate-replay seam in applyCommitted. // // Metrics + log fire AFTER the restore-side work succeeds (coderabbit // Major #934): a header/CRC failure must not register a "successful" // outcome in the soak metrics. -func restoreSnapshotStateFromToken(fsm StateMachine, snapshot raftpb.Snapshot, lastWalIndex uint64, fsmSnapDir string, obs raftengine.ColdStartObserver, logger *zap.Logger) (uint64, error) { +func restoreSnapshotStateFromToken(fsm StateMachine, snapshot raftpb.Snapshot, replayTarget uint64, fsmSnapDir string, obs raftengine.ColdStartObserver, logger *zap.Logger) (uint64, error) { tok, err := decodeSnapshotToken(snapshot.Data) if err != nil { return 0, err } - decision, have := decideSkipOutcome(fsm, lastWalIndex) + decision, have := decideSkipOutcome(fsm, tok.Index) snapPath := fsmSnapPath(fsmSnapDir, tok.Index) if decision == coldStartSkip { if err := applyHeaderStateOnSkip(fsm, snapPath, tok.CRC32C); err != nil { return 0, err } - reportColdStart(obs, logger, decision, tok.Index, lastWalIndex, have) + reportColdStart(obs, logger, decision, tok.Index, replayTarget, have) return have, nil } if err := openAndRestoreFSMSnapshot(fsm, snapPath, tok.CRC32C); err != nil { return 0, err } - reportColdStart(obs, logger, decision, tok.Index, lastWalIndex, have) + reportColdStart(obs, logger, decision, tok.Index, replayTarget, have) return snapshot.GetMetadata().GetIndex(), nil } @@ -461,8 +460,8 @@ func reportColdStart(obs raftengine.ColdStartObserver, logger *zap.Logger, d col case coldStartSkip: // Observer contract (cold_start.go + monitoring/cold_start.go // Prometheus impl): args are (snapshotIndex, haveAppliedIndex); - // gauges compute have-snapIndex. Codex P2 + coderabbit Major - // #934: do NOT pass target/lastWalIndex here or the exported + // gauges compute have-snapIndex. Do NOT pass the replay target + // here or the exported // gauge measures the wrong baseline. if obs != nil { obs.RestoreSkipped(snapIndex, have) @@ -501,21 +500,18 @@ func reportColdStart(obs raftengine.ColdStartObserver, logger *zap.Logger, d col } } -// applyHeaderStateOnSkip mirrors openAndRestoreFSMSnapshot's safety -// contract (size + footer-vs-tokenCRC + full-body-CRC) but applies -// only the header side-effects (HLC ceiling + Stage 8a cutover) -// instead of running the body restore. The body bytes are read for -// CRC coverage but discarded -- fsm.db already holds equivalent -// state, which is precisely the reason we're skipping the restore. +// applyHeaderStateOnSkip validates the cheap snapshot envelope checks +// (size + footer-vs-tokenCRC) and applies only the header side-effects +// (HLC ceiling + Stage 8a cutover) instead of running the body restore. +// The body bytes are not read here: fsm.db already holds equivalent state, +// which is precisely the reason we're skipping the restore. Full-body CRC +// verification still runs on the execute/full-restore path where body +// bytes are actually consumed. // // FSMs that do not implement raftengine.SnapshotHeaderApplier // silently no-op the apply phase -- the FSM has no header state to -// carry forward, and the CRC verification still runs (with no -// observable side-effect on success). On any verification failure -// the typed error propagates and FSM state stays untouched. -// -// See PR #910 design §5 round-7 (two-phase seam) + round-6 -// (three-step CRC mirroring openAndRestoreFSMSnapshot). +// carry forward. On any envelope or header parse failure the typed error +// propagates and FSM state stays untouched. func applyHeaderStateOnSkip(fsm StateMachine, snapPath string, tokenCRC uint32) error { file, err := os.Open(snapPath) if err != nil { @@ -527,61 +523,30 @@ func applyHeaderStateOnSkip(fsm StateMachine, snapPath string, tokenCRC uint32) if err != nil { return errors.WithStack(err) } - footer, err := verifyFSMSnapshotPrefix(file, info.Size(), snapPath, tokenCRC) - if err != nil { + if _, err := verifyFSMSnapshotPrefix(file, info.Size(), snapPath, tokenCRC); err != nil { return err } - // Step 3: full-body CRC. Wrap the payload in a crc32 TeeReader - // and hand it to the FSM's ParseSnapshotHeader for header parse - // + drain. Every payload byte flows through h, matching - // restoreAndComputeCRC's boundary in openAndRestoreFSMSnapshot. - // - // Error-ordering contract (claude #934 R1-F1): header parse - // errors surface BEFORE the body-CRC compare runs, so callers - // (the skip-gate fallback in restoreSnapshotState) may observe - // either an ErrSnapshotHeaderUnknownMagic / InvalidLength chain or - // an ErrFSMSnapshotFileCRC chain depending on which check fails - // first. This is the same ordering openAndRestoreFSMSnapshot has - // — both errors are equally fatal for the skip path (they signal - // snapshot file corruption) and both must propagate without ever - // calling ApplySnapshotHeader. The CRC check stays AFTER the - // header parse so the TeeReader has actually been drained before - // we read h.Sum32(); inverting the order would let a CRC mismatch - // surface on a truncated body even when the header was valid, - // muddying the operator-facing diagnostic. + setter, hasSetter := fsm.(raftengine.SnapshotHeaderApplier) + if !hasSetter { + return nil + } + if _, err := file.Seek(0, io.SeekStart); err != nil { return errors.WithStack(err) } payloadSize := info.Size() - fsmFooterSize - h := crc32.New(crc32cTable) - tee := io.TeeReader(io.LimitReader(file, payloadSize), h) - - setter, hasSetter := fsm.(raftengine.SnapshotHeaderApplier) - ceiling, cutover, err := readSnapshotHeaderOrDrain(setter, hasSetter, tee) + ceiling, cutover, err := setter.ParseSnapshotHeader(io.LimitReader(file, payloadSize)) if err != nil { - return err - } - - if h.Sum32() != footer { - return errors.Wrapf(ErrFSMSnapshotFileCRC, - "path=%s footer=%08x computed=%08x", snapPath, footer, h.Sum32()) - } - - // All three checks passed; apply side-effects (pure assignment - // in the FSM). Skipped silently when the FSM does not expose - // the seam. - if hasSetter { - setter.ApplySnapshotHeader(ceiling, cutover) + return errors.WithStack(err) } + setter.ApplySnapshotHeader(ceiling, cutover) return nil } -// verifyFSMSnapshotPrefix runs the first two cheap checks of -// openAndRestoreFSMSnapshot's three-step contract: size and -// footer-vs-tokenCRC. Returns the on-disk footer value (caller -// reuses it for the step-3 full-body CRC compare). Typed errors -// surface unchanged. +// verifyFSMSnapshotPrefix runs the cheap checks shared with +// openAndRestoreFSMSnapshot: size and footer-vs-tokenCRC. Returns the +// on-disk footer value. Typed errors surface unchanged. func verifyFSMSnapshotPrefix(file *os.File, fileSize int64, snapPath string, tokenCRC uint32) (uint32, error) { if fileSize < fsmMinFileSize { return 0, errors.Wrapf(ErrFSMSnapshotTooSmall, @@ -598,27 +563,6 @@ func verifyFSMSnapshotPrefix(file *os.File, fileSize int64, snapPath string, tok return footer, nil } -// readSnapshotHeaderOrDrain branches on whether the FSM exposes the -// SnapshotHeaderApplier seam: when present, delegate to -// ParseSnapshotHeader (which parses the header AND drains the rest); -// otherwise drain the entire payload through the tee'd reader so the -// CRC pass covers every byte. The (ceiling, cutover) tuple is zero -// in the no-seam case -- the caller's ApplySnapshotHeader branch -// short-circuits on hasSetter, so the zero values are inert. -func readSnapshotHeaderOrDrain(setter raftengine.SnapshotHeaderApplier, hasSetter bool, tee io.Reader) (uint64, uint64, error) { - if hasSetter { - ceiling, cutover, err := setter.ParseSnapshotHeader(tee) - if err != nil { - return 0, 0, errors.WithStack(err) - } - return ceiling, cutover, nil - } - if _, err := io.Copy(io.Discard, tee); err != nil { - return 0, 0, errors.WithStack(err) - } - return 0, 0, nil -} - func walSnapshotFor(snapshot raftpb.Snapshot) walpb.Snapshot { return walpb.Snapshot{ Index: proto.Uint64(snapshot.GetMetadata().GetIndex()), diff --git a/internal/raftengine/etcd/wal_store_skip_gate_test.go b/internal/raftengine/etcd/wal_store_skip_gate_test.go index 212a47ec6..ffde0a0b3 100644 --- a/internal/raftengine/etcd/wal_store_skip_gate_test.go +++ b/internal/raftengine/etcd/wal_store_skip_gate_test.go @@ -53,17 +53,13 @@ func (f *skipGateFSM) ParseSnapshotHeader(r io.Reader) (uint64, uint64, error) { if f.parseErr != nil { return 0, 0, f.parseErr } - // Mimic the real kvFSM contract: parse + drain. We don't actually - // parse a header here; the test fixtures embed magic+ceiling but - // for the gate-level tests we just drain so the CRC matches. + // Mimic the real kvFSM contract: parse only the header. The skip path + // must not drain multi-GiB snapshot bodies just to seed header state. hdrLen := 16 hdr := make([]byte, hdrLen) if n, _ := io.ReadFull(r, hdr); n == hdrLen && bytes.HasPrefix(hdr, []byte("EKVTHLC1")) { f.parsedCeiling = binary.BigEndian.Uint64(hdr[8:16]) } - if _, err := io.Copy(io.Discard, r); err != nil { - return 0, 0, err - } return f.parsedCeiling, 0, nil } @@ -230,13 +226,10 @@ func TestSkipGate_EmitsAfterSuccess(t *testing.T) { require.Empty(t, obs.fallbacks) } -// TestColdStartSkipThreshold pins codex P2 #934 round 3. The -// threshold caps at hardState.Commit so a follower carrying an -// uncommitted WAL suffix is NOT forced to run the full restore -// every restart (the original gate used the WAL tail, which can -// exceed Commit, and raft would not deliver those entries until -// the leader confirmed them). -func TestColdStartSkipThreshold(t *testing.T) { +// TestColdStartReplayTarget verifies the committed replay target caps at +// hardState.Commit so a follower carrying an uncommitted WAL suffix does not +// report or initialize from entries raft cannot deliver yet. +func TestColdStartReplayTarget(t *testing.T) { t.Parallel() mkSnap := func(idx uint64) raftpb.Snapshot { return raftTestSnapshot(idx, 0, nil, nil) @@ -253,36 +246,30 @@ func TestColdStartSkipThreshold(t *testing.T) { {"hardState.Commit zero", mkSnap(100), testHardState(0, 0), 100}, } for _, c := range cases { - got := coldStartSkipThreshold(c.snap, c.hs) + got := coldStartReplayTarget(c.snap, c.hs) if got != c.expected { t.Errorf("%s: got %d, want %d", c.name, got, c.expected) } } } -// TestSkipGate_ExecutesWhenWALCarriesPostSnapshotEntries pins -// codex P1 #934. When the FSM is past tok.Index but the WAL still -// carries entries tok.Index+1 .. have (the normal interval between -// snapshots — metaAppliedIndex advances on each Apply), the skip -// path MUST NOT fire even though have > tok.Index. Those WAL -// entries would re-apply onto a Pebble store that already contains -// them, hitting OCC conflicts and leaving the HLC below timestamps -// already on disk. -// -// Fixture: snap.Index=100, fsm.applied=150, lastWalIndex=150 (the -// WAL has entries 101..150 mirroring the applied tail). Gate -// criterion is have >= lastWalIndex, which holds; that's the -// happy-skip case. To exercise the bug, set lastWalIndex=200 (the -// WAL still has entries 151..200 that have NOT been applied yet); -// have=150 < lastWalIndex=200 must trigger execute, not skip. -func TestSkipGate_ExecutesWhenWALCarriesPostSnapshotEntries(t *testing.T) { +// TestSkipGate_SkipsWhenWALCarriesPostSnapshotTail verifies a node can skip +// the multi-GiB snapshot restore even when the committed WAL has entries past +// the FSM's durable applied index. Engine.Open seeds e.applied with `have`, +// then RawNode/applyCommitted replays only the needed tail and drops +// data-mutating duplicates at or below `have`. +func TestSkipGate_SkipsWhenWALCarriesPostSnapshotTail(t *testing.T) { dir := t.TempDir() const ( snapIndex uint64 = 100 appliedIdx uint64 = 150 - lastWalIndex uint64 = 200 + replayTarget uint64 = 200 + ceilingMs uint64 = 1700_000_000_002 ) - payload := []byte("body-bytes-for-execute") + payload := make([]byte, 16, 16+len("body-bytes-after-header")) + copy(payload[:8], "EKVTHLC1") + binary.BigEndian.PutUint64(payload[8:], ceilingMs) + payload = append(payload, []byte("body-bytes-after-header")...) crc, _ := writeFSMFileForTest(t, dir, snapIndex, payload) fsm := &skipGateFSM{applied: appliedIdx, appliedPresent: true} @@ -291,19 +278,15 @@ func TestSkipGate_ExecutesWhenWALCarriesPostSnapshotEntries(t *testing.T) { Metadata: testSnapshotMetadata(snapIndex, 0, nil), } obs := &recordingObs{} - _, gateErr := restoreSnapshotState(fsm, snap, lastWalIndex, dir, obs, nil) + effective, gateErr := restoreSnapshotState(fsm, snap, replayTarget, dir, obs, nil) require.NoError(t, gateErr) - require.Equal(t, payload, fsm.bodyBytes, - "have(150) < lastWalIndex(200) MUST execute full restore so the WAL replay does not duplicate-apply") - require.False(t, fsm.restoredHeader, "execute path MUST NOT use ApplySnapshotHeader") - require.Empty(t, obs.skipped) - // recordingObs now stores |snapIndex - have| (round-5 fix; mirrors - // monitoring.ColdStartObserver semantics so the FSM-ahead-of- - // snapshot case doesn't underflow). For have(150) > snapIndex(100) - // the absolute gap is 50. - require.Equal(t, []uint64{appliedIdx - snapIndex}, obs.executed, - "observer MUST record the absolute snapshot-relative gap") + require.Empty(t, fsm.bodyBytes, "skip path MUST NOT call fsm.Restore") + require.True(t, fsm.restoredHeader) + require.Equal(t, ceilingMs, fsm.appliedCeiling) + require.Equal(t, appliedIdx, effective) + require.Equal(t, []uint64{appliedIdx - snapIndex}, obs.skipped) + require.Empty(t, obs.executed) require.Empty(t, obs.fallbacks) } @@ -392,30 +375,37 @@ func TestApplyHeaderStateOnSkip_WrongTokenCRC(t *testing.T) { require.False(t, fsm.restoredHeader, "FSM state MUST NOT mutate on verification failure") } -// TestApplyHeaderStateOnSkip_BodyCorruption asserts step 3 catches a -// flipped body byte (CRC mismatch). -func TestApplyHeaderStateOnSkip_BodyCorruption(t *testing.T) { +// TestApplyHeaderStateOnSkip_DoesNotScanBody asserts skip startup cost is +// bounded by the header, not the snapshot body. Body corruption is still +// caught by the full restore path, where the body is actually consumed. +func TestApplyHeaderStateOnSkip_DoesNotScanBody(t *testing.T) { dir := t.TempDir() - crc, path := writeFSMFileForTest(t, dir, 1, []byte("payload-bytes")) + const ceilingMs uint64 = 1700_000_000_001 + payload := make([]byte, 16, 16+len("payload-bytes")) + copy(payload[:8], "EKVTHLC1") + binary.BigEndian.PutUint64(payload[8:], ceilingMs) + payload = append(payload, []byte("payload-bytes")...) + crc, path := writeFSMFileForTest(t, dir, 1, payload) - // Flip the first byte of the body in-place. The footer still + // Flip a byte after the fixed 16-byte header. The footer still // reads as `crc`, but the on-the-wire content no longer matches - // it. Step 2 (footer-vs-token) passes (we pass the same `crc` - // as tokenCRC), step 3 (full-body CRC) fails. + // it. The skip path should still succeed because it never uses body + // bytes; the full restore path remains responsible for body CRC. f, err := os.OpenFile(path, os.O_RDWR, 0) require.NoError(t, err) defer f.Close() var b [1]byte - _, err = f.ReadAt(b[:], 0) + _, err = f.ReadAt(b[:], 16) require.NoError(t, err) b[0] ^= 0x01 - _, err = f.WriteAt(b[:], 0) + _, err = f.WriteAt(b[:], 16) require.NoError(t, err) fsm := &skipGateFSM{} err = applyHeaderStateOnSkip(fsm, path, crc) - require.ErrorIs(t, err, ErrFSMSnapshotFileCRC) - require.False(t, fsm.restoredHeader, "FSM state MUST NOT mutate on verification failure") + require.NoError(t, err) + require.True(t, fsm.restoredHeader) + require.Equal(t, ceilingMs, fsm.appliedCeiling) } // --- kvFSM header preservation contract --- diff --git a/internal/raftengine/statemachine.go b/internal/raftengine/statemachine.go index 9c796f4f8..ec14dc517 100644 --- a/internal/raftengine/statemachine.go +++ b/internal/raftengine/statemachine.go @@ -100,24 +100,22 @@ type AppliedIndexWriter interface { // // The interface is two-phase by design: // -// - ParseSnapshotHeader reads the v1/v2 header from a caller- -// supplied io.Reader (wrapped in a crc32 TeeReader by the -// engine) and drains the remaining bytes so the wrapping CRC -// covers the full payload. It returns the parsed (ceiling, -// cutover) pair WITHOUT mutating FSM state. Errors propagate -// from the underlying header parser -// (ErrSnapshotHeaderUnknownMagic / InvalidLength) or from the -// drain pass (I/O errors); FSM state stays untouched on error. +// - ParseSnapshotHeader reads only the v1/v2 header from a caller- +// supplied io.Reader. It returns the parsed (ceiling, cutover) pair +// WITHOUT mutating FSM state. Errors propagate from the underlying +// header parser (ErrSnapshotHeaderUnknownMagic / InvalidLength); +// FSM state stays untouched on error. // // - ApplySnapshotHeader is pure assignment of the verified header // state. The engine calls this only after ParseSnapshotHeader -// returned successfully AND the wrapping crc32 hash matched -// the file footer. +// returned successfully and the snapshot file's footer matches the +// raft token. // -// Splitting parse from apply lets the CRC verifier stay co-located -// with its private helpers in internal/raftengine/etcd (matching -// the openAndRestoreFSMSnapshot safety contract) while the v1/v2 -// header parser stays inside the kv package where it already lives. +// Splitting parse from apply keeps the v1/v2 header parser inside the +// kv package where it already lives, while the engine remains responsible +// for deciding whether the snapshot body is actually needed. Full-body CRC +// verification still happens on the restore path; the skip path only reads +// the header because the FSM body state is already present locally. // Neither package imports the other in production. type SnapshotHeaderApplier interface { ParseSnapshotHeader(r io.Reader) (ceiling, cutover uint64, err error) diff --git a/kv/coordinator.go b/kv/coordinator.go index 0c1567a26..b32c46716 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -36,15 +36,20 @@ const dispatchLeaderRetryInterval = 25 * time.Millisecond // hlcPhysicalWindowMs is the duration in milliseconds that the Raft-agreed // physical ceiling extends ahead of the current wall clock. Modelled after -// TiDB's TSO 3-second window: the leader commits ceiling = now + window, and +// TiDB's TSO window strategy: the leader commits ceiling = now + window, and // renews before the window expires. A new leader inherits the committed ceiling // so it never issues timestamps that collide with the previous leader's window. -const hlcPhysicalWindowMs int64 = 3_000 +const hlcPhysicalWindowMs int64 = 15_000 // hlcRenewalInterval controls how often the leader proposes a new ceiling. // Must be less than hlcPhysicalWindowMs to guarantee the window never expires. const hlcRenewalInterval = 1 * time.Second +// hlcRenewalTimeout bounds a single renewal proposal. It is intentionally +// longer than hlcRenewalInterval so transient Raft write backlog does not +// cancel the renewal before the physical ceiling has real risk of expiring. +const hlcRenewalTimeout = 5 * time.Second + // CoordinatorOption is a functional option for Coordinate constructors. type CoordinatorOption func(*Coordinate) @@ -813,10 +818,10 @@ func (c *Coordinate) extendLeaseAfterRenewal(dispatchStart monoclock.Instant, ex // RunHLCLeaseRenewal runs a background loop that periodically proposes a new // physical ceiling to the Raft cluster while this node is the leader. // -// The ceiling is set to now + hlcPhysicalWindowMs (3 s) and is renewed every -// hlcRenewalInterval (1 s), mirroring TiDB's TSO window strategy. Because the -// window is always at least 2 s ahead of any real timestamp, a new leader will -// never issue timestamps that overlap with the previous leader's window. +// The ceiling is set to now + hlcPhysicalWindowMs and is renewed every +// hlcRenewalInterval, mirroring TiDB's TSO window strategy. Because the window +// stays ahead of real timestamps, a new leader will never issue timestamps that +// overlap with the previous leader's window. // // RunHLCLeaseRenewal blocks until ctx is cancelled; call it in a goroutine. func (c *Coordinate) RunHLCLeaseRenewal(ctx context.Context) { @@ -835,7 +840,10 @@ func (c *Coordinate) RunHLCLeaseRenewal(ctx context.Context) { } if c.IsLeaderAcceptingWrites() { ceilingMs := time.Now().UnixMilli() + hlcPhysicalWindowMs - if err := c.ProposeHLCLease(ctx, ceilingMs); err != nil { + pctx, cancel := context.WithTimeout(ctx, hlcRenewalTimeout) + err := c.ProposeHLCLease(pctx, ceilingMs) + cancel() + if err != nil { c.log.WarnContext(ctx, "hlc lease renewal failed", slog.Int64("ceiling_ms", ceilingMs), slog.Any("err", err), diff --git a/kv/fsm.go b/kv/fsm.go index dedc7829a..34450dab7 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -687,37 +687,30 @@ func (f *kvFSM) RestoredCutover() uint64 { // ParseSnapshotHeader implements raftengine.SnapshotHeaderApplier // phase 1 — the cold-start skip path's parse-without-side-effect -// step. The engine has wrapped `r` in a crc32 TeeReader sized at -// the body payload (file size minus 4-byte footer), so every byte -// pulled from `r` flows through the engine's hash. We read the -// v1/v2 header via ReadSnapshotHeader, then drain the rest of the -// body so the wrapping hash covers every payload byte — matching -// restoreAndComputeCRC's behaviour in openAndRestoreFSMSnapshot. +// step. The skip path only needs the header state because the FSM body +// is already present locally, so this reads the v1/v2 header and leaves +// the remainder untouched. Full-body CRC verification still happens on +// the restore path where the body bytes are consumed. // // IMPORTANT: this method MUST NOT touch f.hlc or f.restoredCutover. // The engine calls ApplySnapshotHeader separately, only after the -// wrapping CRC verification passes. Mutating FSM state here would -// defeat the "no side-effect on CRC failure" contract that the -// PR #910 design §5 round-7 split is designed to preserve. +// snapshot envelope checks pass. Mutating FSM state here would defeat +// the "no side-effect on parse failure" contract that the PR #910 +// design §5 round-7 split is designed to preserve. func (f *kvFSM) ParseSnapshotHeader(r io.Reader) (uint64, uint64, error) { - br := bufio.NewReaderSize(r, 1<<20) //nolint:mnd // 1 MiB, local to kv - ceiling, cutover, err := ReadSnapshotHeader(br) + const headerReadBufferSize = 4 << 10 + + ceiling, cutover, err := ReadSnapshotHeader(bufio.NewReaderSize(r, headerReadBufferSize)) if err != nil { return 0, 0, errors.WithStack(err) } - // Drain the remainder so the engine's TeeReader-wrapped CRC - // covers every byte of the body (LimitReader exhaustion - // signals "full payload consumed" to the caller). - if _, err := io.Copy(io.Discard, br); err != nil { - return 0, 0, errors.WithStack(err) - } return ceiling, cutover, nil } // ApplySnapshotHeader implements raftengine.SnapshotHeaderApplier // phase 2 — pure assignment of the verified header state. Called -// only after ParseSnapshotHeader returned successfully AND the -// engine's wrapping crc32 hash matched the file footer. Mirrors +// only after ParseSnapshotHeader returned successfully and the +// snapshot file's footer matched the raft token. Mirrors // the two side-effects Restore would have applied for the header // portion (HLC physical ceiling + restoredCutover). See PR #910 // design §5 round-7. diff --git a/kv/lease_read_test.go b/kv/lease_read_test.go index d62c103d2..2fc83a299 100644 --- a/kv/lease_read_test.go +++ b/kv/lease_read_test.go @@ -23,7 +23,8 @@ type fakeLeaseEngine struct { linearizableCalls atomic.Int32 proposeErr error // when set, Propose returns it (warm-up failure tests) proposeCalls atomic.Int32 - proposeHook func() // invoked inside Propose before returning (race injection) + proposeHook func() // invoked inside Propose before returning (race injection) + proposeCtxHook func(context.Context) state atomic.Value // stores raftengine.State; default Leader lastQuorumAckMonoNs atomic.Int64 // 0 = no ack yet. Updated by setQuorumAck(). leaderLossCallbacksMu sync.Mutex @@ -62,8 +63,11 @@ func (e *fakeLeaseEngine) Status() raftengine.Status { func (e *fakeLeaseEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } -func (e *fakeLeaseEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { +func (e *fakeLeaseEngine) Propose(ctx context.Context, _ []byte) (*raftengine.ProposalResult, error) { e.proposeCalls.Add(1) + if e.proposeCtxHook != nil { + e.proposeCtxHook(ctx) + } if e.proposeHook != nil { e.proposeHook() } diff --git a/kv/lease_warmup_test.go b/kv/lease_warmup_test.go index 5558a913c..18d64d2d6 100644 --- a/kv/lease_warmup_test.go +++ b/kv/lease_warmup_test.go @@ -169,6 +169,41 @@ func TestCoordinate_RunHLCLeaseRenewal_BlockerSuppressesProposals(t *testing.T) "HLC renewal should resume after startup rotation blocker clears") } +func TestCoordinate_RunHLCLeaseRenewal_UsesRenewalTimeout(t *testing.T) { + eng := &fakeLeaseEngine{applied: 11, leaseDur: time.Hour} + c := NewCoordinatorWithEngine(nil, eng) + deadline := make(chan time.Duration, 1) + eng.proposeCtxHook = func(ctx context.Context) { + d, ok := ctx.Deadline() + if !ok { + deadline <- 0 + return + } + deadline <- time.Until(d) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + c.RunHLCLeaseRenewal(ctx) + close(done) + }() + t.Cleanup(func() { + cancel() + <-done + }) + + select { + case got := <-deadline: + require.Greater(t, got, hlcRenewalInterval, + "HLC renewal proposal deadline must outlive the renewal cadence") + require.LessOrEqual(t, got, hlcRenewalTimeout, + "HLC renewal proposal deadline must remain bounded") + case <-time.After(2 * hlcRenewalInterval): + t.Fatal("timed out waiting for HLC renewal proposal") + } +} + // TestShardedCoordinator_RenewHLCLease_WarmsGroupLease proves the // sharded renewal path warms the target group's lease on a successful // propose, so LeaseReadForKey on a key owned by that group serves from the @@ -294,6 +329,35 @@ func TestShardedCoordinator_RenewHLCLeases_ProposesToEveryLedGroup(t *testing.T) "the non-default group lease must be warmed by all-group renewal") } +func TestShardedCoordinator_RenewHLCLeases_UsesRenewalTimeout(t *testing.T) { + t.Parallel() + eng1 := newShardedLeaseEngine(100) + eng2 := newShardedLeaseEngine(200) + deadline := make(chan time.Duration, 1) + eng1.proposeCtxHook = func(ctx context.Context) { + d, ok := ctx.Deadline() + if !ok { + deadline <- 0 + return + } + deadline <- time.Until(d) + } + coord := mustShardedLeaseCoord(t, eng1, eng2) + + done := coord.renewHLCLeases(context.Background()) + requireRenewalDone(t, done) + + select { + case got := <-deadline: + require.Greater(t, got, hlcRenewalInterval, + "HLC renewal proposal deadline must outlive the renewal cadence") + require.LessOrEqual(t, got, hlcRenewalTimeout, + "HLC renewal proposal deadline must remain bounded") + default: + t.Fatal("missing HLC renewal proposal deadline sample") + } +} + func TestShardedCoordinator_RenewHLCLeases_SkipsNonLeaders(t *testing.T) { t.Parallel() eng1 := newShardedLeaseEngine(100) diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index ec7c43843..124acfb7b 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -2342,7 +2342,7 @@ func (c *ShardedCoordinator) renewHLCLeases(ctx context.Context) <-chan struct{} go func(gid uint64, group *ShardGroup) { defer wg.Done() defer c.finishHLCLeaseRenewal(gid) - pctx, cancel := context.WithTimeout(ctx, hlcRenewalInterval) + pctx, cancel := context.WithTimeout(ctx, hlcRenewalTimeout) defer cancel() c.renewHLCLease(pctx, gid, group) }(gid, group) diff --git a/monitoring/grafana/dashboards/elastickv-redis-summary.json b/monitoring/grafana/dashboards/elastickv-redis-summary.json index 3d05d6176..ddfce4f7c 100644 --- a/monitoring/grafana/dashboards/elastickv-redis-summary.json +++ b/monitoring/grafana/dashboards/elastickv-redis-summary.json @@ -1683,7 +1683,7 @@ ], "title": "Raft Queue Saturation (stepCh full / outbound drops / errors)", "type": "timeseries", - "description": "Counter rates from the etcd raft engine. step-queue-full means inbound messages from remote peers were dropped because the local raft loop was too slow to consume the selected step queue (the 'etcd raft inbound step queue is full' log line). dispatch-dropped means outbound messages were discarded before transport because the per-peer channel was full. dispatch-errors means transport delivery failed. The pre-#560 seek storm caused all three to spike together; watch for them to fall after the rollout and stay flat." + "description": "Counter rates from the etcd raft engine. step-queue-full means inbound messages from remote peers found the selected step queue full; blocking replication messages now wait for space, while best-effort message classes may still be rejected. dispatch-dropped means outbound messages were discarded before transport because the per-peer channel was full. dispatch-errors means transport delivery failed. The pre-#560 seek storm caused all three to spike together; watch for them to fall after the rollout and stay flat." }, { "datasource": "$datasource", diff --git a/monitoring/hotpath.go b/monitoring/hotpath.go index 98b645b26..3fe7a3b51 100644 --- a/monitoring/hotpath.go +++ b/monitoring/hotpath.go @@ -98,7 +98,7 @@ func newHotPathMetrics(registerer prometheus.Registerer) *HotPathMetrics { stepQueueFullTotal: prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "elastickv_raft_step_queue_full_total", - Help: "Inbound raft messages that could not be enqueued because the selected step queue was full; indicates the raft loop is starved (classic pre-#560 seek-storm symptom).", + Help: "Inbound raft messages that found the selected step queue full. Blocking replication messages wait for space; best-effort messages may still be rejected. Indicates the raft loop is starved.", }, []string{"group"}, ), diff --git a/monitoring/hotpath_test.go b/monitoring/hotpath_test.go index fc10cda8a..d865c5983 100644 --- a/monitoring/hotpath_test.go +++ b/monitoring/hotpath_test.go @@ -147,7 +147,7 @@ elastickv_raft_dispatch_dropped_total{group="1",node_address="10.0.0.1:50051",no # HELP elastickv_raft_dispatch_errors_total Outbound raft dispatches that reached the transport but failed. Mirrors etcd raft Engine.dispatchErrorCount. # TYPE elastickv_raft_dispatch_errors_total counter elastickv_raft_dispatch_errors_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 2 -# HELP elastickv_raft_step_queue_full_total Inbound raft messages that could not be enqueued because the selected step queue was full; indicates the raft loop is starved (classic pre-#560 seek-storm symptom). +# HELP elastickv_raft_step_queue_full_total Inbound raft messages that found the selected step queue full. Blocking replication messages wait for space; best-effort messages may still be rejected. Indicates the raft loop is starved. # TYPE elastickv_raft_step_queue_full_total counter elastickv_raft_step_queue_full_total{group="1",node_address="10.0.0.1:50051",node_id="n1"} 1 `), diff --git a/proxy/config.go b/proxy/config.go index 25cfd2689..f25b240b3 100644 --- a/proxy/config.go +++ b/proxy/config.go @@ -71,7 +71,9 @@ type ProxyConfig struct { SentryEnv string SentrySampleRate float64 MetricsAddr string + PProfAddr string PubSubCompareWindow time.Duration + RedisOnlyRaw bool } // DefaultConfig returns a ProxyConfig with sensible defaults. @@ -86,5 +88,6 @@ func DefaultConfig() ProxyConfig { SentrySampleRate: 1.0, MetricsAddr: ":9191", PubSubCompareWindow: defaultPubSubCompareWindow, + RedisOnlyRaw: true, } } diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index 474f7d89e..1c940e557 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -25,16 +25,14 @@ const ( // (EVAL / EVALSHA). Lua scripts under high load cause write conflicts in the Raft // layer, and each conflict triggers a full script re-execution. Capping the // concurrency reduces contention so individual scripts complete within - // SecondaryTimeout. Excess secondary script writes may be dropped to keep - // contention bounded; this is only tolerable in modes where the script write - // is targeting the non-authoritative backend. + // SecondaryTimeout. Strict dual-write script replays wait for capacity instead + // of being dropped; best-effort users of goScript may still drop. maxScriptWriteGoroutines = 64 - // maxCompactedRetries caps retries when the secondary returns - // "read timestamp has been compacted". Each attempt re-sends the command so - // the secondary re-selects a fresh read snapshot; a small bound is enough - // because the compaction waterline advances slowly relative to SecondaryTimeout. - maxCompactedRetries = 3 + // maxSecondaryTransientRetries caps proxy-level retries when the secondary + // returns a transient OCC/read-snapshot error after exhausting its own retry + // loop. SecondaryTimeout still bounds the whole replay. + maxSecondaryTransientRetries = 3 // compactedRetryInitialBackoff is the first delay before retrying a secondary // command that failed with a compacted-read error. compactedRetryInitialBackoff = 10 * time.Millisecond @@ -67,6 +65,21 @@ func isReadTSCompactedError(err error) bool { return strings.Contains(err.Error(), readTSCompactedMarker) } +func isRetryableSecondaryWriteError(err error) bool { + if err == nil { + return false + } + if isReadTSCompactedError(err) { + return true + } + switch classifySecondaryWriteError(err) { + case "retry_limit", "write_conflict", "txn_locked": + return true + default: + return false + } +} + // DualWriter routes commands to primary and secondary backends based on mode. type DualWriter struct { primary Backend @@ -148,7 +161,9 @@ func (d *DualWriter) Close() { d.wg.Wait() } -// Write sends a write command to the primary synchronously, then to the secondary asynchronously. +// Write sends a write command to the primary synchronously, then to the secondary. +// The secondary write uses a bounded async slot when possible, but applies +// caller backpressure instead of dropping when the slot pool is saturated. // cmd must be the pre-uppercased command name. func (d *DualWriter) Write(ctx context.Context, cmd string, args [][]byte) (any, error) { iArgs := bytesArgsToInterfaces(args) @@ -166,9 +181,8 @@ func (d *DualWriter) Write(ctx context.Context, cmd string, args [][]byte) (any, } d.metrics.CommandTotal.WithLabelValues(cmd, d.primary.Name(), "ok").Inc() - // Secondary: async fire-and-forget (bounded) if d.hasSecondaryWrite() { - d.goWrite(func() { d.writeSecondary(cmd, iArgs) }) + d.runSecondaryWrite(func() { d.writeSecondary(cmd, iArgs) }) } return resp, err //nolint:wrapcheck // redis.Nil must pass through unwrapped for callers to detect nil replies @@ -257,7 +271,9 @@ func (d *DualWriter) Admin(ctx context.Context, cmd string, args [][]byte) (any, return resp, err //nolint:wrapcheck // redis.Nil must pass through unwrapped for callers to detect nil replies } -// Script forwards EVAL/EVALSHA to the primary, and async replays to secondary. +// Script forwards EVAL/EVALSHA to the primary, and replays to secondary. +// Secondary script replays are concurrency-limited and apply caller +// backpressure rather than dropping when the script slot pool is saturated. // cmd must be the pre-uppercased command name. func (d *DualWriter) Script(ctx context.Context, cmd string, args [][]byte) (any, error) { iArgs := bytesArgsToInterfaces(args) @@ -275,23 +291,21 @@ func (d *DualWriter) Script(ctx context.Context, cmd string, args [][]byte) (any d.rememberScript(cmd, args) if d.hasSecondaryWrite() { - d.goScript(func() { d.writeSecondary(cmd, iArgs) }) + d.runSecondaryScript(func() { d.writeSecondary(cmd, iArgs) }) } return resp, err //nolint:wrapcheck // redis.Nil must pass through unwrapped for callers to detect nil replies } // writeSecondary sends the command to the secondary, handling the NOSCRIPT -// → EVAL fallback and transparently retrying when the secondary reports that -// the read snapshot has been compacted. A re-sent command causes the backend -// to re-select a fresh read timestamp, which is the only way to recover once -// the original startTS has fallen behind MinRetainedTS on a peer node. +// → EVAL fallback and transparently retrying transient secondary errors. A +// re-sent command causes the backend to re-select a fresh timestamp and can +// also recover from hot-key OCC retry exhaustion in the secondary Redis adapter. // // The secondary's raw redis error is kept in sErr (not wrapped) so that // writeSecondary can classify it via errors.Is(sErr, redis.Nil), attach the // original message to Sentry and the structured log, and so the retry -// predicate isReadTSCompactedError matches the exact substring coming back -// from gRPC. +// predicate can match the exact substring coming back from gRPC. func (d *DualWriter) writeSecondary(cmd string, iArgs []any) { sCtx, cancel := context.WithTimeout(context.Background(), d.cfg.SecondaryTimeout) defer cancel() @@ -317,14 +331,14 @@ func (d *DualWriter) writeSecondary(cmd string, iArgs []any) { _, sErr = result.Result() } } - if !isReadTSCompactedError(sErr) { + if !isRetryableSecondaryWriteError(sErr) { break } - if attempt >= maxCompactedRetries { + if attempt >= maxSecondaryTransientRetries { break } - d.logger.Debug("retrying secondary write on compacted snapshot", - "cmd", cmd, "attempt", attempt+1, "backoff", backoff, "err", sErr) + d.logger.Debug("retrying secondary write after transient error", + "cmd", cmd, "attempt", attempt+1, "backoff", backoff, "reason", classifySecondaryWriteError(sErr), "err", sErr) if !waitCompactedRetryBackoff(sCtx, backoff) { break } @@ -367,6 +381,19 @@ func (d *DualWriter) recordSecondaryWriteFailure(cmd string, iArgs []any, elapse d.logger.Warn("secondary write failed", warnArgs...) } +func (d *DualWriter) replaySecondaryPipeline(cmds [][]any) { + d.runSecondaryWrite(func() { + sCtx, cancel := context.WithTimeout(context.Background(), d.cfg.SecondaryTimeout) + defer cancel() + _, pErr := d.secondary.Pipeline(sCtx, cmds) + if pErr != nil { + d.logger.Warn("secondary txn replay failed", "err", pErr) + d.metrics.SecondaryWriteErrors.Inc() + d.metrics.SecondaryWriteErrorsByReason.WithLabelValues("PIPELINE", classifySecondaryWriteError(pErr)).Inc() + } + }) +} + // waitCompactedRetryBackoff sleeps for a jittered interval or returns early // when the context is cancelled. Returns false if the caller should abort // the retry loop (context done). @@ -416,13 +443,15 @@ func nextCompactedRetryBackoff(current time.Duration) time.Duration { } // goWrite launches fn in a bounded write goroutine. +// It is best-effort: when the pool is saturated the work is dropped. +// Strict secondary writes should use runSecondaryWrite. func (d *DualWriter) goWrite(fn func()) { d.goAsyncWithSem(d.writeSem, fn) } // goScript launches fn in a bounded Lua-script write goroutine. -// It uses a smaller semaphore than goWrite to cap the number of concurrent -// EVAL/EVALSHA secondary writes. When the cap is reached the write is dropped. +// It is best-effort: when the pool is saturated the work is dropped. +// Strict secondary scripts should use runSecondaryScript. func (d *DualWriter) goScript(fn func()) { d.goAsyncWithSem(d.scriptSem, fn) } @@ -465,6 +494,49 @@ func (d *DualWriter) goAsyncWithSem(sem chan struct{}, fn func()) { } } +// runSecondaryWrite launches a strict secondary write. It uses the async write +// pool while capacity is available, and otherwise blocks the caller until a +// slot is available. This preserves dual-write consistency while still bounding +// secondary backend concurrency. +func (d *DualWriter) runSecondaryWrite(fn func()) { + d.runSecondaryWithBackpressure(d.writeSem, fn) +} + +// runSecondaryScript is the strict variant of goScript for EVAL/EVALSHA replay. +func (d *DualWriter) runSecondaryScript(fn func()) { + d.runSecondaryWithBackpressure(d.scriptSem, fn) +} + +func (d *DualWriter) runSecondaryWithBackpressure(sem chan struct{}, fn func()) { + d.mu.Lock() + if d.closed { + d.mu.Unlock() + return + } + select { + case sem <- struct{}{}: + d.wg.Add(1) + d.mu.Unlock() + go func() { + defer func() { + <-sem + d.wg.Done() + }() + fn() + }() + return + default: + d.metrics.AsyncBackpressure.Inc() + d.wg.Add(1) + d.mu.Unlock() + } + + defer d.wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + fn() +} + func (d *DualWriter) logAsyncDrop() { nowNano := time.Now().UnixNano() diff --git a/proxy/metrics.go b/proxy/metrics.go index cd8b3837e..1436a6c23 100644 --- a/proxy/metrics.go +++ b/proxy/metrics.go @@ -17,7 +17,8 @@ type ProxyMetrics struct { ActiveConnections prometheus.Gauge - AsyncDrops prometheus.Counter + AsyncDrops prometheus.Counter + AsyncBackpressure prometheus.Counter PubSubShadowDivergences *prometheus.CounterVec PubSubShadowErrors prometheus.Counter @@ -78,7 +79,12 @@ func NewProxyMetrics(reg prometheus.Registerer) *ProxyMetrics { AsyncDrops: prometheus.NewCounter(prometheus.CounterOpts{ Namespace: "proxy", Name: "async_drops_total", - Help: "Total async operations dropped due to semaphore backpressure.", + Help: "Total best-effort async operations dropped due to semaphore backpressure.", + }), + AsyncBackpressure: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: "proxy", + Name: "async_backpressure_total", + Help: "Total strict secondary writes that waited for an async semaphore slot instead of being dropped.", }), ActiveConnections: prometheus.NewGauge(prometheus.GaugeOpts{ @@ -110,6 +116,7 @@ func NewProxyMetrics(reg prometheus.Registerer) *ProxyMetrics { m.Divergences, m.MigrationGaps, m.AsyncDrops, + m.AsyncBackpressure, m.ActiveConnections, m.PubSubShadowDivergences, m.PubSubShadowErrors, diff --git a/proxy/noop_backend.go b/proxy/noop_backend.go new file mode 100644 index 000000000..fbe5dc811 --- /dev/null +++ b/proxy/noop_backend.go @@ -0,0 +1,38 @@ +package proxy + +import ( + "context" + + "github.com/redis/go-redis/v9" +) + +// noopBackend is used for modes with no secondary backend. Keeping a concrete +// Backend avoids starting unnecessary leader-discovery goroutines in redis-only +// and elastickv-only modes while preserving DualWriter's simple shape. +type noopBackend struct { + name string +} + +// NewNoopBackend returns a Backend placeholder for an intentionally unused +// side of the proxy. +func NewNoopBackend(name string) Backend { + return noopBackend{name: name} +} + +func (n noopBackend) Do(ctx context.Context, args ...any) *redis.Cmd { + cmd := redis.NewCmd(ctx, args...) + cmd.SetErr(ErrNoLeaderBackend) + return cmd +} + +func (n noopBackend) Pipeline(context.Context, [][]any) ([]*redis.Cmd, error) { + return nil, ErrNoLeaderBackend +} + +func (n noopBackend) Close() error { + return nil +} + +func (n noopBackend) Name() string { + return n.name +} diff --git a/proxy/proxy.go b/proxy/proxy.go index 9ced8559a..9b2add2b1 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -59,6 +59,10 @@ func NewProxyServer(cfg ProxyConfig, dual *DualWriter, metrics *ProxyMetrics, se // ListenAndServe starts the redcon proxy server. func (p *ProxyServer) ListenAndServe(ctx context.Context) error { + if p.cfg.Mode == ModeRedisOnly && p.cfg.RedisOnlyRaw { + return p.listenAndServeRawRedis(ctx) + } + p.shutdownCtx = ctx var lc net.ListenConfig @@ -430,17 +434,8 @@ func (p *ProxyServer) execTxn(conn redcon.Conn, state *proxyConnState) { conn.WriteError("ERR empty transaction response") } - // Async replay to secondary (bounded) if p.dual.hasSecondaryWrite() { - p.dual.goAsync(func() { - sCtx, cancel := context.WithTimeout(context.Background(), p.cfg.SecondaryTimeout) - defer cancel() - _, pErr := p.dual.Secondary().Pipeline(sCtx, cmds) - if pErr != nil { - p.logger.Warn("secondary txn replay failed", "err", pErr) - p.metrics.SecondaryWriteErrors.Inc() - } - }) + p.dual.replaySecondaryPipeline(cmds) } } diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 768e7ee9d..4398aca72 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -688,7 +688,53 @@ func TestDualWriter_GoAsync_DropLogsAreRateLimited(t *testing.T) { d.Close() } -func TestDualWriter_Script_DropsWhenScriptSemFull(t *testing.T) { +func TestDualWriter_Write_WaitsWhenWriteSemFull(t *testing.T) { + primary := newMockBackend("primary") + primary.doFunc = makeCmd("OK", nil) + secondary := newMockBackend("secondary") + secondary.doFunc = makeCmd("OK", nil) + + metrics := newTestMetrics() + cfg := ProxyConfig{Mode: ModeDualWrite, SecondaryWriteConcurrency: 1, SecondaryTimeout: 10 * time.Second} + d := NewDualWriter(primary, secondary, cfg, metrics, newTestSentry(), testLogger) + + blocker := make(chan struct{}) + d.goAsync(func() { + <-blocker + }) + + done := make(chan struct{}) + go func() { + _, err := d.Write(context.Background(), "SET", [][]byte{ + []byte("SET"), []byte("key"), []byte("value"), + }) + assert.NoError(t, err) + close(done) + }() + + select { + case <-done: + t.Fatal("Write returned while the strict secondary write slot was full") + case <-time.After(100 * time.Millisecond): + // good: the primary succeeded, but strict secondary replay is applying backpressure. + } + + assert.Equal(t, 0, secondary.CallCount()) + assert.InDelta(t, 1, testutil.ToFloat64(metrics.AsyncBackpressure), 0.001) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) + + close(blocker) + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Write did not complete after a secondary write slot was released") + } + assert.Equal(t, 1, secondary.CallCount()) + d.Close() +} + +func TestDualWriter_Script_WaitsWhenScriptSemFull(t *testing.T) { primary := newMockBackend("primary") primary.doFunc = makeCmd("OK", nil) secondary := newMockBackend("secondary") @@ -707,7 +753,6 @@ func TestDualWriter_Script_DropsWhenScriptSemFull(t *testing.T) { }) } - // Script should return promptly even when scriptSem is full. done := make(chan struct{}) go func() { _, err := d.Script(context.Background(), "EVALSHA", [][]byte{ @@ -719,16 +764,23 @@ func TestDualWriter_Script_DropsWhenScriptSemFull(t *testing.T) { select { case <-done: - // good — Script returned without blocking on a full scriptSem - case <-time.After(time.Second): - t.Fatal("Script blocked when script semaphore was full") + t.Fatal("Script returned while the strict script replay slot was full") + case <-time.After(100 * time.Millisecond): + // good: strict EVALSHA replay is applying backpressure instead of dropping. } - // The async replay to secondary must be dropped. assert.Equal(t, 0, secondary.CallCount()) - assert.InDelta(t, 1, testutil.ToFloat64(metrics.AsyncDrops), 0.001) + assert.InDelta(t, 1, testutil.ToFloat64(metrics.AsyncBackpressure), 0.001) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) close(blocker) + + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Script did not complete after a secondary script slot was released") + } + assert.Equal(t, 1, secondary.CallCount()) d.Close() } @@ -1024,8 +1076,8 @@ func TestDualWriter_writeSecondary_RetriesReadTSCompacted(t *testing.T) { } func TestDualWriter_writeSecondary_ReadTSCompactedRetriesAreBounded(t *testing.T) { - // When the compacted error is persistent, the retry loop must stop after - // maxCompactedRetries+1 attempts so the secondary goroutine returns + // When the transient error is persistent, the retry loop must stop after + // maxSecondaryTransientRetries+1 attempts so the secondary goroutine returns // instead of burning a scriptSem slot indefinitely. primary := newMockBackend("primary") primary.doFunc = makeCmd("OK", nil) @@ -1039,12 +1091,43 @@ func TestDualWriter_writeSecondary_ReadTSCompactedRetriesAreBounded(t *testing.T d.writeSecondary("EVALSHA", []any{[]byte("EVALSHA"), []byte("deadbeef"), []byte("0")}) - assert.Equal(t, maxCompactedRetries+1, secondary.CallCount(), - "secondary must stop after maxCompactedRetries+1 attempts") + assert.Equal(t, maxSecondaryTransientRetries+1, secondary.CallCount(), + "secondary must stop after maxSecondaryTransientRetries+1 attempts") assert.InDelta(t, 1, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001, "a persistent compacted error must still be reported as a secondary write error") } +func TestDualWriter_writeSecondary_RetriesRetryLimitWriteConflict(t *testing.T) { + // A hot key can exhaust the secondary Redis adapter's internal OCC retry loop. + // Re-sending the command lets the backend choose a fresh timestamp and keeps + // dual-write traffic from permanently diverging on a transient conflict. + primary := newMockBackend("primary") + primary.doFunc = makeCmd("OK", nil) + + secondary := newMockBackend("secondary") + retryLimitErr := testRedisErr("redis txn retry limit exceeded: key: myzset: write conflict") + var calls int + secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { + calls++ + cmd := redis.NewCmd(ctx, args...) + if calls < 3 { + cmd.SetErr(retryLimitErr) + return cmd + } + cmd.SetVal("OK") + return cmd + } + + metrics := newTestMetrics() + d := NewDualWriter(primary, secondary, ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: time.Second}, metrics, newTestSentry(), testLogger) + + d.writeSecondary("ZADD", []any{[]byte("ZADD"), []byte("myzset"), []byte("1"), []byte("member")}) + + assert.Equal(t, 3, calls, "secondary must retry transient retry-limit write conflicts") + assert.InDelta(t, 0, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001, + "a retried success must not count as a secondary write error") +} + func TestDualWriter_writeSecondary_RetriesDoNotRepeatNoScriptProbe(t *testing.T) { // After the EVAL fallback resolves a NOSCRIPT, a compacted-retry must // re-send the resolved EVAL form directly — never the known-missing diff --git a/proxy/pubsub.go b/proxy/pubsub.go index 1ecd9e284..221d085c1 100644 --- a/proxy/pubsub.go +++ b/proxy/pubsub.go @@ -470,15 +470,7 @@ func (s *pubsubSession) execTxn() { s.writeMu.Unlock() if s.proxy.dual.hasSecondaryWrite() { - s.proxy.dual.goAsync(func() { - sCtx, cancel := context.WithTimeout(context.Background(), s.proxy.cfg.SecondaryTimeout) - defer cancel() - _, pErr := s.proxy.dual.Secondary().Pipeline(sCtx, cmds) - if pErr != nil { - s.proxy.logger.Warn("secondary txn replay failed", "err", pErr) - s.proxy.metrics.SecondaryWriteErrors.Inc() - } - }) + s.proxy.dual.replaySecondaryPipeline(cmds) } } diff --git a/proxy/raw_redis_proxy.go b/proxy/raw_redis_proxy.go new file mode 100644 index 000000000..f478d5480 --- /dev/null +++ b/proxy/raw_redis_proxy.go @@ -0,0 +1,156 @@ +package proxy + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + "time" +) + +const ( + rawRedisCopyDirections = 2 + rawRedisHandshakeTimeout = 5 * time.Second +) + +func (p *ProxyServer) listenAndServeRawRedis(ctx context.Context) error { + p.shutdownCtx = ctx + + var lc net.ListenConfig + ln, err := lc.Listen(ctx, "tcp", p.cfg.ListenAddr) + if err != nil { + return fmt.Errorf("raw redis proxy listen: %w", err) + } + defer ln.Close() + + p.logger.Info("raw redis proxy starting", + "addr", p.cfg.ListenAddr, + "mode", p.cfg.Mode.String(), + "primary", p.cfg.PrimaryAddr, + "primary_db", p.cfg.PrimaryDB, + ) + + var wg sync.WaitGroup + defer wg.Wait() + + go func() { + <-ctx.Done() + p.logger.Info("shutting down raw redis proxy") + _ = ln.Close() + }() + + for { + client, err := ln.Accept() + if err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("raw redis proxy accept: %w", err) + } + wg.Add(1) + go func() { + defer wg.Done() + p.handleRawRedisConn(ctx, client) + }() + } +} + +func (p *ProxyServer) handleRawRedisConn(ctx context.Context, client net.Conn) { + p.metrics.ActiveConnections.Inc() + defer p.metrics.ActiveConnections.Dec() + defer client.Close() + + var dialer net.Dialer + upstream, err := dialer.DialContext(ctx, "tcp", p.cfg.PrimaryAddr) + if err != nil { + p.logger.Warn("raw redis upstream dial failed", "addr", p.cfg.PrimaryAddr, "err", err) + return + } + defer upstream.Close() + + upstreamReader := bufio.NewReader(upstream) + if err := p.prepareRawRedisUpstream(upstream, upstreamReader); err != nil { + p.logger.Warn("raw redis upstream prepare failed", "addr", p.cfg.PrimaryAddr, "err", err) + return + } + + stop := make(chan struct{}) + defer close(stop) + go func() { + select { + case <-ctx.Done(): + _ = client.Close() + _ = upstream.Close() + case <-stop: + } + }() + + errCh := make(chan struct{}, rawRedisCopyDirections) + go rawCopy(upstream, client, errCh) + go rawCopy(client, upstreamReader, errCh) + <-errCh +} + +func rawCopy(dst net.Conn, src io.Reader, done chan<- struct{}) { + _, _ = io.Copy(dst, src) + _ = dst.Close() + done <- struct{}{} +} + +func (p *ProxyServer) prepareRawRedisUpstream(upstream net.Conn, upstreamReader *bufio.Reader) error { + if err := upstream.SetDeadline(time.Now().Add(rawRedisHandshakeTimeout)); err != nil { + return fmt.Errorf("set handshake deadline: %w", err) + } + if p.cfg.PrimaryPassword != "" { + if err := rawRedisRoundTrip(upstream, upstreamReader, "AUTH", p.cfg.PrimaryPassword); err != nil { + return fmt.Errorf("auth: %w", err) + } + } + if p.cfg.PrimaryDB != 0 { + if err := rawRedisRoundTrip(upstream, upstreamReader, "SELECT", strconv.Itoa(p.cfg.PrimaryDB)); err != nil { + return fmt.Errorf("select db %d: %w", p.cfg.PrimaryDB, err) + } + } + if err := upstream.SetDeadline(time.Time{}); err != nil { + return fmt.Errorf("clear handshake deadline: %w", err) + } + return nil +} + +func rawRedisRoundTrip(w io.Writer, r *bufio.Reader, args ...string) error { + if _, err := w.Write(rawRedisCommand(args...)); err != nil { + return fmt.Errorf("write command: %w", err) + } + prefix, err := r.ReadByte() + if err != nil { + return fmt.Errorf("read reply prefix: %w", err) + } + line, err := r.ReadString('\n') + if err != nil { + return fmt.Errorf("read reply line: %w", err) + } + line = strings.TrimRight(line, "\r\n") + if prefix == '-' { + return fmt.Errorf("%s", line) + } + return nil +} + +func rawRedisCommand(args ...string) []byte { + var b strings.Builder + b.WriteByte('*') + b.WriteString(strconv.Itoa(len(args))) + b.WriteString("\r\n") + for _, arg := range args { + b.WriteByte('$') + b.WriteString(strconv.Itoa(len(arg))) + b.WriteString("\r\n") + b.WriteString(arg) + b.WriteString("\r\n") + } + return []byte(b.String()) +} diff --git a/proxy/raw_redis_proxy_test.go b/proxy/raw_redis_proxy_test.go new file mode 100644 index 000000000..285c1bd3d --- /dev/null +++ b/proxy/raw_redis_proxy_test.go @@ -0,0 +1,77 @@ +package proxy + +import ( + "bufio" + "io" + "net" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRawRedisCommand(t *testing.T) { + t.Parallel() + + require.Equal(t, + []byte("*2\r\n$4\r\nAUTH\r\n$6\r\nsecret\r\n"), + rawRedisCommand("AUTH", "secret"), + ) +} + +func TestPrepareRawRedisUpstreamAuthenticatesAndSelectsDB(t *testing.T) { + t.Parallel() + + client, server := net.Pipe() + defer client.Close() + defer server.Close() + + seen := make(chan []string, 2) + go func() { + reader := bufio.NewReader(server) + for i := 0; i < 2; i++ { + cmd, err := readRawRedisTestCommand(reader) + if err != nil { + return + } + seen <- cmd + _, _ = server.Write([]byte("+OK\r\n")) + } + }() + + p := &ProxyServer{cfg: ProxyConfig{PrimaryPassword: "secret", PrimaryDB: 1}} + err := p.prepareRawRedisUpstream(client, bufio.NewReader(client)) + require.NoError(t, err) + require.Equal(t, []string{"AUTH", "secret"}, <-seen) + require.Equal(t, []string{"SELECT", "1"}, <-seen) +} + +func readRawRedisTestCommand(r *bufio.Reader) ([]string, error) { + line, err := r.ReadString('\n') + if err != nil { + return nil, err + } + line = strings.TrimRight(line, "\r\n") + n, err := strconv.Atoi(strings.TrimPrefix(line, "*")) + if err != nil { + return nil, err + } + out := make([]string, 0, n) + for range n { + line, err = r.ReadString('\n') + if err != nil { + return nil, err + } + size, err := strconv.Atoi(strings.TrimPrefix(strings.TrimRight(line, "\r\n"), "$")) + if err != nil { + return nil, err + } + arg := make([]byte, size+2) + if _, err := io.ReadFull(r, arg); err != nil { + return nil, err + } + out = append(out, string(arg[:size])) + } + return out, nil +} From fefd809e68c8d5913bb0c24bcf2023a2d86186e5 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 16 Jul 2026 14:55:03 +0900 Subject: [PATCH 103/162] Fix sparse Lua list pop fallback --- adapter/redis_lua_context.go | 76 +++++++++++++--- adapter/redis_lua_list_holes_test.go | 126 ++++++++++++++++++++++++++- kv/leader_routed_store.go | 14 +++ kv/shard_store.go | 30 +++++++ 4 files changed, 235 insertions(+), 11 deletions(-) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 4a57051b5..afb9340de 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -182,6 +182,10 @@ type luaPhysicalLimitedScanStore interface { ReverseScanAtPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, ts uint64) ([]*store.KVPair, bool, error) } +type luaExactScanFallbackDecider interface { + AllowExactScanFallbackAfterPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, ts uint64, reverse bool) bool +} + type luaCommandHandler func(*luaScriptContext, []string) (luaReply, error) type luaRenameHandler func(*luaScriptContext, []byte, []byte) error @@ -2189,6 +2193,66 @@ func (c *luaScriptContext) scanListItemWindow(key []byte, meta store.ListMeta, s if err != nil { return luaLazyListBoundaryItem{}, false, errors.WithStack(err) } + if item, ok := luaListBoundaryItemFromKVs(key, meta, kvs); ok { + return item, true, nil + } + if physicalLimitReached { + if !c.allowExactListScanFallback(startKey, endKey, scanLimit, !left) { + return luaLazyListBoundaryItem{}, false, errors.Wrapf(ErrCollectionTooLarge, + "list %q sparse pop scanned %d physical item rows", string(key), scanLimit) + } + return c.scanListItemWindowExact(key, meta, startKey, endKey, left) + } + if len(kvs) >= scanLimit { + return luaLazyListBoundaryItem{}, false, errors.Wrapf(ErrCollectionTooLarge, + "list %q sparse pop scanned %d non-matching item rows", string(key), scanLimit) + } + return luaLazyListBoundaryItem{}, false, nil +} + +func (c *luaScriptContext) allowExactListScanFallback(startKey, endKey []byte, scanLimit int, reverse bool) bool { + decider, ok := c.server.store.(luaExactScanFallbackDecider) + if !ok { + return true + } + return decider.AllowExactScanFallbackAfterPhysicalLimit(c.ctx, startKey, endKey, scanLimit, scanLimit, c.startTS, reverse) +} + +func (c *luaScriptContext) scanListItemWindowExact(key []byte, meta store.ListMeta, startKey, endKey []byte, left bool) (luaLazyListBoundaryItem, bool, error) { + for { + var ( + kvs []*store.KVPair + err error + ) + if left { + kvs, err = c.server.store.ScanAt(c.ctx, startKey, endKey, 1, c.startTS) + } else { + kvs, err = c.server.store.ReverseScanAt(c.ctx, startKey, endKey, 1, c.startTS) + } + if err != nil { + return luaLazyListBoundaryItem{}, false, errors.WithStack(err) + } + if len(kvs) == 0 { + return luaLazyListBoundaryItem{}, false, nil + } + if item, ok := luaListBoundaryItemFromKVs(key, meta, kvs); ok { + return item, true, nil + } + if left { + startKey = scanStartAfterKey(kvs[0].Key) + } else { + endKey = append([]byte(nil), kvs[0].Key...) + } + } +} + +func scanStartAfterKey(key []byte) []byte { + next := make([]byte, len(key)+1) + copy(next, key) + return next +} + +func luaListBoundaryItemFromKVs(key []byte, meta store.ListMeta, kvs []*store.KVPair) (luaLazyListBoundaryItem, bool) { for _, kvp := range kvs { seq, ok := store.ExtractListItemSeq(kvp.Key, key) if !ok { @@ -2197,17 +2261,9 @@ func (c *luaScriptContext) scanListItemWindow(key []byte, meta store.ListMeta, s return luaLazyListBoundaryItem{ value: string(kvp.Value), index: seq - meta.Head, - }, true, nil - } - if physicalLimitReached { - return luaLazyListBoundaryItem{}, false, errors.Wrapf(ErrCollectionTooLarge, - "list %q sparse pop scanned %d physical item rows", string(key), scanLimit) + }, true } - if len(kvs) >= scanLimit { - return luaLazyListBoundaryItem{}, false, errors.Wrapf(ErrCollectionTooLarge, - "list %q sparse pop scanned %d non-matching item rows", string(key), scanLimit) - } - return luaLazyListBoundaryItem{}, false, nil + return luaLazyListBoundaryItem{}, false } func (c *luaScriptContext) scanAtPhysicalLimit(startKey, endKey []byte, scanLimit int) ([]*store.KVPair, bool, error) { diff --git a/adapter/redis_lua_list_holes_test.go b/adapter/redis_lua_list_holes_test.go index f3211385e..80ac1cd7c 100644 --- a/adapter/redis_lua_list_holes_test.go +++ b/adapter/redis_lua_list_holes_test.go @@ -217,7 +217,7 @@ func TestRedisLua_RPopLPushKeepsRemainingSparseHeadItem(t *testing.T) { require.Equal(t, []string{"job-2"}, dstValues) } -func TestRedisLua_RPopLPushFailsOnTooManyPhysicalTailTombstones(t *testing.T) { +func TestRedisLua_RPopLPushScansPastPhysicalTailTombstones(t *testing.T) { t.Parallel() ctx := context.Background() @@ -226,6 +226,8 @@ func TestRedisLua_RPopLPushFailsOnTooManyPhysicalTailTombstones(t *testing.T) { dst := []byte("bull:test:active:tombstone-tail") largeLen := int64(luaSparseListPopScanLimit) + 2 seedListMeta(t, r, src, store.ListMeta{Head: 0, Len: largeLen, Tail: largeLen}) + require.NoError(t, r.store.PutAt(ctx, listItemKey(src, 0), []byte("job-1"), 2, 0)) + seedListPrefixCollider(t, r, ctx, src, 0) commitTS := uint64(3) for seq := int64(1); seq <= int64(luaSparseListPopScanLimit)+1; seq++ { require.NoError(t, r.store.DeleteAt(ctx, listItemKey(src, seq), commitTS)) @@ -236,6 +238,103 @@ func TestRedisLua_RPopLPushFailsOnTooManyPhysicalTailTombstones(t *testing.T) { require.NoError(t, err) defer scriptCtx.Close() + reply, err := scriptCtx.cmdRPopLPush([]string{string(src), string(dst)}) + require.NoError(t, err) + require.Equal(t, luaReplyString, reply.kind) + require.Equal(t, "job-1", reply.text) + require.NoError(t, scriptCtx.commit()) + + readTS := r.readTS() + typ, err := r.keyTypeAt(ctx, src, readTS) + require.NoError(t, err) + require.Equal(t, redisTypeNone, typ) + dstValues, err := r.listValuesAt(ctx, dst, readTS) + require.NoError(t, err) + require.Equal(t, []string{"job-1"}, dstValues) +} + +func TestRedisLua_LPopScansPastPhysicalHeadTombstones(t *testing.T) { + t.Parallel() + + ctx := context.Background() + r := newListPopTestServer(t) + src := []byte("bull:test:wait:tombstone-head") + largeLen := int64(luaSparseListPopScanLimit) + 2 + seedListMeta(t, r, src, store.ListMeta{Head: 0, Len: largeLen, Tail: largeLen}) + require.NoError(t, r.store.PutAt(ctx, listItemKey(src, largeLen-1), []byte("job-1"), 2, 0)) + seedListPrefixCollider(t, r, ctx, src, 0) + commitTS := uint64(3) + for seq := int64(0); seq <= int64(luaSparseListPopScanLimit); seq++ { + require.NoError(t, r.store.DeleteAt(ctx, listItemKey(src, seq), commitTS)) + commitTS++ + } + + scriptCtx, err := newLuaScriptContext(ctx, r) + require.NoError(t, err) + defer scriptCtx.Close() + + reply, err := scriptCtx.cmdLPop([]string{string(src)}) + require.NoError(t, err) + require.Equal(t, luaReplyString, reply.kind) + require.Equal(t, "job-1", reply.text) + require.NoError(t, scriptCtx.commit()) + + readTS := r.readTS() + typ, err := r.keyTypeAt(ctx, src, readTS) + require.NoError(t, err) + require.Equal(t, redisTypeNone, typ) +} + +func TestRedisLua_RPopLPushDeletesPhysicalTombstoneOnlyList(t *testing.T) { + t.Parallel() + + ctx := context.Background() + r := newListPopTestServer(t) + src := []byte("bull:test:wait:tombstone-only") + dst := []byte("bull:test:active:tombstone-only") + largeLen := int64(luaSparseListPopScanLimit) + 2 + seedListMeta(t, r, src, store.ListMeta{Head: 0, Len: largeLen, Tail: largeLen}) + commitTS := uint64(2) + for seq := int64(0); seq <= int64(luaSparseListPopScanLimit)+1; seq++ { + require.NoError(t, r.store.DeleteAt(ctx, listItemKey(src, seq), commitTS)) + commitTS++ + } + + scriptCtx, err := newLuaScriptContext(ctx, r) + require.NoError(t, err) + defer scriptCtx.Close() + + reply, err := scriptCtx.cmdRPopLPush([]string{string(src), string(dst)}) + require.NoError(t, err) + require.Equal(t, luaReplyNil, reply.kind) + require.NoError(t, scriptCtx.commit()) + + readTS := r.readTS() + typ, err := r.keyTypeAt(ctx, src, readTS) + require.NoError(t, err) + require.Equal(t, redisTypeNone, typ) + dstValues, err := r.listValuesAt(ctx, dst, readTS) + require.NoError(t, err) + require.Empty(t, dstValues) +} + +func TestRedisLua_RPopLPushSyntheticPhysicalLimitFailsClosed(t *testing.T) { + t.Parallel() + + ctx := context.Background() + base := store.NewMVCCStore() + st := noExactFallbackPhysicalLimitStore{MVCCStore: base} + coord := newLocalAdapterCoordinator(st) + r := NewRedisServer(nil, "", st, coord, nil, nil) + src := []byte("bull:test:wait:synthetic-limit") + dst := []byte("bull:test:active:synthetic-limit") + seedListMeta(t, r, src, store.ListMeta{Head: 0, Len: 1, Tail: 1}) + require.NoError(t, r.store.PutAt(ctx, listItemKey(src, 0), []byte("job-1"), 2, 0)) + + scriptCtx, err := newLuaScriptContext(ctx, r) + require.NoError(t, err) + defer scriptCtx.Close() + _, err = scriptCtx.cmdRPopLPush([]string{string(src), string(dst)}) require.ErrorIs(t, err, ErrCollectionTooLarge) } @@ -275,3 +374,28 @@ func seedListMeta(t *testing.T, r *RedisServer, key []byte, meta store.ListMeta) require.NoError(t, err) require.NoError(t, r.store.PutAt(context.Background(), store.ListMetaKey(key), raw, 1, 0)) } + +func seedListPrefixCollider(t *testing.T, r *RedisServer, ctx context.Context, key []byte, seq int64) { + t.Helper() + + collider := append([]byte{}, key...) + collider = append(collider, listItemKey(key, seq)[len(store.ListItemPrefix)+len(key):]...) + collider = append(collider, 'x') + require.NoError(t, r.store.PutAt(ctx, listItemKey(collider, 0), []byte("other-list-job"), 2, 0)) +} + +type noExactFallbackPhysicalLimitStore struct { + store.MVCCStore +} + +func (s noExactFallbackPhysicalLimitStore) ScanAtPhysicalLimit(context.Context, []byte, []byte, int, int, uint64) ([]*store.KVPair, bool, error) { + return nil, true, nil +} + +func (s noExactFallbackPhysicalLimitStore) ReverseScanAtPhysicalLimit(context.Context, []byte, []byte, int, int, uint64) ([]*store.KVPair, bool, error) { + return nil, true, nil +} + +func (s noExactFallbackPhysicalLimitStore) AllowExactScanFallbackAfterPhysicalLimit(context.Context, []byte, []byte, int, int, uint64, bool) bool { + return false +} diff --git a/kv/leader_routed_store.go b/kv/leader_routed_store.go index f92cc39f5..8e169ac81 100644 --- a/kv/leader_routed_store.go +++ b/kv/leader_routed_store.go @@ -392,6 +392,20 @@ func (s *LeaderRoutedStore) ReverseScanAtPhysicalLimit(ctx context.Context, star return s.scanAtPhysicalLimit(ctx, start, end, visibleLimit, physicalLimit, ts, true) } +func (s *LeaderRoutedStore) AllowExactScanFallbackAfterPhysicalLimit(ctx context.Context, start []byte, _ []byte, visibleLimit, physicalLimit int, _ uint64, _ bool) bool { + if s == nil || s.local == nil { + return false + } + if visibleLimit <= 0 || physicalLimit <= 0 { + return false + } + if ok, _ := s.leaderFenceTS(ctx, start); !ok { + return false + } + _, ok := s.local.(physicalLimitedStore) + return ok +} + func (s *LeaderRoutedStore) scanAtPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, ts uint64, reverse bool) ([]*store.KVPair, bool, error) { if s == nil || s.local == nil { return []*store.KVPair{}, false, nil diff --git a/kv/shard_store.go b/kv/shard_store.go index 81b21e28d..2ef4dd868 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -302,6 +302,36 @@ func (s *ShardStore) ReverseScanAtPhysicalLimit(ctx context.Context, start []byt return s.scanRouteAtDirectionPhysicalLimit(ctx, routes[0], start, end, visibleLimit, physicalLimit, ts, true) } +func (s *ShardStore) AllowExactScanFallbackAfterPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, _ uint64, _ bool) bool { + if visibleLimit <= 0 || physicalLimit <= 0 { + return false + } + g := s.exactFallbackPhysicalLimitGroup(start, end) + if g == nil { + return false + } + if _, ok := g.Store.(physicalLimitedStore); !ok { + return false + } + engine := engineForGroup(g) + return engine == nil || isLinearizableRaftLeader(ctx, engine) +} + +func (s *ShardStore) exactFallbackPhysicalLimitGroup(start []byte, end []byte) *ShardGroup { + if s == nil || s.engine == nil { + return nil + } + routes, clampToRoutes := s.routesForScan(start, end) + if len(routes) != 1 || clampToRoutes { + return nil + } + g, ok := s.groupForID(routes[0].GroupID) + if !ok || g == nil || g.Store == nil { + return nil + } + return g +} + func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Route, bool) { if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { return s.engine.GetIntersectingRoutes(routeStart, routeEnd), false From e1b32b5704a2acc875f8dff4a3b9d98d3e208d1c Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 16 Jul 2026 15:48:25 +0900 Subject: [PATCH 104/162] Align ElasticKV proxy socket timeouts --- cmd/redis-proxy/main.go | 20 ++++++++++++++++++++ cmd/redis-proxy/main_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index 4ad067987..b9483ec1e 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -23,6 +23,8 @@ const ( sentryFlushTimeout = 2 * time.Second metricsShutdownTimeout = 5 * time.Second secondaryConcurrencyDivisor = 2 + elasticKVDispatchTimeout = 10 * time.Second + backendTimeoutGrace = time.Second ) func main() { @@ -121,6 +123,7 @@ func newBackends(cfg proxy.ProxyConfig, primaryPoolSize, elasticKVPoolSize int, secondaryOpts.DB = cfg.SecondaryDB secondaryOpts.Password = cfg.SecondaryPassword secondaryOpts.PoolSize = elasticKVPoolSize + alignElasticKVBackendTimeouts(&secondaryOpts, cfg.SecondaryTimeout) secondarySeeds := parseAddrList(cfg.SecondaryAddr) if len(secondarySeeds) == 0 { @@ -145,6 +148,23 @@ func newBackends(cfg proxy.ProxyConfig, primaryPoolSize, elasticKVPoolSize int, } } +func alignElasticKVBackendTimeouts(opts *proxy.BackendOptions, operationTimeout time.Duration) { + if opts == nil { + return + } + floor := elasticKVDispatchTimeout + if operationTimeout > floor { + floor = operationTimeout + } + floor += backendTimeoutGrace + if opts.ReadTimeout > 0 && opts.ReadTimeout < floor { + opts.ReadTimeout = floor + } + if opts.WriteTimeout > 0 && opts.WriteTimeout < floor { + opts.WriteTimeout = floor + } +} + func serveMetrics(ctx context.Context, addr string, reg *prometheus.Registry, logger *slog.Logger) { mux := http.NewServeMux() mux.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{})) diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 29e9a47b7..8690f483c 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -2,6 +2,7 @@ package main import ( "testing" + "time" "github.com/bootjp/elastickv/proxy" "github.com/stretchr/testify/assert" @@ -96,3 +97,34 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { }) } } + +func TestAlignElasticKVBackendTimeouts(t *testing.T) { + t.Run("uses ElasticKV dispatch floor by default", func(t *testing.T) { + opts := proxy.DefaultElasticKVBackendOptions() + + alignElasticKVBackendTimeouts(&opts, 5*time.Second) + + assert.Equal(t, 11*time.Second, opts.ReadTimeout) + assert.Equal(t, 11*time.Second, opts.WriteTimeout) + }) + + t.Run("follows longer secondary timeout", func(t *testing.T) { + opts := proxy.DefaultElasticKVBackendOptions() + + alignElasticKVBackendTimeouts(&opts, 15*time.Second) + + assert.Equal(t, 16*time.Second, opts.ReadTimeout) + assert.Equal(t, 16*time.Second, opts.WriteTimeout) + }) + + t.Run("keeps explicit larger timeout", func(t *testing.T) { + opts := proxy.DefaultElasticKVBackendOptions() + opts.ReadTimeout = 30 * time.Second + opts.WriteTimeout = 31 * time.Second + + alignElasticKVBackendTimeouts(&opts, 15*time.Second) + + assert.Equal(t, 30*time.Second, opts.ReadTimeout) + assert.Equal(t, 31*time.Second, opts.WriteTimeout) + }) +} From cf478580778795fee500f1a48de7461160a41828 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 16 Jul 2026 15:54:20 +0900 Subject: [PATCH 105/162] Replay blocking zset pops deterministically --- proxy/blocking.go | 53 +++++++++++++++++++++++++++++++++++++++++ proxy/dualwrite.go | 9 +++---- proxy/proxy_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 6 deletions(-) diff --git a/proxy/blocking.go b/proxy/blocking.go index 18a42a4ec..8a1bf64be 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -2,6 +2,7 @@ package proxy import ( "context" + "fmt" "strconv" "strings" "time" @@ -52,3 +53,55 @@ func parseBlockingMillisecondsArg(raw []byte) time.Duration { } return time.Duration(millis) * time.Millisecond } + +func blockingReplayCommand(cmd string, _ [][]byte, resp any) (string, []any, bool) { + switch strings.ToUpper(cmd) { + case "BZPOPMIN", "BZPOPMAX": + parts, ok := redisArray(resp) + if !ok || len(parts) < 2 { + return "", nil, false + } + key, keyOK := redisArg(parts[0]) + member, memberOK := redisArg(parts[1]) + if !keyOK || !memberOK { + return "", nil, false + } + return "ZREM", []any{[]byte("ZREM"), key, member}, true + default: + return "", nil, false + } +} + +func redisArray(v any) ([]any, bool) { + switch x := v.(type) { + case []any: + return x, true + case []string: + out := make([]any, len(x)) + for i := range x { + out[i] = x[i] + } + return out, true + case [][]byte: + out := make([]any, len(x)) + for i := range x { + out[i] = x[i] + } + return out, true + default: + return nil, false + } +} + +func redisArg(v any) (any, bool) { + switch x := v.(type) { + case nil: + return nil, false + case []byte: + return append([]byte(nil), x...), true + case string: + return []byte(x), true + default: + return []byte(fmt.Sprint(x)), true + } +} diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index 1c940e557..fc69b6773 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -241,13 +241,10 @@ func (d *DualWriter) Blocking(ctx context.Context, cmd string, args [][]byte) (a } d.metrics.CommandTotal.WithLabelValues(cmd, d.primary.Name(), "ok").Inc() - // Warmup: send to secondary with short timeout (fire-and-forget, bounded) if d.hasSecondaryWrite() { - d.goWrite(func() { - sCtx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - d.secondary.Do(sCtx, iArgs...) - }) + if replayCmd, replayArgs, ok := blockingReplayCommand(cmd, args, resp); ok { + d.runSecondaryWrite(func() { d.writeSecondary(replayCmd, replayArgs) }) + } } return resp, err //nolint:wrapcheck // redis.Nil must pass through unwrapped for callers to detect nil replies diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 4398aca72..8561534f2 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -62,6 +62,16 @@ func (b *mockBackend) CallCount() int { return len(b.calls) } +func (b *mockBackend) Calls() [][]any { + b.mu.Lock() + defer b.mu.Unlock() + out := make([][]any, len(b.calls)) + for i := range b.calls { + out[i] = append([]any(nil), b.calls[i]...) + } + return out +} + // Helper to create a doFunc that returns a specific value. func makeCmd(val any, err error) func(ctx context.Context, args ...any) *redis.Cmd { return func(ctx context.Context, args ...any) *redis.Cmd { @@ -623,6 +633,53 @@ func TestDualWriter_Blocking_UsesTimeoutAwareBackend(t *testing.T) { assert.Equal(t, []any{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}, primary.args) } +func TestDualWriter_Blocking_ReplaysBZPopMinAsZRem(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnValue: []any{[]byte("queue"), []byte("job-1"), []byte("42")}, + } + secondary := newMockBackend("secondary") + + metrics := newTestMetrics() + cfg := ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: 10 * time.Second} + d := NewDualWriter(primary, secondary, cfg, metrics, newTestSentry(), testLogger) + + resp, err := d.Blocking(context.Background(), "BZPOPMIN", [][]byte{[]byte("BZPOPMIN"), []byte("queue"), []byte("5")}) + assert.NoError(t, err) + assert.Equal(t, []any{[]byte("queue"), []byte("job-1"), []byte("42")}, resp) + d.Close() + + assert.Equal(t, [][]any{{[]byte("ZREM"), []byte("queue"), []byte("job-1")}}, secondary.Calls()) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) + assert.InDelta(t, 1, testutil.ToFloat64(metrics.CommandTotal.WithLabelValues("ZREM", "secondary", "ok")), 0.001) +} + +func TestDualWriter_Blocking_XReadDoesNotUseWriteSemaphore(t *testing.T) { + primary := &timeoutCapturingBackend{name: "primary", returnValue: []any{}} + secondary := newMockBackend("secondary") + + metrics := newTestMetrics() + cfg := ProxyConfig{Mode: ModeDualWrite, SecondaryWriteConcurrency: 1, SecondaryTimeout: 10 * time.Second} + d := NewDualWriter(primary, secondary, cfg, metrics, newTestSentry(), testLogger) + + blocker := make(chan struct{}) + d.goWrite(func() { + <-blocker + }) + + resp, err := d.Blocking(context.Background(), "XREAD", [][]byte{ + []byte("XREAD"), []byte("BLOCK"), []byte("1"), []byte("STREAMS"), []byte("jobs"), []byte("0"), + }) + assert.NoError(t, err) + assert.Equal(t, []any{}, resp) + assert.Empty(t, secondary.Calls()) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncBackpressure), 0.001) + + close(blocker) + d.Close() +} + func TestDualWriter_GoAsync_Bounded(t *testing.T) { primary := newMockBackend("primary") primary.doFunc = makeCmd("OK", nil) From f69764063c661d71876867fe190b31d1a65d8a82 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 16 Jul 2026 16:10:00 +0900 Subject: [PATCH 106/162] Cap ElasticKV secondary script concurrency --- cmd/redis-proxy/main.go | 4 ++++ cmd/redis-proxy/main_test.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index b9483ec1e..16117c4cd 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -23,6 +23,7 @@ const ( sentryFlushTimeout = 2 * time.Second metricsShutdownTimeout = 5 * time.Second secondaryConcurrencyDivisor = 2 + elasticKVScriptConcurrency = 1 elasticKVDispatchTimeout = 10 * time.Second backendTimeoutGrace = time.Second ) @@ -229,6 +230,9 @@ func deriveSecondaryConcurrency(mode proxy.ProxyMode, primaryPoolSize, elasticKV } if scriptConcurrency == 0 { scriptConcurrency = defaultSecondaryScriptConcurrency(writeConcurrency) + if mode != proxy.ModeElasticKVPrimary && scriptConcurrency > elasticKVScriptConcurrency { + scriptConcurrency = elasticKVScriptConcurrency + } } return writeConcurrency, scriptConcurrency } diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 8690f483c..0e51ff630 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -44,7 +44,7 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { primaryPoolSize: 128, elasticKVPoolSize: 8, wantWriteConcurrency: 4, - wantScriptConcurrency: 2, + wantScriptConcurrency: 1, }, { name: "ElasticKV primary derives from Redis secondary pool", @@ -61,7 +61,7 @@ func TestDeriveSecondaryConcurrency(t *testing.T) { elasticKVPoolSize: 4, writeConcurrency: 5, wantWriteConcurrency: 5, - wantScriptConcurrency: 2, + wantScriptConcurrency: 1, }, { name: "explicit values win", From 7e409795c6724fe949318a8a4dc712dbdf210f5a Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 16 Jul 2026 16:13:52 +0900 Subject: [PATCH 107/162] Refresh ElasticKV leader after not-leader replies --- proxy/dualwrite.go | 23 +++++++++++++-- proxy/leader_aware_backend.go | 30 ++++++++++++++++++- proxy/leader_aware_backend_test.go | 34 +++++++++++++++++++++ proxy/proxy_test.go | 47 ++++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 3 deletions(-) diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index fc69b6773..c1e284d48 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -52,6 +52,10 @@ const ( asyncDropLogInterval = 5 * time.Second ) +type leaderRefreshingBackend interface { + RefreshLeaderNow(context.Context) +} + // readTSCompactedMarker is the substring produced by // store.ErrReadTSCompacted as it flows through gRPC (wrapped as // FailedPrecondition) and Lua PCall. Matching on substring is necessary @@ -73,13 +77,26 @@ func isRetryableSecondaryWriteError(err error) bool { return true } switch classifySecondaryWriteError(err) { - case "retry_limit", "write_conflict", "txn_locked": + case "retry_limit", "write_conflict", "txn_locked", "not_leader": return true default: return false } } +func isNotLeaderError(err error) bool { + return classifySecondaryWriteError(err) == "not_leader" +} + +func refreshSecondaryLeader(ctx context.Context, backend Backend, err error) { + if !isNotLeaderError(err) { + return + } + if refresher, ok := backend.(leaderRefreshingBackend); ok { + refresher.RefreshLeaderNow(ctx) + } +} + // DualWriter routes commands to primary and secondary backends based on mode. type DualWriter struct { primary Backend @@ -334,8 +351,10 @@ func (d *DualWriter) writeSecondary(cmd string, iArgs []any) { if attempt >= maxSecondaryTransientRetries { break } + reason := classifySecondaryWriteError(sErr) + refreshSecondaryLeader(sCtx, d.secondary, sErr) d.logger.Debug("retrying secondary write after transient error", - "cmd", cmd, "attempt", attempt+1, "backoff", backoff, "reason", classifySecondaryWriteError(sErr), "err", sErr) + "cmd", cmd, "attempt", attempt+1, "backoff", backoff, "reason", reason, "err", sErr) if !waitCompactedRetryBackoff(sCtx, backoff) { break } diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 878d51ebc..3932d4a9d 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -179,6 +179,13 @@ func (b *LeaderAwareRedisBackend) TriggerRefresh() { } } +// RefreshLeaderNow re-probes the cluster before returning. It is used by +// callers that already observed a not-leader response and need the next retry +// to use a fresh target instead of waiting for the background loop. +func (b *LeaderAwareRedisBackend) RefreshLeaderNow(ctx context.Context) { + b.refreshLeader(ctx) +} + // refreshLeader probes INFO replication on the current leader first, then on // each seed, and adopts the first advertised leader address. The current // leader's Redis address is returned by the leader node itself when it's @@ -353,8 +360,19 @@ func (b *LeaderAwareRedisBackend) currentClient() *redis.Client { return b.clients[b.leader] } -// Do forwards a single command to the current leader. +// Do forwards a single command to the current leader. A not-leader Redis reply +// means the command was rejected before applying, so it is safe to refresh the +// leader and retry once. func (b *LeaderAwareRedisBackend) Do(ctx context.Context, args ...any) *redis.Cmd { + cmd := b.doOnce(ctx, args...) + if !isNotLeaderError(cmd.Err()) { + return cmd + } + b.RefreshLeaderNow(ctx) + return b.doOnce(ctx, args...) +} + +func (b *LeaderAwareRedisBackend) doOnce(ctx context.Context, args ...any) *redis.Cmd { cli := b.currentClient() if cli == nil { cmd := redis.NewCmd(ctx, args...) @@ -365,7 +383,17 @@ func (b *LeaderAwareRedisBackend) Do(ctx context.Context, args ...any) *redis.Cm } // DoWithTimeout forwards a blocking command with a per-call socket timeout. +// Like Do, a not-leader rejection is refreshed and retried once. func (b *LeaderAwareRedisBackend) DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { + cmd := b.doWithTimeoutOnce(ctx, timeout, args...) + if !isNotLeaderError(cmd.Err()) { + return cmd + } + b.RefreshLeaderNow(ctx) + return b.doWithTimeoutOnce(ctx, timeout, args...) +} + +func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd { cli := b.currentClient() if cli == nil { cmd := redis.NewCmd(ctx, args...) diff --git a/proxy/leader_aware_backend_test.go b/proxy/leader_aware_backend_test.go index ea4204272..034822d59 100644 --- a/proxy/leader_aware_backend_test.go +++ b/proxy/leader_aware_backend_test.go @@ -127,6 +127,10 @@ func (n *fakeElasticKVNode) handleConn(conn net.Conn) { _, _ = fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(body), body) default: n.commands.Add(1) + if leader := n.Leader(); leader != "" && leader != n.addr { + _, _ = conn.Write([]byte("-ERR etcd raft engine is not leader\r\n")) + continue + } _, _ = conn.Write([]byte("+OK\r\n")) } } @@ -214,6 +218,36 @@ func TestLeaderAwareRedisBackend_FollowsLeaderChange(t *testing.T) { require.Equal(t, beforeB+1, nodeB.commands.Load(), "command must reach new leader B") } +func TestLeaderAwareRedisBackend_RetryRefreshesOnNotLeader(t *testing.T) { + nodeA := newFakeElasticKVNode(t) + nodeB := newFakeElasticKVNode(t) + + nodeA.SetLeader(nodeA.addr) + nodeB.SetLeader(nodeA.addr) + + backend := NewLeaderAwareRedisBackendWithInterval( + []string{nodeA.addr, nodeB.addr}, + "elastickv", + DefaultBackendOptions(), + time.Hour, 500*time.Millisecond, + testLogger, + ) + t.Cleanup(func() { _ = backend.Close() }) + + require.Eventually(t, func() bool { + return backend.CurrentLeader() == nodeA.addr + }, 2*time.Second, 10*time.Millisecond, "initial leader must be A") + + nodeA.SetLeader(nodeB.addr) + nodeB.SetLeader(nodeB.addr) + + res := backend.Do(context.Background(), "SET", "k", "v") + require.NoError(t, res.Err()) + require.Equal(t, nodeB.addr, backend.CurrentLeader(), "not-leader retry must synchronously adopt the advertised leader") + require.Equal(t, int64(1), nodeA.commands.Load(), "first attempt should hit the former leader and be rejected") + require.Equal(t, int64(1), nodeB.commands.Load(), "retry should land on the refreshed leader") +} + func TestLeaderAwareRedisBackend_ConcurrentCloseIsRaceFree(t *testing.T) { // Regression guard: Close() must not race concurrent Do() callers — // currentClient and ensureClientLocked hold the lock consistently with diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 8561534f2..dcfa37ffc 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -72,6 +72,24 @@ func (b *mockBackend) Calls() [][]any { return out } +type refreshableMockBackend struct { + *mockBackend + refreshMu sync.Mutex + refreshes int +} + +func (b *refreshableMockBackend) RefreshLeaderNow(ctx context.Context) { + b.refreshMu.Lock() + defer b.refreshMu.Unlock() + b.refreshes++ +} + +func (b *refreshableMockBackend) RefreshCount() int { + b.refreshMu.Lock() + defer b.refreshMu.Unlock() + return b.refreshes +} + // Helper to create a doFunc that returns a specific value. func makeCmd(val any, err error) func(ctx context.Context, args ...any) *redis.Cmd { return func(ctx context.Context, args ...any) *redis.Cmd { @@ -1185,6 +1203,35 @@ func TestDualWriter_writeSecondary_RetriesRetryLimitWriteConflict(t *testing.T) "a retried success must not count as a secondary write error") } +func TestDualWriter_writeSecondary_RetriesNotLeaderAfterRefresh(t *testing.T) { + primary := newMockBackend("primary") + primary.doFunc = makeCmd("OK", nil) + + secondary := &refreshableMockBackend{mockBackend: newMockBackend("secondary")} + notLeaderErr := testRedisErr("ERR etcd raft engine is not leader") + var calls int + secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { + calls++ + cmd := redis.NewCmd(ctx, args...) + if calls == 1 { + cmd.SetErr(notLeaderErr) + return cmd + } + cmd.SetVal("OK") + return cmd + } + + metrics := newTestMetrics() + d := NewDualWriter(primary, secondary, ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: time.Second}, metrics, newTestSentry(), testLogger) + + d.writeSecondary("EVALSHA", []any{[]byte("EVALSHA"), []byte("deadbeef"), []byte("0")}) + + assert.Equal(t, 2, calls, "secondary must retry after a not-leader rejection") + assert.Equal(t, 1, secondary.RefreshCount(), "not-leader retries must force leader rediscovery before retrying") + assert.InDelta(t, 0, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001, + "a refreshed retry success must not count as a secondary write error") +} + func TestDualWriter_writeSecondary_RetriesDoNotRepeatNoScriptProbe(t *testing.T) { // After the EVAL fallback resolves a NOSCRIPT, a compacted-retry must // re-send the resolved EVAL form directly — never the known-missing From 3df9239e81fe618f2a8c21a7c38c072940ab275c Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 16 Jul 2026 16:36:56 +0900 Subject: [PATCH 108/162] Avoid full stream rewrites for Lua XADD --- adapter/redis_lua_compat_test.go | 69 +++++++++ adapter/redis_lua_context.go | 241 ++++++++++++++++++++++++++++++- 2 files changed, 304 insertions(+), 6 deletions(-) diff --git a/adapter/redis_lua_compat_test.go b/adapter/redis_lua_compat_test.go index a77bc631f..169bc6650 100644 --- a/adapter/redis_lua_compat_test.go +++ b/adapter/redis_lua_compat_test.go @@ -2,6 +2,7 @@ package adapter import ( "context" + "strconv" "testing" "time" @@ -97,6 +98,74 @@ return {eventId, redis.call("HGET", KEYS[1], "name"), cjson.encode(event), redis require.Equal(t, map[string]any{"event": "waiting", "jobId": "job-1"}, events[0].Values) } +func TestRedis_LuaXAddMaxLenExistingStream(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const stream = "bull:test:events-existing" + for i := range 128 { + _, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream, + ID: "*", + Values: []string{"i", strconv.Itoa(i)}, + }).Result() + require.NoError(t, err) + } + + id, err := rdb.Eval(ctx, ` +return redis.call("XADD", KEYS[1], "MAXLEN", "~", 10, "*", "event", "waiting", "jobId", "job-1") +`, []string{stream}).Text() + require.NoError(t, err) + require.NotEmpty(t, id) + + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(10), xlen) + + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Len(t, events, 10) + require.Equal(t, id, events[len(events)-1].ID) + require.Equal(t, map[string]any{"event": "waiting", "jobId": "job-1"}, events[len(events)-1].Values) +} + +func TestRedis_LuaXAddMaxLenZero(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const stream = "bull:test:events-maxlen0" + for i := range 3 { + _, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream, + ID: "*", + Values: []string{"i", strconv.Itoa(i)}, + }).Result() + require.NoError(t, err) + } + + id, err := rdb.Eval(ctx, ` +return redis.call("XADD", KEYS[1], "MAXLEN", "0", "*", "event", "trimmed") +`, []string{stream}).Text() + require.NoError(t, err) + require.NotEmpty(t, id) + + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(0), xlen) + + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Empty(t, events) +} + func TestRedis_LuaReplyHelpers(t *testing.T) { nodes, _, _ := createNode(t, 3) defer shutdown(nodes) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index afb9340de..b145d6e25 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -127,6 +127,22 @@ type luaStreamState struct { exists bool dirty bool value redisStreamValue + delta *luaStreamDeltaState +} + +type luaStreamDeltaState struct { + legacyCleanup []*kv.Elem[kv.OP] + meta store.StreamMeta + metaFound bool + appends []luaStreamDeltaAppend + trimCount int + selfDeletes []redisStreamID + forceEmpty bool +} + +type luaStreamDeltaAppend struct { + id redisStreamID + entry redisStreamEntry } type luaTTLState struct { @@ -399,6 +415,7 @@ func (c *luaScriptContext) deleteLogical(key []byte) { st.exists = false st.dirty = true st.value = redisStreamValue{} + st.delta = nil } } @@ -459,7 +476,7 @@ func hasLoadedZSetValue(st *luaZSetState) bool { } func hasLoadedStreamValue(st *luaStreamState) bool { - return st != nil && st.loaded && st.exists + return st != nil && st.exists && (st.loaded || st.delta != nil) } // maxNegativeTypeCacheEntries caps the size of luaScriptContext.negativeType @@ -943,7 +960,10 @@ func (c *luaScriptContext) zsetStateForRead(key []byte) (*luaZSetState, bool, er func (c *luaScriptContext) streamState(key []byte) (*luaStreamState, error) { k := string(key) if st, ok := c.streams[k]; ok { - return st, nil + if st.loaded { + return st, nil + } + return c.materializeStreamState(key, st) } st := &luaStreamState{} c.streams[k] = st @@ -974,6 +994,49 @@ func (c *luaScriptContext) streamState(key []byte) (*luaStreamState, error) { return st, nil } +func (c *luaScriptContext) materializeStreamState(key []byte, st *luaStreamState) (*luaStreamState, error) { + value, err := c.server.loadStreamAt(context.Background(), key, c.startTS) + if err != nil { + return nil, err + } + if st.delta != nil { + value = applyLuaStreamDelta(value, st.delta) + } + st.loaded = true + st.exists = st.exists || len(value.Entries) > 0 || (st.delta != nil && st.delta.metaFound) + st.value = cloneStreamValue(value) + st.delta = nil + return st, nil +} + +func applyLuaStreamDelta(value redisStreamValue, delta *luaStreamDeltaState) redisStreamValue { + if delta == nil { + return value + } + entries := append([]redisStreamEntry(nil), value.Entries...) + if delta.trimCount > 0 { + if delta.trimCount >= len(entries) { + entries = entries[:0] + } else { + entries = entries[delta.trimCount:] + } + } + selfDeleted := make(map[redisStreamID]struct{}, len(delta.selfDeletes)) + for _, id := range delta.selfDeletes { + selfDeleted[id] = struct{}{} + } + for _, appendOp := range delta.appends { + if _, deleted := selfDeleted[appendOp.id]; deleted { + continue + } + entries = append(entries, appendOp.entry) + } + if delta.forceEmpty { + entries = entries[:0] + } + return redisStreamValue{Entries: entries} +} + func (c *luaScriptContext) markStringValue(key []byte, value []byte) { st := c.ttlState(key) _ = st @@ -1067,6 +1130,7 @@ func (c *luaScriptContext) markStreamValue(key []byte, value redisStreamValue) e st.exists = true st.dirty = true st.value = cloneStreamValue(value) + st.delta = nil c.markTouched(key) c.deleted[string(key)] = false return nil @@ -3206,6 +3270,17 @@ func (c *luaScriptContext) cmdXAdd(args []string) (luaReply, error) { if err != nil { return luaReply{}, err } + deltaID, handled, err := c.cmdXAddDelta(parsed) + if err != nil { + return luaReply{}, err + } + if handled { + return luaStringReply(deltaID), nil + } + return c.cmdXAddMaterialized(parsed) +} + +func (c *luaScriptContext) cmdXAddMaterialized(parsed luaXAddArgs) (luaReply, error) { st, err := c.streamState(parsed.key) if err != nil { return luaReply{}, err @@ -3231,6 +3306,96 @@ func (c *luaScriptContext) cmdXAdd(args []string) (luaReply, error) { return luaStringReply(id), nil } +func (c *luaScriptContext) cmdXAddDelta(parsed luaXAddArgs) (string, bool, error) { + st, handled, err := c.streamDeltaStateForXAdd(parsed.key) + if err != nil || !handled { + return "", handled, err + } + + id, parsedID, err := resolveXAddID(st.delta.meta, st.delta.metaFound, parsed.id) + if err != nil { + return "", true, err + } + if err := xaddEnforceMaxWideColumn(parsed.key, st.delta.meta.Length, parsed.maxLen); err != nil { + return "", true, err + } + st.delta.appends = append(st.delta.appends, luaStreamDeltaAppend{ + id: parsedID, + entry: newRedisStreamEntry(id, append([]string(nil), parsed.fields...)), + }) + c.applyLuaXAddDeltaTrim(st.delta, parsed.maxLen, parsedID) + st.delta.meta.LastMs = parsedID.ms + st.delta.meta.LastSeq = parsedID.seq + st.delta.metaFound = true + st.exists = true + st.dirty = true + c.markTouched(parsed.key) + c.deleted[string(parsed.key)] = false + return id, true, nil +} + +func (c *luaScriptContext) streamDeltaStateForXAdd(key []byte) (*luaStreamState, bool, error) { + keyString := string(key) + st := c.streams[keyString] + if st != nil && st.loaded { + return nil, false, nil + } + if st == nil { + st = &luaStreamState{} + c.streams[keyString] = st + } + if st.delta != nil { + return st, true, nil + } + delta, err := c.newLuaStreamDelta(key) + if err != nil { + return nil, true, err + } + st.delta = delta + return st, true, nil +} + +func (c *luaScriptContext) newLuaStreamDelta(key []byte) (*luaStreamDeltaState, error) { + typ, err := c.server.keyTypeAtExpect(c.scriptCtx(), key, c.startTS, redisTypeStream) + if err != nil { + return nil, err + } + if typ != redisTypeNone && typ != redisTypeStream { + return nil, wrongTypeError() + } + legacyCleanup, meta, metaFound, err := c.server.streamWriteBase(c.scriptCtx(), key, c.startTS) + if err != nil { + return nil, err + } + return &luaStreamDeltaState{ + legacyCleanup: legacyCleanup, + meta: meta, + metaFound: metaFound, + }, nil +} + +func (c *luaScriptContext) applyLuaXAddDeltaTrim(delta *luaStreamDeltaState, maxLen int, appendedID redisStreamID) { + candidateLen := delta.meta.Length + 1 + if maxLen < 0 || candidateLen <= int64(maxLen) { + delta.meta.Length = candidateLen + return + } + diff := candidateLen - int64(maxLen) + if diff > int64(maxWideColumnItems) { + diff = int64(maxWideColumnItems) + } + if diff > 0 { + delta.trimCount += int(diff) + } + if maxLen == 0 { + delta.meta.Length = 0 + delta.selfDeletes = append(delta.selfDeletes, appendedID) + delta.forceEmpty = true + return + } + delta.meta.Length = candidateLen - diff +} + func parseLuaXAddArgs(args []string) (luaXAddArgs, error) { parsed := luaXAddArgs{ key: []byte(args[0]), @@ -3429,7 +3594,7 @@ func (c *luaScriptContext) commitPlanForKey(ctx context.Context, key string, com return luaKeyPlan{}, err } - valuePlan, err := c.valueCommitPlan(key, finalType, commitTS) + valuePlan, err := c.valueCommitPlan(ctx, key, finalType, commitTS) if err != nil { return luaKeyPlan{}, err } @@ -3478,7 +3643,7 @@ func luaWideFenceReadKeysForPlan(key []byte, finalType, startType redisValueType return nil } -func (c *luaScriptContext) valueCommitPlan(key string, finalType redisValueType, commitTS uint64) (luaCommitPlan, error) { +func (c *luaScriptContext) valueCommitPlan(ctx context.Context, key string, finalType redisValueType, commitTS uint64) (luaCommitPlan, error) { switch finalType { case redisTypeNone: return luaCommitPlan{}, nil @@ -3496,8 +3661,7 @@ func (c *luaScriptContext) valueCommitPlan(key string, finalType redisValueType, case redisTypeZSet: return c.zsetCommitPlan(key, commitTS) case redisTypeStream: - elems, err := c.streamCommitElems(key) - return luaCommitPlan{elems: elems}, err + return c.streamCommitPlan(ctx, key) default: return luaCommitPlan{}, errors.New("ERR unsupported final redis type") } @@ -3843,6 +4007,71 @@ func (c *luaScriptContext) zsetDeltaMetaElems(key []byte, st *luaZSetState, comm }} } +func (c *luaScriptContext) streamCommitPlan(ctx context.Context, key string) (luaCommitPlan, error) { + st := c.streams[key] + if st == nil || !st.dirty { + return luaCommitPlan{preserveExisting: true}, nil + } + if st.delta != nil && !st.loaded { + elems, err := c.streamDeltaCommitElems(ctx, key, st.delta) + return luaCommitPlan{preserveExisting: true, elems: elems}, err + } + elems, err := c.streamCommitElems(key) + return luaCommitPlan{elems: elems}, err +} + +func (c *luaScriptContext) streamDeltaCommitElems(ctx context.Context, key string, delta *luaStreamDeltaState) ([]*kv.Elem[kv.OP], error) { + keyBytes := []byte(key) + trim, err := c.server.buildXTrimHeadElems(ctx, keyBytes, c.startTS, delta.trimCount) + if err != nil { + return nil, err + } + + elems := make([]*kv.Elem[kv.OP], 0, len(delta.legacyCleanup)+len(delta.appends)+len(trim)+len(delta.selfDeletes)+1) + elems = append(elems, delta.legacyCleanup...) + for _, appendOp := range delta.appends { + entryValue, err := marshalStreamEntry(appendOp.entry) + if err != nil { + return nil, err + } + elems = append(elems, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: store.StreamEntryKey(keyBytes, appendOp.id.ms, appendOp.id.seq), + Value: entryValue, + }) + } + elems = append(elems, trim...) + for _, id := range delta.selfDeletes { + elems = append(elems, &kv.Elem[kv.OP]{ + Op: kv.Del, + Key: store.StreamEntryKey(keyBytes, id.ms, id.seq), + }) + } + + meta := delta.meta + if delta.forceEmpty { + meta.Length = 0 + } else { + meta.Length = int64(len(delta.appends)) + deltaBaseLength(delta) - int64(len(trim)) - int64(len(delta.selfDeletes)) + if meta.Length < 0 { + meta.Length = 0 + } + } + metaBytes, err := store.MarshalStreamMeta(meta) + if err != nil { + return nil, errors.WithStack(err) + } + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Put, Key: store.StreamMetaKey(keyBytes), Value: metaBytes}) + return elems, nil +} + +func deltaBaseLength(delta *luaStreamDeltaState) int64 { + if delta == nil { + return 0 + } + return delta.meta.Length + int64(delta.trimCount) - int64(len(delta.appends)) +} + // streamCommitElems writes the script's final stream state in the // wide-column layout — one StreamEntryKey Put per entry plus a StreamMetaKey // Put for the aggregate meta. The legacy single-blob path is no longer From d19b2a472394450c37bc2d01d3a2898ef27761d4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 16 Jul 2026 19:55:19 +0900 Subject: [PATCH 109/162] Handle Redis proxy replay edge cases --- adapter/redis_lua_compat_test.go | 74 +++++++++++++++++ adapter/redis_lua_context.go | 41 +++++++--- proxy/blocking.go | 17 ++++ proxy/dualwrite.go | 44 ++++++++++- proxy/leader_aware_backend.go | 22 +++++- proxy/leader_aware_backend_test.go | 123 +++++++++++++++++++++++------ proxy/proxy_test.go | 88 +++++++++++++++++++++ 7 files changed, 373 insertions(+), 36 deletions(-) diff --git a/adapter/redis_lua_compat_test.go b/adapter/redis_lua_compat_test.go index 169bc6650..f9d27c222 100644 --- a/adapter/redis_lua_compat_test.go +++ b/adapter/redis_lua_compat_test.go @@ -166,6 +166,80 @@ return redis.call("XADD", KEYS[1], "MAXLEN", "0", "*", "event", "trimmed") require.Empty(t, events) } +func TestRedis_LuaXAddMultipleMaxLenTrimsInScript(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const stream = "bull:test:events-multi-xadd" + result, err := rdb.Eval(ctx, ` +local first = redis.call("XADD", KEYS[1], "MAXLEN", "1", "*", "event", "first") +local second = redis.call("XADD", KEYS[1], "MAXLEN", "1", "*", "event", "second") +return {first, second} +`, []string{stream}).Result() + require.NoError(t, err) + ids, ok := result.([]any) + require.True(t, ok) + require.Len(t, ids, 2) + + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(1), xlen) + + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, ids[1], events[0].ID) + require.Equal(t, map[string]any{"event": "second"}, events[0].Values) +} + +func TestRedis_LuaXAddRecreatesTTLExpiredStream(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const ( + stream = "bull:test:events-expired" + ttl = 80 * time.Millisecond + ) + _, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream, + ID: "*", + Values: []string{"event", "old"}, + }).Result() + require.NoError(t, err) + require.NoError(t, rdb.PExpire(ctx, stream, ttl).Err()) + eventuallyExpired(t, ttl, func() bool { + xlen, err := rdb.XLen(ctx, stream).Result() + return err == nil && xlen == 0 + }, "stream must be logically expired before Lua XADD recreates it") + + id, err := rdb.Eval(ctx, ` +return redis.call("XADD", KEYS[1], "MAXLEN", "~", 10, "*", "event", "new") +`, []string{stream}).Text() + require.NoError(t, err) + require.NotEmpty(t, id) + + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(1), xlen) + ttlAfter, err := rdb.TTL(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, time.Duration(-1), ttlAfter) + + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, id, events[0].ID) + require.Equal(t, map[string]any{"event": "new"}, events[0].Values) +} + func TestRedis_LuaReplyHelpers(t *testing.T) { nodes, _, _ := createNode(t, 3) defer shutdown(nodes) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index b145d6e25..358718c94 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -3340,38 +3340,61 @@ func (c *luaScriptContext) streamDeltaStateForXAdd(key []byte) (*luaStreamState, if st != nil && st.loaded { return nil, false, nil } + if st != nil && st.delta != nil { + return nil, false, nil + } if st == nil { st = &luaStreamState{} c.streams[keyString] = st } - if st.delta != nil { - return st, true, nil - } - delta, err := c.newLuaStreamDelta(key) + delta, useDelta, err := c.newLuaStreamDelta(key) if err != nil { return nil, true, err } + if !useDelta { + delete(c.streams, keyString) + return nil, false, nil + } st.delta = delta return st, true, nil } -func (c *luaScriptContext) newLuaStreamDelta(key []byte) (*luaStreamDeltaState, error) { +func (c *luaScriptContext) newLuaStreamDelta(key []byte) (*luaStreamDeltaState, bool, error) { typ, err := c.server.keyTypeAtExpect(c.scriptCtx(), key, c.startTS, redisTypeStream) if err != nil { - return nil, err + return nil, false, err } if typ != redisTypeNone && typ != redisTypeStream { - return nil, wrongTypeError() + return nil, false, wrongTypeError() } legacyCleanup, meta, metaFound, err := c.server.streamWriteBase(c.scriptCtx(), key, c.startTS) if err != nil { - return nil, err + return nil, false, err + } + useDelta, err := c.canUseLuaStreamDelta(key, typ, metaFound, legacyCleanup) + if err != nil || !useDelta { + return nil, false, err } return &luaStreamDeltaState{ legacyCleanup: legacyCleanup, meta: meta, metaFound: metaFound, - }, nil + }, true, nil +} + +func (c *luaScriptContext) canUseLuaStreamDelta(key []byte, typ redisValueType, metaFound bool, legacyCleanup []*kv.Elem[kv.OP]) (bool, error) { + hasPhysicalStream := metaFound || len(legacyCleanup) != 0 + if !hasPhysicalStream { + return true, nil + } + expired, err := c.server.hasExpired(c.scriptCtx(), key, c.startTS, true) + if err != nil { + return false, err + } + if expired || typ == redisTypeNone { + return false, nil + } + return true, nil } func (c *luaScriptContext) applyLuaXAddDeltaTrim(delta *luaStreamDeltaState, maxLen int, appendedID redisStreamID) { diff --git a/proxy/blocking.go b/proxy/blocking.go index 8a1bf64be..772c011f2 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -56,6 +56,10 @@ func parseBlockingMillisecondsArg(raw []byte) time.Duration { func blockingReplayCommand(cmd string, _ [][]byte, resp any) (string, []any, bool) { switch strings.ToUpper(cmd) { + case "BLPOP": + return blockingListPopReplay(1, resp) + case "BRPOP": + return blockingListPopReplay(-1, resp) case "BZPOPMIN", "BZPOPMAX": parts, ok := redisArray(resp) if !ok || len(parts) < 2 { @@ -72,6 +76,19 @@ func blockingReplayCommand(cmd string, _ [][]byte, resp any) (string, []any, boo } } +func blockingListPopReplay(count int64, resp any) (string, []any, bool) { + parts, ok := redisArray(resp) + if !ok || len(parts) < 2 { + return "", nil, false + } + key, keyOK := redisArg(parts[0]) + value, valueOK := redisArg(parts[1]) + if !keyOK || !valueOK { + return "", nil, false + } + return "LREM", []any{[]byte("LREM"), key, count, value}, true +} + func redisArray(v any) ([]any, bool) { switch x := v.(type) { case []any: diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index c1e284d48..0963c9ebb 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -76,8 +76,11 @@ func isRetryableSecondaryWriteError(err error) bool { if isReadTSCompactedError(err) { return true } + if isElasticKVNotLeaderError(err) { + return true + } switch classifySecondaryWriteError(err) { - case "retry_limit", "write_conflict", "txn_locked", "not_leader": + case "retry_limit", "write_conflict", "txn_locked": return true default: return false @@ -85,7 +88,23 @@ func isRetryableSecondaryWriteError(err error) bool { } func isNotLeaderError(err error) bool { - return classifySecondaryWriteError(err) == "not_leader" + return isElasticKVNotLeaderError(err) +} + +func isElasticKVNotLeaderError(err error) bool { + if err == nil { + return false + } + msg := strings.TrimSpace(err.Error()) + if strings.HasPrefix(strings.ToUpper(msg), "NOTLEADER ") { + return true + } + if strings.Contains(msg, "etcd raft engine is not leader") || + strings.Contains(msg, "raft engine: not leader") { + return true + } + return msg == "leader not found" || + strings.HasSuffix(msg, "desc = leader not found") } func refreshSecondaryLeader(ctx context.Context, backend Backend, err error) { @@ -401,15 +420,34 @@ func (d *DualWriter) replaySecondaryPipeline(cmds [][]any) { d.runSecondaryWrite(func() { sCtx, cancel := context.WithTimeout(context.Background(), d.cfg.SecondaryTimeout) defer cancel() - _, pErr := d.secondary.Pipeline(sCtx, cmds) + results, pErr := d.secondary.Pipeline(sCtx, cmds) if pErr != nil { d.logger.Warn("secondary txn replay failed", "err", pErr) d.metrics.SecondaryWriteErrors.Inc() d.metrics.SecondaryWriteErrorsByReason.WithLabelValues("PIPELINE", classifySecondaryWriteError(pErr)).Inc() + return + } + if rErr := firstPipelineResultError(results); rErr != nil { + d.logger.Warn("secondary txn replay failed", "err", rErr) + d.metrics.SecondaryWriteErrors.Inc() + d.metrics.SecondaryWriteErrorsByReason.WithLabelValues("PIPELINE", classifySecondaryWriteError(rErr)).Inc() } }) } +func firstPipelineResultError(results []*redis.Cmd) error { + for _, result := range results { + if result == nil { + continue + } + err := result.Err() + if err != nil && !errors.Is(err, redis.Nil) { + return fmt.Errorf("pipeline result: %w", err) + } + } + return nil +} + // waitCompactedRetryBackoff sleeps for a jittered interval or returns early // when the context is cancelled. Returns false if the caller should abort // the retry loop (context done). diff --git a/proxy/leader_aware_backend.go b/proxy/leader_aware_backend.go index 3932d4a9d..a9e372699 100644 --- a/proxy/leader_aware_backend.go +++ b/proxy/leader_aware_backend.go @@ -403,8 +403,19 @@ func (b *LeaderAwareRedisBackend) doWithTimeoutOnce(ctx context.Context, timeout return cli.WithTimeout(effectiveBlockingReadTimeout(timeout)).Do(ctx, args...) } -// Pipeline forwards a batch to the current leader. +// Pipeline forwards a batch to the current leader. A not-leader Redis reply in +// any result means the leader rejected the batch before applying it, so retrying +// once after synchronous discovery is safe. func (b *LeaderAwareRedisBackend) Pipeline(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { + results, err := b.pipelineOnce(ctx, cmds) + if err != nil || !pipelineHasElasticKVNotLeader(results) { + return results, err + } + b.RefreshLeaderNow(ctx) + return b.pipelineOnce(ctx, cmds) +} + +func (b *LeaderAwareRedisBackend) pipelineOnce(ctx context.Context, cmds [][]any) ([]*redis.Cmd, error) { cli := b.currentClient() if cli == nil { return nil, ErrNoLeaderBackend @@ -425,6 +436,15 @@ func (b *LeaderAwareRedisBackend) Pipeline(ctx context.Context, cmds [][]any) ([ return results, nil } +func pipelineHasElasticKVNotLeader(results []*redis.Cmd) bool { + for _, result := range results { + if result != nil && isElasticKVNotLeaderError(result.Err()) { + return true + } + } + return false +} + // NewPubSub opens a subscribe connection on the current leader. func (b *LeaderAwareRedisBackend) NewPubSub(ctx context.Context) *redis.PubSub { cli := b.currentClient() diff --git a/proxy/leader_aware_backend_test.go b/proxy/leader_aware_backend_test.go index 034822d59..ce2f32a6b 100644 --- a/proxy/leader_aware_backend_test.go +++ b/proxy/leader_aware_backend_test.go @@ -62,6 +62,7 @@ type fakeElasticKVNode struct { addr string ln net.Listener leaderAddr atomic.Pointer[string] + commandErr atomic.Pointer[string] commands atomic.Int64 infoCalls atomic.Int64 } @@ -89,6 +90,10 @@ func (n *fakeElasticKVNode) Leader() string { return "" } +func (n *fakeElasticKVNode) SetCommandError(err string) { + n.commandErr.Store(&err) +} + func (n *fakeElasticKVNode) serve() { for { conn, err := n.ln.Accept() @@ -110,32 +115,43 @@ func (n *fakeElasticKVNode) handleConn(conn net.Conn) { if len(args) == 0 { return } - cmd := strings.ToUpper(args[0]) - switch cmd { - case "HELLO": - // Force go-redis to fall back to RESP2 without HELLO: return an - // error the client treats as "HELLO not supported". - _, _ = conn.Write([]byte("-ERR unknown command 'HELLO'\r\n")) - case "CLIENT", "AUTH", "SELECT", "PING": - _, _ = conn.Write([]byte("+OK\r\n")) - case "INFO": - n.infoCalls.Add(1) - body := fmt.Sprintf( - "# Replication\r\nrole:slave\r\nraft_leader_redis:%s\r\n", - n.Leader(), - ) - _, _ = fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(body), body) - default: - n.commands.Add(1) - if leader := n.Leader(); leader != "" && leader != n.addr { - _, _ = conn.Write([]byte("-ERR etcd raft engine is not leader\r\n")) - continue - } - _, _ = conn.Write([]byte("+OK\r\n")) - } + n.handleCommand(conn, args) + } +} + +func (n *fakeElasticKVNode) handleCommand(conn net.Conn, args []string) { + switch strings.ToUpper(args[0]) { + case "HELLO": + // Force go-redis to fall back to RESP2 without HELLO: return an + // error the client treats as "HELLO not supported". + _, _ = conn.Write([]byte("-ERR unknown command 'HELLO'\r\n")) + case "CLIENT", "AUTH", "SELECT", "PING": + _, _ = conn.Write([]byte("+OK\r\n")) + case "INFO": + n.infoCalls.Add(1) + body := fmt.Sprintf( + "# Replication\r\nrole:slave\r\nraft_leader_redis:%s\r\n", + n.Leader(), + ) + _, _ = fmt.Fprintf(conn, "$%d\r\n%s\r\n", len(body), body) + default: + n.writeCommandReply(conn) } } +func (n *fakeElasticKVNode) writeCommandReply(conn net.Conn) { + n.commands.Add(1) + if p := n.commandErr.Load(); p != nil { + _, _ = fmt.Fprintf(conn, "-%s\r\n", *p) + return + } + if leader := n.Leader(); leader != "" && leader != n.addr { + _, _ = conn.Write([]byte("-ERR etcd raft engine is not leader\r\n")) + return + } + _, _ = conn.Write([]byte("+OK\r\n")) +} + // readRESPArray reads a single RESP array of bulk strings from rd. func readRESPArray(rd *bufio.Reader) ([]string, error) { header, err := rd.ReadString('\n') @@ -248,6 +264,67 @@ func TestLeaderAwareRedisBackend_RetryRefreshesOnNotLeader(t *testing.T) { require.Equal(t, int64(1), nodeB.commands.Load(), "retry should land on the refreshed leader") } +func TestLeaderAwareRedisBackend_DoesNotRetryUserNotLeaderError(t *testing.T) { + node := newFakeElasticKVNode(t) + node.SetLeader(node.addr) + node.SetCommandError("ERR not leader") + + backend := NewLeaderAwareRedisBackendWithInterval( + []string{node.addr}, + "elastickv", + DefaultBackendOptions(), + time.Hour, 500*time.Millisecond, + testLogger, + ) + t.Cleanup(func() { _ = backend.Close() }) + + require.Eventually(t, func() bool { + return backend.CurrentLeader() == node.addr + }, 2*time.Second, 10*time.Millisecond, "initial leader must be set") + + res := backend.Do(context.Background(), "EVAL", "return redis.error_reply('not leader')", 0) + require.Error(t, res.Err()) + require.Contains(t, res.Err().Error(), "not leader") + require.Equal(t, int64(1), node.commands.Load(), "user Redis error must not be retried") +} + +func TestLeaderAwareRedisBackend_PipelineRetryRefreshesOnNotLeader(t *testing.T) { + nodeA := newFakeElasticKVNode(t) + nodeB := newFakeElasticKVNode(t) + + nodeA.SetLeader(nodeA.addr) + nodeB.SetLeader(nodeA.addr) + + backend := NewLeaderAwareRedisBackendWithInterval( + []string{nodeA.addr, nodeB.addr}, + "elastickv", + DefaultBackendOptions(), + time.Hour, 500*time.Millisecond, + testLogger, + ) + t.Cleanup(func() { _ = backend.Close() }) + + require.Eventually(t, func() bool { + return backend.CurrentLeader() == nodeA.addr + }, 2*time.Second, 10*time.Millisecond, "initial leader must be A") + + nodeA.SetLeader(nodeB.addr) + nodeB.SetLeader(nodeB.addr) + + results, err := backend.Pipeline(context.Background(), [][]any{ + {"MULTI"}, + {"SET", "k", "v"}, + {"EXEC"}, + }) + require.NoError(t, err) + for _, result := range results { + require.NoError(t, result.Err()) + } + require.Equal(t, nodeB.addr, backend.CurrentLeader(), "pipeline retry must synchronously adopt the advertised leader") + require.Equal(t, int64(3), nodeA.commands.Load(), "first pipeline attempt should hit the former leader") + require.Equal(t, int64(3), nodeB.commands.Load(), "retry should replay the whole pipeline on the refreshed leader") +} + func TestLeaderAwareRedisBackend_ConcurrentCloseIsRaceFree(t *testing.T) { // Regression guard: Close() must not race concurrent Do() callers — // currentClient and ensureClientLocked hold the lock consistently with diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index dcfa37ffc..f092a492d 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -672,6 +672,51 @@ func TestDualWriter_Blocking_ReplaysBZPopMinAsZRem(t *testing.T) { assert.InDelta(t, 1, testutil.ToFloat64(metrics.CommandTotal.WithLabelValues("ZREM", "secondary", "ok")), 0.001) } +func TestDualWriter_Blocking_ReplaysListPopAsLRem(t *testing.T) { + tests := []struct { + name string + cmd string + resp []any + want [][]any + }{ + { + name: "BLPOP removes first matching element", + cmd: "BLPOP", + resp: []any{[]byte("queue"), []byte("job-1")}, + want: [][]any{{[]byte("LREM"), []byte("queue"), int64(1), []byte("job-1")}}, + }, + { + name: "BRPOP removes last matching element", + cmd: "BRPOP", + resp: []any{[]byte("queue"), []byte("job-2")}, + want: [][]any{{[]byte("LREM"), []byte("queue"), int64(-1), []byte("job-2")}}, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnValue: tc.resp, + } + secondary := newMockBackend("secondary") + + metrics := newTestMetrics() + cfg := ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: 10 * time.Second} + d := NewDualWriter(primary, secondary, cfg, metrics, newTestSentry(), testLogger) + + resp, err := d.Blocking(context.Background(), tc.cmd, [][]byte{[]byte(tc.cmd), []byte("queue"), []byte("5")}) + assert.NoError(t, err) + assert.Equal(t, tc.resp, resp) + d.Close() + + assert.Equal(t, tc.want, secondary.Calls()) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) + assert.InDelta(t, 1, testutil.ToFloat64(metrics.CommandTotal.WithLabelValues("LREM", "secondary", "ok")), 0.001) + }) + } +} + func TestDualWriter_Blocking_XReadDoesNotUseWriteSemaphore(t *testing.T) { primary := &timeoutCapturingBackend{name: "primary", returnValue: []any{}} secondary := newMockBackend("secondary") @@ -1232,6 +1277,49 @@ func TestDualWriter_writeSecondary_RetriesNotLeaderAfterRefresh(t *testing.T) { "a refreshed retry success must not count as a secondary write error") } +func TestDualWriter_writeSecondary_DoesNotRetryUserNotLeaderError(t *testing.T) { + primary := newMockBackend("primary") + primary.doFunc = makeCmd("OK", nil) + + secondary := &refreshableMockBackend{mockBackend: newMockBackend("secondary")} + userErr := testRedisErr("ERR not leader") + var calls int + secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { + calls++ + cmd := redis.NewCmd(ctx, args...) + cmd.SetErr(userErr) + return cmd + } + + metrics := newTestMetrics() + d := NewDualWriter(primary, secondary, ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: time.Second}, metrics, newTestSentry(), testLogger) + + d.writeSecondary("EVALSHA", []any{[]byte("EVALSHA"), []byte("deadbeef"), []byte("0")}) + + assert.Equal(t, 1, calls, "user Redis not-leader text must not be retried") + assert.Equal(t, 0, secondary.RefreshCount(), "user Redis not-leader text must not force rediscovery") + assert.InDelta(t, 1, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001) +} + +func TestDualWriter_ReplaySecondaryPipeline_RecordsReplyErrors(t *testing.T) { + primary := newMockBackend("primary") + secondary := newMockBackend("secondary") + secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { + cmd := redis.NewCmd(ctx, args...) + cmd.SetErr(testRedisErr("ERR etcd raft engine is not leader")) + return cmd + } + + metrics := newTestMetrics() + d := NewDualWriter(primary, secondary, ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: time.Second}, metrics, newTestSentry(), testLogger) + + d.replaySecondaryPipeline([][]any{{[]byte("MULTI")}, {[]byte("SET"), []byte("k"), []byte("v")}, {[]byte("EXEC")}}) + d.Close() + + assert.InDelta(t, 1, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001) + assert.InDelta(t, 1, testutil.ToFloat64(metrics.SecondaryWriteErrorsByReason.WithLabelValues("PIPELINE", "not_leader")), 0.001) +} + func TestDualWriter_writeSecondary_RetriesDoNotRepeatNoScriptProbe(t *testing.T) { // After the EVAL fallback resolves a NOSCRIPT, a compacted-retry must // re-send the resolved EVAL form directly — never the known-missing From b8bb6ad605532bd8872583e9b1372e8e628fc7c3 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 02:18:31 +0900 Subject: [PATCH 110/162] Honor Lua XADD cached state --- adapter/redis_lua_compat_test.go | 18 ++++++++ adapter/redis_lua_context.go | 50 ++++++++++++++++---- adapter/redis_lua_list_holes_test.go | 68 ++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 10 deletions(-) diff --git a/adapter/redis_lua_compat_test.go b/adapter/redis_lua_compat_test.go index f9d27c222..8f56dd31b 100644 --- a/adapter/redis_lua_compat_test.go +++ b/adapter/redis_lua_compat_test.go @@ -3,6 +3,7 @@ package adapter import ( "context" "strconv" + "strings" "testing" "time" @@ -240,6 +241,23 @@ return redis.call("XADD", KEYS[1], "MAXLEN", "~", 10, "*", "event", "new") require.Equal(t, map[string]any{"event": "new"}, events[0].Values) } +func TestRedis_LuaXAddHonorsScriptLocalStringType(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const key = "bull:test:events-local-string" + _, err := rdb.Eval(ctx, ` +redis.call("SET", KEYS[1], "string-value") +return redis.call("XADD", KEYS[1], "*", "event", "bad") +`, []string{key}).Result() + require.Error(t, err) + require.Contains(t, strings.ToUpper(err.Error()), "WRONGTYPE") +} + func TestRedis_LuaReplyHelpers(t *testing.T) { nodes, _, _ := createNode(t, 3) defer shutdown(nodes) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 358718c94..9eda2ad57 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -202,6 +202,10 @@ type luaExactScanFallbackDecider interface { AllowExactScanFallbackAfterPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, ts uint64, reverse bool) bool } +type luaRoutePinnedScanStore interface { + ScanAtWithReadFence(ctx context.Context, start []byte, end []byte, limit int, ts uint64, reverse bool, groupID uint64, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) +} + type luaCommandHandler func(*luaScriptContext, []string) (luaReply, error) type luaRenameHandler func(*luaScriptContext, []byte, []byte) error @@ -2283,18 +2287,12 @@ func (c *luaScriptContext) allowExactListScanFallback(startKey, endKey []byte, s } func (c *luaScriptContext) scanListItemWindowExact(key []byte, meta store.ListMeta, startKey, endKey []byte, left bool) (luaLazyListBoundaryItem, bool, error) { + routeStart := append([]byte(nil), startKey...) + routeEnd := append([]byte(nil), endKey...) for { - var ( - kvs []*store.KVPair - err error - ) - if left { - kvs, err = c.server.store.ScanAt(c.ctx, startKey, endKey, 1, c.startTS) - } else { - kvs, err = c.server.store.ReverseScanAt(c.ctx, startKey, endKey, 1, c.startTS) - } + kvs, err := c.scanExactListPage(startKey, endKey, routeStart, routeEnd, left) if err != nil { - return luaLazyListBoundaryItem{}, false, errors.WithStack(err) + return luaLazyListBoundaryItem{}, false, err } if len(kvs) == 0 { return luaLazyListBoundaryItem{}, false, nil @@ -2310,6 +2308,24 @@ func (c *luaScriptContext) scanListItemWindowExact(key []byte, meta store.ListMe } } +func (c *luaScriptContext) scanExactListPage(startKey, endKey, routeStart, routeEnd []byte, left bool) ([]*store.KVPair, error) { + var ( + kvs []*store.KVPair + err error + ) + if scanner, ok := c.server.store.(luaRoutePinnedScanStore); ok { + kvs, err = scanner.ScanAtWithReadFence(c.ctx, startKey, endKey, 1, c.startTS, !left, 0, 0, routeStart, routeEnd) + } else if left { + kvs, err = c.server.store.ScanAt(c.ctx, startKey, endKey, 1, c.startTS) + } else { + kvs, err = c.server.store.ReverseScanAt(c.ctx, startKey, endKey, 1, c.startTS) + } + if err != nil { + return nil, errors.WithStack(err) + } + return kvs, nil +} + func scanStartAfterKey(key []byte) []byte { next := make([]byte, len(key)+1) copy(next, key) @@ -3343,6 +3359,9 @@ func (c *luaScriptContext) streamDeltaStateForXAdd(key []byte) (*luaStreamState, if st != nil && st.delta != nil { return nil, false, nil } + if cached, handled, err := c.streamDeltaDecisionFromCachedType(key); cached { + return nil, handled, err + } if st == nil { st = &luaStreamState{} c.streams[keyString] = st @@ -3359,6 +3378,17 @@ func (c *luaScriptContext) streamDeltaStateForXAdd(key []byte) (*luaStreamState, return st, true, nil } +func (c *luaScriptContext) streamDeltaDecisionFromCachedType(key []byte) (bool, bool, error) { + typ, cached := c.cachedType(key) + if !cached { + return false, false, nil + } + if typ != redisTypeNone && typ != redisTypeStream { + return true, true, wrongTypeError() + } + return true, false, nil +} + func (c *luaScriptContext) newLuaStreamDelta(key []byte) (*luaStreamDeltaState, bool, error) { typ, err := c.server.keyTypeAtExpect(c.scriptCtx(), key, c.startTS, redisTypeStream) if err != nil { diff --git a/adapter/redis_lua_list_holes_test.go b/adapter/redis_lua_list_holes_test.go index 80ac1cd7c..4483540d4 100644 --- a/adapter/redis_lua_list_holes_test.go +++ b/adapter/redis_lua_list_holes_test.go @@ -339,6 +339,41 @@ func TestRedisLua_RPopLPushSyntheticPhysicalLimitFailsClosed(t *testing.T) { require.ErrorIs(t, err, ErrCollectionTooLarge) } +func TestRedisLua_ListExactFallbackPinsRouteBounds(t *testing.T) { + t.Parallel() + + ctx := context.Background() + src := []byte("bull:test:wait:route-pin") + meta := store.ListMeta{Head: 0, Len: 2, Tail: 2} + startKey := listItemKey(src, 0) + endKey := listItemKey(src, 2) + + collider := append([]byte{}, src...) + collider = append(collider, listItemKey(src, 0)[len(store.ListItemPrefix)+len(src):]...) + collider = append(collider, 'x') + st := &routePinnedExactScanStore{ + MVCCStore: store.NewMVCCStore(), + t: t, + wantRouteStart: startKey, + wantRouteEnd: endKey, + pages: [][]*store.KVPair{ + {{Key: listItemKey(collider, 0), Value: []byte("other-list-job")}}, + {{Key: listItemKey(src, 1), Value: []byte("job-1")}}, + }, + } + r := NewRedisServer(nil, "", st, newLocalAdapterCoordinator(st), nil, nil) + scriptCtx, err := newLuaScriptContext(ctx, r) + require.NoError(t, err) + defer scriptCtx.Close() + + item, ok, err := scriptCtx.scanListItemWindowExact(src, meta, startKey, endKey, true) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, int64(1), item.index) + require.Equal(t, "job-1", item.value) + require.Equal(t, 2, st.calls) +} + func TestRedisLua_RPopLPushDeletesLargeSparseListWithoutItems(t *testing.T) { t.Parallel() @@ -399,3 +434,36 @@ func (s noExactFallbackPhysicalLimitStore) ReverseScanAtPhysicalLimit(context.Co func (s noExactFallbackPhysicalLimitStore) AllowExactScanFallbackAfterPhysicalLimit(context.Context, []byte, []byte, int, int, uint64, bool) bool { return false } + +type routePinnedExactScanStore struct { + store.MVCCStore + t *testing.T + wantRouteStart []byte + wantRouteEnd []byte + pages [][]*store.KVPair + calls int +} + +func (s *routePinnedExactScanStore) ScanAtWithReadFence( + _ context.Context, + _ []byte, + _ []byte, + _ int, + _ uint64, + _ bool, + _ uint64, + _ uint64, + routeStart []byte, + routeEnd []byte, +) ([]*store.KVPair, error) { + s.t.Helper() + require.Equal(s.t, s.wantRouteStart, routeStart) + require.Equal(s.t, s.wantRouteEnd, routeEnd) + if s.calls >= len(s.pages) { + s.calls++ + return nil, nil + } + page := s.pages[s.calls] + s.calls++ + return page, nil +} From adf5cdb508928334d651e951aae5860401e8688b Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 02:25:35 +0900 Subject: [PATCH 111/162] Cover Lua XADD maxlen zero append --- adapter/redis_lua_compat_test.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/adapter/redis_lua_compat_test.go b/adapter/redis_lua_compat_test.go index 8f56dd31b..e22234762 100644 --- a/adapter/redis_lua_compat_test.go +++ b/adapter/redis_lua_compat_test.go @@ -167,6 +167,36 @@ return redis.call("XADD", KEYS[1], "MAXLEN", "0", "*", "event", "trimmed") require.Empty(t, events) } +func TestRedis_LuaXAddMaxLenZeroThenAppendInScript(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const stream = "bull:test:events-maxlen0-then-append" + result, err := rdb.Eval(ctx, ` +local trimmed = redis.call("XADD", KEYS[1], "MAXLEN", "0", "*", "event", "trimmed") +local kept = redis.call("XADD", KEYS[1], "MAXLEN", "1", "*", "event", "kept") +return {trimmed, kept} +`, []string{stream}).Result() + require.NoError(t, err) + ids, ok := result.([]any) + require.True(t, ok) + require.Len(t, ids, 2) + + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(1), xlen) + + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, ids[1], events[0].ID) + require.Equal(t, map[string]any{"event": "kept"}, events[0].Values) +} + func TestRedis_LuaXAddMultipleMaxLenTrimsInScript(t *testing.T) { nodes, _, _ := createNode(t, 3) defer shutdown(nodes) From 7407423be243bb6c58f41c6c79789512c32c7d3d Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 03:27:31 +0900 Subject: [PATCH 112/162] Enforce migration write floors during apply --- adapter/internal.go | 21 ++++- adapter/internal_test.go | 38 ++++++++ kv/fsm.go | 110 ++++++++++++++++++++-- kv/fsm_migration_fence_test.go | 73 ++++++++------ kv/shard_store.go | 14 ++- kv/shard_store_test.go | 14 +-- kv/sharded_coordinator.go | 18 ++-- kv/sharded_coordinator_del_prefix_test.go | 37 ++++++++ 8 files changed, 265 insertions(+), 60 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index 85b7caf29..90b9770e3 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -460,18 +460,29 @@ func (i *Internal) rejectWriteTimestampFloorMutations(muts []*pb.Mutation, commi } routes := i.routeEngine.Stats() for _, mut := range muts { - if mut == nil || (len(mut.Key) == 0 && mut.GetOp() != pb.Op_DEL_PREFIX) { + if mut == nil || forwardedTxnControlMutation(mut) || (len(mut.Key) == 0 && mut.GetOp() != pb.Op_DEL_PREFIX) { continue } - for _, route := range routes { - if routeWriteTimestampFloorApplies(route, mut, commitTS) { - return errors.Wrapf(kv.ErrRouteWriteTimestampTooLow, "key %q commit_ts=%d floor=%d", mut.Key, commitTS, route.MinWriteTSExclusive) - } + if err := rejectMutationBelowRouteWriteFloor(routes, mut, commitTS); err != nil { + return err } } return nil } +func rejectMutationBelowRouteWriteFloor(routes []distribution.Route, mut *pb.Mutation, commitTS uint64) error { + for _, route := range routes { + if routeWriteTimestampFloorApplies(route, mut, commitTS) { + return errors.Wrapf(kv.ErrRouteWriteTimestampTooLow, "key %q commit_ts=%d floor=%d", mut.Key, commitTS, route.MinWriteTSExclusive) + } + } + return nil +} + +func forwardedTxnControlMutation(mut *pb.Mutation) bool { + return mut != nil && bytes.HasPrefix(mut.GetKey(), []byte(kv.TxnKeyPrefix)) +} + func routeWriteTimestampFloorApplies(route distribution.Route, mut *pb.Mutation, commitTS uint64) bool { if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { return false diff --git a/adapter/internal_test.go b/adapter/internal_test.go index f7e922505..e4713a4fa 100644 --- a/adapter/internal_test.go +++ b/adapter/internal_test.go @@ -294,3 +294,41 @@ func TestStampTxnTimestampsRejectsRouteWriteFloor(t *testing.T) { err := i.stampTxnTimestamps(context.Background(), reqs) require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) } + +func TestStampTxnTimestampsIgnoresMetadataForRouteWriteFloor(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + }, + }, + })) + i := &Internal{routeEngine: engine} + reqs := []*pb.Request{{ + IsTxn: true, + Phase: pb.Phase_NONE, + Ts: 50, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("z"), CommitTS: 100})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + }} + + require.NoError(t, i.stampTxnTimestamps(context.Background(), reqs)) +} diff --git a/kv/fsm.go b/kv/fsm.go index fba870fc9..1ad886df7 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -504,7 +504,7 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui return f.handleDelPrefix(ctx, prefix, commitTS) } - if err := f.validateRawMutationsForApply(ctx, r); err != nil { + if err := f.validateRawMutationsForApply(ctx, r, commitTS); err != nil { return err } @@ -521,17 +521,17 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui return nil } -func (f *kvFSM) validateRawMutationsForApply(ctx context.Context, r *pb.Request) error { +func (f *kvFSM) validateRawMutationsForApply(ctx context.Context, r *pb.Request, commitTS uint64) error { bypassKeys := writeFenceBypassKeySet(r.GetWriteFenceBypassKeys()) for _, mut := range r.GetMutations() { - if err := f.validateRawMutationForApply(ctx, mut, bypassKeys); err != nil { + if err := f.validateRawMutationForApply(ctx, mut, bypassKeys, commitTS); err != nil { return err } } return nil } -func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutation, writeFenceBypassKeys map[string]struct{}) error { +func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutation, writeFenceBypassKeys map[string]struct{}, commitTS uint64) error { if mut == nil || len(mut.Key) == 0 { return errors.WithStack(ErrInvalidRequest) } @@ -544,6 +544,9 @@ func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutatio return err } } + if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { + return err + } if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { return err } @@ -567,6 +570,9 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { return err } + if err := f.verifyRouteWriteTimestampFloorForPrefix(prefix, commitTS); err != nil { + return err + } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -607,6 +613,74 @@ func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", prefix, start, end) } +func (f *kvFSM) verifyRouteWriteTimestampFloorForKey(key []byte, commitTS uint64) error { + if f.routes == nil || commitTS == 0 { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + if start, end, ok := s3BucketAuxiliaryRouteRange(key); ok { + for _, route := range snap.IntersectingRoutes(start, end) { + if err := verifyRouteWriteTimestampFloorForRange(route, key, start, end, commitTS); err != nil { + return err + } + } + return nil + } + rkey := routeKey(key) + if route, ok := snap.RouteOf(rkey); ok { + if err := verifyRouteWriteTimestampFloorForRoute(route, key, commitTS); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyRouteWriteTimestampFloorsForMutations(muts []*pb.Mutation, commitTS uint64) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyRouteWriteTimestampFloorForPrefix(prefix []byte, commitTS uint64) error { + if f.routes == nil || commitTS == 0 { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + start, end := routePrefixRange(prefix) + for _, route := range snap.IntersectingRoutes(start, end) { + if err := verifyRouteWriteTimestampFloorForRange(route, prefix, start, end, commitTS); err != nil { + return err + } + } + return nil +} + +func verifyRouteWriteTimestampFloorForRoute(route distribution.Route, key []byte, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, routeKey(key), commitTS, route.MinWriteTSExclusive) +} + +func verifyRouteWriteTimestampFloorForRange(route distribution.Route, key, start, end []byte, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) +} + func routePrefixRange(prefix []byte) ([]byte, []byte) { if len(prefix) == 0 { return []byte(""), nil @@ -1102,7 +1176,7 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { } startTS := r.Ts - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueMutationsAboveFloor(muts, startTS) if err != nil { return err } @@ -1172,7 +1246,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueMutationsAboveFloor(muts, commitTS) if err != nil { return err } @@ -1196,6 +1270,28 @@ func uniqueTxnMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { return uniq, nil } +func (f *kvFSM) uniqueMutationsAboveFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + uniq, err := uniqueMutations(muts) + if err != nil { + return nil, err + } + if err := f.verifyRouteWriteTimestampFloorsForMutations(uniq, commitTS); err != nil { + return nil, err + } + return uniq, nil +} + +func (f *kvFSM) uniqueTxnMutationsAboveFloor(muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + uniq, err := uniqueTxnMutations(muts) + if err != nil { + return nil, err + } + if err := f.verifyRouteWriteTimestampFloorsForMutations(uniq, commitTS); err != nil { + return nil, err + } + return uniq, nil +} + // dedupProbeOnePhase decides whether handleOnePhaseTxnRequest should no-op // because the entry is a retry whose prior attempt already landed. Extracted // to keep handleOnePhaseTxnRequest under the cyclop budget; the determinism @@ -1265,7 +1361,7 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - uniq, err := uniqueTxnMutations(muts) + uniq, err := f.uniqueTxnMutationsAboveFloor(muts, commitTS) if err != nil { return err } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 2a620eca2..2c2efd142 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -198,6 +198,33 @@ func TestFSMRejectsObservedWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) require.ErrorIs(t, err, ErrRouteWriteFenced) } +func TestFSMIgnoresRawRouteFloorForS3BucketAuxiliaryWrite(t *testing.T) { + t.Parallel() + + const bucket = "bucket-a" + key := s3keys.BucketMetaKey(bucket) + engine := distribution.NewEngine() + routes := s3BucketAuxiliaryFenceRoutes(bucket, 1, 1) + routes[1].State = distribution.RouteStateActive + routes[2].MinWriteTSExclusive = ^uint64(0) + applyComposed1Snapshot(t, engine, 1, routes) + + rawRoute, ok := engine.GetRoute(routeKey(key)) + require.True(t, ok) + require.Equal(t, ^uint64(0), rawRoute.MinWriteTSExclusive) + auxStart, auxEnd, ok := s3BucketAuxiliaryRouteRange(key) + require.True(t, ok) + auxRoutes := engine.GetIntersectingRoutes(auxStart, auxEnd) + require.NotEmpty(t, auxRoutes) + require.Zero(t, auxRoutes[0].MinWriteTSExclusive) + + fsm := newComposed1FSM(t, engine, 1) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("meta")}}, + }, 100) + require.NoError(t, err) +} + func TestFSMRejectsCurrentWriteFencedDelPrefix(t *testing.T) { t.Parallel() @@ -307,7 +334,7 @@ func TestFSMRejectsObservedWriteFencedPrepareButAllowsAbort(t *testing.T) { require.NotErrorIs(t, fsm.handleTxnRequest(ctx, abort, 11), ErrRouteWriteFenced) } -func TestFSMAllowsRawPointWriteAtMigrationTimestampFloorDuringReplay(t *testing.T) { +func TestFSMRejectsRawPointWriteAtMigrationTimestampFloorDuringApply(t *testing.T) { t.Parallel() ctx := context.Background() @@ -315,13 +342,12 @@ func TestFSMAllowsRawPointWriteAtMigrationTimestampFloorDuringReplay(t *testing. err := fsm.handleRawRequest(ctx, &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("replayed")}}, }, 100) - require.NoError(t, err) - got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("replayed"), got) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) } -func TestFSMAllowsDelPrefixAtMigrationTimestampFloorDuringReplay(t *testing.T) { +func TestFSMRejectsDelPrefixAtMigrationTimestampFloorDuringApply(t *testing.T) { t.Parallel() ctx := context.Background() @@ -331,13 +357,14 @@ func TestFSMAllowsDelPrefixAtMigrationTimestampFloorDuringReplay(t *testing.T) { err := fsm.handleRawRequest(ctx, &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, }, 100) - require.NoError(t, err) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) - _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.ErrorIs(t, getErr, store.ErrKeyNotFound) + got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) } -func TestFSMAllowsOnePhaseTxnAtMigrationTimestampFloorDuringReplay(t *testing.T) { +func TestFSMRejectsOnePhaseTxnAtMigrationTimestampFloorDuringApply(t *testing.T) { t.Parallel() ctx := context.Background() @@ -352,13 +379,12 @@ func TestFSMAllowsOnePhaseTxnAtMigrationTimestampFloorDuringReplay(t *testing.T) }, } err := fsm.handleTxnRequest(ctx, req, 100) - require.NoError(t, err) - got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("low"), got) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) } -func TestFSMAllowsCommitAtMigrationTimestampFloorDuringReplay(t *testing.T) { +func TestFSMRejectsPrepareAtMigrationTimestampFloorDuringApply(t *testing.T) { t.Parallel() ctx := context.Background() @@ -372,20 +398,5 @@ func TestFSMAllowsCommitAtMigrationTimestampFloorDuringReplay(t *testing.T) { {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, }, } - require.NoError(t, fsm.handleTxnRequest(ctx, prepare, 90)) - - commit := &pb.Request{ - IsTxn: true, - Phase: pb.Phase_COMMIT, - Ts: 90, - Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 100})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("low")}, - }, - } - err := fsm.handleTxnRequest(ctx, commit, 100) - require.NoError(t, err) - got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 90), ErrRouteWriteTimestampTooLow) } diff --git a/kv/shard_store.go b/kv/shard_store.go index 257fa8e7e..9f12f499e 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -2443,6 +2443,9 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. if err != nil || group == nil { return err } + if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { + return err + } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) @@ -2456,6 +2459,9 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor if err != nil || group == nil { return err } + if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { + return err + } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) @@ -2473,7 +2479,7 @@ func (s *ShardStore) ensureMutationWriteTimestampFloors(mutations []*store.KVPai return nil } for _, mut := range mutations { - if mut == nil || len(mut.Key) == 0 { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { continue } route, _, ok := s.routeAndGroupForKey(mut.Key) @@ -2607,6 +2613,9 @@ func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS ui // DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { + if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue @@ -2633,6 +2642,9 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl // is the receiver only when an aggregate (admin / coordinator) path // is replaying a global FLUSHALL, which is not raft-applied. func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { + if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + return err + } for _, g := range s.groups { if g == nil || g.Store == nil { continue diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 6aee93eb1..56169148a 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -548,20 +548,20 @@ func TestShardStoreRejectsWritesAtMigrationTimestampFloor(t *testing.T) { require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0)) } -func TestShardStoreRaftApplySkipsMigrationTimestampFloor(t *testing.T) { +func TestShardStoreRaftApplyRejectsMigrationTimestampFloor(t *testing.T) { t.Parallel() ctx := context.Background() st, _ := newStagedVisibilityShardStore(t) - require.NoError(t, st.ApplyMutationsRaft(ctx, []*store.KVPairMutation{ + require.ErrorIs(t, st.ApplyMutationsRaft(ctx, []*store.KVPairMutation{ {Op: store.OpTypePut, Key: []byte("k-raft"), Value: []byte("v")}, - }, nil, 90, 100)) - require.NoError(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ + }, nil, 90, 100), ErrRouteWriteTimestampTooLow) + require.ErrorIs(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ {Op: store.OpTypePut, Key: []byte("k-raft-at"), Value: []byte("v")}, - }, nil, 90, 100, 1)) - require.NoError(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100)) - require.NoError(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2)) + }, nil, 90, 100, 1), ErrRouteWriteTimestampTooLow) + require.ErrorIs(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100), ErrRouteWriteTimestampTooLow) + require.ErrorIs(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2), ErrRouteWriteTimestampTooLow) } func TestShardStoreScanAt_IncludesListKeysAcrossShards(t *testing.T) { diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 8fac71aa8..a16d1e3e5 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1216,18 +1216,18 @@ func (c *ShardedCoordinator) rejectWriteTimestampFloorPointElems(elems []*Elem[O } func (c *ShardedCoordinator) rejectWriteTimestampFloorPointKey(key []byte, commitTS uint64) error { - rkey := routeKey(key) - if route, ok := c.engine.GetRoute(rkey); ok && route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, rkey, commitTS, route.MinWriteTSExclusive) - } start, end, ok := s3BucketAuxiliaryRouteRange(key) - if !ok { + if ok { + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) + } + } return nil } - for _, route := range c.engine.GetIntersectingRoutes(start, end) { - if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) - } + rkey := routeKey(key) + if route, ok := c.engine.GetRoute(rkey); ok && route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, rkey, commitTS, route.MinWriteTSExclusive) } return nil } diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index c1771549b..adebba3bc 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -443,6 +443,43 @@ func TestShardedCoordinatorRoutesS3BucketAuxiliaryWriteToStagedOwner(t *testing. require.Equal(t, key, g2Txn.requests[0].Mutations[0].Key) } +func TestShardedCoordinatorIgnoresRawRouteFloorForS3BucketAuxiliaryWrite(t *testing.T) { + t.Parallel() + + const bucket = "bucket-a" + key := s3keys.BucketMetaKey(bucket) + engine := distribution.NewEngine() + routes := s3BucketAuxiliaryStagedRoutes(bucket, 1, 2) + routes[2].MinWriteTSExclusive = ^uint64(0) + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: routes, + })) + + rawRoute, ok := engine.GetRoute(routeKey(key)) + require.True(t, ok) + require.Equal(t, ^uint64(0), rawRoute.MinWriteTSExclusive) + auxStart, auxEnd, ok := s3BucketAuxiliaryRouteRange(key) + require.True(t, ok) + auxRoutes := engine.GetIntersectingRoutes(auxStart, auxEnd) + require.NotEmpty(t, auxRoutes) + require.Zero(t, auxRoutes[0].MinWriteTSExclusive) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{responses: []*TransactionResponse{{CommitIndex: 22}}} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: key, Value: []byte("meta")}}, + }) + require.NoError(t, err) + require.Empty(t, g1Txn.requests) + require.Len(t, g2Txn.requests, 1) +} + func TestShardedCoordinatorRejectsS3BucketAuxiliaryPointWriteAtMigrationTimestampFloor(t *testing.T) { t.Parallel() From 2c920bcc76b39f908f23ed867dd5e14fa936dc4d Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 03:34:36 +0900 Subject: [PATCH 113/162] Fence migration exports with applied reads --- adapter/internal.go | 16 +++++++- adapter/internal_migration_test.go | 64 ++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/adapter/internal.go b/adapter/internal.go index 90b9770e3..2d1d65469 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -121,7 +121,7 @@ func (i *Internal) ExportRangeVersions(req *pb.ExportRangeVersionsRequest, strea if err := i.validateExportRangeVersionsRequest(req); err != nil { return err } - if err := i.verifyInternalLeader(stream.Context()); err != nil { + if err := i.verifyInternalLeaderApplied(stream.Context()); err != nil { return err } return i.streamExportRangeVersions(req, stream) @@ -211,6 +211,9 @@ func validateImportRangeVersionsRequest(req *pb.ImportRangeVersionsRequest) erro } func (i *Internal) verifyInternalLeader(ctx context.Context) error { + if i.leader == nil { + return errors.WithStack(ErrNotLeader) + } if i.leader.State() != raftengine.StateLeader { return errors.WithStack(ErrNotLeader) } @@ -220,6 +223,17 @@ func (i *Internal) verifyInternalLeader(ctx context.Context) error { return nil } +func (i *Internal) verifyInternalLeaderApplied(ctx context.Context) error { + if i.leader == nil { + return errors.WithStack(ErrNotLeader) + } + if i.leader.State() != raftengine.StateLeader { + return errors.WithStack(ErrNotLeader) + } + _, err := i.leader.LinearizableRead(ctx) + return errors.WithStack(err) +} + func (i *Internal) proposeMigrationImport(ctx context.Context, req *pb.ImportRangeVersionsRequest) (store.ImportVersionsResult, error) { cmd, err := kv.MarshalMigrationImportCommand(req) if err != nil { diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 05e74e76a..58bf6452e 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -36,6 +36,23 @@ func (mockInternalLeader) LinearizableRead(context.Context) (uint64, error) { return 1, nil } +type recordingInternalLeader struct { + mockInternalLeader + linearizableReadErr error + linearizableReadCalls int + verifyLeaderCalls int +} + +func (l *recordingInternalLeader) VerifyLeader(context.Context) error { + l.verifyLeaderCalls++ + return nil +} + +func (l *recordingInternalLeader) LinearizableRead(context.Context) (uint64, error) { + l.linearizableReadCalls++ + return 7, l.linearizableReadErr +} + type applyingMigrationProposer struct { fsm raftengine.StateMachine calls uint64 @@ -113,6 +130,53 @@ func TestInternalExportRangeVersionsUsesStoreAndRouteFilter(t *testing.T) { }, stream.responses[0].GetVersions()) } +func TestInternalExportRangeVersionsUsesAppliedReadFence(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("va"), 10, 0)) + leader := &recordingInternalLeader{} + internal := NewInternalWithEngine(nil, leader, nil, nil, WithInternalStore(st)) + stream := &captureExportRangeVersionsStream{ctx: ctx} + + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + RouteStart: []byte("a"), + RouteEnd: []byte("b"), + KeyFamily: distribution.MigrationFamilyUser, + RangeStart: []byte("a"), + RangeEnd: []byte("b"), + MaxScannedBytes: 1 << 20, + }, stream) + require.NoError(t, err) + require.Equal(t, 1, leader.linearizableReadCalls) + require.Zero(t, leader.verifyLeaderCalls) + require.Len(t, stream.responses, 1) + require.True(t, stream.responses[0].GetDone()) +} + +func TestInternalExportRangeVersionsFailsClosedWhenAppliedReadFenceFails(t *testing.T) { + t.Parallel() + + leader := &recordingInternalLeader{linearizableReadErr: context.Canceled} + internal := NewInternalWithEngine(nil, leader, nil, nil, WithInternalStore(store.NewMVCCStore())) + stream := &captureExportRangeVersionsStream{ctx: context.Background()} + + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + RouteStart: []byte("a"), + RouteEnd: []byte("b"), + KeyFamily: distribution.MigrationFamilyUser, + RangeStart: []byte("a"), + RangeEnd: []byte("b"), + MaxScannedBytes: 1 << 20, + }, stream) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, leader.linearizableReadCalls) + require.Empty(t, stream.responses) +} + func TestInternalExportRangeVersionsRejectsUnboundedExport(t *testing.T) { t.Parallel() From bdf05a645828a2f98521763679ac41497920a5d0 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 03:38:46 +0900 Subject: [PATCH 114/162] Delete staged rows for migrated prefixes --- kv/shard_store.go | 52 ++++++++++++++++++++++++++++++++++++++++++ kv/shard_store_test.go | 25 ++++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/kv/shard_store.go b/kv/shard_store.go index 9f12f499e..77aca0a6a 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -2595,9 +2595,51 @@ func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludeP return errors.WithStack(err) } } + for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { + if err := del.group.Store.DeletePrefixAt(ctx, del.prefix, del.excludePrefix, commitTS); err != nil { + return errors.WithStack(err) + } + } return nil } +type stagedVisibilityPrefixDelete struct { + group *ShardGroup + prefix []byte + excludePrefix []byte +} + +func (s *ShardStore) stagedVisibilityPrefixDeletes(prefix []byte, excludePrefix []byte) []stagedVisibilityPrefixDelete { + if s == nil || s.engine == nil { + return nil + } + start, end := routePrefixRange(prefix) + routes := s.engine.GetIntersectingRoutes(start, end) + out := make([]stagedVisibilityPrefixDelete, 0, len(routes)) + seen := make(map[string]struct{}, len(routes)) + for _, route := range routes { + if !routeHasStagedVisibility(route) { + continue + } + g := s.groups[route.GroupID] + if g == nil || g.Store == nil { + continue + } + stagedPrefix := distribution.MigrationStagedDataKey(route.MigrationJobID, prefix) + var stagedExclude []byte + if excludePrefix != nil { + stagedExclude = distribution.MigrationStagedDataKey(route.MigrationJobID, excludePrefix) + } + dedupeKey := string(stagedPrefix) + "\x00" + string(stagedExclude) + if _, ok := seen[dedupeKey]; ok { + continue + } + seen[dedupeKey] = struct{}{} + out = append(out, stagedVisibilityPrefixDelete{group: g, prefix: stagedPrefix, excludePrefix: stagedExclude}) + } + return out +} + func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS uint64) error { if s == nil || s.engine == nil || commitTS == 0 { return nil @@ -2624,6 +2666,11 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl return errors.WithStack(err) } } + for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { + if err := del.group.Store.DeletePrefixAtRaft(ctx, del.prefix, del.excludePrefix, commitTS); err != nil { + return errors.WithStack(err) + } + } return nil } @@ -2663,6 +2710,11 @@ func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, ex return errors.WithStack(err) } } + for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { + if err := del.group.Store.DeletePrefixAtRaftAt(ctx, del.prefix, del.excludePrefix, commitTS, appliedIndex); err != nil { + return errors.WithStack(err) + } + } return nil } diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 56169148a..3bc7fa859 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -255,6 +255,31 @@ func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { require.Equal(t, uint64(40), ts) } +func TestShardStoreDeletePrefixAtDeletesStagedVisibilityRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + dropKey := []byte("b/drop") + keepKey := []byte("b/keep") + outsideKey := []byte("c/outside") + + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, dropKey), []byte("drop"), 20, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, keepKey), []byte("keep"), 20, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, outsideKey), []byte("outside"), 20, 0)) + + require.NoError(t, st.DeletePrefixAt(ctx, []byte("b/"), []byte("b/keep"), 101)) + + _, err := st.GetAt(ctx, dropKey, 150) + require.ErrorIs(t, err, store.ErrKeyNotFound) + got, err := st.GetAt(ctx, keepKey, 150) + require.NoError(t, err) + require.Equal(t, []byte("keep"), got) + got, err = st.GetAt(ctx, outsideKey, 150) + require.NoError(t, err) + require.Equal(t, []byte("outside"), got) +} + func TestShardStoreRouteFilteredLeaderScanUsesStagedVisibility(t *testing.T) { t.Parallel() From c9d7ddc56c0aec48b425a973b28eab95336ff0d0 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 03:50:51 +0900 Subject: [PATCH 115/162] Preserve staged scan visibility --- adapter/distribution_server.go | 6 + adapter/distribution_server_test.go | 33 ++++- kv/fsm.go | 16 +++ kv/fsm_onephase_dedup_test.go | 34 +++++ kv/shard_store.go | 206 +++++++++++++++++++++------- kv/shard_store_test.go | 113 +++++++++++++++ 6 files changed, 361 insertions(+), 47 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index fce46af57..96742050b 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -162,6 +162,9 @@ func (s *DistributionServer) ListRoutes(ctx context.Context, req *pb.ListRoutesR } func (s *DistributionServer) GetRouteOwnership(ctx context.Context, req *pb.GetRouteOwnershipRequest) (*pb.GetRouteOwnershipResponse, error) { + if err := s.requireReadReady(); err != nil { + return nil, err + } snapshot, err := s.routeSnapshotAt(req.GetCatalogVersion()) if err != nil { return nil, err @@ -181,6 +184,9 @@ func (s *DistributionServer) GetRouteOwnership(ctx context.Context, req *pb.GetR } func (s *DistributionServer) GetIntersectingRoutes(ctx context.Context, req *pb.GetIntersectingRoutesRequest) (*pb.GetIntersectingRoutesResponse, error) { + if err := s.requireReadReady(); err != nil { + return nil, err + } snapshot, err := s.routeSnapshotAt(req.GetCatalogVersion()) if err != nil { return nil, err diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 7669113fa..a9d8da293 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -42,7 +42,12 @@ func TestDistributionServerRouteReadsHonorStartupGate(t *testing.T) { t.Parallel() engine := distribution.NewEngine() - engine.UpdateRoute([]byte("a"), nil, 1) + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) catalog := distribution.NewCatalogStore(store.NewMVCCStore()) _, err := catalog.Save(context.Background(), 0, []distribution.RouteDescriptor{ {RouteID: 1, Start: []byte("a"), End: nil, GroupID: 1, State: distribution.RouteStateActive}, @@ -60,11 +65,37 @@ func TestDistributionServerRouteReadsHonorStartupGate(t *testing.T) { require.Error(t, err) require.Equal(t, codes.Unavailable, status.Code(err)) + _, err = s.GetRouteOwnership(context.Background(), &pb.GetRouteOwnershipRequest{ + Key: []byte("a"), + CatalogVersion: engine.Version(), + }) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + + _, err = s.GetIntersectingRoutes(context.Background(), &pb.GetIntersectingRoutesRequest{ + Start: []byte("a"), + End: []byte("z"), + CatalogVersion: engine.Version(), + }) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + blocked = false _, err = s.GetRoute(context.Background(), &pb.GetRouteRequest{Key: []byte("a")}) require.NoError(t, err) _, err = s.ListRoutes(context.Background(), &pb.ListRoutesRequest{}) require.NoError(t, err) + _, err = s.GetRouteOwnership(context.Background(), &pb.GetRouteOwnershipRequest{ + Key: []byte("a"), + CatalogVersion: engine.Version(), + }) + require.NoError(t, err) + _, err = s.GetIntersectingRoutes(context.Background(), &pb.GetIntersectingRoutesRequest{ + Start: []byte("a"), + End: []byte("z"), + CatalogVersion: engine.Version(), + }) + require.NoError(t, err) } func TestDistributionServerGetTimestamp_IsMonotonic(t *testing.T) { diff --git a/kv/fsm.go b/kv/fsm.go index 1ad886df7..0217c8bd2 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -1334,6 +1334,9 @@ func (f *kvFSM) currentStagedVisibilityRouteForKey(key []byte) (distribution.Rou if !ok { return distribution.Route{}, false } + if route, ok := currentStagedVisibilityRouteForS3BucketAuxiliaryKey(snap, key, f.shardGroupID); ok { + return route, true + } route, ok := snap.RouteOf(routeKey(key)) if !ok || route.GroupID != f.shardGroupID || !routeHasStagedVisibility(route) { return distribution.Route{}, false @@ -1341,6 +1344,19 @@ func (f *kvFSM) currentStagedVisibilityRouteForKey(key []byte) (distribution.Rou return route, true } +func currentStagedVisibilityRouteForS3BucketAuxiliaryKey(snap RouteSnapshot, key []byte, shardGroupID uint64) (distribution.Route, bool) { + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return distribution.Route{}, false + } + for _, route := range snap.IntersectingRoutes(start, end) { + if route.GroupID == shardGroupID && routeHasStagedVisibility(route) { + return route, true + } + } + return distribution.Route{}, false +} + func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { meta, muts, err := extractTxnMeta(r.Mutations) if err != nil { diff --git a/kv/fsm_onephase_dedup_test.go b/kv/fsm_onephase_dedup_test.go index 8b3f50f28..b675e1079 100644 --- a/kv/fsm_onephase_dedup_test.go +++ b/kv/fsm_onephase_dedup_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" @@ -99,6 +100,39 @@ func TestOnePhaseDedup_NoOpsWhenPriorAttemptLandedAsStagedVersion(t *testing.T) require.True(t, stagedAt20) } +func TestOnePhaseDedup_NoOpsWhenS3AuxiliaryPriorAttemptLandedAsStagedVersion(t *testing.T) { + t.Parallel() + ctx := context.Background() + st := store.NewMVCCStore() + engine := distribution.NewEngine() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: prefixScanEnd(routeStart), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }}) + fsmIface := NewKvFSMWithHLC(st, NewHLC(), WithRouteHistory(WrapDistributionEngine(engine), 1)) + fsm, ok := fsmIface.(*kvFSM) + require.True(t, ok) + + key := s3keys.BucketMetaKey(bucket) + require.NoError(t, st.PutAt(ctx, distribution.MigrationStagedDataKey(9, key), []byte("v"), 20, 0)) + + require.NoError(t, applyFSMRequest(t, fsm, onePhaseReq(30, 40, 20, key, []byte("v")))) + + at40, err := st.CommittedVersionAt(ctx, key, 40) + require.NoError(t, err) + require.False(t, at40, "retry must not write a live S3 auxiliary version when the prior attempt is staged") + stagedAt20, err := st.CommittedVersionAt(ctx, distribution.MigrationStagedDataKey(9, key), 20) + require.NoError(t, err) + require.True(t, stagedAt20) +} + // TestOnePhaseDedup_AppliesWhenPriorAttemptDidNotLand covers the truncated / // never-applied case: prev_commit_ts is set but no version exists at exactly // that timestamp (attempt 1's entry lost the log race). The probe misses and diff --git a/kv/shard_store.go b/kv/shard_store.go index 77aca0a6a..486783e80 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -181,7 +181,7 @@ func (s *ShardStore) getAtWithStagedVisibility(ctx context.Context, g *ShardGrou } func latestMVCCVersionAt(ctx context.Context, st store.MVCCStore, key []byte, ts uint64) (store.MVCCVersion, bool, error) { - result, err := st.ExportVersions(ctx, store.ExportVersionsOptions{ + opts := store.ExportVersionsOptions{ StartKey: key, EndKey: prefixScanEnd(key), MaxCommitTSInclusive: ts, @@ -193,16 +193,22 @@ func latestMVCCVersionAt(ctx context.Context, st store.MVCCStore, key []byte, ts AcceptKey: func(rawKey []byte) bool { return bytes.Equal(rawKey, key) }, - }) - if err != nil { - return store.MVCCVersion{}, false, errors.WithStack(err) } - for _, version := range result.Versions { - if bytes.Equal(version.Key, key) { - return version, true, nil + for { + result, err := st.ExportVersions(ctx, opts) + if err != nil { + return store.MVCCVersion{}, false, errors.WithStack(err) + } + for _, version := range result.Versions { + if bytes.Equal(version.Key, key) { + return version, true, nil + } + } + if result.Done || len(result.NextCursor) == 0 { + return store.MVCCVersion{}, false, nil } + opts.Cursor = result.NextCursor } - return store.MVCCVersion{}, false, nil } func newerMigrationVersion(a store.MVCCVersion, aOK bool, b store.MVCCVersion, bOK bool) (store.MVCCVersion, bool) { @@ -324,7 +330,7 @@ func (s *ShardStore) ScanAtWithReadFence(ctx context.Context, start []byte, end if reverse { if groupID != 0 { if routeScanBoundsPresent(routeStart, routeEnd) { - return s.scanRouteAtDirectionWithReadFence(ctx, distribution.Route{GroupID: groupID}, start, end, limit, ts, true, true, readRouteVersion, routeStart, routeEnd) + return s.scanExplicitGroupAtWithReadFence(ctx, groupID, start, end, limit, ts, true, readRouteVersion, routeStart, routeEnd) } return nil, errors.WithStack(store.ErrNotSupported) } @@ -339,7 +345,7 @@ func (s *ShardStore) scanAtWithReadFence(ctx context.Context, start []byte, end } if groupID != 0 { - return s.scanRouteAtDirectionWithReadFence(ctx, distribution.Route{GroupID: groupID}, start, end, limit, ts, false, true, readRouteVersion, routeStart, routeEnd) + return s.scanExplicitGroupAtWithReadFence(ctx, groupID, start, end, limit, ts, false, readRouteVersion, routeStart, routeEnd) } routes, clampToRoutes := s.routesForFencedScan(start, end, routeStart, routeEnd) @@ -378,29 +384,53 @@ func (s *ShardStore) ScanAtPhysicalLimit(ctx context.Context, start []byte, end // Normal callers should use ScanAt so range scans keep following the // distribution engine's route table. func (s *ShardStore) ScanGroupAt(ctx context.Context, groupID uint64, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + return s.scanExplicitGroupAtWithReadFence(ctx, groupID, start, end, limit, ts, false, 0, nil, nil) +} + +func (s *ShardStore) scanExplicitGroupAtWithReadFence(ctx context.Context, groupID uint64, start []byte, end []byte, limit int, ts uint64, reverse bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { if limit <= 0 { return []*store.KVPair{}, nil } - routes, clampToRoutes, err := s.routesForExplicitGroupScan(groupID, start, end) + routes, clampToRoutes, err := s.routesForExplicitGroupScanWithRouteBounds(groupID, start, end, routeStart, routeEnd) if err != nil { return nil, err } + routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) + if !clampToRoutes && !routeFilterPresent { + routes = dedupeRepeatedRawScanRoutes(routes) + } + return s.scanExplicitGroupRoutesAtWithReadFence(ctx, routes, start, end, limit, ts, reverse, readRouteVersion, routeStart, routeEnd, clampToRoutes) +} + +func (s *ShardStore) scanExplicitGroupRoutesAtWithReadFence(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, reverse bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte, clampToRoutes bool) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) - for _, route := range routes { + for i := 0; i < len(routes); i++ { + route := routes[i] + if reverse { + route = routes[len(routes)-1-i] + } scanStart := start scanEnd := end if clampToRoutes { scanStart = clampScanStart(start, route.Start) scanEnd = clampScanEnd(end, route.End) } - kvs, err := s.scanRouteAtDirection(ctx, route, scanStart, scanEnd, limit, ts, false, true) + kvs, err := s.scanRouteAtDirectionWithReadFence(ctx, route, scanStart, scanEnd, limit, ts, reverse, true, readRouteVersion, routeStart, routeEnd) if err != nil { return nil, err } - out = append(out, kvs...) - if len(out) >= limit { - clear(out[limit:]) - return out[:limit], nil + if clampToRoutes { + out = append(out, kvs...) + if len(out) >= limit { + clear(out[limit:]) + return out[:limit], nil + } + continue + } + if reverse { + out = mergeAndTrimReverseScanResults(out, kvs, limit) + } else { + out = mergeAndTrimScanResults(out, kvs, limit) } } return out, nil @@ -445,6 +475,9 @@ func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Rou if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { return s.engine.GetIntersectingRoutes(routeStart, routeEnd), false } + if routes, ok := s.routesForS3BucketAuxiliaryScan(start, end); ok { + return routes, false + } if isBroadLegacyListDeltaScan(start) { return s.engine.GetIntersectingRoutes(nil, nil), false } @@ -472,6 +505,67 @@ func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Rou return routes, true } +func (s *ShardStore) routesForS3BucketAuxiliaryScan(start []byte, end []byte) ([]distribution.Route, bool) { + if s == nil || s.engine == nil || !s3BucketAuxiliaryScanBounds(start, end) { + return nil, false + } + routes := make([]distribution.Route, 0) + routes = append(routes, s.engine.GetIntersectingRoutes(start, end)...) + for _, route := range s.engine.GetIntersectingRoutes([]byte(s3keys.RoutePrefix), prefixScanEnd([]byte(s3keys.RoutePrefix))) { + if routeHasStagedVisibility(route) { + routes = append(routes, route) + } + } + return dedupeRepeatedRawScanRoutes(routes), true +} + +func s3BucketAuxiliaryScanBounds(start []byte, end []byte) bool { + if !bytes.HasPrefix(start, []byte(s3keys.BucketMetaPrefix)) && + !bytes.HasPrefix(start, []byte(s3keys.BucketGenerationPrefix)) { + return false + } + if end == nil { + return true + } + return bytes.Compare(start, end) < 0 +} + +type repeatedRawScanRouteKey struct { + groupID uint64 + staged bool + migrationJobID uint64 +} + +func dedupeRepeatedRawScanRoutes(routes []distribution.Route) []distribution.Route { + if len(routes) <= 1 { + return routes + } + stagedGroups := make(map[uint64]struct{}) + for _, route := range routes { + if routeHasStagedVisibility(route) { + stagedGroups[route.GroupID] = struct{}{} + } + } + out := make([]distribution.Route, 0, len(routes)) + seen := make(map[repeatedRawScanRouteKey]struct{}, len(routes)) + for _, route := range routes { + if _, hasStaged := stagedGroups[route.GroupID]; hasStaged && !routeHasStagedVisibility(route) { + continue + } + key := repeatedRawScanRouteKey{groupID: route.GroupID} + if routeHasStagedVisibility(route) { + key.staged = true + key.migrationJobID = route.MigrationJobID + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, route) + } + return out +} + func routesContainStagedVisibility(routes []distribution.Route) bool { for _, route := range routes { if routeHasStagedVisibility(route) { @@ -481,6 +575,35 @@ func routesContainStagedVisibility(routes []distribution.Route) bool { return false } +func (s *ShardStore) routesForExplicitGroupScanWithRouteBounds(groupID uint64, start []byte, end []byte, routeStart []byte, routeEnd []byte) ([]distribution.Route, bool, error) { + if routeScanBoundsPresent(routeStart, routeEnd) { + return s.routesForExplicitGroupRouteBounds(groupID, start, end, routeStart, routeEnd) + } + return s.routesForExplicitGroupScan(groupID, start, end) +} + +func (s *ShardStore) routesForExplicitGroupRouteBounds(groupID uint64, start []byte, end []byte, routeStart []byte, routeEnd []byte) ([]distribution.Route, bool, error) { + fallback := []distribution.Route{{GroupID: groupID}} + if s == nil || s.engine == nil { + return fallback, false, nil + } + routes := s.engine.GetIntersectingRoutes(routeStart, normalizedRouteScanEnd(routeEnd)) + matched := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID == groupID { + matched = append(matched, route) + continue + } + if routeHasStagedVisibility(route) { + return nil, false, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d range=[%q,%q)", groupID, start, end) + } + } + if len(matched) == 0 { + return fallback, false, nil + } + return matched, false, nil +} + func (s *ShardStore) routesForExplicitGroupScan(groupID uint64, start []byte, end []byte) ([]distribution.Route, bool, error) { fallback := []distribution.Route{{GroupID: groupID}} if s == nil || s.engine == nil { @@ -499,6 +622,9 @@ func (s *ShardStore) routesForExplicitGroupScan(groupID uint64, start []byte, en } } if len(matched) > 0 { + if routeMapped { + matched = dedupeRepeatedRawScanRoutes(matched) + } return matched, !routeMapped, nil } return fallback, false, nil @@ -601,19 +727,16 @@ func normalizedRouteScanEnd(routeEnd []byte) []byte { func (s *ShardStore) scanRoutesAtWithReadFence(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) - seenGroups := make(map[uint64]struct{}) routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) + if !clampToRoutes && !routeFilterPresent { + routes = dedupeRepeatedRawScanRoutes(routes) + } for _, route := range routes { scanStart := start scanEnd := end if clampToRoutes { scanStart = clampScanStart(start, route.Start) scanEnd = clampScanEnd(end, route.End) - } else if !routeFilterPresent { - if _, seen := seenGroups[route.GroupID]; seen { - continue - } - seenGroups[route.GroupID] = struct{}{} } kvs, err := s.scanRouteAtDirectionWithReadFence(ctx, route, scanStart, scanEnd, limit, ts, false, false, readRouteVersion, routeStart, routeEnd) @@ -646,8 +769,10 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( routeEnd []byte, ) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) - seenGroups := make(map[uint64]struct{}) routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) + if !clampToRoutes && !routeFilterPresent { + routes = dedupeRepeatedRawScanRoutes(routes) + } for i := len(routes) - 1; i >= 0; i-- { route := routes[i] if clampToRoutes { @@ -666,15 +791,6 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( // shards), keys from different routes may interleave in descending order. // Fetch up to limit from every route and merge+sort descending so the // result honours the ReverseScanAt contract. - // De-duplicate by GroupID: after a range split both halves share the same - // GroupID (same backing shard store), so only scan each group once unless - // route filters make each descriptor's logical interval distinct. - if !routeFilterPresent { - if _, seen := seenGroups[route.GroupID]; seen { - continue - } - seenGroups[route.GroupID] = struct{}{} - } kvs, err := s.scanRouteAtDirectionWithReadFence(ctx, route, start, end, limit, ts, true, false, readRouteVersion, routeStart, routeEnd) if err != nil { return nil, err @@ -709,19 +825,6 @@ func (s *ShardStore) clampedReverseScanRouteAtWithReadFence( return kvs, false, nil } -func (s *ShardStore) scanRouteAtDirection( - ctx context.Context, - route distribution.Route, - start []byte, - end []byte, - limit int, - ts uint64, - reverse bool, - explicitGroup bool, -) ([]*store.KVPair, error) { - return s.scanRouteAtDirectionWithReadFence(ctx, route, start, end, limit, ts, reverse, explicitGroup, 0, nil, nil) -} - func (s *ShardStore) scanRouteAtDirectionWithReadFence( ctx context.Context, route distribution.Route, @@ -1264,6 +1367,9 @@ func stagedVisibilityCandidateBoundary(liveKVs []*store.KVPair, stagedKVs []*sto if kvp == nil { continue } + if isMigrationStagedDataKey(kvp.Key) { + continue + } liveBoundary.visit(kvp.Key) } stagedBoundary := stagedVisibilityBoundary{reverse: reverse} @@ -1415,6 +1521,9 @@ func stagedVisibilityCandidateKeys(liveKVs []*store.KVPair, stagedKVs []*store.K if kvp == nil { continue } + if isMigrationStagedDataKey(kvp.Key) { + continue + } seen[string(kvp.Key)] = bytes.Clone(kvp.Key) } for _, kvp := range stagedKVs { @@ -1434,6 +1543,11 @@ func stagedVisibilityCandidateKeys(liveKVs []*store.KVPair, stagedKVs []*store.K return out } +func isMigrationStagedDataKey(key []byte) bool { + _, _, ok := distribution.MigrationStagedDataKeyParts(key) + return ok +} + func stagedVisibilityScanBounds(jobID uint64, start []byte, end []byte) ([]byte, []byte) { prefix := distribution.MigrationStagedDataKeyPrefix(jobID) scanStart := prefix diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 3bc7fa859..a2eed3491 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -1,6 +1,7 @@ package kv import ( + "bytes" "context" "fmt" "testing" @@ -255,6 +256,83 @@ func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { require.Equal(t, uint64(40), ts) } +func TestShardStoreScanAt_FiltersStagedShadowRowsFromLiveCandidates(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("b") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 20, 0)) + + kvs, err := st.ScanAt(ctx, []byte(""), nil, 10, 30) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: rawKey, Value: []byte("staged")}}, kvs) +} + +func TestShardStoreScanAt_RoutesS3BucketAuxiliaryStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: routeStart, GroupID: 1, State: distribution.RouteStateActive}, + { + RouteID: 2, + Start: routeStart, + End: routeEnd, + GroupID: 2, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + {RouteID: 3, Start: routeEnd, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + + for _, tc := range []struct { + name string + prefix string + key []byte + value []byte + }{ + {name: "bucket meta", prefix: s3keys.BucketMetaPrefix, key: s3keys.BucketMetaKey(bucket), value: []byte("meta")}, + {name: "bucket generation", prefix: s3keys.BucketGenerationPrefix, key: s3keys.BucketGenerationKey(bucket), value: []byte("generation")}, + } { + t.Run(tc.name, func(t *testing.T) { + require.NoError(t, groups[2].Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, tc.key), tc.value, 20, 0)) + + kvs, err := st.ScanAt(ctx, []byte(tc.prefix), prefixScanEnd([]byte(tc.prefix)), 10, 30) + require.NoError(t, err) + require.Contains(t, kvs, &store.KVPair{Key: tc.key, Value: tc.value}) + }) + } +} + +func TestShardStoreGetAt_ContinuesLatestVersionExportPages(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityPebbleShardStore(t) + rawKey := []byte("k") + large := bytes.Repeat([]byte("x"), 1<<20) + require.NoError(t, group.Store.PutAt(ctx, rawKey, []byte("old"), 20, 0)) + require.NoError(t, group.Store.PutAt(ctx, rawKey, large, 30, 0)) + + got, err := st.GetAt(ctx, rawKey, 25) + require.NoError(t, err) + require.Equal(t, []byte("old"), got) +} + func TestShardStoreDeletePrefixAtDeletesStagedVisibilityRows(t *testing.T) { t.Parallel() @@ -327,6 +405,13 @@ func TestShardStoreExplicitGroupReads_MergeStagedVisibility(t *testing.T) { {Key: []byte("b"), Value: []byte("staged-b")}, {Key: []byte("c"), Value: []byte("staged-c")}, }, kvs) + + kvs, err = st.ScanAtWithReadFence(ctx, []byte("a"), []byte("z"), 10, 35, false, 1, 0, []byte("a"), []byte("z")) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b"), Value: []byte("staged-b")}, + {Key: []byte("c"), Value: []byte("staged-c")}, + }, kvs) } func TestShardStoreExplicitGroupReads_FailClosedWhenRouteMovedToStagedGroup(t *testing.T) { @@ -788,6 +873,34 @@ func TestShardStoreScanGroupAt_DoesNotClampRouteMappedRawBounds(t *testing.T) { require.Equal(t, []*store.KVPair{{Key: key, Value: []byte("msg-2")}}, kvs) } +func TestShardStoreScanGroupAt_DeduplicatesRouteMappedSameGroupSplits(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + routeEnd := prefixScanEnd(sqsGlobalRouteKey) + split := append(bytes.Clone(sqsGlobalRouteKey), 'm') + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: sqsGlobalRouteKey, End: split, GroupID: 42, State: distribution.RouteStateActive}, + {RouteID: 2, Start: split, End: routeEnd, GroupID: 42, State: distribution.RouteStateActive}, + }, + })) + groups := map[uint64]*ShardGroup{ + 42: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + + start := []byte("!sqs|msg|vis|p|") + key := []byte("!sqs|msg|vis|p|orders|partition-2") + require.NoError(t, groups[42].Store.PutAt(ctx, key, []byte("msg-2"), 7, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, start, prefixScanEnd(start), 10, 7) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: key, Value: []byte("msg-2")}}, kvs) +} + func TestShardStoreGetGroupAt_UsesExplicitGroup(t *testing.T) { t.Parallel() From bbc1876dbc18428907a14bc7863ab79f7118e5e1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 04:16:51 +0900 Subject: [PATCH 116/162] migration: harden target readiness guards --- adapter/internal.go | 3 + adapter/internal_migration_test.go | 32 ++++++++++ kv/fsm.go | 22 ++++++- kv/fsm_migration_fence_test.go | 98 ++++++++++++++++++++++++++++++ kv/shard_store.go | 30 ++++----- kv/shard_store_test.go | 75 ++++++++++++++++++++++- main.go | 1 + store/migration_readiness.go | 2 + store/migration_versions_test.go | 17 ++++++ 9 files changed, 257 insertions(+), 23 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index 03986377d..e25447b19 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -270,6 +270,9 @@ func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.Targe if err := i.verifyInternalLeader(ctx); err != nil { return nil, err } + if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { + return nil, err + } if err := i.proposeTargetStagedReadiness(ctx, req); err != nil { return nil, errors.WithStack(err) } diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 13f9a2524..39376005d 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -499,6 +499,37 @@ func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { require.GreaterOrEqual(t, clock.Current(), uint64(30)) } +func TestInternalApplyTargetStagedReadinessRejectsWhenOpcodeGateClosed(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled for test") + }), + ) + + resp, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, uint64(0), proposer.calls) +} + func TestInternalApplyTargetStagedReadinessProposesThroughRaft(t *testing.T) { t.Parallel() @@ -510,6 +541,7 @@ func TestInternalApplyTargetStagedReadinessProposesThroughRaft(t *testing.T) { internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st), WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), ) _, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ diff --git a/kv/fsm.go b/kv/fsm.go index f5214652c..f533b261b 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -824,10 +824,28 @@ func (f *kvFSM) verifyRouteWriteFloorForKey(key []byte, commitTS uint64) error { } rkey := routeKey(key) floor, ok := snap.WriteFloorForKey(rkey) + if ok && commitTS != 0 && commitTS <= floor { + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q routeKey %q", commitTS, floor, key, rkey) + } + if err := f.verifyS3BucketAuxiliaryRouteWriteFloor(snap, key, commitTS); err != nil { + return err + } + return nil +} + +func (f *kvFSM) verifyS3BucketAuxiliaryRouteWriteFloor(snap RouteSnapshot, key []byte, commitTS uint64) error { + if commitTS == 0 { + return nil + } + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return nil + } + floor, ok := snap.WriteFloorIntersects(start, end) if !ok || commitTS > floor { return nil } - return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q routeKey %q", commitTS, floor, key, rkey) + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q route range [%q,%q)", commitTS, floor, key, start, end) } func (f *kvFSM) verifyRouteWriteFloorForPrefix(prefix []byte, commitTS uint64) error { @@ -1281,7 +1299,7 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if meta.CommitTS != 0 { floorTS = meta.CommitTS } - if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, r.ReadKeys, meta.PrimaryKey); err != nil { + if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, r.ReadKeys, nil); err != nil { return err } uniq, err := f.uniqueMutationsNotFenced(ctx, muts, floorTS) diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index f11cf5a4d..c5e100174 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -243,6 +243,62 @@ func TestFSMRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { } } +func TestFSMRejectsS3BucketAuxiliaryPointWriteBelowRouteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}) + fsm := newComposed1FSM(t, engine, 1) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: s3keys.BucketMetaKey(bucket), Value: []byte("v")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) +} + +func TestFSMRejectsS3BucketAuxiliaryPointWriteWithoutTargetReadinessProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: s3keys.BucketGenerationKey(bucket), Value: []byte("v")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { t.Parallel() @@ -479,6 +535,48 @@ func TestFSMPrepareTxnChecksReadKeysForTargetReadiness(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } +func TestFSMPrepareTxnSkipsRemotePrimaryForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("m"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + err := fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("v")}, + }, + }, 10) + require.NoError(t, err) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("n")), ^uint64(0)) + require.NoError(t, getErr) +} + func TestFSMPrepareTxnChecksStagedVisibilityWriteConflicts(t *testing.T) { t.Parallel() diff --git a/kv/shard_store.go b/kv/shard_store.go index e12a73659..d741d0328 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -279,6 +279,9 @@ func targetReadinessAppliesToRoute(route distribution.Route, routeStart []byte, } func readinessRouteRange(start []byte, end []byte) ([]byte, []byte) { + if routeStart, routeEnd, ok := s3BucketAuxiliaryRouteRange(start); ok && (end == nil || bytes.Equal(end, nextScanCursor(start))) { + return routeStart, routeEnd + } routeStart := routeKey(start) if end == nil { return routeStart, nil @@ -590,7 +593,7 @@ func (s *ShardStore) ScanGroupAt(ctx context.Context, groupID uint64, start []by } func (s *ShardStore) scanExplicitGroupRoutesAt(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool) ([]*store.KVPair, error) { - if len(routes) == 1 { + if len(routes) == 1 && !clampToRoutes { return s.scanRouteAtDirection(ctx, routes[0], start, end, limit, ts, false, true) } out := make([]*store.KVPair, 0, limit) @@ -2858,7 +2861,10 @@ func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludeP // DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { - routes := s.stagedVisibilityRoutesForPrefix(prefix) + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { + return err + } for groupID, g := range s.groups { if g == nil || g.Store == nil { continue @@ -2891,7 +2897,10 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl // is the receiver only when an aggregate (admin / coordinator) path // is replaying a global FLUSHALL, which is not raft-applied. func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { - routes := s.stagedVisibilityRoutesForPrefix(prefix) + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { + return err + } for groupID, g := range s.groups { if g == nil || g.Store == nil { continue @@ -2919,21 +2928,6 @@ func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, ex return nil } -func (s *ShardStore) stagedVisibilityRoutesForPrefix(prefix []byte) []distribution.Route { - if s == nil || s.engine == nil { - return nil - } - start, end := routePrefixRange(prefix) - routes := s.engine.GetIntersectingRoutes(start, end) - out := make([]distribution.Route, 0, len(routes)) - for _, route := range routes { - if routeHasStagedVisibility(route) { - out = append(out, route) - } - } - return out -} - func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte, commitTS uint64) ([]distribution.Route, error) { if s == nil || s.engine == nil { return nil, nil diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index a10427273..ce3d09db8 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -553,6 +553,31 @@ func TestShardStoreScanGroupAtChecksEveryCoveredRoute(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreScanGroupAtClampsSingleMatchedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("outside"), 7, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("inside"), 7, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 7) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: []byte("n"), Value: []byte("inside")}}, kvs) +} + func TestShardStoreScanGroupAtSplitsCoveredRoutesForStagedVisibility(t *testing.T) { t.Parallel() @@ -890,6 +915,50 @@ func TestShardStoreDeletePrefixAtChecksTargetReadinessBeforeDelete(t *testing.T) require.Equal(t, []byte("v"), got) } +func TestShardStoreDeletePrefixAtRaftChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAtRaft(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtRaftAtChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAtRaftAt(ctx, []byte("b"), nil, 120, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + func TestShardStoreDeletePrefixAtFailsClosedWithoutRouteProof(t *testing.T) { t.Parallel() @@ -1322,7 +1391,7 @@ func TestShardStoreRejectsWritesAtMigrationTimestampFloor(t *testing.T) { require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0)) } -func TestShardStoreRaftApplySkipsMigrationTimestampFloor(t *testing.T) { +func TestShardStoreRaftApplySkipsPointMigrationTimestampFloorButGuardsPrefixDeletes(t *testing.T) { t.Parallel() ctx := context.Background() @@ -1334,8 +1403,8 @@ func TestShardStoreRaftApplySkipsMigrationTimestampFloor(t *testing.T) { require.NoError(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ {Op: store.OpTypePut, Key: []byte("k-raft-at"), Value: []byte("v")}, }, nil, 90, 100, 1)) - require.NoError(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100)) - require.NoError(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2)) + require.ErrorIs(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100), ErrRouteWriteTimestampTooLow) + require.ErrorIs(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2), ErrRouteWriteTimestampTooLow) } func TestShardStoreScanAt_IncludesListKeysAcrossShards(t *testing.T) { diff --git a/main.go b/main.go index 055327215..8089b7c03 100644 --- a/main.go +++ b/main.go @@ -512,6 +512,7 @@ func run() error { adapter.WithDistributionActiveTimestampTracker(readTracker), adapter.WithDistributionKnownRaftGroups(shardGroupIDs(shardGroups)...), adapter.WithSplitMigrationCapabilityGate(splitMigrationGate), + adapter.WithSplitJobRunnerReady(), ) defaultRuntime := findDefaultGroupRuntime(runtimes, cfg.defaultGroup) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) diff --git a/store/migration_readiness.go b/store/migration_readiness.go index d26d25d36..2799840a8 100644 --- a/store/migration_readiness.go +++ b/store/migration_readiness.go @@ -21,6 +21,8 @@ func validateTargetStagedReadinessState(state TargetStagedReadinessState) error return errors.New("target staged readiness job_id is required") case state.MigrationJobID == 0: return errors.New("target staged readiness migration_job_id is required") + case state.Armed && state.MinWriteTSExclusive == 0: + return errors.New("target staged readiness min_write_ts_exclusive is required when armed") case state.RouteEnd != nil && bytes.Compare(state.RouteStart, state.RouteEnd) >= 0: return errors.New("target staged readiness route range is invalid") default: diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 1aab19768..3cf3f0056 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -1017,6 +1017,23 @@ func TestTargetStagedReadinessStatePersistsAndIsCloned(t *testing.T) { }) } +func TestTargetStagedReadinessRejectsArmedStateWithoutWriteFloor(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + + err := writer.ApplyTargetStagedReadiness(context.Background(), TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + Armed: true, + }) + require.ErrorContains(t, err, "min_write_ts_exclusive") + }) +} + func TestPebbleTargetStagedReadinessPersistsAcrossReopen(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-ready-persist-*") From acc9305a2fcab62203cb441f60eb2c8f50ba7f71 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 04:35:59 +0900 Subject: [PATCH 117/162] migration: fix staged s3 bucket routing --- kv/fsm.go | 17 ++++--- kv/fsm_migration_fence_test.go | 26 +++++++++- kv/shard_store.go | 90 +++++++++++++++++++++++----------- kv/shard_store_test.go | 83 +++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 38 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 0217c8bd2..e570fe6db 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -1100,21 +1100,24 @@ func (f *kvFSM) verifyOwnerFromSnapshot(mutations []*pb.Mutation, snap RouteSnap if isTxnInternalKey(mut.Key) { continue } - // routeKey-normalize before OwnerOf so the gate routes the - // same way as ShardRouter.ResolveGroup — raw adapter keys - // and route catalog ranges live in different lex bands - // (issue #930). - rKey := routeKey(mut.Key) - owner, found := snap.OwnerOf(rKey) + ownerKey := composed1OwnerKey(mut.Key) + owner, found := snap.OwnerOf(ownerKey) if !found || owner != f.shardGroupID { return errors.Wrapf(ErrComposed1Violation, "%s-version v=%d: key %q (routeKey %q) owned by group %d (found=%v); this FSM serves group %d", - phase, snapVer, mut.Key, rKey, owner, found, f.shardGroupID) + phase, snapVer, mut.Key, ownerKey, owner, found, f.shardGroupID) } } return nil } +func composed1OwnerKey(key []byte) []byte { + if start, _, ok := s3BucketAuxiliaryRouteRange(key); ok { + return start + } + return routeKey(key) +} + func (f *kvFSM) validateConflicts(ctx context.Context, muts []*pb.Mutation, startTS uint64) error { seen := make(map[string]struct{}, len(muts)) for _, mut := range muts { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 2c2efd142..c29276bf4 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -169,7 +169,7 @@ func TestFSMRejectsCurrentWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { t.Parallel() ctx := context.Background() - const bucket = "bucket-a" + const bucket = "bucket-b" fsm := newS3BucketAuxiliaryWriteFencedFSM(t, bucket) for _, key := range [][]byte{ @@ -186,7 +186,7 @@ func TestFSMRejectsCurrentWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { func TestFSMRejectsObservedWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { t.Parallel() - const bucket = "bucket-a" + const bucket = "bucket-b" fsm := newS3BucketAuxiliaryWriteFencedFSM(t, bucket) err := fsm.handleRawRequest(context.Background(), &pb.Request{ @@ -198,6 +198,28 @@ func TestFSMRejectsObservedWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) require.ErrorIs(t, err, ErrRouteWriteFenced) } +func TestFSMComposed1UsesS3BucketAuxiliaryRouteOwner(t *testing.T) { + t.Parallel() + + const bucket = "bucket-b" + key := s3keys.BucketMetaKey(bucket) + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, s3BucketAuxiliaryStagedRoutes(bucket, 3, 4)) + fsm := newComposed1FSM(t, engine, 4) + + err := fsm.verifyComposed1(&pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + ObservedRouteVersion: 1, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("meta")}, + }, + }) + require.NoError(t, err) +} + func TestFSMIgnoresRawRouteFloorForS3BucketAuxiliaryWrite(t *testing.T) { t.Parallel() diff --git a/kv/shard_store.go b/kv/shard_store.go index 486783e80..a9086165b 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -396,13 +396,14 @@ func (s *ShardStore) scanExplicitGroupAtWithReadFence(ctx context.Context, group return nil, err } routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) - if !clampToRoutes && !routeFilterPresent { + dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) + if !clampToRoutes && !routeFilterPresent && !dedupeByKey { routes = dedupeRepeatedRawScanRoutes(routes) } - return s.scanExplicitGroupRoutesAtWithReadFence(ctx, routes, start, end, limit, ts, reverse, readRouteVersion, routeStart, routeEnd, clampToRoutes) + return s.scanExplicitGroupRoutesAtWithReadFence(ctx, routes, start, end, limit, ts, reverse, readRouteVersion, routeStart, routeEnd, clampToRoutes, dedupeByKey) } -func (s *ShardStore) scanExplicitGroupRoutesAtWithReadFence(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, reverse bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte, clampToRoutes bool) ([]*store.KVPair, error) { +func (s *ShardStore) scanExplicitGroupRoutesAtWithReadFence(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, reverse bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte, clampToRoutes bool, dedupeByKey bool) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) for i := 0; i < len(routes); i++ { route := routes[i] @@ -427,11 +428,7 @@ func (s *ShardStore) scanExplicitGroupRoutesAtWithReadFence(ctx context.Context, } continue } - if reverse { - out = mergeAndTrimReverseScanResults(out, kvs, limit) - } else { - out = mergeAndTrimScanResults(out, kvs, limit) - } + out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, reverse, dedupeByKey) } return out, nil } @@ -511,12 +508,13 @@ func (s *ShardStore) routesForS3BucketAuxiliaryScan(start []byte, end []byte) ([ } routes := make([]distribution.Route, 0) routes = append(routes, s.engine.GetIntersectingRoutes(start, end)...) - for _, route := range s.engine.GetIntersectingRoutes([]byte(s3keys.RoutePrefix), prefixScanEnd([]byte(s3keys.RoutePrefix))) { + routeStart, routeEnd := s3BucketAuxiliaryScanRouteRange(start, end) + for _, route := range s.engine.GetIntersectingRoutes(routeStart, routeEnd) { if routeHasStagedVisibility(route) { routes = append(routes, route) } } - return dedupeRepeatedRawScanRoutes(routes), true + return routes, true } func s3BucketAuxiliaryScanBounds(start []byte, end []byte) bool { @@ -530,6 +528,14 @@ func s3BucketAuxiliaryScanBounds(start []byte, end []byte) bool { return bytes.Compare(start, end) < 0 } +func s3BucketAuxiliaryScanRouteRange(start []byte, end []byte) ([]byte, []byte) { + if routeStart, routeEnd, ok := s3BucketAuxiliaryRouteRange(start); ok && end != nil && bytes.Compare(end, prefixScanEnd(start)) <= 0 { + return routeStart, routeEnd + } + routeStart := []byte(s3keys.RoutePrefix) + return routeStart, prefixScanEnd(routeStart) +} + type repeatedRawScanRouteKey struct { groupID uint64 staged bool @@ -728,7 +734,8 @@ func normalizedRouteScanEnd(routeEnd []byte) []byte { func (s *ShardStore) scanRoutesAtWithReadFence(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) - if !clampToRoutes && !routeFilterPresent { + dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) + if !clampToRoutes && !routeFilterPresent && !dedupeByKey { routes = dedupeRepeatedRawScanRoutes(routes) } for _, route := range routes { @@ -751,7 +758,7 @@ func (s *ShardStore) scanRoutesAtWithReadFence(ctx context.Context, routes []dis } continue } - out = mergeAndTrimScanResults(out, kvs, limit) + out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, false, dedupeByKey) } return out, nil } @@ -770,7 +777,8 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( ) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) - if !clampToRoutes && !routeFilterPresent { + dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) + if !clampToRoutes && !routeFilterPresent && !dedupeByKey { routes = dedupeRepeatedRawScanRoutes(routes) } for i := len(routes) - 1; i >= 0; i-- { @@ -795,7 +803,7 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( if err != nil { return nil, err } - out = mergeAndTrimReverseScanResults(out, kvs, limit) + out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, true, dedupeByKey) } return out, nil } @@ -1072,6 +1080,9 @@ func minScanEnd(a []byte, b []byte) []byte { } func routeKeyInScanBounds(key []byte, routeStart []byte, routeEnd []byte) bool { + if s3BucketAuxiliaryRouteInRange(key, routeStart, routeEnd) { + return true + } key = routeKey(key) if len(routeStart) > 0 && bytes.Compare(key, routeStart) < 0 { return false @@ -1315,6 +1326,7 @@ func (s *ShardStore) scanRouteWithStagedVisibilityPage( return nil, nil, false, err } out := visibleLogicalKVs(versions, ts, reverse) + out = filterRouteScanKVs(out, route.Start, route.End) liveExhausted := len(liveKVs) < window stagedExhausted := len(stagedKVs) < window boundary, hasBoundary := stagedVisibilityCandidateBoundary(liveKVs, stagedKVs, liveExhausted, stagedExhausted, reverse) @@ -1690,34 +1702,54 @@ func scanUserKey(kvp *store.KVPair) ([]byte, bool) { return txnUserKeyFromLockKey(kvp.Key) } -func mergeAndTrimScanResults(out []*store.KVPair, kvs []*store.KVPair, limit int) []*store.KVPair { +func mergeAndTrimScanResultsWithOptions(out []*store.KVPair, kvs []*store.KVPair, limit int, reverse bool, dedupeByKey bool) []*store.KVPair { if len(kvs) == 0 { return out } - out = append(out, kvs...) - if len(out) <= limit { - return out + if dedupeByKey { + out = appendReplacingKVsByKey(out, kvs) + } else { + out = append(out, kvs...) } sort.Slice(out, func(i, j int) bool { - return bytes.Compare(out[i].Key, out[j].Key) < 0 + cmp := bytes.Compare(out[i].Key, out[j].Key) + if reverse { + return cmp > 0 + } + return cmp < 0 }) + if len(out) <= limit { + return out + } clear(out[limit:]) return out[:limit] } func mergeAndTrimReverseScanResults(out []*store.KVPair, kvs []*store.KVPair, limit int) []*store.KVPair { - if len(kvs) == 0 { - return out + return mergeAndTrimScanResultsWithOptions(out, kvs, limit, true, false) +} + +func appendReplacingKVsByKey(out []*store.KVPair, kvs []*store.KVPair) []*store.KVPair { + indexByKey := make(map[string]int, len(out)+len(kvs)) + for i, kvp := range out { + if kvp == nil { + continue + } + indexByKey[string(kvp.Key)] = i } - out = append(out, kvs...) - sort.Slice(out, func(i, j int) bool { - return bytes.Compare(out[i].Key, out[j].Key) > 0 - }) - if len(out) <= limit { - return out + for _, kvp := range kvs { + if kvp == nil { + continue + } + key := string(kvp.Key) + if idx, ok := indexByKey[key]; ok { + out[idx] = kvp + continue + } + indexByKey[key] = len(out) + out = append(out, kvp) } - clear(out[limit:]) - return out[:limit] + return out } func countNonInternalKVs(kvs []*store.KVPair) int { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index a2eed3491..64c01751d 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -145,6 +145,89 @@ func TestShardStoreGetAt_MergesStagedVisibilityForS3BucketAuxiliary(t *testing.T } } +func TestShardStoreS3BucketAuxiliaryScanFiltersStagedRoutesToBucketRange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const ( + bucketA = "bucket-a" + bucketB = "bucket-b" + bucketC = "bucket-c" + ) + routeStartA := s3keys.RoutePrefixForBucketAnyGeneration(bucketA) + routeEndA := prefixScanEnd(routeStartA) + routeStartB := s3keys.RoutePrefixForBucketAnyGeneration(bucketB) + routeEndB := prefixScanEnd(routeStartB) + require.Less(t, bytes.Compare(routeStartA, routeStartB), 0) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: routeStartA, GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: routeStartA, End: routeEndA, GroupID: 1, State: distribution.RouteStateActive, StagedVisibilityActive: true, MigrationJobID: 9}, + {RouteID: 3, Start: routeEndA, End: routeStartB, GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 4, Start: routeStartB, End: routeEndB, GroupID: 1, State: distribution.RouteStateActive, StagedVisibilityActive: true, MigrationJobID: 10}, + {RouteID: 5, Start: routeEndB, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + + keyA := s3keys.BucketMetaKey(bucketA) + keyB := s3keys.BucketMetaKey(bucketB) + keyC := s3keys.BucketMetaKey(bucketC) + require.NoError(t, group.Store.PutAt(ctx, keyA, []byte("live-a"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, keyB, []byte("live-b"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, keyC, []byte("live-c"), 10, 0)) + + exactA, err := st.ScanAt(ctx, keyA, prefixScanEnd(keyA), 10, 20) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: keyA, Value: []byte("live-a")}}, exactA) + + all, err := st.ScanAt(ctx, []byte(s3keys.BucketMetaPrefix), prefixScanEnd([]byte(s3keys.BucketMetaPrefix)), 10, 20) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: keyA, Value: []byte("live-a")}, + {Key: keyB, Value: []byte("live-b")}, + {Key: keyC, Value: []byte("live-c")}, + }, all) + + reverseAll, err := st.ReverseScanAt(ctx, []byte(s3keys.BucketMetaPrefix), prefixScanEnd([]byte(s3keys.BucketMetaPrefix)), 10, 20) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: keyC, Value: []byte("live-c")}, + {Key: keyB, Value: []byte("live-b")}, + {Key: keyA, Value: []byte("live-a")}, + }, reverseAll) +} + +func TestShardStoreRouteBoundedS3BucketAuxiliaryScanKeepsStagedRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: routeStart, GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: routeStart, End: routeEnd, GroupID: 1, State: distribution.RouteStateActive, StagedVisibilityActive: true, MigrationJobID: 9}, + {RouteID: 3, Start: routeEnd, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + key := s3keys.BucketMetaKey(bucket) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, key), []byte("staged"), 20, 0)) + + kvs, err := st.ScanAtWithReadFence(ctx, []byte(s3keys.BucketMetaPrefix), prefixScanEnd([]byte(s3keys.BucketMetaPrefix)), 10, 25, false, 0, 1, routeStart, routeEnd) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: key, Value: []byte("staged")}}, kvs) +} + func TestShardStoreRejectsS3BucketAuxiliaryWriteAtMigrationTimestampFloor(t *testing.T) { t.Parallel() From f902f34dab2211d0b5e2731d2c5d326e00fba8aa Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 05:16:14 +0900 Subject: [PATCH 118/162] migration: persist promotion timestamp floor --- store/migration_promote.go | 42 +++++++++++----- store/migration_versions.go | 12 ++++- store/migration_versions_test.go | 83 +++++++++++++++++++++++++++++++- store/store.go | 9 ++-- 4 files changed, 127 insertions(+), 19 deletions(-) diff --git a/store/migration_promote.go b/store/migration_promote.go index bfcd1af44..d8a8bc27b 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -9,7 +9,10 @@ import ( "github.com/cockroachdb/pebble/v2" ) -const migrationPromotionDoneFlag byte = 1 +const ( + migrationPromotionDoneFlag byte = 1 + migrationPromotionStateVersion2Flag byte = 1 << 7 +) type promotedVersion struct { staged MVCCVersion @@ -67,17 +70,17 @@ func (s *mvccStore) PromoteVersions(ctx context.Context, opts PromoteVersionsOpt state, cursor := s.promotionStateAndCursorLocked(opts) if opts.JobID != 0 && state.Done { - return PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows}, nil + return PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows, MaxPromotedTS: state.MaxPromotedTS}, nil } exported, toPromote, promoted, err := s.planMemoryPromotionLocked(ctx, opts, cursor) if err != nil { return PromoteVersionsResult{}, err } s.applyMemoryPromotionLocked(toPromote) - if promoted.MaxPromotedTS > s.lastCommitTS { - s.lastCommitTS = promoted.MaxPromotedTS - } result, updatedState := finishPromotionResult(opts, state, exported, promoted) + if result.MaxPromotedTS > s.lastCommitTS { + s.lastCommitTS = result.MaxPromotedTS + } if updatedState != nil { s.migrationPromotions[opts.JobID] = clonePromotionState(*updatedState) } @@ -219,8 +222,8 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO } writeOpts := s.promotionWriteOpts(opts.AppliedIndex) if opts.JobID != 0 && state.Done { - result := PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows} - return s.finishPebblePromotion(nil, opts.JobID, nil, result, opts.AppliedIndex, 0, writeOpts) + result := PromoteVersionsResult{Done: true, TotalPromotedRows: state.PromotedRows, MaxPromotedTS: state.MaxPromotedTS} + return s.finishPebblePromotion(nil, opts.JobID, nil, result, opts.AppliedIndex, state.MaxPromotedTS, writeOpts) } opts.Cursor = cursor exported, toPromote, promoted, err := s.planPebblePromotionLocked(ctx, opts) @@ -234,7 +237,7 @@ func (s *pebbleStore) PromoteVersions(ctx context.Context, opts PromoteVersionsO stateToWrite, result, opts.AppliedIndex, - promoted.MaxPromotedTS, + result.MaxPromotedTS, writeOpts, ) } @@ -306,8 +309,12 @@ func finishPromotionResult( state.Cursor = bytes.Clone(exported.NextCursor) state.Done = exported.Done state.PromotedRows += promoted.PromotedRows + if promoted.MaxPromotedTS > state.MaxPromotedTS { + state.MaxPromotedTS = promoted.MaxPromotedTS + } state.LastError = "" promoted.TotalPromotedRows = state.PromotedRows + promoted.MaxPromotedTS = state.MaxPromotedTS return promoted, &state } @@ -430,13 +437,14 @@ func (s *pebbleStore) commitPebblePromotionBatch( } func encodePromotionState(state PromotionState) []byte { - buf := make([]byte, 0, 1+migrationUint64Bytes+binary.MaxVarintLen64*2+len(state.Cursor)+len(state.LastError)) + buf := make([]byte, 0, 1+2*migrationUint64Bytes+binary.MaxVarintLen64*2+len(state.Cursor)+len(state.LastError)) + flags := migrationPromotionStateVersion2Flag if state.Done { - buf = append(buf, migrationPromotionDoneFlag) - } else { - buf = append(buf, 0) + flags |= migrationPromotionDoneFlag } + buf = append(buf, flags) buf = binary.BigEndian.AppendUint64(buf, state.PromotedRows) + buf = binary.BigEndian.AppendUint64(buf, state.MaxPromotedTS) buf = binary.AppendUvarint(buf, lenAsUint64(len(state.Cursor))) buf = append(buf, state.Cursor...) buf = binary.AppendUvarint(buf, lenAsUint64(len(state.LastError))) @@ -448,11 +456,19 @@ func decodePromotionState(data []byte) (PromotionState, bool) { if len(data) < 1+migrationUint64Bytes { return PromotionState{}, false } + flags := data[0] state := PromotionState{ - Done: data[0]&migrationPromotionDoneFlag != 0, + Done: flags&migrationPromotionDoneFlag != 0, PromotedRows: binary.BigEndian.Uint64(data[1 : 1+migrationUint64Bytes]), } rest := data[1+migrationUint64Bytes:] + if flags&migrationPromotionStateVersion2Flag != 0 { + if len(rest) < migrationUint64Bytes { + return PromotionState{}, false + } + state.MaxPromotedTS = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + } cursorLen, n := binary.Uvarint(rest) if n <= 0 || cursorLen > lenAsUint64(len(rest[n:])) { return PromotionState{}, false diff --git a/store/migration_versions.go b/store/migration_versions.go index ef3ef8254..be764e360 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -147,7 +147,7 @@ func decodeExportCursorForOptions(opts ExportVersionsOptions) (exportCursorPosit if err := validateExportCursorRange(opts, pos); err != nil { return exportCursorPosition{}, err } - return pos, nil + return normalizeExportCursorPositionForRange(opts, pos), nil } func validateExportCursorRange(opts ExportVersionsOptions, pos exportCursorPosition) error { @@ -174,6 +174,16 @@ func exportSkippedCursorOutsideRange(opts ExportVersionsOptions, key []byte) boo (opts.EndKey != nil && bytes.Compare(key, opts.EndKey) >= 0) } +func normalizeExportCursorPositionForRange(opts ExportVersionsOptions, pos exportCursorPosition) exportCursorPosition { + if !pos.hasKey || pos.tag != exportCursorTagSkippedKey || opts.StartKey == nil { + return pos + } + if bytes.Compare(pos.key, opts.StartKey) >= 0 { + return pos + } + return exportCursorPosition{} +} + func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOptions { if opts.EndKey != nil && len(opts.EndKey) == 0 { opts.EndKey = nil diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index c83ab6ca9..0fb58e4bd 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -405,6 +405,24 @@ func TestExportVersionsRejectsCursorOutsideRequestedRange(t *testing.T) { }) } +func TestExportVersionsSkippedCursorBeforeStartResumesAtStartKey(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("a10"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("m"), []byte("m20"), 20, 0)) + + res, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("m"), + EndKey: []byte("z"), + Cursor: encodeExportCursor([]byte("a"), 10, exportCursorTagSkippedKey), + MaxVersions: 10, + }) + require.NoError(t, err) + require.True(t, res.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("m"), CommitTS: 20, Value: []byte("m20")}}, res.Versions) + }) +} + func TestValidateExportCursorForRangeRejectsSkippedCursorInsideRange(t *testing.T) { t.Parallel() @@ -878,6 +896,7 @@ func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { require.False(t, state.Done) require.Equal(t, first.NextCursor, state.Cursor) require.Equal(t, uint64(2), state.PromotedRows) + require.Equal(t, uint64(30), state.MaxPromotedTS) got, err := st.GetAt(ctx, []byte("k"), 25) require.NoError(t, err) @@ -908,13 +927,27 @@ func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { require.Empty(t, second.NextCursor) require.Equal(t, uint64(2), second.PromotedRows) require.Equal(t, uint64(4), second.TotalPromotedRows) - require.Equal(t, uint64(15), second.MaxPromotedTS) + require.Equal(t, uint64(30), second.MaxPromotedTS) state, ok, err = stateReader.MigrationPromotionState(ctx, 99) require.NoError(t, err) require.True(t, ok) require.True(t, state.Done) require.Empty(t, state.Cursor) require.Equal(t, uint64(4), state.PromotedRows) + require.Equal(t, uint64(30), state.MaxPromotedTS) + + retry, err := promoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 99, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, retry.Done) + require.Zero(t, retry.PromotedRows) + require.Equal(t, uint64(4), retry.TotalPromotedRows) + require.Equal(t, uint64(30), retry.MaxPromotedTS) got, err = st.GetAt(ctx, []byte("k"), 10) require.NoError(t, err) @@ -972,6 +1005,7 @@ func TestPromoteVersionsIgnoresClientCursorWhenStateMissing(t *testing.T) { require.True(t, ok) require.True(t, state.Done) require.Equal(t, uint64(2), state.PromotedRows) + require.Equal(t, uint64(20), state.MaxPromotedTS) got, err := st.GetAt(ctx, []byte("a"), 10) require.NoError(t, err) @@ -1028,6 +1062,10 @@ func TestPebblePromoteVersionsAdvancesLastCommitTS(t *testing.T) { require.Equal(t, uint64(1), result.PromotedRows) require.Equal(t, promotedTS, result.MaxPromotedTS) require.Equal(t, promotedTS, ps.LastCommitTS()) + state, ok, err := ps.MigrationPromotionState(ctx, 101) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, promotedTS, state.MaxPromotedTS) metaTS, err := readPebbleUint64(ps.db, metaLastCommitTSBytes) require.NoError(t, err) @@ -1045,6 +1083,49 @@ func TestPebblePromoteVersionsAdvancesLastCommitTS(t *testing.T) { val, err = reopened.GetAt(ctx, []byte("k"), reopened.LastCommitTS()) require.NoError(t, err) require.Equal(t, []byte("v100"), val) + reopenedPromoter, ok := reopened.(MigrationPromoter) + require.True(t, ok) + retry, err := reopenedPromoter.PromoteVersions(ctx, PromoteVersionsOptions{ + JobID: 101, + StartKey: prefix, + EndKey: PrefixScanEnd(prefix), + MaxVersions: 10, + TargetKey: targetKey, + }) + require.NoError(t, err) + require.True(t, retry.Done) + require.Zero(t, retry.PromotedRows) + require.Equal(t, uint64(1), retry.TotalPromotedRows) + require.Equal(t, promotedTS, retry.MaxPromotedTS) +} + +func TestPromotionStateCodecPreservesMaxPromotedTS(t *testing.T) { + t.Parallel() + + state := PromotionState{ + Cursor: []byte("cursor"), + Done: true, + PromotedRows: 7, + MaxPromotedTS: 42, + LastError: "boom", + } + decoded, ok := decodePromotionState(encodePromotionState(state)) + require.True(t, ok) + require.Equal(t, state, decoded) + + old := []byte{migrationPromotionDoneFlag} + old = binary.BigEndian.AppendUint64(old, 3) + old = binary.AppendUvarint(old, lenAsUint64(len("old-cursor"))) + old = append(old, "old-cursor"...) + old = binary.AppendUvarint(old, lenAsUint64(len("old-error"))) + old = append(old, "old-error"...) + decoded, ok = decodePromotionState(old) + require.True(t, ok) + require.True(t, decoded.Done) + require.Equal(t, uint64(3), decoded.PromotedRows) + require.Zero(t, decoded.MaxPromotedTS) + require.Equal(t, []byte("old-cursor"), decoded.Cursor) + require.Equal(t, "old-error", decoded.LastError) } func TestPebbleImportMetadataPersistsAcrossReopen(t *testing.T) { diff --git a/store/store.go b/store/store.go index 700550f6b..1dcd96df8 100644 --- a/store/store.go +++ b/store/store.go @@ -149,10 +149,11 @@ type PromoteVersionsResult struct { // PromotionState is the target-local durable cursor for staged data promotion. type PromotionState struct { - Cursor []byte - Done bool - PromotedRows uint64 - LastError string + Cursor []byte + Done bool + PromotedRows uint64 + MaxPromotedTS uint64 + LastError string } // MigrationPromoter is implemented by stores that can promote staged range From e3e37cebc9328ff9dc2102ec9dd3648b93fd038e Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 05:25:05 +0900 Subject: [PATCH 119/162] migration: keep invalid promote cursors non-halting --- kv/fsm_migration_promote.go | 7 +++++++ kv/fsm_migration_promote_test.go | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/kv/fsm_migration_promote.go b/kv/fsm_migration_promote.go index c4001b634..383426ae2 100644 --- a/kv/fsm_migration_promote.go +++ b/kv/fsm_migration_promote.go @@ -45,6 +45,9 @@ func (f *kvFSM) applyMigrationPromote(ctx context.Context, data []byte) any { } result, err := promoter.PromoteVersions(ctx, migrationPromoteOptionsFromProto(req, f.pendingApplyIdx)) if err != nil { + if isMigrationPromoteOrdinaryApplyError(err) { + return errors.Wrap(err, "kv/fsm: apply migration promote") + } return haltErr(errors.Wrap(errors.Mark(err, ErrMigrationPromoteApply), "kv/fsm: apply migration promote")) } if f.hlc != nil && result.MaxPromotedTS > 0 { @@ -80,6 +83,10 @@ func migrationPromoteOptionsFromProto(req *pb.PromoteStagedVersionsRequest, appl } } +func isMigrationPromoteOrdinaryApplyError(err error) bool { + return errors.Is(err, store.ErrInvalidExportCursor) +} + func migrationPromoteTargetKey(jobID uint64) func([]byte) ([]byte, bool) { return func(stagedKey []byte) ([]byte, bool) { gotJobID, rawKey, ok := distribution.MigrationStagedDataKeyParts(stagedKey) diff --git a/kv/fsm_migration_promote_test.go b/kv/fsm_migration_promote_test.go index 2397243e2..192b4af1a 100644 --- a/kv/fsm_migration_promote_test.go +++ b/kv/fsm_migration_promote_test.go @@ -54,6 +54,7 @@ func TestApplyMigrationPromoteMovesStagedVersions(t *testing.T) { require.True(t, ok) require.True(t, state.Done) require.Equal(t, uint64(2), state.PromotedRows) + require.Equal(t, uint64(20), state.MaxPromotedTS) got, err := st.GetAt(ctx, []byte("user|k"), 10) require.NoError(t, err) @@ -76,3 +77,20 @@ func TestApplyMigrationPromoteMalformedPayloadHalts(t *testing.T) { err := haltApplyOf(fsm.Apply([]byte{raftEncodeMigrationPromote, 0xff, 0xff})) require.True(t, errors.Is(err, ErrMigrationPromoteApply), "got %v", err) } + +func TestApplyMigrationPromoteInvalidCursorReturnsOrdinaryError(t *testing.T) { + t.Parallel() + + fsm := &kvFSM{store: store.NewMVCCStore()} + cmd, err := MarshalMigrationPromoteCommand(&pb.PromoteStagedVersionsRequest{ + Cursor: []byte{0xff}, + MaxVersions: 10, + }) + require.NoError(t, err) + resp := fsm.Apply(cmd) + require.Nil(t, haltApplyOf(resp)) + err, ok := resp.(error) + require.True(t, ok, "got %T: %v", resp, resp) + require.ErrorIs(t, err, store.ErrInvalidExportCursor) + require.False(t, errors.Is(err, ErrMigrationPromoteApply)) +} From 5dec8cb657c1b738fd86212cfcc58c73878daff5 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 06:10:43 +0900 Subject: [PATCH 120/162] migration: fix promotion complete review findings --- distribution/migration_promotion_complete.go | 13 +++++++++- .../migration_promotion_complete_test.go | 24 +++++++++++++++++++ distribution/split_job_catalog_test.go | 7 ++---- main.go | 2 +- main_catalog_test.go | 16 +++++++++++++ 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go index 88ee72b14..61fd643ee 100644 --- a/distribution/migration_promotion_complete.go +++ b/distribution/migration_promotion_complete.go @@ -204,13 +204,24 @@ func (s *CatalogStore) applyPromotionCompleteMutations(ctx context.Context, plan } if err := s.store.ApplyMutations(ctx, mutations, readKeys, plan.readTS, commitTS); err != nil { if errors.Is(err, store.ErrWriteConflict) { - return errors.WithStack(ErrCatalogSplitJobConflict) + return s.promotionCompleteWriteConflict(ctx, plan) } return errors.WithStack(err) } return nil } +func (s *CatalogStore) promotionCompleteWriteConflict(ctx context.Context, plan savePlan) error { + currentVersion, err := s.versionAt(ctx, s.store.LastCommitTS()) + if err != nil { + return err + } + if currentVersion != plan.nextVersion-1 { + return errors.WithStack(ErrCatalogVersionMismatch) + } + return errors.WithStack(ErrCatalogSplitJobConflict) +} + func targetClearedDescriptorPresent(job SplitJob, routes []RouteDescriptor) bool { for _, route := range routes { if route.GroupID != job.TargetGroupID { diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go index b04045bfd..7d6f710e9 100644 --- a/distribution/migration_promotion_complete_test.go +++ b/distribution/migration_promotion_complete_test.go @@ -140,6 +140,30 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionRejectsStaleInputs(t *testin require.True(t, errors.Is(err, ErrCatalogSplitJobConflict), "got %v", err) } +func TestCatalogStoreCompleteSplitJobTargetPromotionPreservesVersionConflict(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + readTS, _, routes, err := cs.loadPromotionCompleteInputs(ctx, saved.Version, job) + require.NoError(t, err) + completion, err := CompleteTargetPromotionState(job, routes, 1000) + require.NoError(t, err) + plan, mutations, commitTS, err := cs.buildPromotionCompleteMutations(ctx, readTS, saved.Version, job.JobID, &completion) + require.NoError(t, err) + + _, err = cs.Save(ctx, saved.Version, promotionCompleteTestRoutes()) + require.NoError(t, err) + + err = cs.applyPromotionCompleteMutations(ctx, plan, mutations, job.JobID, commitTS) + require.True(t, errors.Is(err, ErrCatalogVersionMismatch), "got %v", err) +} + func promotionCompleteTestJob() SplitJob { return SplitJob{ JobID: 99, diff --git a/distribution/split_job_catalog_test.go b/distribution/split_job_catalog_test.go index f8bbd5317..9e161c22e 100644 --- a/distribution/split_job_catalog_test.go +++ b/distribution/split_job_catalog_test.go @@ -376,11 +376,8 @@ func TestCatalogStoreRetrySplitJobRejectsMissingRetryWitness(t *testing.T) { job.Phase = SplitJobPhaseFailed job.RetryPhase = SplitJobPhaseNone - if err := cs.CreateSplitJob(ctx, job); err != nil { - t.Fatalf("create split job: %v", err) - } - if _, err := cs.RetrySplitJob(ctx, job.JobID, 1200); !errors.Is(err, ErrCatalogSplitJobCannotRetry) { - t.Fatalf("expected ErrCatalogSplitJobCannotRetry, got %v", err) + if err := cs.CreateSplitJob(ctx, job); !errors.Is(err, ErrCatalogInvalidSplitJobPhase) { + t.Fatalf("expected ErrCatalogInvalidSplitJobPhase, got %v", err) } } diff --git a/main.go b/main.go index 457ed1d5b..294dd5ee7 100644 --- a/main.go +++ b/main.go @@ -2643,7 +2643,7 @@ func distributionCatalogStoreForGroup(runtimes []*raftGroupRuntime, groupID uint continue } if rt.spec.id == groupID { - return distribution.NewCatalogStore(rt.store) + return distribution.NewCatalogStore(rt.store, distribution.WithCatalogRouteDescriptorV2Writes(true)) } } return nil diff --git a/main_catalog_test.go b/main_catalog_test.go index b4227a9ee..2b712234f 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -78,6 +78,22 @@ func TestSetupDistributionCatalog_UsesResolvedCatalogGroup(t *testing.T) { require.NotNil(t, catalog) } +func TestDistributionCatalogStoreForGroupEnablesRouteDescriptorV2Writes(t *testing.T) { + t.Parallel() + + rt := &raftGroupRuntime{ + spec: groupSpec{id: 5}, + store: store.NewMVCCStore(), + } + t.Cleanup(func() { + rt.Close() + }) + + catalog := distributionCatalogStoreForGroup([]*raftGroupRuntime{rt}, 5) + require.NotNil(t, catalog) + require.True(t, catalog.AllowsRouteDescriptorV2Writes()) +} + func TestSplitMigrationCapabilityGateChecksAllPeers(t *testing.T) { t.Parallel() From 00da46c88ff664899ae6e40cc924104be64af4ff Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 06:23:06 +0900 Subject: [PATCH 121/162] migration: make promotion completion retry idempotent --- distribution/migration_promotion_complete.go | 105 ++++++++++++++++-- .../migration_promotion_complete_test.go | 25 ++++- 2 files changed, 119 insertions(+), 11 deletions(-) diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go index 61fd643ee..e4c064d33 100644 --- a/distribution/migration_promotion_complete.go +++ b/distribution/migration_promotion_complete.go @@ -103,10 +103,17 @@ func (s *CatalogStore) CompleteSplitJobTargetPromotion( } ctx = contextOrBackground(ctx) - readTS, currentVersion, routes, err := s.loadPromotionCompleteInputs(ctx, expectedVersion, expected) + readTS, currentVersion, routes, currentJob, alreadyApplied, err := s.loadPromotionCompleteInputs(ctx, expectedVersion, expected) if err != nil { return CatalogSnapshot{}, SplitJob{}, err } + if alreadyApplied { + return CatalogSnapshot{ + Version: currentVersion, + Routes: cloneRouteDescriptors(routes), + ReadTS: readTS, + }, currentJob, nil + } completion, err := CompleteTargetPromotionState(expected, routes, nowMs) if err != nil { return CatalogSnapshot{}, SplitJob{}, err @@ -133,27 +140,105 @@ func (s *CatalogStore) CompleteSplitJobTargetPromotion( }, completion.Job, nil } -func (s *CatalogStore) loadPromotionCompleteInputs(ctx context.Context, expectedVersion uint64, expected SplitJob) (uint64, uint64, []RouteDescriptor, error) { +func (s *CatalogStore) loadPromotionCompleteInputs(ctx context.Context, expectedVersion uint64, expected SplitJob) (uint64, uint64, []RouteDescriptor, SplitJob, bool, error) { expectedRaw, err := EncodeSplitJob(expected) if err != nil { - return 0, 0, nil, err + return 0, 0, nil, SplitJob{}, false, err } readTS := s.store.LastCommitTS() currentVersion, err := s.versionAt(ctx, readTS) if err != nil { - return 0, 0, nil, err + return 0, 0, nil, SplitJob{}, false, err } - if currentVersion != expectedVersion { - return 0, 0, nil, errors.WithStack(ErrCatalogVersionMismatch) + raw, currentJob, err := s.livePromotionCompleteJobAt(ctx, expected.JobID, readTS, expectedVersion, currentVersion) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err } - if err := s.expectLiveSplitJobAt(ctx, expected.JobID, expectedRaw, readTS); err != nil { - return 0, 0, nil, err + if currentVersion != expectedVersion || !bytes.Equal(raw, expectedRaw) { + return s.resolvePromotionCompleteInputConflict(ctx, readTS, currentVersion, expectedVersion, expected, expectedRaw, currentJob) } routes, err := s.routesAt(ctx, readTS) if err != nil { - return 0, 0, nil, err + return 0, 0, nil, SplitJob{}, false, err + } + return readTS, currentVersion, routes, currentJob, false, nil +} + +func (s *CatalogStore) livePromotionCompleteJobAt(ctx context.Context, jobID uint64, ts uint64, expectedVersion uint64, currentVersion uint64) ([]byte, SplitJob, error) { + raw, err := s.store.GetAt(ctx, CatalogSplitJobKey(jobID), ts) + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + return nil, SplitJob{}, promotionCompleteMissingJobError(expectedVersion, currentVersion) + } + return nil, SplitJob{}, errors.WithStack(err) + } + currentJob, err := DecodeSplitJob(raw) + if err != nil { + return nil, SplitJob{}, err + } + if currentJob.JobID != jobID { + return nil, SplitJob{}, errors.WithStack(ErrCatalogSplitJobKeyIDMismatch) + } + return raw, currentJob, nil +} + +func promotionCompleteMissingJobError(expectedVersion uint64, currentVersion uint64) error { + if currentVersion != expectedVersion { + return errors.WithStack(ErrCatalogVersionMismatch) + } + return errors.WithStack(ErrCatalogSplitJobConflict) +} + +func (s *CatalogStore) resolvePromotionCompleteInputConflict( + ctx context.Context, + readTS uint64, + currentVersion uint64, + expectedVersion uint64, + expected SplitJob, + expectedRaw []byte, + currentJob SplitJob, +) (uint64, uint64, []RouteDescriptor, SplitJob, bool, error) { + alreadyApplied, routes, err := s.promotionCompleteAlreadyAppliedAt(ctx, readTS, expected, expectedRaw, currentJob) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + if alreadyApplied { + return readTS, currentVersion, routes, currentJob, true, nil + } + if currentVersion != expectedVersion { + return 0, 0, nil, SplitJob{}, false, errors.WithStack(ErrCatalogVersionMismatch) + } + return 0, 0, nil, SplitJob{}, false, errors.WithStack(ErrCatalogSplitJobConflict) +} + +func (s *CatalogStore) promotionCompleteAlreadyAppliedAt(ctx context.Context, ts uint64, expected SplitJob, expectedRaw []byte, current SplitJob) (bool, []RouteDescriptor, error) { + matches, err := promotionCompleteJobMatchesExpected(expected, expectedRaw, current) + if err != nil || !matches { + return false, nil, err + } + routes, err := s.routesAt(ctx, ts) + if err != nil { + return false, nil, err + } + if !targetClearedDescriptorPresent(current, routes) { + return false, nil, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + return true, routes, nil +} + +func promotionCompleteJobMatchesExpected(expected SplitJob, expectedRaw []byte, current SplitJob) (bool, error) { + if expected.TargetPromotionDone || !current.TargetPromotionDone || current.PromotionCompletedTS == 0 { + return false, nil + } + normalized := CloneSplitJob(current) + normalized.TargetPromotionDone = expected.TargetPromotionDone + normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.UpdatedAtMs = expected.UpdatedAtMs + raw, err := EncodeSplitJob(normalized) + if err != nil { + return false, err } - return readTS, currentVersion, routes, nil + return bytes.Equal(raw, expectedRaw), nil } func (s *CatalogStore) buildPromotionCompleteMutations( diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go index 7d6f710e9..b134c2dd1 100644 --- a/distribution/migration_promotion_complete_test.go +++ b/distribution/migration_promotion_complete_test.go @@ -92,6 +92,29 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionCommitsRouteAndJobTogether(t require.Equal(t, uint64(777), target.MinWriteTSExclusive) } +func TestCatalogStoreCompleteSplitJobTargetPromotionRetryAfterCommit(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + firstSnapshot, firstCompleted, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.NoError(t, err) + + retrySnapshot, retryCompleted, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 2000) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, retrySnapshot.Version) + assertSplitJobEqual(t, firstCompleted, retryCompleted) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, loaded.Version) +} + func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDescriptor(t *testing.T) { t.Parallel() @@ -150,7 +173,7 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionPreservesVersionConflict(t * job := promotionCompleteTestJob() require.NoError(t, cs.CreateSplitJob(ctx, job)) - readTS, _, routes, err := cs.loadPromotionCompleteInputs(ctx, saved.Version, job) + readTS, _, routes, _, _, err := cs.loadPromotionCompleteInputs(ctx, saved.Version, job) require.NoError(t, err) completion, err := CompleteTargetPromotionState(job, routes, 1000) require.NoError(t, err) From 19961d3f3572a93b4a064f062a9d346b72f0b230 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 06:41:57 +0900 Subject: [PATCH 122/162] migration: disambiguate legacy list exports --- distribution/migrator.go | 17 +++++++++++++---- distribution/migrator_export_plan_test.go | 22 ++++++++++++++++++---- store/lsm_migration.go | 8 ++++++++ store/migration_versions.go | 4 ++++ store/migration_versions_test.go | 18 ++++++++++++++++++ store/store.go | 1 + 6 files changed, 62 insertions(+), 8 deletions(-) diff --git a/distribution/migrator.go b/distribution/migrator.go index 097bfbe2e..949c53eaf 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -232,6 +232,13 @@ func (b MigrationBracket) ContainsRawKey(rawKey []byte) bool { // route ownership predicate. S3 bucket-level auxiliary rows do not encode an // object route key, so they are matched by bucket route-prefix intersection. func (b MigrationBracket) ContainsRoutedKey(rawKey, routeStart, routeEnd []byte, routeKey func([]byte) []byte) bool { + return b.ContainsRoutedVersion(rawKey, nil, routeStart, routeEnd, routeKey) +} + +// ContainsRoutedVersion is the value-aware variant of ContainsRoutedKey. It is +// needed for legacy list metadata because old delta keys overlap byte-for-byte +// with base metadata keys whose user key begins with "d|". +func (b MigrationBracket) ContainsRoutedVersion(rawKey, value, routeStart, routeEnd []byte, routeKey func([]byte) []byte) bool { if !b.ContainsRawKey(rawKey) { return false } @@ -240,7 +247,7 @@ func (b MigrationBracket) ContainsRoutedKey(rawKey, routeStart, routeEnd []byte, return b.containsDecodedS3Route(rawKey, routeStart, routeEnd) } if b.Family == MigrationFamilyLegacyListMetaDelta { - return b.containsLegacyListMetaDeltaRoute(rawKey, routeStart, routeEnd) + return b.containsLegacyListMetaDeltaRoute(rawKey, value, routeStart, routeEnd) } if !b.RequiresRouteKeyCheck { return true @@ -251,9 +258,11 @@ func (b MigrationBracket) ContainsRoutedKey(rawKey, routeStart, routeEnd []byte, return routeKeyInRange(routeKey(rawKey), routeStart, routeEnd) } -func (b MigrationBracket) containsLegacyListMetaDeltaRoute(rawKey, routeStart, routeEnd []byte) bool { - return routeKeyInRange(store.ExtractLegacyListUserKeyFromDelta(rawKey), routeStart, routeEnd) || - routeKeyInRange(store.ExtractListUserKey(rawKey), routeStart, routeEnd) +func (b MigrationBracket) containsLegacyListMetaDeltaRoute(rawKey, value, routeStart, routeEnd []byte) bool { + if value != nil && !store.IsListMetaDeltaValue(value) { + return routeKeyInRange(store.ExtractListUserKey(rawKey), routeStart, routeEnd) + } + return routeKeyInRange(store.ExtractLegacyListUserKeyFromDelta(rawKey), routeStart, routeEnd) } func (b MigrationBracket) containsFamilyShape(rawKey []byte) bool { diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index d87ed277d..0869ea0b1 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -280,15 +280,25 @@ func TestMigrationBracketContainsRoutedKeyUsesLegacyListDeltaUserKey(t *testing. require.NoError(t, err) legacy := bracketsByFamily(brackets)[MigrationFamilyLegacyListMetaDelta] raw := legacyListMetaDeltaKey([]byte("target-list"), 10, 0) + value := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) - require.True(t, legacy.ContainsRoutedKey( + require.True(t, legacy.ContainsRoutedVersion( raw, + value, []byte("target"), []byte("target-list\x00"), store.ExtractListUserKey, )) - require.False(t, legacy.ContainsRoutedKey( + require.False(t, legacy.ContainsRoutedVersion( raw, + value, + []byte("d|"), + []byte("d}"), + store.ExtractListUserKey, + )) + require.False(t, legacy.ContainsRoutedVersion( + raw, + value, []byte("zzz"), nil, store.ExtractListUserKey, @@ -303,15 +313,19 @@ func TestMigrationBracketContainsRoutedKeyRoutesAmbiguousLegacyListMetaByBaseKey legacy := bracketsByFamily(brackets)[MigrationFamilyLegacyListMetaDelta] userKey := deltaLookingListMetaUserKey([]byte("target-list"), 10, 0) raw := store.ListMetaKey(userKey) + value, err := store.MarshalListMeta(store.ListMeta{Head: 1, Tail: 2, Len: 1}) + require.NoError(t, err) - require.True(t, legacy.ContainsRoutedKey( + require.True(t, legacy.ContainsRoutedVersion( raw, + value, []byte("d|"), []byte("d}"), store.ExtractListUserKey, )) - require.False(t, legacy.ContainsRoutedKey( + require.False(t, legacy.ContainsRoutedVersion( raw, + value, []byte("zzz"), nil, store.ExtractListUserKey, diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 5a6d22f55..56623fda5 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -253,6 +253,14 @@ func (s *pebbleStore) exportPebbleVersion( if err != nil { return false, err } + if opts.AcceptVersion != nil && !opts.AcceptVersion(version.Key, version.Value) { + result.NextCursor = encodeExportCursor(userKey, commitTS, exportCursorTagScanned) + if finishExportIfLimited(opts, result) { + result.Done = false + return false, nil + } + return true, nil + } result.Versions = append(result.Versions, version) result.ExportedBytes += versionExportSize(userKey, len(version.Value)) result.AcceptedRows++ diff --git a/store/migration_versions.go b/store/migration_versions.go index 6eb5f65db..4e42af1db 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -136,6 +136,7 @@ func normalizeExportVersionsOptions(opts ExportVersionsOptions) ExportVersionsOp func exportUsesSparseScanBudget(opts ExportVersionsOptions) bool { return opts.AcceptKey != nil || + opts.AcceptVersion != nil || opts.MaxCommitTSInclusive != 0 || opts.MinCommitTSExclusive != 0 || opts.StartKey != nil || @@ -406,6 +407,9 @@ func appendMemoryExportVersion(opts ExportVersionsOptions, key []byte, version V if opts.AcceptKey != nil && !opts.AcceptKey(key) { return exportCursorTagScanned } + if opts.AcceptVersion != nil && !opts.AcceptVersion(key, version.Value) { + return exportCursorTagScanned + } if opts.MaxCommitTSInclusive != 0 && version.TS > opts.MaxCommitTSInclusive { return exportCursorTagScanned } diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 9035dd692..1609da8d8 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -70,6 +70,24 @@ func TestExportVersionsExcludesTxnLocks(t *testing.T) { }) } +func TestExportVersionsAcceptVersionFiltersByValue(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("drop"), []byte("legacy-meta"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("keep"), []byte("legacy-delta"), 20, 0)) + + result, err := st.ExportVersions(ctx, ExportVersionsOptions{ + MaxVersions: 10, + AcceptVersion: func(_ []byte, value []byte) bool { + return bytes.Equal(value, []byte("legacy-delta")) + }, + }) + require.NoError(t, err) + require.True(t, result.Done) + require.Equal(t, []MVCCVersion{{Key: []byte("keep"), CommitTS: 20, Value: []byte("legacy-delta")}}, result.Versions) + }) +} + func TestExportVersionsCursorResumesWithinHotKey(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() diff --git a/store/store.go b/store/store.go index bc80d6293..37ad9b426 100644 --- a/store/store.go +++ b/store/store.go @@ -93,6 +93,7 @@ type ExportVersionsOptions struct { MaxScannedBytes uint64 KeyFamily uint32 AcceptKey func([]byte) bool + AcceptVersion func(key []byte, value []byte) bool } // ExportVersionsResult is one resumable chunk of raw MVCC versions. From bd55701b99be8786c5ff1bf2dead0707b10596b1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 06:52:37 +0900 Subject: [PATCH 123/162] store: preserve empty-key migration exports --- store/lsm_migration.go | 4 ++-- store/migration_versions_test.go | 32 ++++++++++++++------------------ 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 56623fda5..63878bb0b 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -212,12 +212,12 @@ func advancePebbleExportPastCurrentUserKey( } func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { - if startKey == nil { + if len(startKey) == 0 { return false } for prefixLen := 1; prefixLen <= len(userKey); prefixLen++ { prefix := userKey[:prefixLen] - if len(startKey) != 0 && bytes.Compare(prefix, startKey) < 0 { + if bytes.Compare(prefix, startKey) < 0 { continue } if bytes.Compare(prefix, endKey) < 0 { diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 1609da8d8..3dc418888 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -624,7 +624,7 @@ func TestPebbleExportStopsAtEndKeyWhenNoLaterInRangeKeyCanTrail(t *testing.T) { require.Zero(t, result.ScannedBytes) } -func TestPebbleExportStopsLeadingRangeWithExplicitEmptyStartAtEndKey(t *testing.T) { +func TestPebbleExportDoesNotStopLeadingRangeWithExplicitEmptyStartAtEndKey(t *testing.T) { ctx := context.Background() dir, err := os.MkdirTemp("", "migration-leading-end-key-*") require.NoError(t, err) @@ -632,25 +632,21 @@ func TestPebbleExportStopsLeadingRangeWithExplicitEmptyStartAtEndKey(t *testing. st, err := NewPebbleStore(dir) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, st.Close()) }) - ps, ok := st.(*pebbleStore) - require.True(t, ok) - require.NoError(t, ps.PutAt(ctx, []byte("b"), []byte("later"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("later"), 10, 0)) + require.NoError(t, st.PutAt(ctx, nil, []byte("empty"), 20, 0)) - iter, err := ps.db.NewIter(nil) + res, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte{}, + EndKey: []byte("b"), + MaxVersions: 10, + }) require.NoError(t, err) - defer func() { require.NoError(t, iter.Close()) }() - require.True(t, iter.SeekGE(encodeKey([]byte("b"), math.MaxUint64))) - - result := newExportVersionsResult(10) - advance, done, err := ps.exportPebbleIteratorPosition(ctx, iter, ExportVersionsOptions{ - StartKey: []byte{}, - EndKey: []byte("b"), - }, exportCursorPosition{}, &result) - require.ErrorIs(t, err, errExportReachedEnd) - require.False(t, advance) - require.True(t, done) - require.Empty(t, result.Versions) - require.Zero(t, result.ScannedBytes) + require.True(t, res.Done) + require.Equal(t, []MVCCVersion{{ + Key: []byte{}, + CommitTS: 20, + Value: []byte("empty"), + }}, res.Versions) } func TestPebbleExportOutOfRangeEndSkipReturnsResumableCursor(t *testing.T) { From 8a687cad92b3eb7671f513e7cad410756e85fe41 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:04:29 +0900 Subject: [PATCH 124/162] migration: tighten routed cleanup safety --- adapter/redis_txn.go | 47 ++++++++++++----- adapter/redis_txn_test.go | 45 ++++++++++++++++ internal/raftengine/etcd/fsm_snapshot_file.go | 12 +++++ internal/raftengine/etcd/wal_store.go | 23 ++++++--- .../etcd/wal_store_skip_gate_test.go | 22 ++++---- kv/sharded_coordinator.go | 19 +++++-- kv/sharded_coordinator_txn_test.go | 51 +++++++++++++++++++ 7 files changed, 181 insertions(+), 38 deletions(-) diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index 739e43bb8..d06e6b312 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -98,11 +98,12 @@ type txnValue struct { } type stringReplacement struct { - key []byte - value []byte - ttl *time.Time - rawTyp redisValueType - rawTypKnown bool + key []byte + value []byte + ttl *time.Time + rawTyp redisValueType + rawTypKnown bool + rawPrefixedString bool } type txnContext struct { @@ -778,7 +779,11 @@ func (t *txnContext) applySet(cmd redcon.Command) (redisResult, error) { return redisResult{}, err } t.trackWideCollectionFenceReads(cmd.Args[1]) - t.stageStringReplacementWithRawType(cmd.Args[1], cmd.Args[2], opts.ttl, typeView.rawTyp, typeView.rawTypKnown) + rawPrefixedString, err := t.rawPrefixedStringAtStart(cmd.Args[1], typeView) + if err != nil { + return redisResult{}, err + } + t.stageStringReplacementWithRawType(cmd.Args[1], cmd.Args[2], opts.ttl, typeView.rawTyp, typeView.rawTypKnown, rawPrefixedString) return applySetResult(opts, oldValue), nil } @@ -816,10 +821,10 @@ func cloneTimePtr(in *time.Time) *time.Time { } func (t *txnContext) stageStringReplacement(key, value []byte, ttl *time.Time) { - t.stageStringReplacementWithRawType(key, value, ttl, redisTypeNone, false) + t.stageStringReplacementWithRawType(key, value, ttl, redisTypeNone, false, false) } -func (t *txnContext) stageStringReplacementWithRawType(key, value []byte, ttl *time.Time, rawTyp redisValueType, rawTypKnown bool) { +func (t *txnContext) stageStringReplacementWithRawType(key, value []byte, ttl *time.Time, rawTyp redisValueType, rawTypKnown bool, rawPrefixedString bool) { if t.replacers == nil { t.replacers = map[string]*stringReplacement{} } @@ -830,21 +835,34 @@ func (t *txnContext) stageStringReplacementWithRawType(key, value []byte, ttl *t if rawTypKnown { repl.rawTyp = rawTyp repl.rawTypKnown = true + repl.rawPrefixedString = rawPrefixedString } delete(t.deletedKeys, k) return } repl := &stringReplacement{ - key: bytes.Clone(key), - value: bytes.Clone(value), - ttl: cloneTimePtr(ttl), - rawTyp: rawTyp, - rawTypKnown: rawTypKnown, + key: bytes.Clone(key), + value: bytes.Clone(value), + ttl: cloneTimePtr(ttl), + rawTyp: rawTyp, + rawTypKnown: rawTypKnown, + rawPrefixedString: rawPrefixedString, } t.replacers[k] = repl delete(t.deletedKeys, k) } +func (t *txnContext) rawPrefixedStringAtStart(key []byte, view txnKeyTypeView) (bool, error) { + if !view.rawTypKnown || view.rawTyp != redisTypeString { + return false, nil + } + exists, err := t.server.store.ExistsAt(t.ctxOrBackground(), redisStrKey(key), t.startTS) + if err != nil { + return false, errors.WithStack(err) + } + return exists, nil +} + func (t *txnContext) updateStringReplacementTTL(key []byte, ttl *time.Time) bool { repl, ok := t.replacers[string(key)] if !ok { @@ -1658,6 +1676,9 @@ func (r *stringReplacement) needsFullLogicalDelete() bool { if !r.rawTypKnown { return true } + if r.rawTyp == redisTypeString { + return !r.rawPrefixedString + } return isNonStringCollectionType(r.rawTyp) } diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index bbd245e3a..b6c621195 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -766,6 +766,51 @@ func TestRedisTxnSetReplacementSkipsWideCleanupForRawStringOrMissing(t *testing. } } +func TestRedisTxnSetReplacementDeletesNonPrefixedStringEncodings(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cases := []struct { + name string + seedKey func([]byte) []byte + seedValue []byte + expectedDel func([]byte) []byte + }{ + { + name: "hll", + seedKey: redisHLLKey, + seedValue: []byte("hll-payload"), + expectedDel: redisHLLKey, + }, + { + name: "legacy-bare-string", + seedKey: func(key []byte) []byte { return key }, + seedValue: []byte("old"), + expectedDel: func(key []byte) []byte { return key }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + server, st := newRedisStorageMigrationTestServer(t) + key := []byte("set-replace:non-prefixed-string:" + tc.name) + require.NoError(t, st.PutAt(ctx, tc.seedKey(key), tc.seedValue, redisTxnTestStartTS, 0)) + + txn := newRedisTxnTestContext(server) + res, err := txn.applySet(redcon.Command{Args: [][]byte{[]byte(cmdSet), key, []byte("string")}}) + require.NoError(t, err) + require.Equal(t, "OK", res.str) + + elems, err := txn.buildReplacementElems(ctx) + require.NoError(t, err) + require.True(t, elemKeysContain(elems, tc.expectedDel(key))) + require.True(t, elemKeysContain(elems, redisStrKey(key))) + }) + } +} + func TestRedisTxnSetReplacementDeletesExpiredRawHash(t *testing.T) { t.Parallel() diff --git a/internal/raftengine/etcd/fsm_snapshot_file.go b/internal/raftengine/etcd/fsm_snapshot_file.go index e27bf0b0b..13f3d3b0c 100644 --- a/internal/raftengine/etcd/fsm_snapshot_file.go +++ b/internal/raftengine/etcd/fsm_snapshot_file.go @@ -361,6 +361,18 @@ func restoreAndComputeCRC(f *os.File, fileSize int64, fsm StateMachine) (uint32, return h.Sum32(), nil } +func computeFSMSnapshotPayloadCRC(f *os.File, fileSize int64) (uint32, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return 0, errors.WithStack(err) + } + payloadSize := fileSize - fsmFooterSize + h := crc32.New(crc32cTable) + if _, err := io.CopyN(h, f, payloadSize); err != nil { + return 0, errors.WithStack(err) + } + return h.Sum32(), nil +} + // verifyFSMSnapshotFile performs a read-only CRC check without restoring the FSM. // Used for startup orphan detection. Pass tokenCRC=0 to skip the token comparison. func verifyFSMSnapshotFile(path string, tokenCRC uint32) error { diff --git a/internal/raftengine/etcd/wal_store.go b/internal/raftengine/etcd/wal_store.go index 2a38ac3d0..acd8edb65 100644 --- a/internal/raftengine/etcd/wal_store.go +++ b/internal/raftengine/etcd/wal_store.go @@ -500,13 +500,11 @@ func reportColdStart(obs raftengine.ColdStartObserver, logger *zap.Logger, d col } } -// applyHeaderStateOnSkip validates the cheap snapshot envelope checks -// (size + footer-vs-tokenCRC) and applies only the header side-effects -// (HLC ceiling + Stage 8a cutover) instead of running the body restore. -// The body bytes are not read here: fsm.db already holds equivalent state, -// which is precisely the reason we're skipping the restore. Full-body CRC -// verification still runs on the execute/full-restore path where body -// bytes are actually consumed. +// applyHeaderStateOnSkip validates the snapshot envelope and full payload CRC, +// then applies only the header side-effects (HLC ceiling + Stage 8a cutover) +// instead of running the body restore. The FSM body is already durable enough +// to skip restoring, but header side-effects still come from this file and +// must not be applied from corrupt bytes. // // FSMs that do not implement raftengine.SnapshotHeaderApplier // silently no-op the apply phase -- the FSM has no header state to @@ -523,9 +521,18 @@ func applyHeaderStateOnSkip(fsm StateMachine, snapPath string, tokenCRC uint32) if err != nil { return errors.WithStack(err) } - if _, err := verifyFSMSnapshotPrefix(file, info.Size(), snapPath, tokenCRC); err != nil { + footer, err := verifyFSMSnapshotPrefix(file, info.Size(), snapPath, tokenCRC) + if err != nil { return err } + computed, err := computeFSMSnapshotPayloadCRC(file, info.Size()) + if err != nil { + return err + } + if computed != footer { + return errors.Wrapf(ErrFSMSnapshotFileCRC, + "path=%s footer=%08x computed=%08x", snapPath, footer, computed) + } setter, hasSetter := fsm.(raftengine.SnapshotHeaderApplier) if !hasSetter { diff --git a/internal/raftengine/etcd/wal_store_skip_gate_test.go b/internal/raftengine/etcd/wal_store_skip_gate_test.go index ffde0a0b3..e121ee396 100644 --- a/internal/raftengine/etcd/wal_store_skip_gate_test.go +++ b/internal/raftengine/etcd/wal_store_skip_gate_test.go @@ -375,10 +375,9 @@ func TestApplyHeaderStateOnSkip_WrongTokenCRC(t *testing.T) { require.False(t, fsm.restoredHeader, "FSM state MUST NOT mutate on verification failure") } -// TestApplyHeaderStateOnSkip_DoesNotScanBody asserts skip startup cost is -// bounded by the header, not the snapshot body. Body corruption is still -// caught by the full restore path, where the body is actually consumed. -func TestApplyHeaderStateOnSkip_DoesNotScanBody(t *testing.T) { +// TestApplyHeaderStateOnSkip_VerifiesPayloadBeforeHeaderApply asserts the skip +// path checks payload integrity before applying header side-effects. +func TestApplyHeaderStateOnSkip_VerifiesPayloadBeforeHeaderApply(t *testing.T) { dir := t.TempDir() const ceilingMs uint64 = 1700_000_000_001 payload := make([]byte, 16, 16+len("payload-bytes")) @@ -387,25 +386,22 @@ func TestApplyHeaderStateOnSkip_DoesNotScanBody(t *testing.T) { payload = append(payload, []byte("payload-bytes")...) crc, path := writeFSMFileForTest(t, dir, 1, payload) - // Flip a byte after the fixed 16-byte header. The footer still - // reads as `crc`, but the on-the-wire content no longer matches - // it. The skip path should still succeed because it never uses body - // bytes; the full restore path remains responsible for body CRC. + // Flip a byte in the fixed 16-byte header. The footer still reads as `crc`, + // but the bytes used to derive header side-effects no longer match it. f, err := os.OpenFile(path, os.O_RDWR, 0) require.NoError(t, err) defer f.Close() var b [1]byte - _, err = f.ReadAt(b[:], 16) + _, err = f.ReadAt(b[:], 8) require.NoError(t, err) b[0] ^= 0x01 - _, err = f.WriteAt(b[:], 16) + _, err = f.WriteAt(b[:], 8) require.NoError(t, err) fsm := &skipGateFSM{} err = applyHeaderStateOnSkip(fsm, path, crc) - require.NoError(t, err) - require.True(t, fsm.restoredHeader) - require.Equal(t, ceilingMs, fsm.appliedCeiling) + require.ErrorIs(t, err, ErrFSMSnapshotFileCRC) + require.False(t, fsm.restoredHeader) } // --- kvFSM header preservation contract --- diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 056aa63ab..e2cd989c5 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1299,7 +1299,7 @@ func (c *ShardedCoordinator) dispatchMultiShardTxn(ctx context.Context, startTS, return nil, err } - primaryGid, maxIndex, err := c.commitPrimaryTxn(ctx, startTS, primaryKey, grouped, commitTS, observedRouteVersion) + primaryGid, maxIndex, err := c.commitPrimaryTxn(ctx, startTS, primaryKey, grouped, gids, commitTS, observedRouteVersion) if err != nil { // abortPreparedTxn must run even when ctx was the reason // commitPrimaryTxn failed — otherwise prewrite intents on @@ -1439,9 +1439,9 @@ func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS return prepared, nil } -func (c *ShardedCoordinator) commitPrimaryTxn(ctx context.Context, startTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, commitTS uint64, observedRouteVersion uint64) (uint64, uint64, error) { - primaryGid := c.engineGroupIDForKey(primaryKey) - if primaryGid == 0 { +func (c *ShardedCoordinator) commitPrimaryTxn(ctx context.Context, startTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, commitTS uint64, observedRouteVersion uint64) (uint64, uint64, error) { + primaryGid, ok := primaryGroupIDForKey(primaryKey, grouped, gids) + if !ok { return 0, 0, errors.WithStack(ErrInvalidRequest) } @@ -1471,6 +1471,17 @@ func (c *ShardedCoordinator) commitPrimaryTxn(ctx context.Context, startTS uint6 return primaryGid, r.CommitIndex, nil } +func primaryGroupIDForKey(primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64) (uint64, bool) { + for _, gid := range gids { + for _, mut := range grouped[gid] { + if mut != nil && bytes.Equal(mut.Key, primaryKey) { + return gid, true + } + } + } + return 0, false +} + func (c *ShardedCoordinator) commitSecondaryTxns(ctx context.Context, startTS uint64, primaryGid uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, commitTS uint64, maxIndex uint64, observedRouteVersion uint64) (uint64, error) { // Secondary commits are best-effort for non-Composed-1 errors: // if a shard is unavailable after the primary commits, read-time diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index d47871e91..163897fb0 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -80,6 +80,57 @@ func TestShardedCoordinatorGroupMutationsUsesExplicitElemGroup(t *testing.T) { require.Equal(t, []byte("a-key"), grouped[2][0].Key) } +func TestShardedCoordinatorDispatchTxn_CommitPrimaryUsesPinnedGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), nil, 1) + + g1Txn := &recordingTransactional{ + responses: []*TransactionResponse{ + {CommitIndex: 3}, + {CommitIndex: 11}, + }, + } + g2Txn := &recordingTransactional{ + responses: []*TransactionResponse{ + {CommitIndex: 5}, + {CommitIndex: 27}, + }, + } + + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + startTS := uint64(10) + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + IsTxn: true, + StartTS: startTS, + Elems: []*Elem[OP]{ + {Op: Del, Key: []byte("a-key"), GroupID: 2}, + {Op: Put, Key: []byte("z-key"), Value: []byte("v"), GroupID: 1}, + }, + }) + require.NoError(t, err) + require.Len(t, g1Txn.requests, 2) + require.Len(t, g2Txn.requests, 2) + + g1Commit := g1Txn.requests[1] + g2Commit := g2Txn.requests[1] + require.Equal(t, pb.Phase_COMMIT, g1Commit.Phase) + require.Equal(t, pb.Phase_COMMIT, g2Commit.Phase) + require.Equal(t, []byte("z-key"), g1Commit.Mutations[1].Key) + require.Equal(t, pb.Op_PUT, g1Commit.Mutations[1].Op) + require.Equal(t, []byte("a-key"), g2Commit.Mutations[1].Key) + require.Equal(t, pb.Op_PUT, g2Commit.Mutations[1].Op) + + primaryCommitMeta := requestTxnMeta(t, g2Commit) + require.Equal(t, []byte("a-key"), primaryCommitMeta.PrimaryKey) + require.Greater(t, primaryCommitMeta.CommitTS, startTS) +} + func requestTxnMeta(t *testing.T, req *pb.Request) TxnMeta { t.Helper() require.NotNil(t, req) From fd179f994d4e0e7c88e6a1c6834fb7add4b8642e Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:10:23 +0900 Subject: [PATCH 125/162] migration: preserve ambiguous list tombstones --- distribution/migrator.go | 4 +++ distribution/migrator_export_plan_test.go | 32 +++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/distribution/migrator.go b/distribution/migrator.go index 949c53eaf..000da7803 100644 --- a/distribution/migrator.go +++ b/distribution/migrator.go @@ -259,6 +259,10 @@ func (b MigrationBracket) ContainsRoutedVersion(rawKey, value, routeStart, route } func (b MigrationBracket) containsLegacyListMetaDeltaRoute(rawKey, value, routeStart, routeEnd []byte) bool { + if value == nil { + return routeKeyInRange(store.ExtractListUserKey(rawKey), routeStart, routeEnd) || + routeKeyInRange(store.ExtractLegacyListUserKeyFromDelta(rawKey), routeStart, routeEnd) + } if value != nil && !store.IsListMetaDeltaValue(value) { return routeKeyInRange(store.ExtractListUserKey(rawKey), routeStart, routeEnd) } diff --git a/distribution/migrator_export_plan_test.go b/distribution/migrator_export_plan_test.go index 0869ea0b1..104625499 100644 --- a/distribution/migrator_export_plan_test.go +++ b/distribution/migrator_export_plan_test.go @@ -332,6 +332,38 @@ func TestMigrationBracketContainsRoutedKeyRoutesAmbiguousLegacyListMetaByBaseKey )) } +func TestMigrationBracketContainsRoutedKeyRoutesAmbiguousListMetaTombstoneConservatively(t *testing.T) { + t.Parallel() + + brackets, err := PlanMigrationBrackets([]byte("a"), []byte("z")) + require.NoError(t, err) + legacy := bracketsByFamily(brackets)[MigrationFamilyLegacyListMetaDelta] + baseUserKey := deltaLookingListMetaUserKey([]byte("target-list"), 10, 0) + raw := store.ListMetaKey(baseUserKey) + + require.True(t, legacy.ContainsRoutedVersion( + raw, + nil, + []byte("d|"), + []byte("d}"), + store.ExtractListUserKey, + )) + require.True(t, legacy.ContainsRoutedVersion( + raw, + nil, + []byte("target"), + []byte("target-list\x00"), + store.ExtractListUserKey, + )) + require.False(t, legacy.ContainsRoutedVersion( + raw, + nil, + []byte("zzz"), + nil, + store.ExtractListUserKey, + )) +} + func TestMigrationKnownInternalPrefixesAreConcreteOnly(t *testing.T) { t.Parallel() From c2750223348084d1824b571b64e0eb95395700e6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:23:26 +0900 Subject: [PATCH 126/162] adapter: preserve scan route groups for list cleanup --- adapter/redis_compat_helpers.go | 4 +- adapter/redis_txn.go | 16 ++++++-- adapter/redis_txn_test.go | 72 ++++++++++++++++++++++++++++++++- 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/adapter/redis_compat_helpers.go b/adapter/redis_compat_helpers.go index de96882aa..f396fd0db 100644 --- a/adapter/redis_compat_helpers.go +++ b/adapter/redis_compat_helpers.go @@ -991,7 +991,7 @@ func (r *RedisServer) scanListItemDelElems(ctx context.Context, key []byte, read if len(elems)+1 > maxWideColumnItems { return nil, errors.Wrapf(ErrCollectionTooLarge, "list %q exceeds %d items", key, maxWideColumnItems) } - elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: pair.Key}) + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: bytes.Clone(pair.Key), GroupID: pair.RouteGroupID}) } if len(itemKVs) < store.MaxDeltaScanLimit { break @@ -1041,7 +1041,7 @@ func (r *RedisServer) scanAllDeltaElemsFiltered( if len(elems)+1 > maxWideColumnItems { return nil, errors.Wrapf(ErrCollectionTooLarge, "delta key count exceeds %d", maxWideColumnItems) } - elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: pair.Key}) + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: bytes.Clone(pair.Key), GroupID: pair.RouteGroupID}) } if len(deltaKVs) < store.MaxDeltaScanLimit { break diff --git a/adapter/redis_txn.go b/adapter/redis_txn.go index d06e6b312..6cdfd64c7 100644 --- a/adapter/redis_txn.go +++ b/adapter/redis_txn.go @@ -141,7 +141,12 @@ type listTxnState struct { deleted bool purge bool purgeMeta store.ListMeta - existingDeltas [][]byte // delta key bytes present at load time; deleted on purge/delete + existingDeltas []listDeltaRef // delta keys present at load time; deleted on purge/delete +} + +type listDeltaRef struct { + key []byte + groupID uint64 } type hashTxnState struct { @@ -354,7 +359,7 @@ func (t *txnContext) loadListState(key []byte) (*listTxnState, error) { // truncation: if >MaxDeltaScanLimit deltas exist the transaction cannot // safely enumerate all of them for deletion, so we return ErrDeltaScanTruncated // and let the caller retry after the background compactor has caught up. - var existingDeltas [][]byte + var existingDeltas []listDeltaRef acceptedDeltas := 0 for _, deltaPrefix := range store.ListMetaDeltaScanPrefixes(key) { deltaKVs, truncated, err := scanAcceptedDeltaKVsAt( @@ -376,7 +381,10 @@ func (t *txnContext) loadListState(key []byte) (*listTxnState, error) { if acceptedDeltas > store.MaxDeltaScanLimit { return nil, ErrDeltaScanTruncated } - existingDeltas = append(existingDeltas, kv.Key) + existingDeltas = append(existingDeltas, listDeltaRef{ + key: bytes.Clone(kv.Key), + groupID: kv.RouteGroupID, + }) } } @@ -1866,7 +1874,7 @@ func appendListDeletionElems(elems []*kv.Elem[kv.OP], userKey []byte, st *listTx } // Delete existing delta keys so they do not survive logical delete/purge. for _, dk := range st.existingDeltas { - elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: dk}) + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: dk.key, GroupID: dk.groupID}) } elems = append(elems, redisTxnWideListFenceElem(userKey)) return elems diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index b6c621195..9411337ee 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -41,6 +41,22 @@ func newRedisTxnTestContext(server *RedisServer) *txnContext { } } +type routeGroupScanStore struct { + store.MVCCStore + groupID uint64 +} + +func (s *routeGroupScanStore) ScanAt(ctx context.Context, start []byte, end []byte, limit int, ts uint64) ([]*store.KVPair, error) { + kvs, err := s.MVCCStore.ScanAt(ctx, start, end, limit, ts) + if err != nil { + return nil, err + } + for _, kvp := range kvs { + kvp.RouteGroupID = s.groupID + } + return kvs, nil +} + func TestRedisTxnLoadListStateFiltersLegacyDeltaPrefixCollisions(t *testing.T) { t.Parallel() @@ -64,7 +80,48 @@ func TestRedisTxnLoadListStateFiltersLegacyDeltaPrefixCollisions(t *testing.T) { txn.startTS = 4 stState, err := txn.loadListState(key) require.NoError(t, err) - require.Equal(t, [][]byte{deltaKey}, stState.existingDeltas) + require.Equal(t, []listDeltaRef{{key: deltaKey}}, stState.existingDeltas) +} + +func TestRedisTxnLoadListStatePreservesDeltaRouteGroupID(t *testing.T) { + t.Parallel() + + baseStore := store.NewMVCCStore() + scanStore := &routeGroupScanStore{MVCCStore: baseStore, groupID: 7} + server := NewRedisServer(nil, "", scanStore, newLocalAdapterCoordinator(baseStore), nil, nil) + ctx := context.Background() + key := []byte("txn-list-route-group") + base, err := store.MarshalListMeta(store.ListMeta{Head: 0, Tail: 1, Len: 1}) + require.NoError(t, err) + require.NoError(t, baseStore.PutAt(ctx, store.ListMetaKey(key), base, 1, 0)) + deltaKey := store.ListMetaDeltaKey(key, 2, 0) + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) + require.NoError(t, baseStore.PutAt(ctx, deltaKey, delta, 2, 0)) + + txn := newRedisTxnTestContext(server) + txn.startTS = 3 + stState, err := txn.loadListState(key) + require.NoError(t, err) + require.Equal(t, []listDeltaRef{{key: deltaKey, groupID: 7}}, stState.existingDeltas) +} + +func TestRedisScanAllDeltaElemsFilteredPreservesRouteGroupID(t *testing.T) { + t.Parallel() + + baseStore := store.NewMVCCStore() + scanStore := &routeGroupScanStore{MVCCStore: baseStore, groupID: 11} + server := NewRedisServer(nil, "", scanStore, newLocalAdapterCoordinator(baseStore), nil, nil) + ctx := context.Background() + key := []byte("scan-list-route-group") + deltaKey := store.ListMetaDeltaKey(key, 2, 0) + delta := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) + require.NoError(t, baseStore.PutAt(ctx, deltaKey, delta, 2, 0)) + + elems, err := server.scanAllDeltaElemsFiltered(ctx, store.ListMetaDeltaScanPrefix(key), 3, nil) + require.NoError(t, err) + require.Len(t, elems, 1) + require.Equal(t, deltaKey, elems[0].Key) + require.Equal(t, uint64(11), elems[0].GroupID) } func TestRedisTxnLoadListStateEnforcesDeltaLimitAcrossCurrentAndLegacyPrefixes(t *testing.T) { @@ -1180,6 +1237,19 @@ func TestRedisTxnListDeletionElemsWriteFence(t *testing.T) { require.True(t, elemKeysContain(elems, redisTxnWideListFenceKey(key))) } +func TestRedisTxnListDeletionElemsPreserveDeltaRouteGroupID(t *testing.T) { + t.Parallel() + + key := []byte("delete-route-group:list") + deltaKey := store.ListMetaDeltaKey(key, 9, 0) + elems := appendListDeletionElems(nil, key, &listTxnState{ + existingDeltas: []listDeltaRef{{key: deltaKey, groupID: 17}}, + deleted: true, + }) + require.Equal(t, uint64(17), requireElemByKey(t, elems, deltaKey).GroupID) + require.True(t, elemKeysContain(elems, redisTxnWideListFenceKey(key))) +} + func TestRedisTxnHashLegacyRewriteWritesFence(t *testing.T) { t.Parallel() From 5cf01fdb6eb90a3a7ac359ddc57241297858eab4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:37:14 +0900 Subject: [PATCH 127/162] adapter: backtrack list compactor accepted tails --- adapter/redis_delta_compactor.go | 55 ++++++++++++++++++++++++--- adapter/redis_delta_compactor_test.go | 16 ++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/adapter/redis_delta_compactor.go b/adapter/redis_delta_compactor.go index 69a13c413..ead6e32e6 100644 --- a/adapter/redis_delta_compactor.go +++ b/adapter/redis_delta_compactor.go @@ -469,7 +469,7 @@ func (c *DeltaCompactor) compactHandler(ctx context.Context, h collectionDeltaHa kvs = filterDeltaKVs(rawKVs, h.acceptDeltaKV) byKey, ukOrder := c.groupByUserKey(kvs, h.extractUserKey) lastScannedKey := c.splitGuardCursor(byKey, ukOrder, kvs, truncated) - lastScannedKey = deltaCompactorCursorAfterFiltering(rawKVs, kvs, lastScannedKey, truncated) + lastScannedKey = deltaCompactorCursorAfterFiltering(rawKVs, kvs, lastScannedKey, truncated, h.extractUserKey) allElems := c.buildBatchElems(ctx, h, byKey, ukOrder, readTS) @@ -483,7 +483,12 @@ func (c *DeltaCompactor) compactHandler(ctx context.Context, h collectionDeltaHa return nil } -func deltaCompactorCursorAfterFiltering(rawKVs, acceptedKVs []*store.KVPair, lastScannedKey []byte, truncated bool) []byte { +func deltaCompactorCursorAfterFiltering( + rawKVs, acceptedKVs []*store.KVPair, + lastScannedKey []byte, + truncated bool, + extractUserKey func([]byte) []byte, +) []byte { if !truncated || len(rawKVs) == 0 { return lastScannedKey } @@ -493,7 +498,7 @@ func deltaCompactorCursorAfterFiltering(rawKVs, acceptedKVs []*store.KVPair, las lastAcceptedKey := acceptedKVs[len(acceptedKVs)-1].Key switch { case len(lastScannedKey) == 0: - if prev := rawKeyBeforeAcceptedTail(rawKVs, lastAcceptedKey); len(prev) > 0 { + if prev := rawKeyBeforeAcceptedTail(rawKVs, acceptedKVs, lastAcceptedKey, extractUserKey); len(prev) > 0 { return prev } if rawPageHasRejectedTailAfter(rawKVs, lastAcceptedKey) { @@ -517,7 +522,11 @@ func rawPageHasRejectedTailAfter(rawKVs []*store.KVPair, acceptedKey []byte) boo return false } -func rawKeyBeforeAcceptedTail(rawKVs []*store.KVPair, acceptedKey []byte) []byte { +func rawKeyBeforeAcceptedTail( + rawKVs, acceptedKVs []*store.KVPair, + acceptedKey []byte, + extractUserKey func([]byte) []byte, +) []byte { if len(rawKVs) < 2 || len(acceptedKey) == 0 { return nil } @@ -525,14 +534,48 @@ func rawKeyBeforeAcceptedTail(rawKVs []*store.KVPair, acceptedKey []byte) []byte if rawKVs[last] == nil || !bytes.Equal(rawKVs[last].Key, acceptedKey) { return nil } + acceptedUserKey := []byte(nil) + if extractUserKey != nil { + acceptedUserKey = extractUserKey(acceptedKey) + } + acceptedKeys := deltaCompactorAcceptedKeySet(acceptedKVs) for i := last - 1; i >= 0; i-- { - if rawKVs[i] != nil { - return rawKVs[i].Key + if rawKVs[i] == nil { + continue + } + if deltaCompactorSameAcceptedTailUser(rawKVs[i].Key, acceptedKeys, acceptedUserKey, extractUserKey) { + continue } + return rawKVs[i].Key } return nil } +func deltaCompactorAcceptedKeySet(acceptedKVs []*store.KVPair) map[string]struct{} { + acceptedKeys := make(map[string]struct{}, len(acceptedKVs)) + for _, kvp := range acceptedKVs { + if kvp != nil { + acceptedKeys[string(kvp.Key)] = struct{}{} + } + } + return acceptedKeys +} + +func deltaCompactorSameAcceptedTailUser( + rawKey []byte, + acceptedKeys map[string]struct{}, + acceptedUserKey []byte, + extractUserKey func([]byte) []byte, +) bool { + if len(acceptedUserKey) == 0 || extractUserKey == nil { + return false + } + if _, accepted := acceptedKeys[string(rawKey)]; !accepted { + return false + } + return bytes.Equal(extractUserKey(rawKey), acceptedUserKey) +} + // groupByUserKey groups KVPairs by their user key, returning both the map and // the unique user keys in lexicographic (scan) order. func filterDeltaKVs(kvs []*store.KVPair, accept func(*store.KVPair) bool) []*store.KVPair { diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index 137884419..6eba9c25b 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -451,6 +451,22 @@ func TestDeltaCompactor_LegacyListCursorAdvancesPastRejectedPrefixBeforeAccepted require.NoError(t, err) } +func TestDeltaCompactor_CursorBacktracksBeforeWholeAcceptedTail(t *testing.T) { + t.Parallel() + + userKey := []byte("compactor-accepted-tail") + delta := store.MarshalListMetaDelta(store.ListMetaDelta{HeadDelta: 0, LenDelta: 1}) + d1 := legacyListMetaDeltaKey(userKey, 1) + d2 := legacyListMetaDeltaKey(userKey, 2) + rawKVs := []*store.KVPair{ + {Key: d1, Value: delta}, + {Key: d2, Value: delta}, + } + + got := deltaCompactorCursorAfterFiltering(rawKVs, rawKVs, nil, true, store.ExtractLegacyListUserKeyFromDelta) + require.Nil(t, got) +} + func TestDeltaCompactor_ListDeltaDeletesPreserveScanRouteGroup(t *testing.T) { t.Parallel() From 7fe4112a75d495b0538ebd78c7d8d5333d09afcf Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:41:42 +0900 Subject: [PATCH 128/162] kv: cover pinned primary transaction commits --- kv/sharded_coordinator_txn_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 163897fb0..170dacbd9 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -655,6 +655,35 @@ func TestShardedCoordinatorDispatchTxn_SingleShardIncludesReadKeysInRaftEntry(t require.Equal(t, [][]byte{[]byte("rk1"), []byte("rk2")}, g1Txn.requests[0].ReadKeys) } +func TestShardedCoordinatorCommitPrimaryUsesPinnedMutationGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), nil, 2) + + g1Txn := &recordingTransactional{responses: []*TransactionResponse{{CommitIndex: 7}}} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 2, NewHLC(), nil) + + primaryKey := []byte("!lst|meta|d|pinned") + grouped := map[uint64][]*pb.Mutation{ + 1: {{Op: pb.Op_DEL, Key: primaryKey}}, + 2: {{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, + } + primaryGid, commitIndex, err := coord.commitPrimaryTxn(context.Background(), 10, primaryKey, grouped, []uint64{1, 2}, 20, 0) + require.NoError(t, err) + require.Equal(t, uint64(1), primaryGid) + require.Equal(t, uint64(7), commitIndex) + require.Len(t, g1Txn.requests, 1) + require.Empty(t, g2Txn.requests) + require.Equal(t, pb.Phase_COMMIT, g1Txn.requests[0].Phase) + require.Len(t, g1Txn.requests[0].Mutations, 2) + require.Equal(t, primaryKey, g1Txn.requests[0].Mutations[1].Key) +} + // TestShardedCoordinatorDispatchTxn_CrossShardPropagatesObservedRouteVersion // is the gemini-critical regression from PR #881. Contract: // every PREPARE and COMMIT envelope across the 2PC paths From 83cd5bfe7fa39070449ce2fc04531e6c99160d70 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:45:49 +0900 Subject: [PATCH 129/162] ci: generate redis proxy image metadata locally --- .github/workflows/redis-proxy-docker.yml | 30 ++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/.github/workflows/redis-proxy-docker.yml b/.github/workflows/redis-proxy-docker.yml index 6e79ebe25..b79ad0454 100644 --- a/.github/workflows/redis-proxy-docker.yml +++ b/.github/workflows/redis-proxy-docker.yml @@ -45,13 +45,29 @@ jobs: - name: Docker metadata id: meta - uses: docker/metadata-action@v6 - with: - images: ghcr.io/${{ github.repository }}/redis-proxy - tags: | - type=sha - type=ref,event=branch - type=raw,value=latest,enable={{is_default_branch}} + shell: bash + env: + IMAGE: ghcr.io/${{ github.repository }}/redis-proxy + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + short_sha="${GITHUB_SHA::7}" + ref_tag="$(printf '%s' "$GITHUB_REF_NAME" | sed -E 's/[^A-Za-z0-9_.-]+/-/g; s/^-+//; s/-+$//')" + ref_tag="${ref_tag:0:128}" + { + echo "tags<> "$GITHUB_OUTPUT" - name: Build and push uses: docker/build-push-action@v7 From 0e761816229ad49dcb24202a95c656255a16541d Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:50:05 +0900 Subject: [PATCH 130/162] proxy: allow redis-only without secondary seeds --- cmd/redis-proxy/main.go | 12 +++++++++--- cmd/redis-proxy/main_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cmd/redis-proxy/main.go b/cmd/redis-proxy/main.go index 4ad067987..b0c76af86 100644 --- a/cmd/redis-proxy/main.go +++ b/cmd/redis-proxy/main.go @@ -123,21 +123,27 @@ func newBackends(cfg proxy.ProxyConfig, primaryPoolSize, elasticKVPoolSize int, secondaryOpts.PoolSize = elasticKVPoolSize secondarySeeds := parseAddrList(cfg.SecondaryAddr) - if len(secondarySeeds) == 0 { - return nil, nil, fmt.Errorf("at least one secondary address is required") - } switch cfg.Mode { case proxy.ModeElasticKVPrimary: + if len(secondarySeeds) == 0 { + return nil, nil, fmt.Errorf("at least one secondary address is required") + } return proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger), proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts), nil case proxy.ModeElasticKVOnly: + if len(secondarySeeds) == 0 { + return nil, nil, fmt.Errorf("at least one secondary address is required") + } return proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger), proxy.NewNoopBackend("redis"), nil case proxy.ModeRedisOnly: return proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts), proxy.NewNoopBackend("elastickv"), nil case proxy.ModeDualWrite, proxy.ModeDualWriteShadow: + if len(secondarySeeds) == 0 { + return nil, nil, fmt.Errorf("at least one secondary address is required") + } return proxy.NewRedisBackendWithOptions(cfg.PrimaryAddr, "redis", primaryOpts), proxy.NewLeaderAwareRedisBackend(secondarySeeds, "elastickv", secondaryOpts, logger), nil default: diff --git a/cmd/redis-proxy/main_test.go b/cmd/redis-proxy/main_test.go index 29e9a47b7..b10cffa66 100644 --- a/cmd/redis-proxy/main_test.go +++ b/cmd/redis-proxy/main_test.go @@ -18,6 +18,33 @@ func TestParseRuntimeOptionsRejectsNegativeSecondaryConcurrency(t *testing.T) { assert.Contains(t, err.Error(), "secondary-script-concurrency") } +func TestNewBackendsAllowsRedisOnlyWithoutSecondarySeeds(t *testing.T) { + cfg := proxy.DefaultConfig() + cfg.Mode = proxy.ModeRedisOnly + cfg.PrimaryAddr = "127.0.0.1:6379" + cfg.SecondaryAddr = "" + + primary, secondary, err := newBackends(cfg, 2, 2, nil) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, primary.Close()) + require.NoError(t, secondary.Close()) + }) + assert.Equal(t, "redis", primary.Name()) + assert.Equal(t, "elastickv", secondary.Name()) +} + +func TestNewBackendsRejectsDualWriteWithoutSecondarySeeds(t *testing.T) { + cfg := proxy.DefaultConfig() + cfg.Mode = proxy.ModeDualWrite + cfg.PrimaryAddr = "127.0.0.1:6379" + cfg.SecondaryAddr = "" + + _, _, err := newBackends(cfg, 2, 2, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "secondary address") +} + func TestDeriveSecondaryConcurrency(t *testing.T) { tests := []struct { name string From ab8244766a20c781c406ee36088983a0fc3209cc Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 07:59:45 +0900 Subject: [PATCH 131/162] ci: avoid failing lint on reviewdog API errors --- .github/workflows/golangci-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 458931293..d4a3c6d1b 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -28,4 +28,4 @@ jobs: filter_mode: nofilter reporter: github-pr-review cache: false - fail_on_error: true + fail_on_error: false From 5189172e19c39d224b8c2dc10bd2bdd836d212b1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 08:04:33 +0900 Subject: [PATCH 132/162] ci: keep lint required when reviewdog is unavailable --- .github/workflows/golangci-lint.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index d4a3c6d1b..e0cbccc4e 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -18,7 +18,18 @@ jobs: steps: - name: Check out code into the Go module directory uses: actions/checkout@v7 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version: "1.x" + - name: Install golangci-lint + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b "$(go env GOPATH)/bin" v2.9.0 + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + - name: Run golangci-lint + run: golangci-lint run --config=.golangci.yaml - name: golangci-lint + continue-on-error: true uses: reviewdog/action-golangci-lint@v2 with: github_token: ${{ secrets.GITHUB_TOKEN }} From 319b442c2ee96c19fa83a2a63d57db951e9013a8 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 08:36:41 +0900 Subject: [PATCH 133/162] migration: persist applied index on imports --- kv/fsm_migration_import.go | 11 +++--- kv/fsm_migration_import_test.go | 37 ++++++++++++++++++ store/lsm_migration.go | 25 +++++++++++++ store/lsm_store_applied_index_test.go | 54 +++++++++++++++++++++++++++ store/store.go | 15 +++++--- 5 files changed, 132 insertions(+), 10 deletions(-) diff --git a/kv/fsm_migration_import.go b/kv/fsm_migration_import.go index 91dca3070..d146b6fea 100644 --- a/kv/fsm_migration_import.go +++ b/kv/fsm_migration_import.go @@ -35,11 +35,12 @@ func (f *kvFSM) applyMigrationImport(ctx context.Context, data []byte) any { return errors.WithStack(err) } result, err := f.store.ImportVersionsRaft(ctx, store.ImportVersionsOptions{ - JobID: req.GetJobId(), - BracketID: req.GetBracketId(), - BatchSeq: req.GetBatchSeq(), - Cursor: req.GetCursor(), - Versions: migrationStoreVersionsFromProto(req.GetJobId(), req.GetVersions()), + JobID: req.GetJobId(), + AppliedIndex: f.pendingApplyIdx, + BracketID: req.GetBracketId(), + BatchSeq: req.GetBatchSeq(), + Cursor: req.GetCursor(), + Versions: migrationStoreVersionsFromProto(req.GetJobId(), req.GetVersions()), }) if err != nil { return errors.WithStack(err) diff --git a/kv/fsm_migration_import_test.go b/kv/fsm_migration_import_test.go index 589b6e1b1..dafd726d9 100644 --- a/kv/fsm_migration_import_test.go +++ b/kv/fsm_migration_import_test.go @@ -73,3 +73,40 @@ func TestApplyMigrationImportWritesOnlyStagedKeys(t *testing.T) { _, err = st.GetAt(ctx, []byte("user|k"), 10) require.ErrorIs(t, err, store.ErrKeyNotFound) } + +type captureMigrationImportStore struct { + store.MVCCStore + opts store.ImportVersionsOptions +} + +func (s *captureMigrationImportStore) ImportVersionsRaft(_ context.Context, opts store.ImportVersionsOptions) (store.ImportVersionsResult, error) { + s.opts = opts + return store.ImportVersionsResult{AckedCursor: opts.Cursor, MaxImportedTS: 10}, nil +} + +func TestApplyMigrationImportThreadsPendingApplyIndex(t *testing.T) { + t.Parallel() + + capturing := &captureMigrationImportStore{} + fsm := &kvFSM{store: capturing, pendingApplyIdx: 1234} + req := &pb.ImportRangeVersionsRequest{ + JobId: 9, + BracketId: 1, + BatchSeq: 1, + Cursor: []byte("cursor"), + Versions: []*pb.MVCCVersion{ + {Key: []byte("user|k"), CommitTs: 10, Value: []byte("v")}, + }, + } + data, err := proto.Marshal(req) + require.NoError(t, err) + + applied := fsm.applyMigrationImport(context.Background(), data) + result, ok := applied.(store.ImportVersionsResult) + require.True(t, ok, "got %T: %v", applied, applied) + require.Equal(t, []byte("cursor"), result.AckedCursor) + require.Equal(t, uint64(1234), capturing.opts.AppliedIndex) + require.Equal(t, uint64(9), capturing.opts.JobID) + require.Len(t, capturing.opts.Versions, 1) + require.Equal(t, distribution.MigrationStagedDataKey(9, []byte("user|k")), capturing.opts.Versions[0].Key) +} diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 96ad1e082..b1e6182f2 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -322,6 +322,9 @@ func (s *pebbleStore) importVersionsWithOpts(ctx context.Context, opts ImportVer return ImportVersionsResult{}, err } if duplicate { + if err := s.commitPebbleImportAppliedIndex(opts.AppliedIndex, writeOpts); err != nil { + return ImportVersionsResult{}, err + } return ImportVersionsResult{AckedCursor: ackedCursor, Duplicate: true}, nil } @@ -339,6 +342,18 @@ func (s *pebbleStore) importVersionsWithOpts(ctx context.Context, opts ImportVer return ImportVersionsResult{AckedCursor: bytes.Clone(opts.Cursor), MaxImportedTS: batchMax}, nil } +func (s *pebbleStore) commitPebbleImportAppliedIndex(appliedIndex uint64, writeOpts *pebble.WriteOptions) error { + if appliedIndex == 0 { + return nil + } + batch := s.db.NewBatch() + defer batch.Close() + if err := setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex); err != nil { + return err + } + return errors.WithStack(batch.Commit(writeOpts)) +} + func (s *pebbleStore) validatePebbleImportBatch(opts ImportVersionsOptions) (bool, []byte, error) { existing, hasExisting, err := s.readMigrationImportAck(opts.JobID, opts.BracketID) if err != nil { @@ -376,6 +391,9 @@ func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchM return err } defer unlock() + if err := stagePebbleAppliedIndex(batch, opts.AppliedIndex); err != nil { + return err + } if err := batch.Commit(writeOpts); err != nil { return errors.WithStack(err) } @@ -385,6 +403,13 @@ func (s *pebbleStore) commitPebbleImportBatch(opts ImportVersionsOptions, batchM return nil } +func stagePebbleAppliedIndex(batch *pebble.Batch, appliedIndex uint64) error { + if appliedIndex == 0 { + return nil + } + return setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex) +} + func (s *pebbleStore) stageMigrationImportAck(batch *pebble.Batch, jobID, bracketID uint64, ack migrationImportAck) error { acks, err := s.readMigrationImportAcks() if err != nil { diff --git a/store/lsm_store_applied_index_test.go b/store/lsm_store_applied_index_test.go index 4d4ecd4da..dee62b6fb 100644 --- a/store/lsm_store_applied_index_test.go +++ b/store/lsm_store_applied_index_test.go @@ -155,6 +155,60 @@ func TestPromoteVersions_BundlesMetaAppliedIndex(t *testing.T) { require.Equal(t, retryEntryIdx, got) } +func TestImportVersionsRaft_BundlesMetaAppliedIndex(t *testing.T) { + ctx := context.Background() + st := newApplyIndexPebbleStore(t) + ps := pebbleStoreApplied(t, st) + + const entryIdx uint64 = 123 + result, err := ps.ImportVersionsRaft(ctx, ImportVersionsOptions{ + JobID: 9, + AppliedIndex: entryIdx, + BracketID: 1, + BatchSeq: 1, + Cursor: []byte("c1"), + Versions: []MVCCVersion{ + {Key: []byte("stage|k"), CommitTS: 100, Value: []byte("v100")}, + }, + }) + require.NoError(t, err) + require.Equal(t, []byte("c1"), result.AckedCursor) + require.Equal(t, uint64(100), result.MaxImportedTS) + + got, present, err := ps.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present, "ImportVersionsRaft must persist metaAppliedIndex") + require.Equal(t, entryIdx, got) + + val, err := ps.GetAt(ctx, []byte("stage|k"), 100) + require.NoError(t, err) + require.Equal(t, []byte("v100"), val) + + const retryEntryIdx uint64 = 124 + duplicate, err := ps.ImportVersionsRaft(ctx, ImportVersionsOptions{ + JobID: 9, + AppliedIndex: retryEntryIdx, + BracketID: 1, + BatchSeq: 1, + Cursor: []byte("ignored"), + Versions: []MVCCVersion{ + {Key: []byte("stage|k"), CommitTS: 100, Value: []byte("changed")}, + }, + }) + require.NoError(t, err) + require.True(t, duplicate.Duplicate) + require.Equal(t, []byte("c1"), duplicate.AckedCursor) + + got, present, err = ps.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present, "duplicate ImportVersionsRaft retry must still advance metaAppliedIndex") + require.Equal(t, retryEntryIdx, got) + + val, err = ps.GetAt(ctx, []byte("stage|k"), 100) + require.NoError(t, err) + require.Equal(t, []byte("v100"), val, "duplicate import must not rewrite the acknowledged batch") +} + // TestApplyMutationsRaftAt_ZeroIndexLeavesMetaKey covers the // appliedIndex==0 escape hatch — callers without a raft entry index // (test fakes, legacy ApplyMutationsRaft) MUST NOT bump the meta key. diff --git a/store/store.go b/store/store.go index 41b660bbf..ecc7109d8 100644 --- a/store/store.go +++ b/store/store.go @@ -107,11 +107,14 @@ type ExportVersionsResult struct { // ImportVersionsOptions applies one idempotent migration-import batch. type ImportVersionsOptions struct { - JobID uint64 - BracketID uint64 - BatchSeq uint64 - Versions []MVCCVersion - Cursor []byte + JobID uint64 + // AppliedIndex is the optional Raft entry index to bundle with Pebble + // import batches as metaAppliedIndex. Zero leaves the meta key unchanged. + AppliedIndex uint64 + BracketID uint64 + BatchSeq uint64 + Versions []MVCCVersion + Cursor []byte } // ImportVersionsResult reports the cursor durably acknowledged by the target. @@ -319,6 +322,8 @@ type MVCCStore interface { ImportVersions(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) // ImportVersionsRaft is the raft-apply variant of ImportVersions. It // preserves the same idempotency contract while using the FSM write path. + // When opts.AppliedIndex is non-zero, the implementation must durably + // bundle metaAppliedIndex with the import batch. ImportVersionsRaft(ctx context.Context, opts ImportVersionsOptions) (ImportVersionsResult, error) // MigrationHLCFloor returns the full-HLC target-local migration floor // persisted by ImportVersions for jobID. From 2c29191e72b60f258bfecb7cd1352bfdf4a348c1 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 08:56:35 +0900 Subject: [PATCH 134/162] kv: preserve broad scans with staged routes --- kv/shard_store.go | 97 ++++++++++++++++++++++++++++-------------- kv/shard_store_test.go | 49 +++++++++++++++++++++ 2 files changed, 115 insertions(+), 31 deletions(-) diff --git a/kv/shard_store.go b/kv/shard_store.go index a9086165b..d87de47e0 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -397,8 +397,8 @@ func (s *ShardStore) scanExplicitGroupAtWithReadFence(ctx context.Context, group } routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) - if !clampToRoutes && !routeFilterPresent && !dedupeByKey { - routes = dedupeRepeatedRawScanRoutes(routes) + if !clampToRoutes && !routeFilterPresent { + routes, dedupeByKey = prepareUnclampedRawScanRoutes(routes, dedupeByKey) } return s.scanExplicitGroupRoutesAtWithReadFence(ctx, routes, start, end, limit, ts, reverse, readRouteVersion, routeStart, routeEnd, clampToRoutes, dedupeByKey) } @@ -407,7 +407,7 @@ func (s *ShardStore) scanExplicitGroupRoutesAtWithReadFence(ctx context.Context, out := make([]*store.KVPair, 0) for i := 0; i < len(routes); i++ { route := routes[i] - if reverse { + if reverse && clampToRoutes { route = routes[len(routes)-1-i] } scanStart := start @@ -540,29 +540,18 @@ type repeatedRawScanRouteKey struct { groupID uint64 staged bool migrationJobID uint64 + routeStart string + routeEnd string } func dedupeRepeatedRawScanRoutes(routes []distribution.Route) []distribution.Route { if len(routes) <= 1 { return routes } - stagedGroups := make(map[uint64]struct{}) - for _, route := range routes { - if routeHasStagedVisibility(route) { - stagedGroups[route.GroupID] = struct{}{} - } - } out := make([]distribution.Route, 0, len(routes)) seen := make(map[repeatedRawScanRouteKey]struct{}, len(routes)) for _, route := range routes { - if _, hasStaged := stagedGroups[route.GroupID]; hasStaged && !routeHasStagedVisibility(route) { - continue - } - key := repeatedRawScanRouteKey{groupID: route.GroupID} - if routeHasStagedVisibility(route) { - key.staged = true - key.migrationJobID = route.MigrationJobID - } + key := repeatedRawScanRouteDedupeKey(route) if _, ok := seen[key]; ok { continue } @@ -572,6 +561,44 @@ func dedupeRepeatedRawScanRoutes(routes []distribution.Route) []distribution.Rou return out } +func repeatedRawScanRouteDedupeKey(route distribution.Route) repeatedRawScanRouteKey { + key := repeatedRawScanRouteKey{groupID: route.GroupID} + if routeHasStagedVisibility(route) { + key.staged = true + key.migrationJobID = route.MigrationJobID + key.routeStart = string(route.Start) + key.routeEnd = string(route.End) + } + return key +} + +func prepareUnclampedRawScanRoutes(routes []distribution.Route, dedupeByKey bool) ([]distribution.Route, bool) { + if routesContainStagedVisibility(routes) { + routes = dedupeRepeatedRawScanRoutes(routes) + return orderRawScanRoutesForStagedVisibility(routes), true + } + if !dedupeByKey { + routes = dedupeRepeatedRawScanRoutes(routes) + } + return routes, dedupeByKey +} + +func orderRawScanRoutesForStagedVisibility(routes []distribution.Route) []distribution.Route { + if !routesContainStagedVisibility(routes) { + return routes + } + out := make([]distribution.Route, 0, len(routes)) + staged := make([]distribution.Route, 0) + for _, route := range routes { + if routeHasStagedVisibility(route) { + staged = append(staged, route) + continue + } + out = append(out, route) + } + return append(out, staged...) +} + func routesContainStagedVisibility(routes []distribution.Route) bool { for _, route := range routes { if routeHasStagedVisibility(route) { @@ -735,8 +762,8 @@ func (s *ShardStore) scanRoutesAtWithReadFence(ctx context.Context, routes []dis out := make([]*store.KVPair, 0) routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) - if !clampToRoutes && !routeFilterPresent && !dedupeByKey { - routes = dedupeRepeatedRawScanRoutes(routes) + if !clampToRoutes && !routeFilterPresent { + routes, dedupeByKey = prepareUnclampedRawScanRoutes(routes, dedupeByKey) } for _, route := range routes { scanStart := start @@ -778,12 +805,13 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( out := make([]*store.KVPair, 0) routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) - if !clampToRoutes && !routeFilterPresent && !dedupeByKey { - routes = dedupeRepeatedRawScanRoutes(routes) + if !clampToRoutes && !routeFilterPresent { + routes, dedupeByKey = prepareUnclampedRawScanRoutes(routes, dedupeByKey) } - for i := len(routes) - 1; i >= 0; i-- { + for i := 0; i < len(routes); i++ { route := routes[i] if clampToRoutes { + route = routes[len(routes)-1-i] kvs, done, err := s.clampedReverseScanRouteAtWithReadFence(ctx, route, start, end, limit, len(out), ts, readRouteVersion, routeStart, routeEnd) if err != nil { return nil, err @@ -932,7 +960,7 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceRouteFilterPage( if err != nil { return nil, nil, errors.WithStack(err) } - return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID, markRouteGroup), markScanRouteGroup(kvs, route.GroupID, markRouteGroup), nil + return markScanRouteGroup(filterScanInternalKVs(kvs), route.GroupID, markRouteGroup), markScanRouteGroup(kvs, route.GroupID, markRouteGroup), nil } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { @@ -945,7 +973,7 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceRouteFilterPage( if err != nil { return nil, nil, err } - filtered := filterTxnInternalKVs(kvs) + filtered := filterScanInternalKVs(kvs) return markScanRouteGroup(filtered, route.GroupID, markRouteGroup), markScanRouteGroup(kvs, route.GroupID, markRouteGroup), nil } @@ -980,7 +1008,7 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceOnce( if err != nil { return nil, errors.WithStack(err) } - return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID, markRouteGroup), nil + return markScanRouteGroup(filterScanInternalKVs(kvs), route.GroupID, markRouteGroup), nil } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { @@ -995,7 +1023,7 @@ func (s *ShardStore) scanRouteAtDirectionWithReadFenceOnce( } // The leader's RawScanAt is expected to perform lock resolution and filtering // via ShardStore.ScanAt, so avoid N+1 proxy gets here. - return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID, markRouteGroup), nil + return markScanRouteGroup(filterScanInternalKVs(kvs), route.GroupID, markRouteGroup), nil } const routeFilteredScanBatchMin = 128 @@ -1122,7 +1150,7 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( if err != nil { return nil, limitReached, errors.WithStack(err) } - return markScanRouteGroup(filterTxnInternalKVs(kvs), route.GroupID, markRouteGroup), limitReached, nil + return markScanRouteGroup(filterScanInternalKVs(kvs), route.GroupID, markRouteGroup), limitReached, nil } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { @@ -1696,6 +1724,9 @@ func scanUserKey(kvp *store.KVPair) ([]byte, bool) { if kvp == nil || len(kvp.Key) == 0 { return nil, false } + if isMigrationStagedDataKey(kvp.Key) { + return nil, false + } if !isTxnInternalKey(kvp.Key) { return kvp.Key, true } @@ -1755,7 +1786,7 @@ func appendReplacingKVsByKey(out []*store.KVPair, kvs []*store.KVPair) []*store. func countNonInternalKVs(kvs []*store.KVPair) int { count := 0 for _, kvp := range kvs { - if kvp == nil || isTxnInternalKey(kvp.Key) { + if kvp == nil || isScanInternalKey(kvp.Key) { continue } count++ @@ -2084,7 +2115,7 @@ func (s *ShardStore) planScanLockFromLockKVP(ctx context.Context, plan *scanLock } func (s *ShardStore) planScanLockItem(ctx context.Context, g *ShardGroup, ts uint64, plan *scanLockPlan, kvp *store.KVPair) error { - if kvp == nil || isTxnInternalKey(kvp.Key) { + if kvp == nil || isScanInternalKey(kvp.Key) { plan.items = append(plan.items, scanItem{skip: true}) return nil } @@ -2365,7 +2396,7 @@ func (s *ShardStore) materializeScanLockResults(ctx context.Context, g *ShardGro return out, nil } -func filterTxnInternalKVs(kvs []*store.KVPair) []*store.KVPair { +func filterScanInternalKVs(kvs []*store.KVPair) []*store.KVPair { if len(kvs) == 0 { return kvs } @@ -2374,7 +2405,7 @@ func filterTxnInternalKVs(kvs []*store.KVPair) []*store.KVPair { if kvp == nil { continue } - if isTxnInternalKey(kvp.Key) { + if isScanInternalKey(kvp.Key) { continue } out = append(out, kvp) @@ -2382,6 +2413,10 @@ func filterTxnInternalKVs(kvs []*store.KVPair) []*store.KVPair { return out } +func isScanInternalKey(key []byte) bool { + return isTxnInternalKey(key) || isMigrationStagedDataKey(key) +} + func markScanRouteGroup(kvs []*store.KVPair, groupID uint64, mark bool) []*store.KVPair { if !mark || groupID == 0 { return kvs diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 64c01751d..dcc4a086b 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -352,6 +352,55 @@ func TestShardStoreScanAt_FiltersStagedShadowRowsFromLiveCandidates(t *testing.T require.Equal(t, []*store.KVPair{{Key: rawKey, Value: []byte("staged")}}, kvs) } +func TestShardStoreScanAt_PreservesNonStagedRoutesDuringBroadStagedVisibilityScan(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("t"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + }, + {RouteID: 3, Start: []byte("t"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive}, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live-b"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live-n"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("u"), []byte("live-u"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("n")), []byte("staged-n"), 20, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("o")), []byte("staged-o"), 20, 0)) + + kvs, err := st.ScanAt(ctx, nil, nil, 10, 30) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b"), Value: []byte("live-b")}, + {Key: []byte("n"), Value: []byte("staged-n")}, + {Key: []byte("o"), Value: []byte("staged-o")}, + {Key: []byte("u"), Value: []byte("live-u")}, + }, kvs) + + kvs, err = st.ReverseScanAt(ctx, nil, nil, 10, 30) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("u"), Value: []byte("live-u")}, + {Key: []byte("o"), Value: []byte("staged-o")}, + {Key: []byte("n"), Value: []byte("staged-n")}, + {Key: []byte("b"), Value: []byte("live-b")}, + }, kvs) +} + func TestShardStoreScanAt_RoutesS3BucketAuxiliaryStagedVisibility(t *testing.T) { t.Parallel() From aaa96da3768ca456018f447f9ca0427c96dc3b73 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 09:07:32 +0900 Subject: [PATCH 135/162] migration: gate import proposals during rollouts --- adapter/internal.go | 38 +++++++++++++++++++++++++++++- adapter/internal_migration_test.go | 33 ++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/adapter/internal.go b/adapter/internal.go index dd466144d..8dcf2bb18 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -37,6 +37,12 @@ func WithInternalMigrationProposer(proposer raftengine.Proposer) InternalOption } } +func WithInternalMigrationImportGate(gate func(context.Context) error) InternalOption { + return func(i *Internal) { + i.migrationImportGate = gate + } +} + func WithInternalMigrationPromoteGate(gate func(context.Context) error) InternalOption { return func(i *Internal) { i.migrationPromoteGate = gate @@ -55,6 +61,7 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c transactionManager: txm, clock: clock, relay: relay, + migrationImportGate: defaultMigrationImportGate, migrationPromoteGate: defaultMigrationPromoteGate, } for _, opt := range opts { @@ -71,6 +78,7 @@ type Internal struct { relay *RedisPubSubRelay store store.MVCCStore migrationProposer raftengine.Proposer + migrationImportGate func(context.Context) error migrationPromoteGate func(context.Context) error routeEngine *distribution.Engine @@ -200,6 +208,9 @@ func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeV if err := i.verifyInternalLeader(ctx); err != nil { return nil, err } + if err := i.verifyMigrationImportEnabled(ctx); err != nil { + return nil, err + } result, err := i.proposeMigrationImport(ctx, req) if err != nil { return nil, errors.WithStack(err) @@ -220,6 +231,16 @@ func validateImportRangeVersionsRequest(req *pb.ImportRangeVersionsRequest) erro return nil } +func (i *Internal) verifyMigrationImportEnabled(ctx context.Context) error { + if i.migrationImportGate == nil { + return nil + } + if err := i.migrationImportGate(ctx); err != nil { + return errors.WithStack(err) + } + return nil +} + func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteStagedVersionsRequest) (*pb.PromoteStagedVersionsResponse, error) { if req == nil { return nil, errors.WithStack(status.Error(codes.InvalidArgument, "promote staged versions request is nil")) @@ -275,6 +296,17 @@ func validatePromoteStagedVersionsRequest(req *pb.PromoteStagedVersionsRequest) return nil } +func defaultMigrationImportGate(context.Context) error { + if migrationImportOpcodeEnabledFromEnv() { + return nil + } + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration import opcode is disabled; enable after every voter is running a build that supports migration import")) +} + +func migrationImportOpcodeEnabledFromEnv() bool { + return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE") +} + func defaultMigrationPromoteGate(context.Context) error { if migrationPromoteOpcodeEnabledFromEnv() { return nil @@ -283,7 +315,11 @@ func defaultMigrationPromoteGate(context.Context) error { } func migrationPromoteOpcodeEnabledFromEnv() bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv("ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE"))) { + return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE") +} + +func envFlagEnabled(name string) bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(name))) { case "1", "true", "yes", "on": return true default: diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index a3344b75b..a83c82833 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -397,6 +397,7 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, WithInternalStore(st), WithInternalMigrationProposer(proposer), + WithInternalMigrationImportGate(func(context.Context) error { return nil }), ) resp, err := internal.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ @@ -424,6 +425,38 @@ func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { require.GreaterOrEqual(t, clock.Current(), uint64(30)) } +func TestInternalImportRangeVersionsRejectsWhenOpcodeGateClosed(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + clock := kv.NewHLC() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, clock), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, clock, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationImportGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration import disabled for test") + }), + ) + + resp, err := internal.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ + JobId: 7, + BracketId: 3, + BatchSeq: 1, + Cursor: []byte("cursor-1"), + Versions: []*pb.MVCCVersion{ + {Key: []byte("k"), CommitTs: 30, Value: []byte("v")}, + }, + }) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, uint64(0), proposer.calls) +} + func TestInternalImportRangeVersionsRejectsMissingIdentifiers(t *testing.T) { t.Parallel() From d34d103ff316e35e5006f79827d8ad65bd00a194 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 09:16:47 +0900 Subject: [PATCH 136/162] kv: reject raw prefix writes below floors --- kv/sharded_coordinator.go | 28 ++++++++++++++++++----- kv/sharded_coordinator_del_prefix_test.go | 16 +++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index a16d1e3e5..26e63cb1d 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1240,11 +1240,18 @@ func (c *ShardedCoordinator) rejectWriteTimestampFloorDelPrefixes(elems []*Elem[ if elem == nil { continue } - start, end := routePrefixRange(elem.Key) - for _, route := range c.engine.GetIntersectingRoutes(start, end) { - if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", elem.Key, start, end, commitTS, route.MinWriteTSExclusive) - } + if err := c.rejectWriteTimestampFloorDelPrefix(elem.Key, commitTS); err != nil { + return err + } + } + return nil +} + +func (c *ShardedCoordinator) rejectWriteTimestampFloorDelPrefix(prefix []byte, commitTS uint64) error { + start, end := routePrefixRange(prefix) + for _, route := range c.engine.GetIntersectingRoutes(start, end) { + if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { + return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", prefix, start, end, commitTS, route.MinWriteTSExclusive) } } return nil @@ -2342,7 +2349,16 @@ func (c *ShardedCoordinator) rejectWriteTimestampFloorMutations(muts []*pb.Mutat return nil } for _, mut := range muts { - if mut == nil || len(mut.Key) == 0 { + if mut == nil { + continue + } + if mut.GetOp() == pb.Op_DEL_PREFIX { + if err := c.rejectWriteTimestampFloorDelPrefix(mut.Key, commitTS); err != nil { + return err + } + continue + } + if len(mut.Key) == 0 { continue } if err := c.rejectWriteTimestampFloorPointKey(mut.Key, commitTS); err != nil { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index adebba3bc..1970379a9 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -172,6 +172,22 @@ func TestShardedCoordinatorRejectsDelPrefixAtMigrationTimestampFloor(t *testing. require.Empty(t, g1Txn.requests, "coordinator must reject before broadcasting a floor-violating prefix delete") } +func TestShardedCoordinatorRejectsRawDelPrefixMutationAtMigrationTimestampFloor(t *testing.T) { + t.Parallel() + + coord := NewShardedCoordinator(newMigrationFloorEngine(t, ^uint64(0)), map[uint64]*ShardGroup{ + 1: {Txn: &recordingTransactional{}}, + }, 1, NewHLC(), nil) + + for _, mut := range []*pb.Mutation{ + {Op: pb.Op_DEL_PREFIX, Key: []byte("z")}, + {Op: pb.Op_DEL_PREFIX, Key: nil}, + } { + err := coord.rejectWriteTimestampFloorMutations([]*pb.Mutation{mut}, 100) + require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + } +} + func TestShardedCoordinator_DelPrefixDoesNotAutoPinObservedRouteVersion(t *testing.T) { t.Parallel() From 78a040056b4492bcdbe9fcd333920fe0b0d42d95 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 09:31:55 +0900 Subject: [PATCH 137/162] migration: route partitioned exports by group --- adapter/internal.go | 46 ++++++++++++++++++++---------- adapter/internal_migration_test.go | 36 +++++++++++++++++++++++ main.go | 7 +++++ main_bootstrap_e2e_test.go | 6 ++-- 4 files changed, 78 insertions(+), 17 deletions(-) diff --git a/adapter/internal.go b/adapter/internal.go index 8dcf2bb18..7cfaaa5d6 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -55,6 +55,13 @@ func WithInternalRouteEngine(engine *distribution.Engine) InternalOption { } } +func WithInternalMigrationExportRouting(groupID uint64, resolver kv.PartitionResolver) InternalOption { + return func(i *Internal) { + i.migrationExportGroupID = groupID + i.migrationExportResolver = resolver + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ leader: leader, @@ -71,16 +78,18 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c } type Internal struct { - leader raftengine.LeaderView - transactionManager kv.Transactional - clock *kv.HLC - tsAllocator kv.TimestampAllocator - relay *RedisPubSubRelay - store store.MVCCStore - migrationProposer raftengine.Proposer - migrationImportGate func(context.Context) error - migrationPromoteGate func(context.Context) error - routeEngine *distribution.Engine + leader raftengine.LeaderView + transactionManager kv.Transactional + clock *kv.HLC + tsAllocator kv.TimestampAllocator + relay *RedisPubSubRelay + store store.MVCCStore + migrationProposer raftengine.Proposer + migrationImportGate func(context.Context) error + migrationPromoteGate func(context.Context) error + routeEngine *distribution.Engine + migrationExportGroupID uint64 + migrationExportResolver kv.PartitionResolver pb.UnimplementedInternalServer } @@ -172,7 +181,7 @@ func exportRangeVersionsRequestFullyUnbounded(req *pb.ExportRangeVersionsRequest } func (i *Internal) streamExportRangeVersions(req *pb.ExportRangeVersionsRequest, stream pb.Internal_ExportRangeVersionsServer) error { - opts := exportRangeVersionsOptions(req) + opts := i.exportRangeVersionsOptions(req) for { result, err := i.store.ExportVersions(stream.Context(), opts) if err != nil { @@ -410,7 +419,7 @@ func (i *Internal) proposeMigrationCommand(ctx context.Context, cmd []byte, labe return result.Response, nil } -func exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.ExportVersionsOptions { +func (i *Internal) exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.ExportVersionsOptions { chunkBytes := uint64(req.GetChunkBytes()) if chunkBytes == 0 { chunkBytes = defaultMigrationExportChunkBytes @@ -429,13 +438,13 @@ func exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.Export MaxBytes: chunkBytes, MaxScannedBytes: maxScannedBytes, KeyFamily: req.GetKeyFamily(), - AcceptKey: migrationExportFilter(req), + AcceptKey: i.migrationExportFilter(req), } return opts } -func migrationExportFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool { - routeFilter := kv.RouteKeyFilter(req.GetRouteStart(), req.GetRouteEnd()) +func (i *Internal) migrationExportFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool { + routeFilter := i.migrationExportRouteFilter(req) if migrationFamilyRequiresDecodedS3(req.GetKeyFamily()) { routeFilter = decodedS3BucketRouteFilter(req.GetKeyFamily(), req.GetRouteStart(), req.GetRouteEnd()) } @@ -452,6 +461,13 @@ func migrationExportFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool } } +func (i *Internal) migrationExportRouteFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool { + if i != nil && i.migrationExportGroupID != 0 && i.migrationExportResolver != nil { + return kv.RouteKeyFilterForGroup(req.GetRouteStart(), req.GetRouteEnd(), i.migrationExportGroupID, i.migrationExportResolver) + } + return kv.RouteKeyFilter(req.GetRouteStart(), req.GetRouteEnd()) +} + func migrationFamilyRequiresDecodedS3(family uint32) bool { return family == distribution.MigrationFamilyS3BucketMeta || family == distribution.MigrationFamilyS3BucketGeneration diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index a83c82833..c3d552dac 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -385,6 +385,42 @@ func TestInternalExportRangeVersionsPreservesS3BucketRawRouteMatches(t *testing. }, stream.responses[0].GetVersions()) } +func TestInternalExportRangeVersionsUsesPartitionResolverGroup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + resolver := NewSQSPartitionResolver(map[string][]uint64{ + "orders.fifo": {10, 11}, + }) + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationExportRouting(11, resolver), + ) + + p0 := sqsPartitionedMsgDataKey("orders.fifo", 0, 1, "msg-0") + p1 := sqsPartitionedMsgDataKey("orders.fifo", 1, 1, "msg-1") + unknown := sqsPartitionedMsgDataKey("unknown.fifo", 0, 1, "msg-unknown") + require.NoError(t, st.PutAt(ctx, p0, []byte("p0"), 10, 0)) + require.NoError(t, st.PutAt(ctx, p1, []byte("p1"), 10, 0)) + require.NoError(t, st.PutAt(ctx, unknown, []byte("unknown"), 10, 0)) + + stream := &captureExportRangeVersionsStream{ctx: ctx} + prefix := []byte(SqsPartitionedMsgDataPrefix) + err := internal.ExportRangeVersions(&pb.ExportRangeVersionsRequest{ + MaxCommitTs: 20, + KeyFamily: distribution.MigrationFamilySQSPartitionedMessageData, + RangeStart: prefix, + RangeEnd: testPrefixScanEnd(prefix), + MaxScannedBytes: 1 << 20, + }, stream) + require.NoError(t, err) + require.Len(t, stream.responses, 1) + require.Equal(t, []*pb.MVCCVersion{ + {Key: p1, CommitTs: 10, Value: []byte("p1"), KeyFamily: distribution.MigrationFamilySQSPartitionedMessageData}, + }, stream.responses[0].GetVersions()) +} + func TestInternalImportRangeVersionsAppliesStoreBatch(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 4a26e67c9..3142f76a7 100644 --- a/main.go +++ b/main.go @@ -2199,6 +2199,7 @@ func startRaftServers( forwardDeps adminForwardServerDeps, confChangeInterceptor internalraftadmin.MembershipChangeInterceptor, encWiring encryptionWriteWiring, + sqsPartitionResolver kv.PartitionResolver, ) error { forwardLogger := slog.Default().With(slog.String("component", "admin")) // extraOptsCap reserves slots for the unary + stream admin interceptor @@ -2237,6 +2238,7 @@ func startRaftServers( adapter.WithInternalStore(rt.store), adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), adapter.WithInternalRouteEngine(routeEngine), + adapter.WithInternalMigrationExportRouting(rt.spec.id, sqsPartitionResolver), )..., )) pb.RegisterDistributionServer(gs, distServer) @@ -2680,6 +2682,10 @@ func (r *runtimeServerRunner) startRaftTransport() error { buckets: newBucketsSource(r.s3Server), roles: r.roleStore, } + var sqsPartitionResolver kv.PartitionResolver + if r.sqsPartitionResolver != nil { + sqsPartitionResolver = r.sqsPartitionResolver + } if err := startRaftServers( r.ctx, r.lc, @@ -2700,6 +2706,7 @@ func (r *runtimeServerRunner) startRaftTransport() error { forwardDeps, r.encryptionConfChangeInterceptor, r.encWiring, + sqsPartitionResolver, ); err != nil { return r.startupFailure(err) } diff --git a/main_bootstrap_e2e_test.go b/main_bootstrap_e2e_test.go index f4483c137..98a7ddb74 100644 --- a/main_bootstrap_e2e_test.go +++ b/main_bootstrap_e2e_test.go @@ -705,7 +705,7 @@ func startRuntimeServersWithBoundListeners( if err := startBoundRedisServer(ctx, eg, listeners.redis, shardStore, coordinate, leaderRedis, redisAddr, relay); err != nil { return waitErrgroupAfterStartupFailure(cancel, eg, err) } - if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, routeEngine, relay, listeners.grpc); err != nil { + if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, routeEngine, relay, nil, listeners.grpc); err != nil { return waitErrgroupAfterStartupFailure(cancel, eg, err) } if err := startBoundDynamoDBServer(ctx, eg, listeners.dynamo, shardStore, coordinate); err != nil { @@ -735,7 +735,7 @@ func startRuntimeServersWithBoundMultiGroupListeners( return waitErrgroupAfterStartupFailure(cancel, eg, err) } for _, rt := range runtimes { - if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, routeEngine, relay, listeners.grpc[rt.spec.id]); err != nil { + if err := startBoundGRPCServer(ctx, eg, rt, shardStore, coordinate, distServer, routeEngine, relay, nil, listeners.grpc[rt.spec.id]); err != nil { return waitErrgroupAfterStartupFailure(cancel, eg, err) } } @@ -754,6 +754,7 @@ func startBoundGRPCServer( distServer *adapter.DistributionServer, routeEngine *distribution.Engine, relay *adapter.RedisPubSubRelay, + sqsPartitionResolver kv.PartitionResolver, listener net.Listener, ) error { if rt == nil || rt.engine == nil { @@ -776,6 +777,7 @@ func startBoundGRPCServer( adapter.WithInternalStore(rt.store), adapter.WithInternalMigrationProposer(rt.engine), adapter.WithInternalRouteEngine(routeEngine), + adapter.WithInternalMigrationExportRouting(rt.spec.id, sqsPartitionResolver), )) pb.RegisterDistributionServer(gs, distServer) rt.registerGRPC(gs) From 4851c6fb3c97b1df31a223bfe48652918e5553cd Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 09:52:22 +0900 Subject: [PATCH 138/162] kv: scan staged routes for physical limits --- kv/shard_store.go | 12 ++++++++---- kv/shard_store_test.go | 20 ++++++++++++++------ 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/kv/shard_store.go b/kv/shard_store.go index d87de47e0..4f9fa66d9 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -369,7 +369,8 @@ func (s *ShardStore) ScanAtPhysicalLimit(ctx context.Context, start []byte, end } routes, clampToRoutes := s.routesForScan(start, end) if routesContainStagedVisibility(routes) { - return nil, true, nil + kvs, err := s.ScanAt(ctx, start, end, visibleLimit, ts) + return kvs, false, err } if len(routes) != 1 || clampToRoutes { kvs, err := s.ScanAt(ctx, start, end, visibleLimit, ts) @@ -459,7 +460,8 @@ func (s *ShardStore) ReverseScanAtPhysicalLimit(ctx context.Context, start []byt } routes, clampToRoutes := s.routesForScan(start, end) if routesContainStagedVisibility(routes) { - return nil, true, nil + kvs, err := s.ReverseScanAt(ctx, start, end, visibleLimit, ts) + return kvs, false, err } if len(routes) != 1 || clampToRoutes { kvs, err := s.ReverseScanAt(ctx, start, end, visibleLimit, ts) @@ -1144,7 +1146,8 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( if engineForGroup(g) == nil { if routeHasStagedVisibility(route) { - return nil, true, nil + kvs, err := s.scanRouteLocal(ctx, g, route, start, end, visibleLimit, ts, reverse) + return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), false, err } kvs, limitReached, err := scanLocalPhysicalLimit(ctx, g.Store, start, end, visibleLimit, physicalLimit, ts, reverse) if err != nil { @@ -1155,7 +1158,8 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( if isLinearizableRaftLeader(ctx, engineForGroup(g)) { if routeHasStagedVisibility(route) { - return nil, true, nil + kvs, err := s.scanRouteAtLeader(ctx, g, route, start, end, visibleLimit, ts, reverse) + return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), false, err } kvs, limitReached, err := s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), limitReached, err diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index dcc4a086b..d94724e82 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -762,21 +762,29 @@ func TestShardStoreApplyMutations_ValidatesStagedWriteKeys(t *testing.T) { } } -func TestShardStorePhysicalLimitFailsClosedBeforeStagedVisibilityFallback(t *testing.T) { +func TestShardStorePhysicalLimitFallsBackToStagedVisibilityScan(t *testing.T) { t.Parallel() ctx := context.Background() - st, _ := newStagedVisibilityShardStore(t) + st, group := newStagedVisibilityShardStore(t) + require.NoError(t, group.Store.PutAt(ctx, []byte("b/live"), []byte("live"), 10, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b/staged")), []byte("staged"), 20, 0)) kvs, limitReached, err := st.ScanAtPhysicalLimit(ctx, []byte("b"), []byte("c"), 10, 10, 50) require.NoError(t, err) - require.True(t, limitReached) - require.Nil(t, kvs) + require.False(t, limitReached) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b/live"), Value: []byte("live")}, + {Key: []byte("b/staged"), Value: []byte("staged")}, + }, kvs) kvs, limitReached, err = st.ReverseScanAtPhysicalLimit(ctx, []byte("b"), []byte("c"), 10, 10, 50) require.NoError(t, err) - require.True(t, limitReached) - require.Nil(t, kvs) + require.False(t, limitReached) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b/staged"), Value: []byte("staged")}, + {Key: []byte("b/live"), Value: []byte("live")}, + }, kvs) } func TestShardStoreRejectsWritesAtMigrationTimestampFloor(t *testing.T) { From 2b7c3af03dc55d89262d0b23b6bfa484edbdf63a Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 10:16:27 +0900 Subject: [PATCH 139/162] migration: keep split capability gated --- kv/fsm.go | 9 +++++--- kv/fsm_migration_fence_test.go | 42 ++++++++++++++++++++++++++++++++++ main.go | 1 - 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 8c96e898b..cbbbb245a 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -828,14 +828,17 @@ func (f *kvFSM) verifyRouteWriteFloorForKey(key []byte, commitTS uint64) error { if !ok { return nil } + if err := f.verifyS3BucketAuxiliaryRouteWriteFloor(snap, key, commitTS); err != nil { + return err + } + if _, _, ok := s3BucketAuxiliaryRouteRange(key); ok { + return nil + } rkey := routeKey(key) floor, ok := snap.WriteFloorForKey(rkey) if ok && commitTS != 0 && commitTS <= floor { return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q routeKey %q", commitTS, floor, key, rkey) } - if err := f.verifyS3BucketAuxiliaryRouteWriteFloor(snap, key, commitTS); err != nil { - return err - } return nil } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 8235c57ba..de73c635a 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -1,6 +1,7 @@ package kv import ( + "bytes" "context" "testing" @@ -295,6 +296,47 @@ func TestFSMRejectsS3BucketAuxiliaryPointWriteBelowRouteFloor(t *testing.T) { require.ErrorIs(t, err, ErrRouteWriteBelowFloor) } +func TestFSMIgnoresRawRouteFloorForS3BucketAuxiliaryPointWrite(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const bucket = "bucket-a" + key := s3keys.BucketMetaKey(bucket) + auxStart, auxEnd, ok := s3BucketAuxiliaryRouteRange(key) + require.True(t, ok) + rawStart := []byte(s3keys.BucketMetaPrefix) + rawEnd := prefixScanEnd(rawStart) + require.Less(t, bytes.Compare(auxEnd, rawStart), 0) + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: auxStart, + End: auxEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: rawStart, + End: rawEnd, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + }) + fsm := newComposed1FSM(t, engine, 1) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 100) + require.NoError(t, err) + got, err := fsm.store.GetAt(ctx, key, 100) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) +} + func TestFSMRejectsS3BucketAuxiliaryPointWriteWithoutTargetReadinessProof(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 6bc6b412d..205f970f4 100644 --- a/main.go +++ b/main.go @@ -1623,7 +1623,6 @@ func prepareDistributionRuntimeServer( splitMigrationCapabilityProbeTimeout, nil, )), - adapter.WithSplitJobRunnerReady(), ) preparedServer, err := prepareRuntimeServerRunner(waitRotateOnStartup, in) if err != nil { From eea5dec556fbb74d895ee0f475352a450c4255b4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 10:31:25 +0900 Subject: [PATCH 140/162] adapter: pin sparse list fallback route bounds --- adapter/redis_lua_context.go | 7 +++++-- adapter/redis_lua_list_holes_test.go | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 9eda2ad57..fb7a757a8 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -2287,8 +2287,7 @@ func (c *luaScriptContext) allowExactListScanFallback(startKey, endKey []byte, s } func (c *luaScriptContext) scanListItemWindowExact(key []byte, meta store.ListMeta, startKey, endKey []byte, left bool) (luaLazyListBoundaryItem, bool, error) { - routeStart := append([]byte(nil), startKey...) - routeEnd := append([]byte(nil), endKey...) + routeStart, routeEnd := luaListRouteScanBounds(key) for { kvs, err := c.scanExactListPage(startKey, endKey, routeStart, routeEnd, left) if err != nil { @@ -2308,6 +2307,10 @@ func (c *luaScriptContext) scanListItemWindowExact(key []byte, meta store.ListMe } } +func luaListRouteScanBounds(key []byte) ([]byte, []byte) { + return append([]byte(nil), key...), prefixScanEnd(key) +} + func (c *luaScriptContext) scanExactListPage(startKey, endKey, routeStart, routeEnd []byte, left bool) ([]*store.KVPair, error) { var ( kvs []*store.KVPair diff --git a/adapter/redis_lua_list_holes_test.go b/adapter/redis_lua_list_holes_test.go index 4483540d4..fd4f215d2 100644 --- a/adapter/redis_lua_list_holes_test.go +++ b/adapter/redis_lua_list_holes_test.go @@ -354,8 +354,8 @@ func TestRedisLua_ListExactFallbackPinsRouteBounds(t *testing.T) { st := &routePinnedExactScanStore{ MVCCStore: store.NewMVCCStore(), t: t, - wantRouteStart: startKey, - wantRouteEnd: endKey, + wantRouteStart: src, + wantRouteEnd: prefixScanEnd(src), pages: [][]*store.KVPair{ {{Key: listItemKey(collider, 0), Value: []byte("other-list-job")}}, {{Key: listItemKey(src, 1), Value: []byte("job-1")}}, From 1b70e3909cb7cc195721545b58ccedfd98d887e6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 11:21:14 +0900 Subject: [PATCH 141/162] redis: fix Lua XADD and blocking move replay --- adapter/redis_lua_compat_test.go | 39 ++++++++++++++ adapter/redis_lua_context.go | 24 +++++++-- proxy/blocking.go | 90 +++++++++++++++++++++++++++----- proxy/proxy_test.go | 69 ++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 15 deletions(-) diff --git a/adapter/redis_lua_compat_test.go b/adapter/redis_lua_compat_test.go index e22234762..547142eee 100644 --- a/adapter/redis_lua_compat_test.go +++ b/adapter/redis_lua_compat_test.go @@ -271,6 +271,45 @@ return redis.call("XADD", KEYS[1], "MAXLEN", "~", 10, "*", "event", "new") require.Equal(t, map[string]any{"event": "new"}, events[0].Values) } +func TestRedis_LuaXAddRecreatesTTLExpiredHash(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const ( + stream = "bull:test:events-expired-hash" + ttl = 80 * time.Millisecond + ) + require.NoError(t, rdb.HSet(ctx, stream, "event", "old").Err()) + require.NoError(t, rdb.PExpire(ctx, stream, ttl).Err()) + eventuallyExpired(t, ttl, func() bool { + hlen, err := rdb.HLen(ctx, stream).Result() + return err == nil && hlen == 0 + }, "hash must be logically expired before Lua XADD recreates it") + + id, err := rdb.Eval(ctx, ` +return redis.call("XADD", KEYS[1], "MAXLEN", "~", 10, "*", "event", "new") +`, []string{stream}).Text() + require.NoError(t, err) + require.NotEmpty(t, id) + + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(1), xlen) + ttlAfter, err := rdb.TTL(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, time.Duration(-1), ttlAfter) + + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, id, events[0].ID) + require.Equal(t, map[string]any{"event": "new"}, events[0].Values) +} + func TestRedis_LuaXAddHonorsScriptLocalStringType(t *testing.T) { nodes, _, _ := createNode(t, 3) defer shutdown(nodes) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index fb7a757a8..eae9d8fbf 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -3417,12 +3417,30 @@ func (c *luaScriptContext) newLuaStreamDelta(key []byte) (*luaStreamDeltaState, func (c *luaScriptContext) canUseLuaStreamDelta(key []byte, typ redisValueType, metaFound bool, legacyCleanup []*kv.Elem[kv.OP]) (bool, error) { hasPhysicalStream := metaFound || len(legacyCleanup) != 0 + var ( + expired bool + expiredChecked bool + ) + if typ == redisTypeNone { + var err error + expired, err = c.server.hasExpired(c.scriptCtx(), key, c.startTS, true) + if err != nil { + return false, err + } + expiredChecked = true + if expired { + return false, nil + } + } if !hasPhysicalStream { return true, nil } - expired, err := c.server.hasExpired(c.scriptCtx(), key, c.startTS, true) - if err != nil { - return false, err + if !expiredChecked { + var err error + expired, err = c.server.hasExpired(c.scriptCtx(), key, c.startTS, true) + if err != nil { + return false, err + } } if expired || typ == redisTypeNone { return false, nil diff --git a/proxy/blocking.go b/proxy/blocking.go index 772c011f2..a73a5704e 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -10,7 +10,23 @@ import ( "github.com/redis/go-redis/v9" ) -const blockingMultiPopMinArgs = 2 +const ( + blockingMultiPopMinArgs = 2 + blockingListMoveMinArgs = 3 + blockingBLMoveArgs = 6 + blockingListMoveReplayKeyCount = int64(2) +) + +const blockingListMoveReplayScript = ` +local removed = redis.call("LREM", KEYS[1], tonumber(ARGV[1]), ARGV[3]) +if removed == 0 then + return 0 +end +if ARGV[2] == "LEFT" then + return redis.call("LPUSH", KEYS[2], ARGV[3]) +end +return redis.call("RPUSH", KEYS[2], ARGV[3]) +` type blockingTimeoutBackend interface { DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd @@ -54,23 +70,18 @@ func parseBlockingMillisecondsArg(raw []byte) time.Duration { return time.Duration(millis) * time.Millisecond } -func blockingReplayCommand(cmd string, _ [][]byte, resp any) (string, []any, bool) { +func blockingReplayCommand(cmd string, args [][]byte, resp any) (string, []any, bool) { switch strings.ToUpper(cmd) { case "BLPOP": return blockingListPopReplay(1, resp) case "BRPOP": return blockingListPopReplay(-1, resp) + case "BRPOPLPUSH": + return blockingListMoveReplay(args, resp, -1, "LEFT") + case "BLMOVE": + return blockingBLMoveReplay(args, resp) case "BZPOPMIN", "BZPOPMAX": - parts, ok := redisArray(resp) - if !ok || len(parts) < 2 { - return "", nil, false - } - key, keyOK := redisArg(parts[0]) - member, memberOK := redisArg(parts[1]) - if !keyOK || !memberOK { - return "", nil, false - } - return "ZREM", []any{[]byte("ZREM"), key, member}, true + return blockingZSetPopReplay(resp) default: return "", nil, false } @@ -89,6 +100,61 @@ func blockingListPopReplay(count int64, resp any) (string, []any, bool) { return "LREM", []any{[]byte("LREM"), key, count, value}, true } +func blockingZSetPopReplay(resp any) (string, []any, bool) { + parts, ok := redisArray(resp) + if !ok || len(parts) < blockingMultiPopMinArgs { + return "", nil, false + } + key, keyOK := redisArg(parts[0]) + member, memberOK := redisArg(parts[1]) + if !keyOK || !memberOK { + return "", nil, false + } + return "ZREM", []any{[]byte("ZREM"), key, member}, true +} + +func blockingBLMoveReplay(args [][]byte, resp any) (string, []any, bool) { + if len(args) < blockingBLMoveArgs { + return "", nil, false + } + var count int64 + switch strings.ToUpper(string(args[3])) { + case "LEFT": + count = 1 + case "RIGHT": + count = -1 + default: + return "", nil, false + } + to := strings.ToUpper(string(args[4])) + if to != "LEFT" && to != "RIGHT" { + return "", nil, false + } + return blockingListMoveReplay(args, resp, count, to) +} + +func blockingListMoveReplay(args [][]byte, resp any, count int64, to string) (string, []any, bool) { + if len(args) < blockingListMoveMinArgs { + return "", nil, false + } + value, ok := redisArg(resp) + if !ok { + return "", nil, false + } + source := append([]byte(nil), args[1]...) + destination := append([]byte(nil), args[2]...) + return "EVAL", []any{ + []byte("EVAL"), + blockingListMoveReplayScript, + blockingListMoveReplayKeyCount, + source, + destination, + count, + []byte(to), + value, + }, true +} + func redisArray(v any) ([]any, bool) { switch x := v.(type) { case []any: diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index f092a492d..d0d1e5286 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -717,6 +717,75 @@ func TestDualWriter_Blocking_ReplaysListPopAsLRem(t *testing.T) { } } +func TestDualWriter_Blocking_ReplaysListMoveAsEval(t *testing.T) { + tests := []struct { + name string + cmd string + args [][]byte + resp any + want []any + }{ + { + name: "BRPOPLPUSH removes source tail and pushes destination head", + cmd: "BRPOPLPUSH", + args: [][]byte{[]byte("BRPOPLPUSH"), []byte("source"), []byte("dest"), []byte("5")}, + resp: []byte("job-1"), + want: []any{ + []byte("EVAL"), + blockingListMoveReplayScript, + int64(2), + []byte("source"), + []byte("dest"), + int64(-1), + []byte("LEFT"), + []byte("job-1"), + }, + }, + { + name: "BLMOVE mirrors requested source and destination sides", + cmd: "BLMOVE", + args: [][]byte{ + []byte("BLMOVE"), []byte("source"), []byte("dest"), + []byte("LEFT"), []byte("RIGHT"), []byte("5"), + }, + resp: "job-2", + want: []any{ + []byte("EVAL"), + blockingListMoveReplayScript, + int64(2), + []byte("source"), + []byte("dest"), + int64(1), + []byte("RIGHT"), + []byte("job-2"), + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnValue: tc.resp, + } + secondary := newMockBackend("secondary") + + metrics := newTestMetrics() + cfg := ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: 10 * time.Second} + d := NewDualWriter(primary, secondary, cfg, metrics, newTestSentry(), testLogger) + + resp, err := d.Blocking(context.Background(), tc.cmd, tc.args) + assert.NoError(t, err) + assert.Equal(t, tc.resp, resp) + d.Close() + + assert.Equal(t, [][]any{tc.want}, secondary.Calls()) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) + assert.InDelta(t, 1, testutil.ToFloat64(metrics.CommandTotal.WithLabelValues("EVAL", "secondary", "ok")), 0.001) + }) + } +} + func TestDualWriter_Blocking_XReadDoesNotUseWriteSemaphore(t *testing.T) { primary := &timeoutCapturingBackend{name: "primary", returnValue: []any{}} secondary := newMockBackend("secondary") From 8df8acf8ba2edb3d3843f7b99ada494d30d76f10 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 11:55:33 +0900 Subject: [PATCH 142/162] proxy: replay blocking multipop writes --- proxy/blocking.go | 124 ++++++++++++++++++++++++++++++++++++++++---- proxy/proxy_test.go | 67 ++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 9 deletions(-) diff --git a/proxy/blocking.go b/proxy/blocking.go index a73a5704e..eab445fa1 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -1,6 +1,7 @@ package proxy import ( + "bytes" "context" "fmt" "strconv" @@ -12,11 +13,20 @@ import ( const ( blockingMultiPopMinArgs = 2 + blockingBLMPopMinArgs = 5 + blockingBLMPopNumKeysArgIndex = 2 + blockingBLMPopFirstKeyArgIndex = 3 blockingListMoveMinArgs = 3 blockingBLMoveArgs = 6 + blockingListPopReplayKeyCount = int64(1) blockingListMoveReplayKeyCount = int64(2) ) +const ( + blockingListSideLeft = "LEFT" + blockingListSideRight = "RIGHT" +) + const blockingListMoveReplayScript = ` local removed = redis.call("LREM", KEYS[1], tonumber(ARGV[1]), ARGV[3]) if removed == 0 then @@ -28,6 +38,17 @@ end return redis.call("RPUSH", KEYS[2], ARGV[3]) ` +const blockingListMultiPopReplayScript = ` +local removed = 0 +local count = tonumber(ARGV[1]) +for i = 2, #ARGV do + if redis.call("LREM", KEYS[1], count, ARGV[i]) > 0 then + removed = removed + 1 + end +end +return removed +` + type blockingTimeoutBackend interface { DoWithTimeout(ctx context.Context, timeout time.Duration, args ...any) *redis.Cmd } @@ -77,9 +98,11 @@ func blockingReplayCommand(cmd string, args [][]byte, resp any) (string, []any, case "BRPOP": return blockingListPopReplay(-1, resp) case "BRPOPLPUSH": - return blockingListMoveReplay(args, resp, -1, "LEFT") + return blockingListMoveReplay(args, resp, -1, blockingListSideLeft) case "BLMOVE": return blockingBLMoveReplay(args, resp) + case "BLMPOP": + return blockingBLMPopReplay(args, resp) case "BZPOPMIN", "BZPOPMAX": return blockingZSetPopReplay(resp) default: @@ -117,22 +140,96 @@ func blockingBLMoveReplay(args [][]byte, resp any) (string, []any, bool) { if len(args) < blockingBLMoveArgs { return "", nil, false } - var count int64 - switch strings.ToUpper(string(args[3])) { - case "LEFT": - count = 1 - case "RIGHT": - count = -1 - default: + count, ok := blockingListPopCount(args[3]) + if !ok { return "", nil, false } to := strings.ToUpper(string(args[4])) - if to != "LEFT" && to != "RIGHT" { + if to != blockingListSideLeft && to != blockingListSideRight { return "", nil, false } return blockingListMoveReplay(args, resp, count, to) } +func blockingBLMPopReplay(args [][]byte, resp any) (string, []any, bool) { + numKeys, count, ok := parseBlockingBLMPopArgs(args) + if !ok { + return "", nil, false + } + key, values, ok := blockingBLMPopResponse(resp) + if !ok || !blockingBLMPopKeyListed(args, numKeys, key) { + return "", nil, false + } + + replay := []any{ + []byte("EVAL"), + blockingListMultiPopReplayScript, + blockingListPopReplayKeyCount, + key, + count, + } + for _, value := range values { + arg, ok := redisArg(value) + if !ok { + return "", nil, false + } + replay = append(replay, arg) + } + return "EVAL", replay, true +} + +func parseBlockingBLMPopArgs(args [][]byte) (int, int64, bool) { + if len(args) < blockingBLMPopMinArgs { + return 0, 0, false + } + numKeys, err := strconv.Atoi(string(args[blockingBLMPopNumKeysArgIndex])) + if err != nil || numKeys <= 0 { + return 0, 0, false + } + directionArgIndex := blockingBLMPopFirstKeyArgIndex + numKeys + if directionArgIndex >= len(args) { + return 0, 0, false + } + count, ok := blockingListPopCount(args[directionArgIndex]) + return numKeys, count, ok +} + +func blockingBLMPopResponse(resp any) ([]byte, []any, bool) { + parts, ok := redisArray(resp) + if !ok || len(parts) != blockingMultiPopMinArgs { + return nil, nil, false + } + key, ok := redisArgBytes(parts[0]) + if !ok { + return nil, nil, false + } + values, ok := redisArray(parts[1]) + if !ok || len(values) == 0 { + return nil, nil, false + } + return key, values, true +} + +func blockingBLMPopKeyListed(args [][]byte, numKeys int, key []byte) bool { + for i := 0; i < numKeys; i++ { + if bytes.Equal(args[blockingBLMPopFirstKeyArgIndex+i], key) { + return true + } + } + return false +} + +func blockingListPopCount(side []byte) (int64, bool) { + switch strings.ToUpper(string(side)) { + case blockingListSideLeft: + return 1, true + case blockingListSideRight: + return -1, true + default: + return 0, false + } +} + func blockingListMoveReplay(args [][]byte, resp any, count int64, to string) (string, []any, bool) { if len(args) < blockingListMoveMinArgs { return "", nil, false @@ -176,6 +273,15 @@ func redisArray(v any) ([]any, bool) { } } +func redisArgBytes(v any) ([]byte, bool) { + arg, ok := redisArg(v) + if !ok { + return nil, false + } + b, ok := arg.([]byte) + return b, ok +} + func redisArg(v any) (any, bool) { switch x := v.(type) { case nil: diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index d0d1e5286..8559dfb88 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -786,6 +786,73 @@ func TestDualWriter_Blocking_ReplaysListMoveAsEval(t *testing.T) { } } +func TestDualWriter_Blocking_ReplaysBLMPopAsEval(t *testing.T) { + tests := []struct { + name string + args [][]byte + resp any + want []any + }{ + { + name: "left pop removes returned values from head side", + args: [][]byte{ + []byte("BLMPOP"), []byte("5"), []byte("2"), + []byte("queue-a"), []byte("queue-b"), []byte("LEFT"), + []byte("COUNT"), []byte("2"), + }, + resp: []any{[]byte("queue-b"), []any{[]byte("job-1"), []byte("job-2")}}, + want: []any{ + []byte("EVAL"), + blockingListMultiPopReplayScript, + int64(1), + []byte("queue-b"), + int64(1), + []byte("job-1"), + []byte("job-2"), + }, + }, + { + name: "right pop removes returned values from tail side", + args: [][]byte{ + []byte("BLMPOP"), []byte("5"), []byte("1"), + []byte("queue"), []byte("RIGHT"), + }, + resp: []any{"queue", []string{"job-3"}}, + want: []any{ + []byte("EVAL"), + blockingListMultiPopReplayScript, + int64(1), + []byte("queue"), + int64(-1), + []byte("job-3"), + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + primary := &timeoutCapturingBackend{ + name: "primary", + returnValue: tc.resp, + } + secondary := newMockBackend("secondary") + + metrics := newTestMetrics() + cfg := ProxyConfig{Mode: ModeDualWrite, SecondaryTimeout: 10 * time.Second} + d := NewDualWriter(primary, secondary, cfg, metrics, newTestSentry(), testLogger) + + resp, err := d.Blocking(context.Background(), "BLMPOP", tc.args) + assert.NoError(t, err) + assert.Equal(t, tc.resp, resp) + d.Close() + + assert.Equal(t, [][]any{tc.want}, secondary.Calls()) + assert.InDelta(t, 0, testutil.ToFloat64(metrics.AsyncDrops), 0.001) + assert.InDelta(t, 1, testutil.ToFloat64(metrics.CommandTotal.WithLabelValues("EVAL", "secondary", "ok")), 0.001) + }) + } +} + func TestDualWriter_Blocking_XReadDoesNotUseWriteSemaphore(t *testing.T) { primary := &timeoutCapturingBackend{name: "primary", returnValue: []any{}} secondary := newMockBackend("secondary") From 9a0286dc2c5ad120e467ce43fe2519530b64e669 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 12:48:46 +0900 Subject: [PATCH 143/162] proxy: avoid retrying user not-leader errors --- adapter/redis_lua_compat_test.go | 41 ++++++++++++++++++++++++++++++ adapter/redis_lua_context.go | 2 +- proxy/blocking.go | 8 +++--- proxy/dualwrite.go | 13 +++++++--- proxy/leader_aware_backend_test.go | 8 +++--- proxy/proxy_test.go | 26 ++++++++++++++++--- 6 files changed, 82 insertions(+), 16 deletions(-) diff --git a/adapter/redis_lua_compat_test.go b/adapter/redis_lua_compat_test.go index 547142eee..18393cff4 100644 --- a/adapter/redis_lua_compat_test.go +++ b/adapter/redis_lua_compat_test.go @@ -310,6 +310,47 @@ return redis.call("XADD", KEYS[1], "MAXLEN", "~", 10, "*", "event", "new") require.Equal(t, map[string]any{"event": "new"}, events[0].Values) } +func TestRedis_LuaXAddRecreatesTTLExpiredString(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const ( + stream = "bull:test:events-expired-string" + ttl = 80 * time.Millisecond + ) + require.NoError(t, rdb.Set(ctx, stream, "old", ttl).Err()) + eventuallyExpired(t, ttl, func() bool { + exists, err := rdb.Exists(ctx, stream).Result() + return err == nil && exists == 0 + }, "string must be logically expired before Lua XADD recreates it") + + id, err := rdb.Eval(ctx, ` +return redis.call("XADD", KEYS[1], "MAXLEN", "~", 10, "*", "event", "new") +`, []string{stream}).Text() + require.NoError(t, err) + require.NotEmpty(t, id) + + typ, err := rdb.Type(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, "stream", typ) + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(1), xlen) + ttlAfter, err := rdb.TTL(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, time.Duration(-1), ttlAfter) + + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Len(t, events, 1) + require.Equal(t, id, events[0].ID) + require.Equal(t, map[string]any{"event": "new"}, events[0].Values) +} + func TestRedis_LuaXAddHonorsScriptLocalStringType(t *testing.T) { nodes, _, _ := createNode(t, 3) defer shutdown(nodes) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index eae9d8fbf..44dc62523 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -3423,7 +3423,7 @@ func (c *luaScriptContext) canUseLuaStreamDelta(key []byte, typ redisValueType, ) if typ == redisTypeNone { var err error - expired, err = c.server.hasExpired(c.scriptCtx(), key, c.startTS, true) + expired, err = c.server.hasExpired(c.scriptCtx(), key, c.startTS, false) if err != nil { return false, err } diff --git a/proxy/blocking.go b/proxy/blocking.go index eab445fa1..8c1902605 100644 --- a/proxy/blocking.go +++ b/proxy/blocking.go @@ -162,7 +162,7 @@ func blockingBLMPopReplay(args [][]byte, resp any) (string, []any, bool) { } replay := []any{ - []byte("EVAL"), + []byte(cmdEval), blockingListMultiPopReplayScript, blockingListPopReplayKeyCount, key, @@ -175,7 +175,7 @@ func blockingBLMPopReplay(args [][]byte, resp any) (string, []any, bool) { } replay = append(replay, arg) } - return "EVAL", replay, true + return cmdEval, replay, true } func parseBlockingBLMPopArgs(args [][]byte) (int, int64, bool) { @@ -240,8 +240,8 @@ func blockingListMoveReplay(args [][]byte, resp any, count int64, to string) (st } source := append([]byte(nil), args[1]...) destination := append([]byte(nil), args[2]...) - return "EVAL", []any{ - []byte("EVAL"), + return cmdEval, []any{ + []byte(cmdEval), blockingListMoveReplayScript, blockingListMoveReplayKeyCount, source, diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index 0963c9ebb..4c45720ef 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -96,15 +96,20 @@ func isElasticKVNotLeaderError(err error) bool { return false } msg := strings.TrimSpace(err.Error()) - if strings.HasPrefix(strings.ToUpper(msg), "NOTLEADER ") { + upper := strings.ToUpper(msg) + if upper == "NOTLEADER" || strings.HasPrefix(upper, "NOTLEADER ") { return true } - if strings.Contains(msg, "etcd raft engine is not leader") || - strings.Contains(msg, "raft engine: not leader") { + if strings.HasPrefix(upper, "ERR ") { + return false + } + if msg == "etcd raft engine is not leader" || + msg == "raft engine: not leader" { return true } return msg == "leader not found" || - strings.HasSuffix(msg, "desc = leader not found") + strings.HasSuffix(msg, "desc = leader not found") || + strings.HasSuffix(msg, "desc = raft engine: not leader") } func refreshSecondaryLeader(ctx context.Context, backend Backend, err error) { diff --git a/proxy/leader_aware_backend_test.go b/proxy/leader_aware_backend_test.go index ce2f32a6b..c3f1d035c 100644 --- a/proxy/leader_aware_backend_test.go +++ b/proxy/leader_aware_backend_test.go @@ -146,7 +146,7 @@ func (n *fakeElasticKVNode) writeCommandReply(conn net.Conn) { return } if leader := n.Leader(); leader != "" && leader != n.addr { - _, _ = conn.Write([]byte("-ERR etcd raft engine is not leader\r\n")) + _, _ = conn.Write([]byte("-NOTLEADER etcd raft engine is not leader\r\n")) return } _, _ = conn.Write([]byte("+OK\r\n")) @@ -267,7 +267,7 @@ func TestLeaderAwareRedisBackend_RetryRefreshesOnNotLeader(t *testing.T) { func TestLeaderAwareRedisBackend_DoesNotRetryUserNotLeaderError(t *testing.T) { node := newFakeElasticKVNode(t) node.SetLeader(node.addr) - node.SetCommandError("ERR not leader") + node.SetCommandError("ERR raft engine: not leader") backend := NewLeaderAwareRedisBackendWithInterval( []string{node.addr}, @@ -282,9 +282,9 @@ func TestLeaderAwareRedisBackend_DoesNotRetryUserNotLeaderError(t *testing.T) { return backend.CurrentLeader() == node.addr }, 2*time.Second, 10*time.Millisecond, "initial leader must be set") - res := backend.Do(context.Background(), "EVAL", "return redis.error_reply('not leader')", 0) + res := backend.Do(context.Background(), "EVAL", "return redis.error_reply('raft engine: not leader')", 0) require.Error(t, res.Err()) - require.Contains(t, res.Err().Error(), "not leader") + require.Contains(t, res.Err().Error(), "raft engine: not leader") require.Equal(t, int64(1), node.commands.Load(), "user Redis error must not be retried") } diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 8559dfb88..5c50b6f13 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -1389,7 +1389,7 @@ func TestDualWriter_writeSecondary_RetriesNotLeaderAfterRefresh(t *testing.T) { primary.doFunc = makeCmd("OK", nil) secondary := &refreshableMockBackend{mockBackend: newMockBackend("secondary")} - notLeaderErr := testRedisErr("ERR etcd raft engine is not leader") + notLeaderErr := testRedisErr("NOTLEADER etcd raft engine is not leader") var calls int secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { calls++ @@ -1418,7 +1418,7 @@ func TestDualWriter_writeSecondary_DoesNotRetryUserNotLeaderError(t *testing.T) primary.doFunc = makeCmd("OK", nil) secondary := &refreshableMockBackend{mockBackend: newMockBackend("secondary")} - userErr := testRedisErr("ERR not leader") + userErr := testRedisErr("ERR raft engine: not leader") var calls int secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { calls++ @@ -1437,12 +1437,32 @@ func TestDualWriter_writeSecondary_DoesNotRetryUserNotLeaderError(t *testing.T) assert.InDelta(t, 1, testutil.ToFloat64(metrics.SecondaryWriteErrors), 0.001) } +func TestIsElasticKVNotLeaderErrorClassifiesWireNotLeaderOnly(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "redis wire notleader", err: testRedisErr("NOTLEADER raft engine: not leader"), want: true}, + {name: "grpc wrapped no redis err prefix", err: errors.New("rpc error: code = FailedPrecondition desc = raft engine: not leader"), want: true}, + {name: "bare leader not found", err: errors.New("leader not found"), want: true}, + {name: "lua user err exact raft phrase", err: testRedisErr("ERR raft engine: not leader"), want: false}, + {name: "lua user err etcd phrase", err: testRedisErr("ERR etcd raft engine is not leader"), want: false}, + {name: "lua user err leader not found", err: testRedisErr("ERR leader not found"), want: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isElasticKVNotLeaderError(tc.err)) + }) + } +} + func TestDualWriter_ReplaySecondaryPipeline_RecordsReplyErrors(t *testing.T) { primary := newMockBackend("primary") secondary := newMockBackend("secondary") secondary.doFunc = func(ctx context.Context, args ...any) *redis.Cmd { cmd := redis.NewCmd(ctx, args...) - cmd.SetErr(testRedisErr("ERR etcd raft engine is not leader")) + cmd.SetErr(testRedisErr("NOTLEADER etcd raft engine is not leader")) return cmd } From 7b124825f451a6b94cce28c15522f5b79fa7672f Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 13:10:14 +0900 Subject: [PATCH 144/162] proxy: classify not-leader redis replies safely --- proxy/dualwrite.go | 6 ++++-- proxy/proxy_test.go | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/proxy/dualwrite.go b/proxy/dualwrite.go index 4c45720ef..160e78295 100644 --- a/proxy/dualwrite.go +++ b/proxy/dualwrite.go @@ -100,7 +100,8 @@ func isElasticKVNotLeaderError(err error) bool { if upper == "NOTLEADER" || strings.HasPrefix(upper, "NOTLEADER ") { return true } - if strings.HasPrefix(upper, "ERR ") { + var redisErr redis.Error + if errors.As(err, &redisErr) { return false } if msg == "etcd raft engine is not leader" || @@ -109,7 +110,8 @@ func isElasticKVNotLeaderError(err error) bool { } return msg == "leader not found" || strings.HasSuffix(msg, "desc = leader not found") || - strings.HasSuffix(msg, "desc = raft engine: not leader") + strings.HasSuffix(msg, "desc = raft engine: not leader") || + strings.HasSuffix(msg, "desc = etcd raft engine is not leader") } func refreshSecondaryLeader(ctx context.Context, backend Backend, err error) { diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index 5c50b6f13..a1304f4ab 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -1445,7 +1445,10 @@ func TestIsElasticKVNotLeaderErrorClassifiesWireNotLeaderOnly(t *testing.T) { }{ {name: "redis wire notleader", err: testRedisErr("NOTLEADER raft engine: not leader"), want: true}, {name: "grpc wrapped no redis err prefix", err: errors.New("rpc error: code = FailedPrecondition desc = raft engine: not leader"), want: true}, + {name: "grpc wrapped etcd sentinel no redis err prefix", err: errors.New("rpc error: code = FailedPrecondition desc = etcd raft engine is not leader"), want: true}, {name: "bare leader not found", err: errors.New("leader not found"), want: true}, + {name: "lua user err bare raft phrase", err: testRedisErr("raft engine: not leader"), want: false}, + {name: "lua user err bare etcd phrase", err: testRedisErr("etcd raft engine is not leader"), want: false}, {name: "lua user err exact raft phrase", err: testRedisErr("ERR raft engine: not leader"), want: false}, {name: "lua user err etcd phrase", err: testRedisErr("ERR etcd raft engine is not leader"), want: false}, {name: "lua user err leader not found", err: testRedisErr("ERR leader not found"), want: false}, From 5d78d7ad1c0c529bdb92169cdf43bd1623d578cd Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 14:07:40 +0900 Subject: [PATCH 145/162] migration: reject armed readiness without write floor --- adapter/internal.go | 3 +++ adapter/internal_migration_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/adapter/internal.go b/adapter/internal.go index 53222d9ce..60ccfb632 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -294,6 +294,9 @@ func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.Targe if req.GetMigrationJobId() == 0 { return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness migration_job_id is required")) } + if req.GetArmed() && req.GetMinWriteTsExclusive() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness min_write_ts_exclusive is required when armed")) + } if i.migrationProposer == nil { return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "target staged readiness proposer is not configured")) } diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 74ad94f45..b9ac70b74 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -703,3 +703,31 @@ func TestInternalApplyTargetStagedReadinessProposesThroughRaft(t *testing.T) { Armed: true, }}, states) } + +func TestInternalApplyTargetStagedReadinessRejectsArmedZeroMinWriteTS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), + ) + + resp, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + Armed: true, + }) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.Equal(t, uint64(0), proposer.calls) +} From 178fda6e23afd9fb89f7853c401903975d3e71f8 Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 16:34:10 +0900 Subject: [PATCH 146/162] docs: mark hotspot m2 migration partial --- docs/design/2026_02_18_partial_hotspot_shard_split.md | 7 +++++++ ...6_11_partial_hotspot_split_milestone2_migration.md} | 10 ++++++---- docs/design/2026_06_12_proposed_scaling_roadmap.md | 4 ++-- docs/design/2026_06_23_proposed_scaling_roadmap.md | 5 +++-- 4 files changed, 18 insertions(+), 8 deletions(-) rename docs/design/{2026_06_11_proposed_hotspot_split_milestone2_migration.md => 2026_06_11_partial_hotspot_split_milestone2_migration.md} (99%) diff --git a/docs/design/2026_02_18_partial_hotspot_shard_split.md b/docs/design/2026_02_18_partial_hotspot_shard_split.md index e2c26ca01..55ecfe1b0 100644 --- a/docs/design/2026_02_18_partial_hotspot_shard_split.md +++ b/docs/design/2026_02_18_partial_hotspot_shard_split.md @@ -293,6 +293,13 @@ Add RPCs: 2. Job phases: BACKFILL/FENCE/DELTA/CUTOVER 3. Manual split with target-group relocation +Status: partial. SplitJob catalog/codec, MVCC export/import primitives, staged +visibility, write-fence checks, target readiness, and target-promotion catalog +components exist in the M2 stack, but no production runner advances cross-group +jobs to `DONE` yet. M2 remains open in +[`2026_06_11_partial_hotspot_split_milestone2_migration.md`](2026_06_11_partial_hotspot_split_milestone2_migration.md) +until the cross-group acceptance criteria and Jepsen workload pass. + ### Milestone 3: Automation 1. Access aggregation diff --git a/docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md b/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md similarity index 99% rename from docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md rename to docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md index 1b5592f7f..dc13a3044 100644 --- a/docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md +++ b/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md @@ -1,12 +1,14 @@ # Hotspot Shard Split — Milestone 2: Migration Plane -Status: Proposed +Status: Partial Author: bootjp Date: 2026-06-11 Parent: [2026_02_18_partial_hotspot_shard_split.md](2026_02_18_partial_hotspot_shard_split.md). M1 (control plane) is as-built in [2026_02_18_implemented_hotspot_split_milestone1_pr.md](2026_02_18_implemented_hotspot_split_milestone1_pr.md). +Current implementation status: partial. The SplitJob catalog/codec, versioned route descriptor storage, range export/import primitives, staged-read visibility, write-fence checks, target readiness, and target-promotion catalog pieces have landed or are covered by the active M2 stack. The feature is not implemented end to end until a production runner advances cross-group jobs to `DONE`, `ListRoutes` exposes the moved child on its target group after cutover, and the Jepsen cross-group split workload passes the acceptance criteria in §13. + ## 1. Background M1 landed the control plane: @@ -1281,9 +1283,9 @@ This is independent of the existing rolling-upgrade protocol for unrelated subsy ## 14. Lifecycle -This document begins as `*_proposed_*`. Per CLAUDE.md: +This document is `*_partial_*` because M2-PR1 and later component slices have landed, but the cross-group migration plane is not complete end to end. -- Rename to `*_partial_*` after M2-PR1 lands; track per-PR landing under §11. -- Rename to `*_implemented_*` after M2-PR7 ships and the parent partial doc's M2 row is checked off. +- Track per-PR landing under §11. +- Rename to `*_implemented_*` only after the acceptance criteria in §13 pass: cross-group `StartSplitMigration` reaches `phase=DONE`, `ListRoutes` shows the target-group child after cutover, leader-kill recovery is automatic, and the Jepsen split workload is green. `git mv` is used so history follows. diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 4002f2209..6a7968cc5 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -337,7 +337,7 @@ control-plane (`*_proposed_*` doc TBD).** - M1 standalone but doesn't enable cross-region writes — it just makes Raft survive cross-WAN partition. - M2 depends on the M2 hotspot-split migration contract - (`2026_06_11_proposed_hotspot_split_milestone2_migration.md`) + (`2026_06_11_partial_hotspot_split_milestone2_migration.md`) being implemented so the monotone-merge primitive exists. - M3 depends on M1's region-aware membership and M2's per-region ceiling. @@ -531,7 +531,7 @@ the ceiling shape: Composability invariant: **every monotone-merge happens via the same `SetPhysicalCeiling` + `Observe` primitive**. The M2 hotspot-split contract (§6.2.1 of -`2026_06_11_proposed_hotspot_split_milestone2_migration.md`) is the +`2026_06_11_partial_hotspot_split_milestone2_migration.md`) is the reference implementation; per-region and per-group merges reuse it. ### 7.2 Capability bits diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index d30405c20..0cfb6447c 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -98,8 +98,9 @@ memory each group's private cache/memtable pins. - **Range split — distribute a range across groups.** Same-group split shipped in M1 (`distribution/`). Cross-group migration (the part that - actually relocates data and reduces per-node volume) is **PR #945** - (`docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md`, + actually relocates data and reduces per-node volume) is tracked by **PR #945** + and the active M2 stack + (`docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md`, branch `docs/hotspot-split-m2-proposal`): a resumable `SplitJob` with `PLANNED → BACKFILL → FENCE → DELTA_COPY → CUTOVER → CLEANUP → DONE` phases driven by a migrator on the default-group leader. M2 is the required From c065ad98fe0f8f5d931da83e6d3fd0544c626d1c Mon Sep 17 00:00:00 2001 From: bootjp Date: Fri, 17 Jul 2026 19:13:05 +0900 Subject: [PATCH 147/162] migration: run split target promotion --- adapter/distribution_server.go | 397 +++++++++++++++++++++++++++- adapter/distribution_server_test.go | 240 +++++++++++++++++ main.go | 44 ++- 3 files changed, 667 insertions(+), 14 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 839c7c45f..b547cc4bd 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/binary" + "log/slog" "math" "sort" "strings" @@ -16,22 +17,24 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // DistributionServer serves distribution related gRPC APIs. type DistributionServer struct { - mu sync.Mutex - engine *distribution.Engine - catalog *distribution.CatalogStore - coordinator kv.Coordinator - readTracker *kv.ActiveTimestampTracker - readBlocked func() bool - migrationCapabilityGate SplitMigrationCapabilityGate - splitJobRunnerReady bool - knownRaftGroups map[uint64]struct{} - reloadRetry struct { + mu sync.Mutex + engine *distribution.Engine + catalog *distribution.CatalogStore + coordinator kv.Coordinator + readTracker *kv.ActiveTimestampTracker + readBlocked func() bool + migrationCapabilityGate SplitMigrationCapabilityGate + splitJobRunnerReady bool + splitPromotionClientFactory SplitPromotionClientFactory + knownRaftGroups map[uint64]struct{} + reloadRetry struct { attempts int interval time.Duration } @@ -45,6 +48,16 @@ type DistributionServerOption func(*DistributionServer) // migration-only side effects. A nil gate keeps StartSplitMigration fail-closed. type SplitMigrationCapabilityGate func(context.Context) error +// SplitPromotionClient is the subset of the internal gRPC client the split +// job runner needs to promote target-local staged rows. +type SplitPromotionClient interface { + PromoteStagedVersions(context.Context, *pb.PromoteStagedVersionsRequest, ...grpc.CallOption) (*pb.PromoteStagedVersionsResponse, error) +} + +// SplitPromotionClientFactory dials the node currently leading a split job's +// target range and returns the internal client used by the runner. +type SplitPromotionClientFactory func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) + // WithDistributionCoordinator configures the coordinator used for Raft-backed // catalog mutations in SplitRange. func WithDistributionCoordinator(coordinator kv.Coordinator) DistributionServerOption { @@ -79,6 +92,14 @@ func WithSplitJobRunnerReady() DistributionServerOption { } } +// WithSplitPromotionClientFactory configures the client factory used by the +// split job runner to promote target-local staged versions. +func WithSplitPromotionClientFactory(factory SplitPromotionClientFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitPromotionClientFactory = factory + } +} + // WithDistributionKnownRaftGroups configures the Raft group IDs this node can // route migration work to. func WithDistributionKnownRaftGroups(groupIDs ...uint64) DistributionServerOption { @@ -117,11 +138,21 @@ const ( splitJobListCursorEncodedBytes = splitJobListCursorJobIDOff + 8 SplitMigrationCapabilityV2 = "cap_migration_v2" splitMigrationCapabilityV2 = SplitMigrationCapabilityV2 + + splitPromotionDefaultMaxVersions = 1024 + splitPromotionBytesPerMiB = 1024 * 1024 + splitPromotionMaxBytesMiB = 4 + splitPromotionMaxScannedMiB = 16 + promotionCompleteBaseOpCount = 2 ) var ( defaultCatalogReloadRetryAttempts = 20 defaultCatalogReloadRetryInterval = 10 * time.Millisecond + defaultSplitJobRunnerInterval = time.Second + defaultSplitPromotionMaxVersions = uint32(splitPromotionDefaultMaxVersions) + defaultSplitPromotionMaxBytes = uint64(splitPromotionMaxBytesMiB * splitPromotionBytesPerMiB) + defaultSplitPromotionMaxScanned = uint64(splitPromotionMaxScannedMiB * splitPromotionBytesPerMiB) errDistributionCatalogNotConfigured = errors.New("route catalog is not configured") errDistributionUnknownRoute = errors.New("unknown route") @@ -169,6 +200,115 @@ func (s *DistributionServer) requireReadReady() error { return nil } +func (s *DistributionServer) RunSplitJobRunner(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + ticker := time.NewTicker(defaultSplitJobRunnerInterval) + defer ticker.Stop() + for { + if err := s.RunSplitJobRunnerOnce(ctx); err != nil { + slog.Warn("split job runner tick failed", "err", err) + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func (s *DistributionServer) RunSplitJobRunnerOnce(ctx context.Context) error { + if !s.splitJobRunnerConfigured() { + return nil + } + leader, err := s.verifySplitJobRunnerLeader(ctx) + if err != nil || !leader { + return err + } + jobs, err := s.catalog.ListSplitJobs(ctx) + if err != nil { + return splitJobCatalogStatusError(err) + } + if job, ok := nextCleanupSplitJob(jobs); ok { + return s.promoteSplitJobTargetAndComplete(ctx, job) + } + return nil +} + +func (s *DistributionServer) splitJobRunnerConfigured() bool { + return s != nil && s.catalog != nil && s.coordinator != nil && s.splitPromotionClientFactory != nil +} + +func (s *DistributionServer) verifySplitJobRunnerLeader(ctx context.Context) (bool, error) { + if !s.coordinator.IsLeaderForKey(distribution.CatalogVersionKey()) { + return false, nil + } + if err := s.coordinator.VerifyLeaderForKey(ctx, distribution.CatalogVersionKey()); err != nil { + return false, errors.WithStack(err) + } + return true, nil +} + +func nextCleanupSplitJob(jobs []distribution.SplitJob) (distribution.SplitJob, bool) { + for _, job := range jobs { + if job.Phase == distribution.SplitJobPhaseCleanup && !job.TargetPromotionDone { + return job, true + } + } + return distribution.SplitJob{}, false +} + +func (s *DistributionServer) promoteSplitJobTargetAndComplete(ctx context.Context, job distribution.SplitJob) error { + client, err := s.splitPromotionClientFactory(ctx, job) + if err != nil { + return errors.WithStack(err) + } + if client == nil { + return errors.New("split promotion client is nil") + } + if err := promoteSplitJobTarget(ctx, client, job); err != nil { + return errors.WithStack(err) + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + current, found, err := s.catalog.SplitJobAt(ctx, job.JobID, snapshot.ReadTS) + if err != nil { + return splitJobPromotionStatusError(err) + } + if !found { + return splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) + } + _, _, err = s.completeSplitJobTargetPromotionViaCoordinator(ctx, snapshot.Version, current, time.Now().UnixMilli()) + return err +} + +func promoteSplitJobTarget(ctx context.Context, client SplitPromotionClient, job distribution.SplitJob) error { + var cursor []byte + for { + resp, err := client.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{ + JobId: job.JobID, + Cursor: cursor, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + }) + if err != nil { + return errors.WithStack(err) + } + if resp.GetDone() { + return nil + } + next := resp.GetNextCursor() + if len(next) == 0 || bytes.Equal(next, cursor) { + return errors.New("split promotion made no cursor progress") + } + cursor = distribution.CloneBytes(next) + } +} + // UpdateRoute allows updating route information. func (s *DistributionServer) UpdateRoute(start, end []byte, group uint64) { s.engine.UpdateRoute(start, end, group) @@ -730,6 +870,226 @@ func (s *DistributionServer) updateSplitJobViaCoordinator( return nil } +func (s *DistributionServer) completeSplitJobTargetPromotionViaCoordinator( + ctx context.Context, + expectedVersion uint64, + expected distribution.SplitJob, + nowMs int64, +) (distribution.CatalogSnapshot, distribution.SplitJob, error) { + if err := s.requirePromotionCompletionDeps(); err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + snapshot, current, alreadyDone, err := s.loadPromotionCompletionCandidate(ctx, expectedVersion, expected, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + if alreadyDone { + return snapshot, current, nil + } + completion, err := distribution.CompleteTargetPromotionState(expected, snapshot.Routes, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) + } + return s.dispatchPromotionCompletion(ctx, snapshot, expectedVersion, expected.JobID, completion) +} + +func (s *DistributionServer) requirePromotionCompletionDeps() error { + if s.catalog == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if s.coordinator == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionCoordinatorRequired.Error()) + } + return nil +} + +func (s *DistributionServer) loadPromotionCompletionCandidate( + ctx context.Context, + expectedVersion uint64, + expected distribution.SplitJob, + nowMs int64, +) (distribution.CatalogSnapshot, distribution.SplitJob, bool, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, err + } + current, found, err := s.catalog.SplitJobAt(ctx, expected.JobID, snapshot.ReadTS) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(err) + } + if !found { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) + } + if snapshot.Version == expectedVersion && distribution.SplitJobsEquivalent(current, expected) { + return snapshot, current, false, nil + } + job, err := completedPromotionRetryJob(snapshot.Routes, expected, current, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(err) + } + if job.TargetPromotionDone { + return snapshot, job, true, nil + } + if snapshot.Version != expectedVersion { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogVersionMismatch) + } + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) +} + +func completedPromotionRetryJob( + routes []distribution.RouteDescriptor, + expected distribution.SplitJob, + current distribution.SplitJob, + nowMs int64, +) (distribution.SplitJob, error) { + matches, err := splitJobPromotionMatchesExpected(expected, current) + if err != nil { + return distribution.SplitJob{}, errors.WithStack(err) + } + if !matches { + return distribution.SplitJob{}, nil + } + completion, err := distribution.CompleteTargetPromotionState(current, routes, nowMs) + if err != nil { + return distribution.SplitJob{}, errors.WithStack(err) + } + return completion.Job, nil +} + +func (s *DistributionServer) dispatchPromotionCompletion( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + expectedVersion uint64, + jobID uint64, + completion distribution.TargetPromotionCompletion, +) (distribution.CatalogSnapshot, distribution.SplitJob, error) { + if !completion.Changed { + return snapshot, completion.Job, nil + } + commitTS, err := s.nextPromotionCompleteCommitTS(snapshot.ReadTS) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + completion.Job.PromotionCompletedTS = commitTS + ops, err := s.buildPromotionCompleteOps(expectedVersion, completion) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) + } + jobKey := distribution.CatalogSplitJobKey(jobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: ops, + IsTxn: true, + StartTS: snapshot.ReadTS, + CommitTS: commitTS, + ReadKeys: [][]byte{ + distribution.CatalogVersionKey(), + jobKey, + }, + }); err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobCoordinatorStatusError(err) + } + loaded, err := s.loadCatalogSnapshotAtLeastVersion(ctx, expectedVersion+1) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + return loaded, completion.Job, nil +} + +func (s *DistributionServer) nextPromotionCompleteCommitTS(readTS uint64) (uint64, error) { + if readTS == math.MaxUint64 { + return 0, grpcStatusError(codes.Internal, distribution.ErrCatalogVersionOverflow.Error()) + } + clock := s.coordinator.Clock() + if clock == nil { + return readTS + 1, nil + } + clock.Observe(readTS) + commitTS, err := clock.NextFenced() + if err != nil { + return 0, grpcStatusErrorf(codes.FailedPrecondition, "allocate promotion completion timestamp: %v", err) + } + if commitTS > readTS { + return commitTS, nil + } + clock.Observe(readTS) + commitTS, err = clock.NextFenced() + if err != nil { + return 0, grpcStatusErrorf(codes.FailedPrecondition, "allocate promotion completion timestamp: %v", err) + } + if commitTS <= readTS { + return 0, grpcStatusError(codes.Internal, "promotion completion timestamp did not advance") + } + return commitTS, nil +} + +func (s *DistributionServer) buildPromotionCompleteOps( + expectedVersion uint64, + completion distribution.TargetPromotionCompletion, +) ([]*kv.Elem[kv.OP], error) { + if expectedVersion == math.MaxUint64 { + return nil, errors.WithStack(distribution.ErrCatalogVersionOverflow) + } + ops := make([]*kv.Elem[kv.OP], 0, len(completion.ClearedRouteIDs)+promotionCompleteBaseOpCount) + for _, routeID := range completion.ClearedRouteIDs { + route, ok := promotionCompleteRouteByID(completion.Routes, routeID) + if !ok { + return nil, errors.WithStack(distribution.ErrMigrationPromotionTargetAbsent) + } + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return nil, errors.WithStack(err) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogRouteKey(route.RouteID), + Value: encoded, + }) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogVersionKey(), + Value: distribution.EncodeCatalogVersion(expectedVersion + 1), + }) + encodedJob, err := distribution.EncodeSplitJob(completion.Job) + if err != nil { + return nil, errors.WithStack(err) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogSplitJobKey(completion.Job.JobID), + Value: encodedJob, + }) + return ops, nil +} + +func promotionCompleteRouteByID(routes []distribution.RouteDescriptor, routeID uint64) (distribution.RouteDescriptor, bool) { + for _, route := range routes { + if route.RouteID == routeID { + return route, true + } + } + return distribution.RouteDescriptor{}, false +} + +func splitJobPromotionMatchesExpected(expected, current distribution.SplitJob) (bool, error) { + if expected.TargetPromotionDone || !current.TargetPromotionDone || current.PromotionCompletedTS == 0 { + return false, nil + } + normalized := distribution.CloneSplitJob(current) + normalized.TargetPromotionDone = expected.TargetPromotionDone + normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.UpdatedAtMs = expected.UpdatedAtMs + expectedRaw, err := distribution.EncodeSplitJob(expected) + if err != nil { + return false, errors.WithStack(err) + } + normalizedRaw, err := distribution.EncodeSplitJob(normalized) + if err != nil { + return false, errors.WithStack(err) + } + return bytes.Equal(expectedRaw, normalizedRaw), nil +} + func (s *DistributionServer) verifyKnownTargetGroup(groupID uint64) error { if len(s.knownRaftGroups) == 0 { return grpcStatusError(codes.FailedPrecondition, errDistributionRaftGroupsNotKnown.Error()) @@ -983,6 +1343,23 @@ func splitJobCoordinatorStatusError(err error) error { } } +func splitJobPromotionStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrCatalogVersionMismatch), + errors.Is(err, distribution.ErrCatalogSplitJobConflict), + errors.Is(err, store.ErrWriteConflict): + return grpcStatusError(codes.Aborted, err.Error()) + case errors.Is(err, distribution.ErrMigrationPromotionNotReady), + errors.Is(err, distribution.ErrMigrationPromotionTargetAbsent): + return grpcStatusError(codes.FailedPrecondition, err.Error()) + case errors.Is(err, distribution.ErrMigrationInvalidRoute), + errors.Is(err, distribution.ErrCatalogRouteV2WriteDisabled): + return grpcStatusError(codes.InvalidArgument, err.Error()) + default: + return splitJobCatalogStatusError(err) + } +} + func splitJobListPage(jobs []distribution.SplitJob, req *pb.ListSplitJobsRequest, phaseFilter string) (*pb.ListSplitJobsResponse, error) { filtered := make([]distribution.SplitJob, 0, len(jobs)) for _, job := range jobs { diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index cdd4e1142..c9c265265 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -13,6 +13,7 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -787,6 +788,169 @@ func TestDistributionServerAbandonSplitJob_RecordsAbandoningViaCoordinator(t *te require.Equal(t, distribution.SplitJobPhaseBackfill, got.AbandonFromPhase) } +func TestDistributionServerCompleteSplitJobTargetPromotion_UsesCoordinatorCAS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + before, err := catalog.Snapshot(ctx) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + snapshot, completed, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 2000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.True(t, completed.TargetPromotionDone) + require.Greater(t, completed.PromotionCompletedTS, before.ReadTS) + require.Equal(t, int64(2000), completed.UpdatedAtMs) + require.Equal(t, 1, coordinator.dispatchCalls) + require.Equal(t, before.ReadTS, coordinator.lastStartTS) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogVersionKey()) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogSplitJobKey(job.JobID)) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, completed, loadedJob) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + target := distributionRouteByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestDistributionServerCompleteSplitJobTargetPromotion_RetryAfterCommitIsIdempotent(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + firstSnapshot, firstCompleted, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 2000) + require.NoError(t, err) + retrySnapshot, retryCompleted, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 3000) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, retrySnapshot.Version) + require.Equal(t, firstCompleted, retryCompleted) + require.Equal(t, 1, coordinator.dispatchCalls) +} + +func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{ + {NextCursor: []byte("cursor-1")}, + {Done: true}, + }, + } + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return client, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + require.Equal(t, 2, client.calls) + require.Equal(t, job.JobID, client.requests[0].GetJobId()) + require.Empty(t, client.requests[0].GetCursor()) + require.Equal(t, defaultSplitPromotionMaxVersions, client.requests[0].GetMaxVersions()) + require.Equal(t, defaultSplitPromotionMaxBytes, client.requests[0].GetMaxBytes()) + require.Equal(t, defaultSplitPromotionMaxScanned, client.requests[0].GetMaxScannedBytes()) + require.Equal(t, []byte("cursor-1"), client.requests[1].GetCursor()) + require.Equal(t, 1, coordinator.dispatchCalls) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, loadedJob.TargetPromotionDone) + require.NotZero(t, loadedJob.PromotionCompletedTS) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version+1, loaded.Version) + target := distributionRouteByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestDistributionServerRunSplitJobRunnerOnce_SkipsFollower(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, promotionCompleteDistributionJob())) + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{{Done: true}}, + } + coordinator := newDistributionCoordinatorStub(baseStore, false) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return client, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + require.Zero(t, client.calls) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestPromoteSplitJobTargetRejectsNoCursorProgress(t *testing.T) { + t.Parallel() + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{{NextCursor: []byte("same")}}, + } + err := promoteSplitJobTarget(context.Background(), client, promotionCompleteDistributionJob()) + require.NoError(t, err) + + client = &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{ + {NextCursor: []byte("same")}, + {NextCursor: []byte("same")}, + }, + } + err = promoteSplitJobTarget(context.Background(), client, promotionCompleteDistributionJob()) + require.Error(t, err) + require.ErrorContains(t, err, "split promotion made no cursor progress") +} + func TestDistributionServerRetrySplitJob_RequiresCatalogLeader(t *testing.T) { t.Parallel() @@ -1546,6 +1710,51 @@ func sampleDistributionSplitJob(jobID uint64) distribution.SplitJob { } } +func promotionCompleteDistributionJob() distribution.SplitJob { + return distribution.SplitJob{ + JobID: 99, + SourceRouteID: 42, + SplitKey: []byte("m"), + TargetGroupID: 2, + Phase: distribution.SplitJobPhaseCleanup, + } +} + +func promotionCompleteDistributionRoutes() []distribution.RouteDescriptor { + return []distribution.RouteDescriptor{ + { + RouteID: 2, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 42, + }, + { + RouteID: 3, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + ParentRouteID: 42, + StagedVisibilityActive: true, + MigrationJobID: 99, + MinWriteTSExclusive: 777, + }, + } +} + +func distributionRouteByID(t *testing.T, routes []distribution.RouteDescriptor, routeID uint64) distribution.RouteDescriptor { + t.Helper() + for _, route := range routes { + if route.RouteID == routeID { + return route + } + } + t.Fatalf("route %d not found in %+v", routeID, routes) + return distribution.RouteDescriptor{} +} + func splitJobIDs(jobs []*pb.SplitJob) []uint64 { ids := make([]uint64, 0, len(jobs)) for _, job := range jobs { @@ -1562,6 +1771,37 @@ func byteSliceStrings(in [][]byte) []string { return out } +type splitPromotionClientStub struct { + responses []*pb.PromoteStagedVersionsResponse + err error + requests []*pb.PromoteStagedVersionsRequest + calls int +} + +func (s *splitPromotionClientStub) PromoteStagedVersions( + _ context.Context, + req *pb.PromoteStagedVersionsRequest, + _ ...grpc.CallOption, +) (*pb.PromoteStagedVersionsResponse, error) { + s.calls++ + s.requests = append(s.requests, &pb.PromoteStagedVersionsRequest{ + JobId: req.GetJobId(), + Cursor: distribution.CloneBytes(req.GetCursor()), + MaxVersions: req.GetMaxVersions(), + MaxBytes: req.GetMaxBytes(), + MaxScannedBytes: req.GetMaxScannedBytes(), + }) + if s.err != nil { + return nil, s.err + } + if len(s.responses) == 0 { + return &pb.PromoteStagedVersionsResponse{Done: true}, nil + } + resp := s.responses[0] + s.responses = s.responses[1:] + return resp, nil +} + type distributionCoordinatorStub struct { store store.MVCCStore leader bool diff --git a/main.go b/main.go index 205f970f4..8e4663641 100644 --- a/main.go +++ b/main.go @@ -1612,9 +1612,15 @@ func prepareDistributionRuntimeServer( if err != nil { return nil, err } - in.distServer = adapter.NewDistributionServer( - engine, - distCatalog, + splitPromotionConnCache := &kv.GRPCConnCache{} + in.eg.Go(func() error { + <-in.ctx.Done() + if err := splitPromotionConnCache.Close(); err != nil { + return errors.Wrap(err, "close split promotion gRPC connection cache") + } + return nil + }) + distOptions := []adapter.DistributionServerOption{ adapter.WithDistributionCoordinator(coordinate), adapter.WithDistributionActiveTimestampTracker(readTracker), adapter.WithDistributionKnownRaftGroups(shardGroupIDs(in.shardGroups)...), @@ -1623,7 +1629,10 @@ func prepareDistributionRuntimeServer( splitMigrationCapabilityProbeTimeout, nil, )), - ) + adapter.WithSplitPromotionClientFactory(splitPromotionClientFactory(coordinate, splitPromotionConnCache)), + adapter.WithSplitJobRunnerReady(), + } + in.distServer = adapter.NewDistributionServer(engine, distCatalog, distOptions...) preparedServer, err := prepareRuntimeServerRunner(waitRotateOnStartup, in) if err != nil { return nil, err @@ -1664,10 +1673,37 @@ func startDistributionRuntimeAfterTransport(in distributionRuntimeStartupInput) startMemoryWatchdog(in.runCtx, in.eg, in.cancel) startMonitoringCollectors(in.runCtx, in.metricsRegistry, in.runtimes, in.clock) startFSMCompactorIfEnabled(in.runCtx, in.eg, in.runtimes, in.readTracker) + startDistributionSplitJobRunner(in.runCtx, in.eg, prepared.serverInput.distServer) return startPreparedServerAfterStartupRotation( in.waitRotateOnStartup, prepared.serverInput, prepared.preparedServer) } +func splitPromotionClientFactory(coordinate kv.Coordinator, connCache *kv.GRPCConnCache) adapter.SplitPromotionClientFactory { + return func(_ context.Context, job distribution.SplitJob) (adapter.SplitPromotionClient, error) { + if coordinate == nil { + return nil, errors.New("distribution coordinator is not configured") + } + if connCache == nil { + return nil, errors.New("split promotion gRPC connection cache is not configured") + } + addr := coordinate.RaftLeaderForKey(job.SplitKey) + conn, err := connCache.ConnFor(addr) + if err != nil { + return nil, errors.WithStack(err) + } + return pb.NewInternalClient(conn), nil + } +} + +func startDistributionSplitJobRunner(ctx context.Context, eg *errgroup.Group, server *adapter.DistributionServer) { + if eg == nil || server == nil { + return + } + eg.Go(func() error { + return server.RunSplitJobRunner(ctx) + }) +} + func prepareRuntimeServerRunner(waitRotateOnStartup startupRotationWaiter, in serversInput) (*preparedRuntimeServer, error) { adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler) if err != nil { From 252bd64d6ac19c36c5b03f2589ce743d3950d425 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 00:52:52 +0900 Subject: [PATCH 148/162] migration: harden split runner promotion --- adapter/distribution_server.go | 59 ++++++++++++++++++++++++----- adapter/distribution_server_test.go | 17 +++++++++ adapter/internal.go | 25 +++++++++++- main.go | 45 +++++++++++++++++++--- main_catalog_test.go | 5 ++- main_proposer_for_group_test.go | 36 ++++++++++++++++++ 6 files changed, 167 insertions(+), 20 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index b547cc4bd..25829fe16 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -32,6 +32,7 @@ type DistributionServer struct { readBlocked func() bool migrationCapabilityGate SplitMigrationCapabilityGate splitJobRunnerReady bool + splitJobRunnerReadinessGate SplitMigrationCapabilityGate splitPromotionClientFactory SplitPromotionClientFactory knownRaftGroups map[uint64]struct{} reloadRetry struct { @@ -92,6 +93,14 @@ func WithSplitJobRunnerReady() DistributionServerOption { } } +// WithSplitJobRunnerReadinessGate configures local runtime gates that must be +// open before this node advertises split migration capability. +func WithSplitJobRunnerReadinessGate(gate SplitMigrationCapabilityGate) DistributionServerOption { + return func(s *DistributionServer) { + s.splitJobRunnerReadinessGate = gate + } +} + // WithSplitPromotionClientFactory configures the client factory used by the // split job runner to promote target-local staged versions. func WithSplitPromotionClientFactory(factory SplitPromotionClientFactory) DistributionServerOption { @@ -143,16 +152,18 @@ const ( splitPromotionBytesPerMiB = 1024 * 1024 splitPromotionMaxBytesMiB = 4 splitPromotionMaxScannedMiB = 16 + splitPromotionAttemptTimeoutSecs = 2 promotionCompleteBaseOpCount = 2 ) var ( - defaultCatalogReloadRetryAttempts = 20 - defaultCatalogReloadRetryInterval = 10 * time.Millisecond - defaultSplitJobRunnerInterval = time.Second - defaultSplitPromotionMaxVersions = uint32(splitPromotionDefaultMaxVersions) - defaultSplitPromotionMaxBytes = uint64(splitPromotionMaxBytesMiB * splitPromotionBytesPerMiB) - defaultSplitPromotionMaxScanned = uint64(splitPromotionMaxScannedMiB * splitPromotionBytesPerMiB) + defaultCatalogReloadRetryAttempts = 20 + defaultCatalogReloadRetryInterval = 10 * time.Millisecond + defaultSplitJobRunnerInterval = time.Second + defaultSplitPromotionMaxVersions = uint32(splitPromotionDefaultMaxVersions) + defaultSplitPromotionMaxBytes = uint64(splitPromotionMaxBytesMiB * splitPromotionBytesPerMiB) + defaultSplitPromotionMaxScanned = uint64(splitPromotionMaxScannedMiB * splitPromotionBytesPerMiB) + defaultSplitPromotionAttemptTimeout = time.Duration(splitPromotionAttemptTimeoutSecs) * time.Second errDistributionCatalogNotConfigured = errors.New("route catalog is not configured") errDistributionUnknownRoute = errors.New("unknown route") @@ -208,7 +219,9 @@ func (s *DistributionServer) RunSplitJobRunner(ctx context.Context) error { defer ticker.Stop() for { if err := s.RunSplitJobRunnerOnce(ctx); err != nil { - slog.Warn("split job runner tick failed", "err", err) + if !splitJobRunnerContextDone(err) { + slog.Warn("split job runner tick failed", "err", err) + } } select { case <-ctx.Done(): @@ -288,13 +301,15 @@ func (s *DistributionServer) promoteSplitJobTargetAndComplete(ctx context.Contex func promoteSplitJobTarget(ctx context.Context, client SplitPromotionClient, job distribution.SplitJob) error { var cursor []byte for { - resp, err := client.PromoteStagedVersions(ctx, &pb.PromoteStagedVersionsRequest{ + attemptCtx, cancel := splitPromotionAttemptContext(ctx) + resp, err := client.PromoteStagedVersions(attemptCtx, &pb.PromoteStagedVersionsRequest{ JobId: job.JobID, Cursor: cursor, MaxVersions: defaultSplitPromotionMaxVersions, MaxBytes: defaultSplitPromotionMaxBytes, MaxScannedBytes: defaultSplitPromotionMaxScanned, }) + cancel() if err != nil { return errors.WithStack(err) } @@ -309,6 +324,20 @@ func promoteSplitJobTarget(ctx context.Context, client SplitPromotionClient, job } } +func splitPromotionAttemptContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + if defaultSplitPromotionAttemptTimeout <= 0 { + return ctx, func() {} + } + return context.WithTimeout(ctx, defaultSplitPromotionAttemptTimeout) +} + +func splitJobRunnerContextDone(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) +} + // UpdateRoute allows updating route information. func (s *DistributionServer) UpdateRoute(start, end []byte, group uint64) { s.engine.UpdateRoute(start, end, group) @@ -431,8 +460,8 @@ func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.St }, nil } -func (s *DistributionServer) GetSplitMigrationCapability(context.Context, *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { - if !s.splitJobRunnerReady { +func (s *DistributionServer) GetSplitMigrationCapability(ctx context.Context, _ *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + if !s.splitMigrationCapabilityReady(ctx) { return &pb.GetSplitMigrationCapabilityResponse{}, nil } return &pb.GetSplitMigrationCapabilityResponse{ @@ -441,6 +470,16 @@ func (s *DistributionServer) GetSplitMigrationCapability(context.Context, *pb.Ge }, nil } +func (s *DistributionServer) splitMigrationCapabilityReady(ctx context.Context) bool { + if !s.splitJobRunnerReady { + return false + } + if s.splitJobRunnerReadinessGate == nil { + return true + } + return s.splitJobRunnerReadinessGate(ctx) == nil +} + func (s *DistributionServer) verifyStartSplitMigrationPreflight(ctx context.Context, req *pb.StartSplitMigrationRequest) error { if req.GetTargetGroupId() == 0 { return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index c9c265265..6aeec9c87 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -231,6 +231,23 @@ func TestDistributionServerGetSplitMigrationCapabilityReportsReadyWhenRunnerRead require.Contains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) } +func TestDistributionServerGetSplitMigrationCapabilityRespectsReadinessGate(t *testing.T) { + t.Parallel() + + s := NewDistributionServer( + distribution.NewEngine(), + nil, + WithSplitJobRunnerReady(), + WithSplitJobRunnerReadinessGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled") + }), + ) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.False(t, resp.GetMigrationCapable()) + require.NotContains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + func TestDistributionServerStartSplitMigration_FailsClosedUntilCapabilityGate(t *testing.T) { t.Parallel() diff --git a/adapter/internal.go b/adapter/internal.go index 60ccfb632..c6ac0eeb0 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -17,6 +17,15 @@ import ( "google.golang.org/grpc/status" ) +const ( + // MigrationImportOpcodeEnv enables the internal staged import opcode after + // every voter is running a compatible build. + MigrationImportOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE" + // MigrationPromoteOpcodeEnv enables the internal staged promotion opcode + // after every voter is running a compatible build. + MigrationPromoteOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE" +) + type InternalOption func(*Internal) func WithInternalTimestampAllocator(alloc kv.TimestampAllocator) InternalOption { @@ -341,7 +350,7 @@ func defaultMigrationImportGate(context.Context) error { } func migrationImportOpcodeEnabledFromEnv() bool { - return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE") + return MigrationImportOpcodeEnabledFromEnv() } func defaultMigrationPromoteGate(context.Context) error { @@ -352,7 +361,19 @@ func defaultMigrationPromoteGate(context.Context) error { } func migrationPromoteOpcodeEnabledFromEnv() bool { - return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE") + return MigrationPromoteOpcodeEnabledFromEnv() +} + +// MigrationImportOpcodeEnabledFromEnv reports whether the staged import opcode +// is enabled for this process. +func MigrationImportOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationImportOpcodeEnv) +} + +// MigrationPromoteOpcodeEnabledFromEnv reports whether the staged promotion +// opcode is enabled for this process. +func MigrationPromoteOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationPromoteOpcodeEnv) } func envFlagEnabled(name string) bool { diff --git a/main.go b/main.go index 8e4663641..1867a29bf 100644 --- a/main.go +++ b/main.go @@ -1629,7 +1629,11 @@ func prepareDistributionRuntimeServer( splitMigrationCapabilityProbeTimeout, nil, )), - adapter.WithSplitPromotionClientFactory(splitPromotionClientFactory(coordinate, splitPromotionConnCache)), + adapter.WithSplitPromotionClientFactory(splitPromotionClientFactory( + splitPromotionTargetLeaderResolver(in.shardGroups), + splitPromotionConnCache, + )), + adapter.WithSplitJobRunnerReadinessGate(splitMigrationLocalReadinessGate), adapter.WithSplitJobRunnerReady(), } in.distServer = adapter.NewDistributionServer(engine, distCatalog, distOptions...) @@ -1673,20 +1677,38 @@ func startDistributionRuntimeAfterTransport(in distributionRuntimeStartupInput) startMemoryWatchdog(in.runCtx, in.eg, in.cancel) startMonitoringCollectors(in.runCtx, in.metricsRegistry, in.runtimes, in.clock) startFSMCompactorIfEnabled(in.runCtx, in.eg, in.runtimes, in.readTracker) - startDistributionSplitJobRunner(in.runCtx, in.eg, prepared.serverInput.distServer) return startPreparedServerAfterStartupRotation( in.waitRotateOnStartup, prepared.serverInput, prepared.preparedServer) } -func splitPromotionClientFactory(coordinate kv.Coordinator, connCache *kv.GRPCConnCache) adapter.SplitPromotionClientFactory { +type splitPromotionLeaderResolver func(distribution.SplitJob) (string, error) + +func splitPromotionTargetLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) splitPromotionLeaderResolver { + return func(job distribution.SplitJob) (string, error) { + sg := shardGroups[job.TargetGroupID] + if sg == nil || sg.Engine == nil { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split promotion target group %d", job.TargetGroupID) + } + addr := strings.TrimSpace(sg.Engine.Leader().Address) + if addr == "" { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split promotion target group %d", job.TargetGroupID) + } + return addr, nil + } +} + +func splitPromotionClientFactory(resolve splitPromotionLeaderResolver, connCache *kv.GRPCConnCache) adapter.SplitPromotionClientFactory { return func(_ context.Context, job distribution.SplitJob) (adapter.SplitPromotionClient, error) { - if coordinate == nil { - return nil, errors.New("distribution coordinator is not configured") + if resolve == nil { + return nil, errors.New("split promotion leader resolver is not configured") } if connCache == nil { return nil, errors.New("split promotion gRPC connection cache is not configured") } - addr := coordinate.RaftLeaderForKey(job.SplitKey) + addr, err := resolve(job) + if err != nil { + return nil, errors.WithStack(err) + } conn, err := connCache.ConnFor(addr) if err != nil { return nil, errors.WithStack(err) @@ -1695,6 +1717,16 @@ func splitPromotionClientFactory(coordinate kv.Coordinator, connCache *kv.GRPCCo } } +func splitMigrationLocalReadinessGate(context.Context) error { + if !adapter.MigrationImportOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationImportOpcodeEnv) + } + if !adapter.MigrationPromoteOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationPromoteOpcodeEnv) + } + return nil +} + func startDistributionSplitJobRunner(ctx context.Context, eg *errgroup.Group, server *adapter.DistributionServer) { if eg == nil || server == nil { return @@ -1844,6 +1876,7 @@ func startPreparedServerAfterStartupRotation(waitRotateOnStartup startupRotation } startHLCLeaseRenewal(in.ctx, in.eg, in.coordinate) runner.publicKVGate.markReady() + startDistributionSplitJobRunner(in.ctx, in.eg, in.distServer) if err := runner.startPublicServices(); err != nil { return err } diff --git a/main_catalog_test.go b/main_catalog_test.go index 2b712234f..47cd6b1d1 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -294,7 +294,8 @@ func startSplitMigrationCapabilityTestServer(t *testing.T, resp *pb.GetSplitMigr } type capabilityConfigEngine struct { - cfg raftengine.Configuration + cfg raftengine.Configuration + leader raftengine.LeaderInfo } func (e capabilityConfigEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { @@ -310,7 +311,7 @@ func (e capabilityConfigEngine) State() raftengine.State { } func (e capabilityConfigEngine) Leader() raftengine.LeaderInfo { - return raftengine.LeaderInfo{} + return e.leader } func (e capabilityConfigEngine) VerifyLeader(context.Context) error { diff --git a/main_proposer_for_group_test.go b/main_proposer_for_group_test.go index 017344040..f40f01ee0 100644 --- a/main_proposer_for_group_test.go +++ b/main_proposer_for_group_test.go @@ -3,8 +3,10 @@ package main import ( "testing" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" ) // stubEngineForProposerTest satisfies raftengine.Engine at the @@ -92,3 +94,37 @@ func TestProposerForGroup_FallsBackToEngineWhenShardGroupNil(t *testing.T) { t.Errorf("proposerForGroup did not fall back to rt.engine for nil shardGroup entry; got %T, want raw engine", got) } } + +func TestSplitPromotionTargetLeaderResolverUsesTargetGroup(t *testing.T) { + t.Parallel() + + source := capabilityConfigEngine{leader: raftengine.LeaderInfo{Address: "source:50051"}} + target := capabilityConfigEngine{leader: raftengine.LeaderInfo{Address: "target:50051"}} + resolve := splitPromotionTargetLeaderResolver(map[uint64]*kv.ShardGroup{ + 1: {Engine: source}, + 2: {Engine: target}, + }) + + addr, err := resolve(distribution.SplitJob{ + SplitKey: []byte("m"), + TargetGroupID: 2, + }) + if err != nil { + t.Fatalf("resolve returned error: %v", err) + } + if addr != "target:50051" { + t.Fatalf("resolve returned %q, want target leader address", addr) + } +} + +func TestSplitPromotionTargetLeaderResolverRejectsMissingLeader(t *testing.T) { + t.Parallel() + + resolve := splitPromotionTargetLeaderResolver(map[uint64]*kv.ShardGroup{ + 2: {Engine: capabilityConfigEngine{}}, + }) + _, err := resolve(distribution.SplitJob{TargetGroupID: 2}) + if !errors.Is(err, kv.ErrLeaderNotFound) { + t.Fatalf("resolve error = %v, want ErrLeaderNotFound", err) + } +} From 938a2842ecaa5f0c295e8174cd704371b3301ac5 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 01:20:45 +0900 Subject: [PATCH 149/162] distribution: keep split migration gates closed --- adapter/distribution_server.go | 6 +++- adapter/distribution_server_test.go | 11 +++++++ main.go | 13 ++++++++- main_catalog_test.go | 45 ++++++++++++++++++++++++++--- 4 files changed, 69 insertions(+), 6 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 25829fe16..1cdc040a1 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -335,7 +335,11 @@ func splitPromotionAttemptContext(ctx context.Context) (context.Context, context } func splitJobRunnerContextDone(err error) bool { - return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return true + } + code := status.Code(errors.Cause(err)) + return code == codes.Canceled || code == codes.DeadlineExceeded } // UpdateRoute allows updating route information. diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 6aeec9c87..015910b72 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -12,6 +12,7 @@ import ( "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" + crdberrors "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -948,6 +949,16 @@ func TestDistributionServerRunSplitJobRunnerOnce_SkipsFollower(t *testing.T) { require.Zero(t, coordinator.dispatchCalls) } +func TestSplitJobRunnerContextDoneRecognizesWrappedGRPCStatus(t *testing.T) { + t.Parallel() + + require.True(t, splitJobRunnerContextDone(context.Canceled)) + require.True(t, splitJobRunnerContextDone(context.DeadlineExceeded)) + require.True(t, splitJobRunnerContextDone(crdberrors.WithStack(status.Error(codes.Canceled, "stopping")))) + require.True(t, splitJobRunnerContextDone(crdberrors.WithStack(status.Error(codes.DeadlineExceeded, "stopping")))) + require.False(t, splitJobRunnerContextDone(crdberrors.WithStack(status.Error(codes.Unavailable, "not leader")))) +} + func TestPromoteSplitJobTargetRejectsNoCursorProgress(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 1867a29bf..84fddf44f 100644 --- a/main.go +++ b/main.go @@ -735,11 +735,21 @@ func splitMigrationCapabilityPeersFromConfiguration(cfg raftengine.Configuration return peers } -func newSplitMigrationCapabilityGate(source splitMigrationCapabilityPeerSource, timeout time.Duration, probe splitMigrationCapabilityProbe) adapter.SplitMigrationCapabilityGate { +func newSplitMigrationCapabilityGate( + source splitMigrationCapabilityPeerSource, + timeout time.Duration, + probe splitMigrationCapabilityProbe, + localGate adapter.SplitMigrationCapabilityGate, +) adapter.SplitMigrationCapabilityGate { if probe == nil { probe = probeSplitMigrationCapabilityPeer } return func(ctx context.Context) error { + if localGate != nil { + if err := localGate(ctx); err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration local readiness is not available: %v", err) + } + } if source == nil { return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") } @@ -1628,6 +1638,7 @@ func prepareDistributionRuntimeServer( splitMigrationCapabilityPeerSourceForRuntimes(runtimes), splitMigrationCapabilityProbeTimeout, nil, + splitMigrationLocalReadinessGate, )), adapter.WithSplitPromotionClientFactory(splitPromotionClientFactory( splitPromotionTargetLeaderResolver(in.shardGroups), diff --git a/main_catalog_test.go b/main_catalog_test.go index 47cd6b1d1..742697653 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -105,7 +105,7 @@ func TestSplitMigrationCapabilityGateChecksAllPeers(t *testing.T) { gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(_ context.Context, address string) error { probed = append(probed, address) return nil - }) + }, nil) require.NoError(t, gate(context.Background())) require.Equal(t, []string{"10.0.0.11:50051", "10.0.0.12:50051"}, probed) @@ -117,7 +117,7 @@ func TestSplitMigrationCapabilityGateFailsClosedWithoutPeers(t *testing.T) { gate := newSplitMigrationCapabilityGate(nil, time.Second, func(context.Context, string) error { t.Fatal("probe must not run without peers") return nil - }) + }, nil) err := gate(context.Background()) require.Error(t, err) @@ -137,7 +137,7 @@ func TestSplitMigrationCapabilityGateFailsClosedWhenPeerMissingCapability(t *tes return status.Error(codes.Unimplemented, "method not found") } return nil - }) + }, nil) err := gate(context.Background()) require.Error(t, err) @@ -155,7 +155,7 @@ func TestSplitMigrationCapabilityGateFailsClosedWhenPeerSourceErrors(t *testing. }, time.Second, func(context.Context, string) error { t.Fatal("probe must not run when peer source fails") return nil - }) + }, nil) err := gate(context.Background()) require.Error(t, err) @@ -163,6 +163,43 @@ func TestSplitMigrationCapabilityGateFailsClosedWhenPeerSourceErrors(t *testing. require.ErrorContains(t, err, "peers are not available") } +func TestSplitMigrationCapabilityGateFailsClosedWhenLocalReadinessFails(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + } + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(context.Context, string) error { + t.Fatal("probe must not run when local readiness is closed") + return nil + }, func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled") + }) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "local readiness") + require.ErrorContains(t, err, "migration opcode disabled") +} + +func TestSplitMigrationLocalReadinessGateRequiresImportAndPromoteOpcodes(t *testing.T) { + t.Setenv(adapter.MigrationImportOpcodeEnv, "") + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") + err := splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationImportOpcodeEnv) + + t.Setenv(adapter.MigrationImportOpcodeEnv, "1") + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "") + err = splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationPromoteOpcodeEnv) + + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") + require.NoError(t, splitMigrationLocalReadinessGate(context.Background())) +} + func TestSplitMigrationCapabilityPeersFromConfigurationUsesCurrentMembers(t *testing.T) { t.Parallel() From 1c7ea5b3430631b372c48b16af91b087c941d09a Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 01:58:59 +0900 Subject: [PATCH 150/162] distribution: finish split promotion cleanup --- adapter/distribution_server.go | 58 ++++++----- adapter/distribution_server_test.go | 95 ++++++++++++++++++- distribution/migration_promotion_complete.go | 19 +++- .../migration_promotion_complete_test.go | 17 ++-- 4 files changed, 148 insertions(+), 41 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 1cdc040a1..6b16a591f 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -265,7 +265,7 @@ func (s *DistributionServer) verifySplitJobRunnerLeader(ctx context.Context) (bo func nextCleanupSplitJob(jobs []distribution.SplitJob) (distribution.SplitJob, bool) { for _, job := range jobs { - if job.Phase == distribution.SplitJobPhaseCleanup && !job.TargetPromotionDone { + if job.Phase == distribution.SplitJobPhaseCleanup { return job, true } } @@ -273,15 +273,17 @@ func nextCleanupSplitJob(jobs []distribution.SplitJob) (distribution.SplitJob, b } func (s *DistributionServer) promoteSplitJobTargetAndComplete(ctx context.Context, job distribution.SplitJob) error { - client, err := s.splitPromotionClientFactory(ctx, job) - if err != nil { - return errors.WithStack(err) - } - if client == nil { - return errors.New("split promotion client is nil") - } - if err := promoteSplitJobTarget(ctx, client, job); err != nil { - return errors.WithStack(err) + if !job.TargetPromotionDone { + client, err := s.splitPromotionClientFactory(ctx, job) + if err != nil { + return errors.WithStack(err) + } + if client == nil { + return errors.New("split promotion client is nil") + } + if err := promoteSplitJobTarget(ctx, client, job); err != nil { + return errors.WithStack(err) + } } snapshot, err := s.loadCatalogSnapshot(ctx) if err != nil { @@ -294,8 +296,11 @@ func (s *DistributionServer) promoteSplitJobTargetAndComplete(ctx context.Contex if !found { return splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) } - _, _, err = s.completeSplitJobTargetPromotionViaCoordinator(ctx, snapshot.Version, current, time.Now().UnixMilli()) - return err + completed, _, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, snapshot.Version, current, time.Now().UnixMilli()) + if err != nil { + return err + } + return s.applyEngineSnapshot(completed) } func promoteSplitJobTarget(ctx context.Context, client SplitPromotionClient, job distribution.SplitJob) error { @@ -1009,11 +1014,13 @@ func (s *DistributionServer) dispatchPromotionCompletion( if !completion.Changed { return snapshot, completion.Job, nil } - commitTS, err := s.nextPromotionCompleteCommitTS(snapshot.ReadTS) + commitTS, err := s.nextPromotionCompleteCommitTS(ctx, snapshot.ReadTS) if err != nil { return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err } - completion.Job.PromotionCompletedTS = commitTS + if completion.Job.PromotionCompletedTS == 0 { + completion.Job.PromotionCompletedTS = commitTS + } ops, err := s.buildPromotionCompleteOps(expectedVersion, completion) if err != nil { return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) @@ -1038,24 +1045,11 @@ func (s *DistributionServer) dispatchPromotionCompletion( return loaded, completion.Job, nil } -func (s *DistributionServer) nextPromotionCompleteCommitTS(readTS uint64) (uint64, error) { +func (s *DistributionServer) nextPromotionCompleteCommitTS(ctx context.Context, readTS uint64) (uint64, error) { if readTS == math.MaxUint64 { return 0, grpcStatusError(codes.Internal, distribution.ErrCatalogVersionOverflow.Error()) } - clock := s.coordinator.Clock() - if clock == nil { - return readTS + 1, nil - } - clock.Observe(readTS) - commitTS, err := clock.NextFenced() - if err != nil { - return 0, grpcStatusErrorf(codes.FailedPrecondition, "allocate promotion completion timestamp: %v", err) - } - if commitTS > readTS { - return commitTS, nil - } - clock.Observe(readTS) - commitTS, err = clock.NextFenced() + commitTS, err := kv.NextTimestampAfterThrough(ctx, s.coordinator, readTS, "allocate promotion completion timestamp") if err != nil { return 0, grpcStatusErrorf(codes.FailedPrecondition, "allocate promotion completion timestamp: %v", err) } @@ -1115,10 +1109,14 @@ func promotionCompleteRouteByID(routes []distribution.RouteDescriptor, routeID u } func splitJobPromotionMatchesExpected(expected, current distribution.SplitJob) (bool, error) { - if expected.TargetPromotionDone || !current.TargetPromotionDone || current.PromotionCompletedTS == 0 { + if expected.TargetPromotionDone || + !current.TargetPromotionDone || + current.PromotionCompletedTS == 0 || + current.Phase != distribution.SplitJobPhaseDone { return false, nil } normalized := distribution.CloneSplitJob(current) + normalized.Phase = expected.Phase normalized.TargetPromotionDone = expected.TargetPromotionDone normalized.PromotionCompletedTS = expected.PromotionCompletedTS normalized.UpdatedAtMs = expected.UpdatedAtMs diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 015910b72..2a8512e1a 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -820,16 +820,20 @@ func TestDistributionServerCompleteSplitJobTargetPromotion_UsesCoordinatorCAS(t require.NoError(t, err) coordinator := newDistributionCoordinatorStub(baseStore, true) + coordinator.timestampNext = before.ReadTS + 10 s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) snapshot, completed, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 2000) require.NoError(t, err) require.Equal(t, saved.Version+1, snapshot.Version) require.True(t, completed.TargetPromotionDone) - require.Greater(t, completed.PromotionCompletedTS, before.ReadTS) + require.Equal(t, distribution.SplitJobPhaseDone, completed.Phase) + require.Equal(t, before.ReadTS+10, completed.PromotionCompletedTS) require.Equal(t, int64(2000), completed.UpdatedAtMs) require.Equal(t, 1, coordinator.dispatchCalls) + require.Equal(t, 1, coordinator.timestampCalls) require.Equal(t, before.ReadTS, coordinator.lastStartTS) + require.Equal(t, completed.PromotionCompletedTS, coordinator.lastCommitTS) requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogVersionKey()) requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogSplitJobKey(job.JobID)) @@ -910,6 +914,7 @@ func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T require.NoError(t, err) require.True(t, found) require.True(t, loadedJob.TargetPromotionDone) + require.Equal(t, distribution.SplitJobPhaseDone, loadedJob.Phase) require.NotZero(t, loadedJob.PromotionCompletedTS) loaded, err := catalog.Snapshot(ctx) @@ -919,6 +924,59 @@ func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T require.False(t, target.StagedVisibilityActive) require.Equal(t, uint64(0), target.MigrationJobID) require.Equal(t, uint64(777), target.MinWriteTSExclusive) + + route, ok := s.engine.GetRoute([]byte("z")) + require.True(t, ok) + require.False(t, route.StagedVisibilityActive) + require.Equal(t, uint64(0), route.MigrationJobID) + require.Equal(t, saved.Version+1, s.engine.Version()) +} + +func TestDistributionServerRunSplitJobRunnerOnce_TerminalizesPromotedCleanupJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + job.TargetPromotionDone = true + job.PromotionCompletedTS = saved.ReadTS + 1 + job.UpdatedAtMs = 2000 + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{{Done: true}}, + } + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return client, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + require.Zero(t, client.calls) + require.Equal(t, 1, coordinator.dispatchCalls) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseDone, loadedJob.Phase) + require.True(t, loadedJob.TargetPromotionDone) + require.Equal(t, job.PromotionCompletedTS, loadedJob.PromotionCompletedTS) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version+1, loaded.Version) + require.Equal(t, saved.Version+1, s.engine.Version()) } func TestDistributionServerRunSplitJobRunnerOnce_SkipsFollower(t *testing.T) { @@ -1835,7 +1893,11 @@ type distributionCoordinatorStub struct { leader bool dispatchErr error nextTS uint64 + timestampNext uint64 + timestampErr error + timestampCalls int lastStartTS uint64 + lastCommitTS uint64 lastReadKeys [][]byte beforeApply func(context.Context, store.MVCCStore) error afterDispatch func(context.Context, store.MVCCStore, uint64) error @@ -1859,8 +1921,9 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope if s.dispatchErr != nil { return nil, s.dispatchErr } - startTS, commitTS := s.nextTimestamps(reqs.StartTS) + startTS, commitTS := s.nextTimestamps(reqs.StartTS, reqs.CommitTS) s.lastStartTS = startTS + s.lastCommitTS = commitTS readKeys := cloneDistributionReadKeys(reqs.ReadKeys) s.lastReadKeys = readKeys @@ -1896,7 +1959,13 @@ func (s *distributionCoordinatorStub) validateDispatch(reqs *kv.OperationGroup[k return nil } -func (s *distributionCoordinatorStub) nextTimestamps(startTS uint64) (uint64, uint64) { +func (s *distributionCoordinatorStub) nextTimestamps(startTS uint64, requestedCommitTS uint64) (uint64, uint64) { + if requestedCommitTS != 0 { + if s.nextTS <= requestedCommitTS { + s.nextTS = requestedCommitTS + 1 + } + return startTS, requestedCommitTS + } if s.nextTS == 0 { s.nextTS = s.store.LastCommitTS() + 1 } @@ -2030,6 +2099,26 @@ func (s *distributionCoordinatorStub) Clock() *kv.HLC { return nil } +func (s *distributionCoordinatorStub) Next(ctx context.Context) (uint64, error) { + return s.NextAfter(ctx, 0) +} + +func (s *distributionCoordinatorStub) NextAfter(_ context.Context, min uint64) (uint64, error) { + s.timestampCalls++ + if s.timestampErr != nil { + return 0, s.timestampErr + } + next := s.timestampNext + if next == 0 { + next = s.store.LastCommitTS() + 1 + } + if next <= min { + next = min + 1 + } + s.timestampNext = next + 1 + return next, nil +} + func (s *distributionCoordinatorStub) LinearizableRead(_ context.Context) (uint64, error) { return 0, nil } diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go index e4c064d33..0420dec5e 100644 --- a/distribution/migration_promotion_complete.go +++ b/distribution/migration_promotion_complete.go @@ -39,6 +39,11 @@ func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs } if out.Job.TargetPromotionDone { if targetClearedDescriptorPresent(out.Job, out.Routes) { + if out.Job.Phase != SplitJobPhaseDone { + out.Changed = true + out.Job.Phase = SplitJobPhaseDone + out.Job.UpdatedAtMs = nowMs + } return out, nil } return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) @@ -55,6 +60,7 @@ func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs out.Changed = true out.ClearedRouteIDs = cleared out.Job.TargetPromotionDone = true + out.Job.Phase = SplitJobPhaseDone out.Job.UpdatedAtMs = nowMs return out, nil } @@ -63,6 +69,9 @@ func normalizePromotionCompletionInput(job SplitJob, routes []RouteDescriptor) ( if err := validateSplitJob(job); err != nil { return nil, err } + if job.Phase == SplitJobPhaseDone && job.TargetPromotionDone { + return normalizeRoutes(routes) + } if job.Phase != SplitJobPhaseCleanup { return nil, errors.WithStack(ErrMigrationPromotionNotReady) } @@ -227,10 +236,14 @@ func (s *CatalogStore) promotionCompleteAlreadyAppliedAt(ctx context.Context, ts } func promotionCompleteJobMatchesExpected(expected SplitJob, expectedRaw []byte, current SplitJob) (bool, error) { - if expected.TargetPromotionDone || !current.TargetPromotionDone || current.PromotionCompletedTS == 0 { + if expected.TargetPromotionDone || + !current.TargetPromotionDone || + current.PromotionCompletedTS == 0 || + current.Phase != SplitJobPhaseDone { return false, nil } normalized := CloneSplitJob(current) + normalized.Phase = expected.Phase normalized.TargetPromotionDone = expected.TargetPromotionDone normalized.PromotionCompletedTS = expected.PromotionCompletedTS normalized.UpdatedAtMs = expected.UpdatedAtMs @@ -270,7 +283,9 @@ func (s *CatalogStore) buildPromotionCompleteMutations( if err != nil { return savePlan{}, nil, 0, err } - completion.Job.PromotionCompletedTS = commitTS + if completion.Job.PromotionCompletedTS == 0 { + completion.Job.PromotionCompletedTS = commitTS + } encodedJob, err := EncodeSplitJob(completion.Job) if err != nil { return savePlan{}, nil, 0, err diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go index b134c2dd1..1973c19e9 100644 --- a/distribution/migration_promotion_complete_test.go +++ b/distribution/migration_promotion_complete_test.go @@ -22,7 +22,7 @@ func TestCompleteTargetPromotionStateClearsStagedFieldsAndRetainsFloor(t *testin require.True(t, result.Job.TargetPromotionDone) require.Zero(t, result.Job.PromotionCompletedTS) require.Equal(t, int64(1000), result.Job.UpdatedAtMs) - require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) + require.Equal(t, SplitJobPhaseDone, result.Job.Phase) target := routeByID(t, result.Routes, 3) require.False(t, target.StagedVisibilityActive) @@ -49,9 +49,10 @@ func TestCompleteTargetPromotionStateAcceptsAlreadyClearedDescriptor(t *testing. result, err := CompleteTargetPromotionState(job, routes, 1100) require.NoError(t, err) - require.False(t, result.Changed) + require.True(t, result.Changed) + require.Equal(t, SplitJobPhaseDone, result.Job.Phase) require.Equal(t, uint64(900), result.Job.PromotionCompletedTS) - require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Equal(t, int64(1100), result.Job.UpdatedAtMs) target := routeByID(t, result.Routes, 3) require.False(t, target.StagedVisibilityActive) @@ -75,6 +76,7 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionCommitsRouteAndJobTogether(t require.NoError(t, err) require.Equal(t, saved.Version+1, snapshot.Version) require.True(t, completed.TargetPromotionDone) + require.Equal(t, SplitJobPhaseDone, completed.Phase) require.Greater(t, completed.PromotionCompletedTS, before.ReadTS) require.Equal(t, int64(1000), completed.UpdatedAtMs) @@ -127,18 +129,21 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDesc require.NoError(t, err) job := promotionCompleteTestJob() job.TargetPromotionDone = true + job.Phase = SplitJobPhaseCleanup job.PromotionCompletedTS = 900 job.UpdatedAtMs = 1000 require.NoError(t, cs.CreateSplitJob(ctx, job)) snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1100) require.NoError(t, err) - require.Equal(t, saved.Version, snapshot.Version) - assertSplitJobEqual(t, job, completed) + require.Equal(t, saved.Version+1, snapshot.Version) + require.Equal(t, SplitJobPhaseDone, completed.Phase) + require.Equal(t, uint64(900), completed.PromotionCompletedTS) + require.Equal(t, int64(1100), completed.UpdatedAtMs) loaded, err := cs.Snapshot(ctx) require.NoError(t, err) - require.Equal(t, saved.Version, loaded.Version) + require.Equal(t, saved.Version+1, loaded.Version) } func TestCatalogStoreCompleteSplitJobTargetPromotionRejectsStaleInputs(t *testing.T) { From 3a7c3c27f09d8a5b8151e094f11ca50a6add1abf Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 20:06:17 +0900 Subject: [PATCH 151/162] Fix pinned migration cleanup routing --- adapter/redis_txn_test.go | 4 +- kv/fsm.go | 10 +- kv/fsm_migration_fence_test.go | 24 +++++ kv/sharded_coordinator.go | 120 +++++++++++++++------- kv/sharded_coordinator_del_prefix_test.go | 25 +++++ kv/sharded_coordinator_txn_test.go | 45 +++++++- proto/internal.pb.go | 9 +- proto/internal.proto | 9 +- 8 files changed, 198 insertions(+), 48 deletions(-) diff --git a/adapter/redis_txn_test.go b/adapter/redis_txn_test.go index 197fbbb30..872d728a1 100644 --- a/adapter/redis_txn_test.go +++ b/adapter/redis_txn_test.go @@ -837,6 +837,8 @@ func TestRedisTxnSetReplacementDeletesNonPrefixedStringEncodings(t *testing.T) { t.Parallel() ctx := context.Background() + hllValue, err := encodeRedisHLL(redisSetValue{Members: []string{"member"}}, nil) + require.NoError(t, err) cases := []struct { name string seedKey func([]byte) []byte @@ -846,7 +848,7 @@ func TestRedisTxnSetReplacementDeletesNonPrefixedStringEncodings(t *testing.T) { { name: "hll", seedKey: redisHLLKey, - seedValue: []byte("hll-payload"), + seedValue: hllValue, expectedDel: redisHLLKey, }, { diff --git a/kv/fsm.go b/kv/fsm.go index f85528650..89599a845 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -833,13 +833,14 @@ func (f *kvFSM) verifyComposed1(r *pb.Request) error { if observedVer == 0 { return nil } + bypassKeys := writeFenceBypassKeySet(r.GetWriteFenceBypassKeys()) // (a) Observed-version check. observedSnap, ok := f.routes.SnapshotAt(observedVer) if !ok { return errors.WithStack(ErrComposed1VersionGCd) } - if err := f.verifyOwnerFromSnapshot(r.GetMutations(), observedSnap, observedVer, "observed"); err != nil { + if err := f.verifyOwnerFromSnapshot(r.GetMutations(), bypassKeys, observedSnap, observedVer, "observed"); err != nil { return err } @@ -851,7 +852,7 @@ func (f *kvFSM) verifyComposed1(r *pb.Request) error { // short-circuit posture of an unwired FSM). return nil } - return f.verifyOwnerFromSnapshot(r.GetMutations(), currentSnap, currentSnap.Version(), "current") + return f.verifyOwnerFromSnapshot(r.GetMutations(), bypassKeys, currentSnap, currentSnap.Version(), "current") } func (f *kvFSM) verifyWriteFence(r *pb.Request) error { @@ -957,7 +958,7 @@ func writeFenceBypassKeySet(keys [][]byte) map[string]struct{} { // "current") that ends up in the wrapped error. isTxnInternalKey // mutations (the TxnMeta marker prefix) are skipped — they are // always on every shard and have no Composed-1 ownership. -func (f *kvFSM) verifyOwnerFromSnapshot(mutations []*pb.Mutation, snap RouteSnapshot, snapVer uint64, phase string) error { +func (f *kvFSM) verifyOwnerFromSnapshot(mutations []*pb.Mutation, bypassKeys map[string]struct{}, snap RouteSnapshot, snapVer uint64, phase string) error { for _, mut := range mutations { if mut == nil || len(mut.Key) == 0 { continue @@ -965,6 +966,9 @@ func (f *kvFSM) verifyOwnerFromSnapshot(mutations []*pb.Mutation, snap RouteSnap if isTxnInternalKey(mut.Key) { continue } + if _, ok := bypassKeys[string(mut.Key)]; ok { + continue + } // routeKey-normalize before OwnerOf so the gate routes the // same way as ShardRouter.ResolveGroup — raw adapter keys // and route catalog ranges live in different lex bands diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index ecb771b38..807a5f551 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -97,6 +97,30 @@ func TestFSMWriteFenceBypassAllowsMarkedRawPointWrite(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMWriteFenceBypassAllowsPinnedTxnOnNonOwningGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }) + fsm := newComposed1FSM(t, engine, 1) + key := []byte("z") + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + ObservedRouteVersion: 1, + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_DEL, Key: key}, + }, + }, 10) + require.NoError(t, err) +} + func TestFSMWriteFenceBypassDoesNotAllowDelPrefix(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index e2cd989c5..2427e3375 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -984,6 +984,9 @@ func isComposed1RetryableError(err error) bool { // shard router. Extracted from Dispatch to keep that method's branch // count within the cyclop budget after the 7a registration gate landed. func (c *ShardedCoordinator) dispatchNonTxn(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, error) { + if hasExplicitGroupElem(reqs.Elems) { + return c.dispatchExplicitGroupNonTxn(ctx, reqs) + } logs, err := c.requestLogs(ctx, reqs) if err != nil { return nil, err @@ -995,6 +998,37 @@ func (c *ShardedCoordinator) dispatchNonTxn(ctx context.Context, reqs *Operation return &CoordinateResponse{CommitIndex: r.CommitIndex}, nil } +func hasExplicitGroupElem(elems []*Elem[OP]) bool { + for _, elem := range elems { + if elem != nil && elem.GroupID != 0 { + return true + } + } + return false +} + +func (c *ShardedCoordinator) dispatchExplicitGroupNonTxn(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, error) { + logs, gids, err := c.rawLogsWithGroups(ctx, reqs) + if err != nil { + return nil, err + } + var maxIndex uint64 + for i, gid := range gids { + g, err := c.txnGroupForID(gid) + if err != nil { + return nil, err + } + resp, err := g.Txn.Commit(ctx, []*pb.Request{logs[i]}) + if err != nil { + return nil, errors.WithStack(err) + } + if resp != nil && resp.CommitIndex > maxIndex { + maxIndex = resp.CommitIndex + } + } + return &CoordinateResponse{CommitIndex: maxIndex}, nil +} + func (c *ShardedCoordinator) dispatchRawWithComposed1Retry(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, error) { for attempt := 0; attempt <= composed1RetryAttempts; attempt++ { resp, handled, err := c.dispatchBeforeShardRouting(ctx, reqs) @@ -1024,10 +1058,7 @@ func (c *ShardedCoordinator) canRetryRawVersionGC(reqs *OperationGroup[OP]) bool seen bool ) for _, elem := range reqs.Elems { - if elem == nil { - return false - } - gid, ok := c.router.ResolveGroup(elem.Key) + gid, ok := c.rawElemGroupID(elem) if !ok { return false } @@ -1043,6 +1074,16 @@ func (c *ShardedCoordinator) canRetryRawVersionGC(reqs *OperationGroup[OP]) bool return seen } +func (c *ShardedCoordinator) rawElemGroupID(elem *Elem[OP]) (uint64, bool) { + if elem == nil { + return 0, false + } + if elem.GroupID != 0 { + return elem.GroupID, true + } + return c.router.ResolveGroup(elem.Key) +} + // hasDelPrefixElem returns true if any element is a DelPrefix operation. func hasDelPrefixElem(elems []*Elem[OP]) bool { for _, e := range elems { @@ -1103,7 +1144,7 @@ func (c *ShardedCoordinator) rejectWriteFencedPointElems(elems []*Elem[OP]) erro return nil } for _, elem := range elems { - if elem == nil || len(elem.Key) == 0 { + if elem == nil || elem.GroupID != 0 { continue } if err := c.rejectWriteFencedPointKey(elem.Key); err != nil { @@ -1143,13 +1184,13 @@ func (c *ShardedCoordinator) partitionResolverRecognisesPointKey(key []byte) boo return c.router.partitionResolver.RecognisesPartitionedKey(key) } -func (c *ShardedCoordinator) resolverClaimedWriteFenceBypassKeysForElems(elems []*Elem[OP]) [][]byte { +func (c *ShardedCoordinator) writeFenceBypassKeysForElems(elems []*Elem[OP]) [][]byte { if len(elems) == 0 { return nil } out := make([][]byte, 0, len(elems)) for _, elem := range elems { - if elem == nil || elem.Op == DelPrefix || !c.partitionResolverClaimsPointKey(elem.Key) { + if elem == nil || elem.Op == DelPrefix || (elem.GroupID == 0 && !c.partitionResolverClaimsPointKey(elem.Key)) { continue } out = append(out, bytes.Clone(elem.Key)) @@ -1157,16 +1198,21 @@ func (c *ShardedCoordinator) resolverClaimedWriteFenceBypassKeysForElems(elems [ return out } -func (c *ShardedCoordinator) resolverClaimedWriteFenceBypassKeys(muts []*pb.Mutation) [][]byte { - if len(muts) == 0 { - return nil - } - out := make([][]byte, 0, len(muts)) - for _, mut := range muts { - if mut == nil || mut.GetOp() == pb.Op_DEL_PREFIX || !c.partitionResolverClaimsPointKey(mut.Key) { +func (c *ShardedCoordinator) writeFenceBypassKeysByGroup(elems []*Elem[OP]) map[uint64][][]byte { + out := make(map[uint64][][]byte) + for _, elem := range elems { + if elem == nil || elem.Op == DelPrefix { continue } - out = append(out, bytes.Clone(mut.Key)) + gid := elem.GroupID + if gid == 0 { + var ok bool + gid, ok = c.router.ResolveGroup(elem.Key) + if !ok || !c.partitionResolverClaimsPointKey(elem.Key) { + continue + } + } + out[gid] = append(out[gid], bytes.Clone(elem.Key)) } return out } @@ -1245,6 +1291,7 @@ func (c *ShardedCoordinator) dispatchTxn(ctx context.Context, startTS uint64, co if err != nil { return nil, err } + bypassKeysByGroup := c.writeFenceBypassKeysByGroup(elems) primaryKey := primaryKeyForElems(elems) if len(primaryKey) == 0 { return nil, errors.WithStack(ErrTxnPrimaryKeyRequired) @@ -1263,7 +1310,7 @@ func (c *ShardedCoordinator) dispatchTxn(ctx context.Context, startTS uint64, co // preserving SSI. return c.dispatchSingleShardTxn(ctx, startTS, commitTS, prevCommitTS, primaryKey, gids[0], elems, readKeys, observedRouteVersion) } - return c.dispatchMultiShardTxn(ctx, startTS, commitTS, prevCommitTS, primaryKey, grouped, gids, readKeys, observedRouteVersion) + return c.dispatchMultiShardTxn(ctx, startTS, commitTS, prevCommitTS, primaryKey, grouped, gids, readKeys, observedRouteVersion, bypassKeysByGroup) } // dispatchMultiShardTxn runs the 2PC path. Extracted from dispatchTxn to keep @@ -1271,7 +1318,7 @@ func (c *ShardedCoordinator) dispatchTxn(ctx context.Context, startTS uint64, co // P2 round-10) was added; the multi-shard branch already carries five linear // error checks (groupReadKeys, prewrite, commitPrimary, abortCleanup, // commitSecondaries) that pushed the parent over the 10-edge limit. -func (c *ShardedCoordinator) dispatchMultiShardTxn(ctx context.Context, startTS, commitTS, prevCommitTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, readKeys [][]byte, observedRouteVersion uint64) (*CoordinateResponse, error) { +func (c *ShardedCoordinator) dispatchMultiShardTxn(ctx context.Context, startTS, commitTS, prevCommitTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, readKeys [][]byte, observedRouteVersion uint64, bypassKeysByGroup map[uint64][][]byte) (*CoordinateResponse, error) { // Fail-closed when a retry carries the option-2 dedup probe key but its // write set / read set spans shards (codex P2 round-10 "reject retries // that leave the one-phase path"). The 2PC log builders only encode @@ -1294,12 +1341,12 @@ func (c *ShardedCoordinator) dispatchMultiShardTxn(ctx context.Context, startTS, if err != nil { return nil, err } - prepared, err := c.prewriteTxn(ctx, startTS, commitTS, primaryKey, grouped, gids, groupedReadKeys, observedRouteVersion) + prepared, err := c.prewriteTxn(ctx, startTS, commitTS, primaryKey, grouped, gids, groupedReadKeys, observedRouteVersion, bypassKeysByGroup) if err != nil { return nil, err } - primaryGid, maxIndex, err := c.commitPrimaryTxn(ctx, startTS, primaryKey, grouped, gids, commitTS, observedRouteVersion) + primaryGid, maxIndex, err := c.commitPrimaryTxn(ctx, startTS, primaryKey, grouped, gids, commitTS, observedRouteVersion, bypassKeysByGroup) if err != nil { // abortPreparedTxn must run even when ctx was the reason // commitPrimaryTxn failed — otherwise prewrite intents on @@ -1326,7 +1373,7 @@ func (c *ShardedCoordinator) dispatchMultiShardTxn(ctx context.Context, startTS, // reachable by readers on the new owner. Both directions are // silent partial commits; surfacing the error is the only honest // posture (codex P1 on d8487672 + 6202b964, PR #900). - maxIndex, err = c.commitSecondaryTxns(ctx, startTS, primaryGid, primaryKey, grouped, gids, commitTS, maxIndex, observedRouteVersion) + maxIndex, err = c.commitSecondaryTxns(ctx, startTS, primaryGid, primaryKey, grouped, gids, commitTS, maxIndex, observedRouteVersion, bypassKeysByGroup) if err != nil { return nil, errors.WithStack(err) } @@ -1373,7 +1420,7 @@ func (c *ShardedCoordinator) dispatchSingleShardTxn(ctx context.Context, startTS // carries the one-phase dedup probe key for a retry that reuses a failed // attempt's write set. resp, err := g.Txn.Commit(ctx, []*pb.Request{ - onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS, primaryKey, elems, readKeys, observedRouteVersion, c.resolverClaimedWriteFenceBypassKeysForElems(elems)), + onePhaseTxnRequestWithPrevCommit(startTS, commitTS, prevCommitTS, primaryKey, elems, readKeys, observedRouteVersion, c.writeFenceBypassKeysForElems(elems)), }) if err != nil { return nil, errors.WithStack(err) @@ -1389,7 +1436,7 @@ type preparedGroup struct { keys []*pb.Mutation } -func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, groupedReadKeys map[uint64][][]byte, observedRouteVersion uint64) ([]preparedGroup, error) { +func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, groupedReadKeys map[uint64][][]byte, observedRouteVersion uint64, bypassKeysByGroup map[uint64][][]byte) ([]preparedGroup, error) { prepareMeta := txnMetaMutation(primaryKey, defaultTxnLockTTLms, 0) prepared := make([]preparedGroup, 0, len(gids)) @@ -1405,7 +1452,7 @@ func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS Mutations: append([]*pb.Mutation{prepareMeta}, grouped[gid]...), ReadKeys: groupedReadKeys[gid], ObservedRouteVersion: observedRouteVersion, - WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(grouped[gid]), + WriteFenceBypassKeys: bypassKeysByGroup[gid], } if _, err := g.Txn.Commit(ctx, []*pb.Request{req}); err != nil { // Same WithoutCancel pattern as dispatchTxn's @@ -1439,7 +1486,7 @@ func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS return prepared, nil } -func (c *ShardedCoordinator) commitPrimaryTxn(ctx context.Context, startTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, commitTS uint64, observedRouteVersion uint64) (uint64, uint64, error) { +func (c *ShardedCoordinator) commitPrimaryTxn(ctx context.Context, startTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, commitTS uint64, observedRouteVersion uint64, bypassKeysByGroup map[uint64][][]byte) (uint64, uint64, error) { primaryGid, ok := primaryGroupIDForKey(primaryKey, grouped, gids) if !ok { return 0, 0, errors.WithStack(ErrInvalidRequest) @@ -1458,7 +1505,7 @@ func (c *ShardedCoordinator) commitPrimaryTxn(ctx context.Context, startTS uint6 Ts: startTS, Mutations: append([]*pb.Mutation{meta}, keys...), ObservedRouteVersion: observedRouteVersion, - WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(keys), + WriteFenceBypassKeys: bypassKeysByGroup[primaryGid], } r, err := g.Txn.Commit(ctx, []*pb.Request{req}) @@ -1482,7 +1529,7 @@ func primaryGroupIDForKey(primaryKey []byte, grouped map[uint64][]*pb.Mutation, return 0, false } -func (c *ShardedCoordinator) commitSecondaryTxns(ctx context.Context, startTS uint64, primaryGid uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, commitTS uint64, maxIndex uint64, observedRouteVersion uint64) (uint64, error) { +func (c *ShardedCoordinator) commitSecondaryTxns(ctx context.Context, startTS uint64, primaryGid uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, commitTS uint64, maxIndex uint64, observedRouteVersion uint64, bypassKeysByGroup map[uint64][][]byte) (uint64, error) { // Secondary commits are best-effort for non-Composed-1 errors: // if a shard is unavailable after the primary commits, read-time // lock resolution will commit the remaining secondaries based on @@ -1519,7 +1566,7 @@ func (c *ShardedCoordinator) commitSecondaryTxns(ctx context.Context, startTS ui Ts: startTS, Mutations: append([]*pb.Mutation{meta}, keyMutations(grouped[gid])...), ObservedRouteVersion: observedRouteVersion, - WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(grouped[gid]), + WriteFenceBypassKeys: bypassKeysByGroup[gid], } r, err := commitSecondaryWithRetry(ctx, g, req) if err != nil { @@ -2100,16 +2147,22 @@ func (c *ShardedCoordinator) requestLogs(ctx context.Context, reqs *OperationGro } func (c *ShardedCoordinator) rawLogs(ctx context.Context, reqs *OperationGroup[OP]) ([]*pb.Request, error) { + logs, _, err := c.rawLogsWithGroups(ctx, reqs) + return logs, err +} + +func (c *ShardedCoordinator) rawLogsWithGroups(ctx context.Context, reqs *OperationGroup[OP]) ([]*pb.Request, []uint64, error) { grouped, gids, err := c.groupMutations(reqs.Elems, reqs.KeyVizLabel) if err != nil { - return nil, err + return nil, nil, err } + bypassKeysByGroup := c.writeFenceBypassKeysByGroup(reqs.Elems) logs := make([]*pb.Request, 0, len(gids)) for _, gid := range gids { ts, err := c.rawLogTimestamp(ctx) if err != nil { - return nil, err + return nil, nil, err } logs = append(logs, &pb.Request{ IsTxn: false, @@ -2117,10 +2170,10 @@ func (c *ShardedCoordinator) rawLogs(ctx context.Context, reqs *OperationGroup[O Ts: ts, Mutations: grouped[gid], ObservedRouteVersion: reqs.ObservedRouteVersion, - WriteFenceBypassKeys: c.resolverClaimedWriteFenceBypassKeys(grouped[gid]), + WriteFenceBypassKeys: bypassKeysByGroup[gid], }) } - return logs, nil + return logs, gids, nil } func (c *ShardedCoordinator) rawLogTimestamp(ctx context.Context) (uint64, error) { @@ -2158,10 +2211,7 @@ func (c *ShardedCoordinator) txnLogs(ctx context.Context, reqs *OperationGroup[O if err != nil { return nil, err } - bypassKeysByGroup := make(map[uint64][][]byte, len(grouped)) - for gid, muts := range grouped { - bypassKeysByGroup[gid] = c.resolverClaimedWriteFenceBypassKeys(muts) - } + bypassKeysByGroup := c.writeFenceBypassKeysByGroup(reqs.Elems) return buildTxnLogs(reqs.StartTS, commitTS, grouped, gids, reqs.ObservedRouteVersion, bypassKeysByGroup) } diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index cd314eef6..fecd59c9a 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -324,6 +324,31 @@ func TestShardedCoordinatorRejectsPointWriteOnWriteFencedRoute(t *testing.T) { require.Empty(t, g2Txn.requests, "coordinator must reject before proposing to the fenced shard") } +func TestShardedCoordinatorRejectsEmptyKeyWriteOnLeadingWriteFencedRoute(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateWriteFenced}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + + g1Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: &recordingTransactional{}}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: Put, Key: []byte{}, Value: []byte("v")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteFenced) + require.Empty(t, g1Txn.requests, "coordinator must reject the empty key before proposing to the fenced shard") +} + func TestShardedCoordinatorRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 170dacbd9..e1ba18a4e 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -119,6 +119,10 @@ func TestShardedCoordinatorDispatchTxn_CommitPrimaryUsesPinnedGroup(t *testing.T g1Commit := g1Txn.requests[1] g2Commit := g2Txn.requests[1] + require.Equal(t, [][]byte{[]byte("z-key")}, g1Txn.requests[0].WriteFenceBypassKeys) + require.Equal(t, [][]byte{[]byte("z-key")}, g1Commit.WriteFenceBypassKeys) + require.Equal(t, [][]byte{[]byte("a-key")}, g2Txn.requests[0].WriteFenceBypassKeys) + require.Equal(t, [][]byte{[]byte("a-key")}, g2Commit.WriteFenceBypassKeys) require.Equal(t, pb.Phase_COMMIT, g1Commit.Phase) require.Equal(t, pb.Phase_COMMIT, g2Commit.Phase) require.Equal(t, []byte("z-key"), g1Commit.Mutations[1].Key) @@ -131,6 +135,45 @@ func TestShardedCoordinatorDispatchTxn_CommitPrimaryUsesPinnedGroup(t *testing.T require.Greater(t, primaryCommitMeta.CommitTS, startTS) } +func TestShardedCoordinatorPinnedWritesBypassLogicalRouteFence(t *testing.T) { + t.Parallel() + + for _, isTxn := range []bool{false, true} { + t.Run(map[bool]string{false: "raw", true: "txn"}[isTxn], func(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }, + })) + + g1Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: &recordingTransactional{}}, + }, 1, NewHLC(), nil) + key := []byte("z-key") + reqs := &OperationGroup[OP]{ + IsTxn: isTxn, + Elems: []*Elem[OP]{{Op: Del, Key: key, GroupID: 1}}, + } + if isTxn { + reqs.StartTS = 10 + } + + _, err := coord.Dispatch(context.Background(), reqs) + require.NoError(t, err) + require.Len(t, g1Txn.requests, 1) + require.Equal(t, [][]byte{key}, g1Txn.requests[0].WriteFenceBypassKeys) + require.Equal(t, key, g1Txn.requests[0].Mutations[len(g1Txn.requests[0].Mutations)-1].Key) + }) + } +} + func requestTxnMeta(t *testing.T, req *pb.Request) TxnMeta { t.Helper() require.NotNil(t, req) @@ -673,7 +716,7 @@ func TestShardedCoordinatorCommitPrimaryUsesPinnedMutationGroup(t *testing.T) { 1: {{Op: pb.Op_DEL, Key: primaryKey}}, 2: {{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, } - primaryGid, commitIndex, err := coord.commitPrimaryTxn(context.Background(), 10, primaryKey, grouped, []uint64{1, 2}, 20, 0) + primaryGid, commitIndex, err := coord.commitPrimaryTxn(context.Background(), 10, primaryKey, grouped, []uint64{1, 2}, 20, 0, nil) require.NoError(t, err) require.Equal(t, uint64(1), primaryGid) require.Equal(t, uint64(7), commitIndex) diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 7490809ce..72c3be73d 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -208,10 +208,11 @@ type Request struct { // is plumbing only — the FSM ignores the value, so all existing // callers see no behaviour change. ObservedRouteVersion uint64 `protobuf:"varint,6,opt,name=observed_route_version,json=observedRouteVersion,proto3" json:"observed_route_version,omitempty"` - // write_fence_bypass_keys carries point keys whose owner was resolved by a - // higher-level partition resolver before the request was appended to Raft. - // The FSM uses it only to skip the byte-route write fence for those exact - // point mutations; prefix deletes and unmarked keys still fail closed. + // write_fence_bypass_keys carries point keys whose target group was resolved + // outside the byte-route catalog, either by a partition resolver or an + // explicit maintenance pin. The FSM skips byte-route write-fence and owner + // checks only for those exact point mutations; prefixes and unmarked keys + // still fail closed. WriteFenceBypassKeys [][]byte `protobuf:"bytes,7,rep,name=write_fence_bypass_keys,json=writeFenceBypassKeys,proto3" json:"write_fence_bypass_keys,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache diff --git a/proto/internal.proto b/proto/internal.proto index f58257cec..55abcc256 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -55,10 +55,11 @@ message Request { // is plumbing only — the FSM ignores the value, so all existing // callers see no behaviour change. uint64 observed_route_version = 6; - // write_fence_bypass_keys carries point keys whose owner was resolved by a - // higher-level partition resolver before the request was appended to Raft. - // The FSM uses it only to skip the byte-route write fence for those exact - // point mutations; prefix deletes and unmarked keys still fail closed. + // write_fence_bypass_keys carries point keys whose target group was resolved + // outside the byte-route catalog, either by a partition resolver or an + // explicit maintenance pin. The FSM skips byte-route write-fence and owner + // checks only for those exact point mutations; prefixes and unmarked keys + // still fail closed. repeated bytes write_fence_bypass_keys = 7; } From 903e5896c10eb9f747d7d82024014940014797f4 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 21:12:12 +0900 Subject: [PATCH 152/162] adapter: preserve stream metadata on Lua expiry --- adapter/redis_lua_compat_test.go | 52 ++++++++++++++++++++++++++++++++ adapter/redis_lua_context.go | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/adapter/redis_lua_compat_test.go b/adapter/redis_lua_compat_test.go index 18393cff4..85abb76d2 100644 --- a/adapter/redis_lua_compat_test.go +++ b/adapter/redis_lua_compat_test.go @@ -227,6 +227,58 @@ return {first, second} require.Equal(t, map[string]any{"event": "second"}, events[0].Values) } +func TestRedis_LuaXAddAndExpirePreservesUpdatedStreamMeta(t *testing.T) { + nodes, _, _ := createNode(t, 3) + defer shutdown(nodes) + + ctx := context.Background() + rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) + defer func() { _ = rdb.Close() }() + + const stream = "bull:test:events-xadd-expire" + firstID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream, + ID: "*", + Values: []string{"event", "first"}, + }).Result() + require.NoError(t, err) + + result, err := rdb.Eval(ctx, ` +local id = redis.call("XADD", KEYS[1], "*", "event", "second") +local applied = redis.call("PEXPIRE", KEYS[1], 60000) +return {id, applied} +`, []string{stream}).Result() + require.NoError(t, err) + values, ok := result.([]any) + require.True(t, ok) + require.Len(t, values, 2) + secondID, ok := values[0].(string) + require.True(t, ok) + require.Equal(t, int64(1), values[1]) + require.Greater(t, secondID, firstID) + + xlen, err := rdb.XLen(ctx, stream).Result() + require.NoError(t, err) + require.Equal(t, int64(2), xlen) + events, err := rdb.XRange(ctx, stream, "-", "+").Result() + require.NoError(t, err) + require.Len(t, events, 2) + require.Equal(t, secondID, events[1].ID) + require.Equal(t, map[string]any{"event": "second"}, events[1].Values) + + ttl, err := rdb.PTTL(ctx, stream).Result() + require.NoError(t, err) + require.Positive(t, ttl) + + thirdID, err := rdb.XAdd(ctx, &redis.XAddArgs{ + Stream: stream, + ID: "*", + Values: []string{"event", "third"}, + }).Result() + require.NoError(t, err) + require.Greater(t, thirdID, secondID) +} + func TestRedis_LuaXAddRecreatesTTLExpiredStream(t *testing.T) { nodes, _, _ := createNode(t, 3) defer shutdown(nodes) diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 126d4d2e3..4d1319700 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -4216,7 +4216,7 @@ func (c *luaScriptContext) streamCommitPlan(ctx context.Context, key string) (lu } if st.delta != nil && !st.loaded { elems, err := c.streamDeltaCommitElems(ctx, key, st.delta) - return luaCommitPlan{preserveExisting: true, elems: elems}, err + return luaCommitPlan{preserveExisting: true, inlineMetaRewritten: true, elems: elems}, err } elems, err := c.streamCommitElems(ctx, key) return luaCommitPlan{elems: elems}, err From d9da086eb359cc9e988efa2847493beaa6139a5b Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 21:31:30 +0900 Subject: [PATCH 153/162] raft: report cold-start replay gaps safely --- internal/raftengine/etcd/wal_store.go | 17 ++++++++++---- .../etcd/wal_store_skip_gate_test.go | 23 +++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/internal/raftengine/etcd/wal_store.go b/internal/raftengine/etcd/wal_store.go index acd8edb65..2b11cf49c 100644 --- a/internal/raftengine/etcd/wal_store.go +++ b/internal/raftengine/etcd/wal_store.go @@ -467,19 +467,21 @@ func reportColdStart(obs raftengine.ColdStartObserver, logger *zap.Logger, d col obs.RestoreSkipped(snapIndex, have) } if logger != nil { - // Two named gap fields so an operator correlating the - // log against the Prometheus gauge sees consistent - // magnitudes (claude #934 round 5): + gapAheadCommitted, gapBehindCommitted := coldStartCommittedGaps(target, have) + // Separate gap fields let operators correlate the log with + // the Prometheus gauge without unsigned underflow: // - gap_ahead_snapshot mirrors monitoring.ColdStartObserver // (have - snapIndex), the metric baseline. // - gap_ahead_committed measures how far past the WAL // committed tail (target) the FSM is. + // - gap_behind_committed measures the replay tail still pending. logger.Info("restoreSnapshotState skipped", zap.Uint64("fsm_applied", have), zap.Uint64("snapshot_index", snapIndex), zap.Uint64("last_committed_index", target), zap.Uint64("gap_ahead_snapshot", have-snapIndex), - zap.Uint64("gap_ahead_committed", have-target), + zap.Uint64("gap_ahead_committed", gapAheadCommitted), + zap.Uint64("gap_behind_committed", gapBehindCommitted), ) } case coldStartExecute: @@ -500,6 +502,13 @@ func reportColdStart(obs raftengine.ColdStartObserver, logger *zap.Logger, d col } } +func coldStartCommittedGaps(target, have uint64) (ahead, behind uint64) { + if have >= target { + return have - target, 0 + } + return 0, target - have +} + // applyHeaderStateOnSkip validates the snapshot envelope and full payload CRC, // then applies only the header side-effects (HLC ceiling + Stage 8a cutover) // instead of running the body restore. The FSM body is already durable enough diff --git a/internal/raftengine/etcd/wal_store_skip_gate_test.go b/internal/raftengine/etcd/wal_store_skip_gate_test.go index e121ee396..5f331156f 100644 --- a/internal/raftengine/etcd/wal_store_skip_gate_test.go +++ b/internal/raftengine/etcd/wal_store_skip_gate_test.go @@ -253,6 +253,29 @@ func TestColdStartReplayTarget(t *testing.T) { } } +func TestColdStartCommittedGaps(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + target uint64 + have uint64 + wantAhead uint64 + wantBehind uint64 + }{ + {name: "ahead", target: 100, have: 150, wantAhead: 50}, + {name: "equal", target: 100, have: 100}, + {name: "replay tail remains", target: 150, have: 100, wantBehind: 50}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ahead, behind := coldStartCommittedGaps(tc.target, tc.have) + require.Equal(t, tc.wantAhead, ahead) + require.Equal(t, tc.wantBehind, behind) + }) + } +} + // TestSkipGate_SkipsWhenWALCarriesPostSnapshotTail verifies a node can skip // the multi-GiB snapshot restore even when the committed WAL has entries past // the FSM's durable applied index. Engine.Open seeds e.applied with `have`, From 7c1f0aa43151236ef6003b8607a217940654f793 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 21:45:20 +0900 Subject: [PATCH 154/162] distribution: timestamp completed split jobs --- adapter/distribution_server.go | 1 + adapter/distribution_server_test.go | 2 ++ distribution/migration_promotion_complete.go | 30 +++++++++++++------ .../migration_promotion_complete_test.go | 20 +++++++++++++ 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 6b16a591f..b80d450e6 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -1119,6 +1119,7 @@ func splitJobPromotionMatchesExpected(expected, current distribution.SplitJob) ( normalized.Phase = expected.Phase normalized.TargetPromotionDone = expected.TargetPromotionDone normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.TerminalAtMs = expected.TerminalAtMs normalized.UpdatedAtMs = expected.UpdatedAtMs expectedRaw, err := distribution.EncodeSplitJob(expected) if err != nil { diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 2a8512e1a..42373f024 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -830,6 +830,7 @@ func TestDistributionServerCompleteSplitJobTargetPromotion_UsesCoordinatorCAS(t require.Equal(t, distribution.SplitJobPhaseDone, completed.Phase) require.Equal(t, before.ReadTS+10, completed.PromotionCompletedTS) require.Equal(t, int64(2000), completed.UpdatedAtMs) + require.Equal(t, int64(2000), completed.TerminalAtMs) require.Equal(t, 1, coordinator.dispatchCalls) require.Equal(t, 1, coordinator.timestampCalls) require.Equal(t, before.ReadTS, coordinator.lastStartTS) @@ -916,6 +917,7 @@ func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T require.True(t, loadedJob.TargetPromotionDone) require.Equal(t, distribution.SplitJobPhaseDone, loadedJob.Phase) require.NotZero(t, loadedJob.PromotionCompletedTS) + require.Positive(t, loadedJob.TerminalAtMs) loaded, err := catalog.Snapshot(ctx) require.NoError(t, err) diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go index 0420dec5e..a0ffe801a 100644 --- a/distribution/migration_promotion_complete.go +++ b/distribution/migration_promotion_complete.go @@ -38,15 +38,7 @@ func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs Routes: normalized, } if out.Job.TargetPromotionDone { - if targetClearedDescriptorPresent(out.Job, out.Routes) { - if out.Job.Phase != SplitJobPhaseDone { - out.Changed = true - out.Job.Phase = SplitJobPhaseDone - out.Job.UpdatedAtMs = nowMs - } - return out, nil - } - return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + return completeAlreadyPromotedTarget(out, nowMs) } cleared, err := clearTargetPromotionRoutes(job, out.Routes) @@ -61,10 +53,29 @@ func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs out.ClearedRouteIDs = cleared out.Job.TargetPromotionDone = true out.Job.Phase = SplitJobPhaseDone + out.Job.TerminalAtMs = nowMs out.Job.UpdatedAtMs = nowMs return out, nil } +func completeAlreadyPromotedTarget(out TargetPromotionCompletion, nowMs int64) (TargetPromotionCompletion, error) { + if !targetClearedDescriptorPresent(out.Job, out.Routes) { + return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + if out.Job.Phase != SplitJobPhaseDone { + out.Changed = true + out.Job.Phase = SplitJobPhaseDone + } + if out.Job.TerminalAtMs <= 0 { + out.Changed = true + out.Job.TerminalAtMs = nowMs + } + if out.Changed { + out.Job.UpdatedAtMs = nowMs + } + return out, nil +} + func normalizePromotionCompletionInput(job SplitJob, routes []RouteDescriptor) ([]RouteDescriptor, error) { if err := validateSplitJob(job); err != nil { return nil, err @@ -246,6 +257,7 @@ func promotionCompleteJobMatchesExpected(expected SplitJob, expectedRaw []byte, normalized.Phase = expected.Phase normalized.TargetPromotionDone = expected.TargetPromotionDone normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.TerminalAtMs = expected.TerminalAtMs normalized.UpdatedAtMs = expected.UpdatedAtMs raw, err := EncodeSplitJob(normalized) if err != nil { diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go index 1973c19e9..d68cd7c00 100644 --- a/distribution/migration_promotion_complete_test.go +++ b/distribution/migration_promotion_complete_test.go @@ -22,6 +22,7 @@ func TestCompleteTargetPromotionStateClearsStagedFieldsAndRetainsFloor(t *testin require.True(t, result.Job.TargetPromotionDone) require.Zero(t, result.Job.PromotionCompletedTS) require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Equal(t, int64(1000), result.Job.TerminalAtMs) require.Equal(t, SplitJobPhaseDone, result.Job.Phase) target := routeByID(t, result.Routes, 3) @@ -53,6 +54,7 @@ func TestCompleteTargetPromotionStateAcceptsAlreadyClearedDescriptor(t *testing. require.Equal(t, SplitJobPhaseDone, result.Job.Phase) require.Equal(t, uint64(900), result.Job.PromotionCompletedTS) require.Equal(t, int64(1100), result.Job.UpdatedAtMs) + require.Equal(t, int64(1100), result.Job.TerminalAtMs) target := routeByID(t, result.Routes, 3) require.False(t, target.StagedVisibilityActive) @@ -140,12 +142,30 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDesc require.Equal(t, SplitJobPhaseDone, completed.Phase) require.Equal(t, uint64(900), completed.PromotionCompletedTS) require.Equal(t, int64(1100), completed.UpdatedAtMs) + require.Equal(t, int64(1100), completed.TerminalAtMs) loaded, err := cs.Snapshot(ctx) require.NoError(t, err) require.Equal(t, saved.Version+1, loaded.Version) } +func TestCompleteTargetPromotionStateBackfillsMissingTerminalTime(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.Phase = SplitJobPhaseDone + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + + result, err := CompleteTargetPromotionState(job, routes, 1200) + require.NoError(t, err) + require.True(t, result.Changed) + require.Equal(t, int64(1200), result.Job.TerminalAtMs) + require.Equal(t, int64(1200), result.Job.UpdatedAtMs) +} + func TestCatalogStoreCompleteSplitJobTargetPromotionRejectsStaleInputs(t *testing.T) { t.Parallel() From 737b9301fe70893976deb4e22c25292d6b255f8c Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 22:08:01 +0900 Subject: [PATCH 155/162] kv: preserve migration fence bypasses --- kv/fsm.go | 28 ++++++++++++++++++++++------ kv/fsm_migration_fence_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/kv/fsm.go b/kv/fsm.go index 7d9ca2710..5221a008a 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -641,11 +641,14 @@ func (f *kvFSM) stagedVisibilityRoutesForPrefix(prefix []byte) []distribution.Ro return out } -func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation) error { +func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation, bypassKeys map[string]struct{}) error { for _, mut := range muts { if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { continue } + if _, bypass := bypassKeys[string(mut.Key)]; bypass { + continue + } if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { return err } @@ -1452,7 +1455,7 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, r.ReadKeys, nil); err != nil { return err } - uniq, err := f.uniqueMutationsNotFenced(ctx, muts, floorTS) + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, r.GetWriteFenceBypassKeys(), floorTS) if err != nil { return err } @@ -1522,7 +1525,14 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, storeMuts, err := f.onePhaseStoreMutations(ctx, muts, r.ReadKeys, startTS, commitTS) + uniq, storeMuts, err := f.onePhaseStoreMutations( + ctx, + muts, + r.ReadKeys, + r.GetWriteFenceBypassKeys(), + startTS, + commitTS, + ) if err != nil { return err } @@ -1537,10 +1547,11 @@ func (f *kvFSM) onePhaseStoreMutations( ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, + writeFenceBypassKeys [][]byte, startTS uint64, commitTS uint64, ) ([]*pb.Mutation, []*store.KVPairMutation, error) { - uniq, err := f.uniqueMutationsNotFenced(ctx, muts, commitTS) + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, writeFenceBypassKeys, commitTS) if err != nil { return nil, nil, err } @@ -1554,12 +1565,17 @@ func (f *kvFSM) onePhaseStoreMutations( return uniq, storeMuts, nil } -func (f *kvFSM) uniqueMutationsNotFenced(ctx context.Context, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { +func (f *kvFSM) uniqueMutationsNotFenced( + ctx context.Context, + muts []*pb.Mutation, + writeFenceBypassKeys [][]byte, + commitTS uint64, +) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { return nil, err } - if err := f.verifyRouteNotFencedForMutations(uniq); err != nil { + if err := f.verifyRouteNotFencedForMutations(uniq, writeFenceBypassKeySet(writeFenceBypassKeys)); err != nil { return nil, err } if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 7c3d1c494..787ccefce 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -190,6 +190,33 @@ func TestFSMWriteFenceBypassAllowsPinnedTxnOnNonOwningGroup(t *testing.T) { require.NoError(t, err) } +func TestFSMWriteFenceBypassAllowsPinnedOnePhaseTxnOnNonOwningGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }) + fsm := newComposed1FSM(t, engine, 1) + key := []byte("z") + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Ts: 10, + ObservedRouteVersion: 1, + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("v")}, + }, + }, 20) + require.NoError(t, err) + + got, err := fsm.store.GetAt(context.Background(), key, 20) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) +} + func TestFSMWriteFenceBypassDoesNotAllowDelPrefix(t *testing.T) { t.Parallel() From 0c74b6043d4a9942cbb788156886713ec3f21714 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 23:15:08 +0900 Subject: [PATCH 156/162] Complete split migration runner phases --- adapter/distribution_server.go | 42 ++- adapter/distribution_server_test.go | 164 +++++++++ adapter/internal.go | 73 +++- adapter/split_job_runner.go | 552 ++++++++++++++++++++++++++++ kv/compactor.go | 24 ++ kv/compactor_test.go | 72 ++++ kv/fsm.go | 67 +++- kv/fsm_migration_fence_test.go | 130 ++++++- kv/fsm_migration_readiness.go | 90 ++++- kv/shard_store.go | 15 + kv/shard_store_test.go | 59 +++ main.go | 47 +++ proto/internal.pb.go | 207 ++++++++++- proto/internal.proto | 22 +- proto/internal_grpc.pb.go | 38 ++ store/migration_readiness.go | 85 ++++- store/migration_readiness_test.go | 79 ++++ store/store.go | 5 + 18 files changed, 1717 insertions(+), 54 deletions(-) create mode 100644 adapter/split_job_runner.go create mode 100644 store/migration_readiness_test.go diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index b80d450e6..25837a8fd 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -34,6 +34,7 @@ type DistributionServer struct { splitJobRunnerReady bool splitJobRunnerReadinessGate SplitMigrationCapabilityGate splitPromotionClientFactory SplitPromotionClientFactory + splitMigrationClientFactory SplitMigrationClientFactory knownRaftGroups map[uint64]struct{} reloadRetry struct { attempts int @@ -59,6 +60,19 @@ type SplitPromotionClient interface { // target range and returns the internal client used by the runner. type SplitPromotionClientFactory func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) +// SplitMigrationClient is the internal data-plane client used for a split +// job's source export, target import, and durable source/target guards. +type SplitMigrationClient interface { + SplitPromotionClient + ExportRangeVersions(context.Context, *pb.ExportRangeVersionsRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) + ImportRangeVersions(context.Context, *pb.ImportRangeVersionsRequest, ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) + ApplyTargetStagedReadiness(context.Context, *pb.TargetStagedReadinessRequest, ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) + ProbeMigrationLocks(context.Context, *pb.ProbeMigrationLocksRequest, ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) +} + +// SplitMigrationClientFactory resolves both participating group leaders. +type SplitMigrationClientFactory func(context.Context, distribution.SplitJob, uint64) (source SplitMigrationClient, target SplitMigrationClient, err error) + // WithDistributionCoordinator configures the coordinator used for Raft-backed // catalog mutations in SplitRange. func WithDistributionCoordinator(coordinator kv.Coordinator) DistributionServerOption { @@ -109,6 +123,14 @@ func WithSplitPromotionClientFactory(factory SplitPromotionClientFactory) Distri } } +// WithSplitMigrationClientFactory configures the source/target clients used by +// the production split job state machine. +func WithSplitMigrationClientFactory(factory SplitMigrationClientFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitMigrationClientFactory = factory + } +} + // WithDistributionKnownRaftGroups configures the Raft group IDs this node can // route migration work to. func WithDistributionKnownRaftGroups(groupIDs ...uint64) DistributionServerOption { @@ -243,8 +265,8 @@ func (s *DistributionServer) RunSplitJobRunnerOnce(ctx context.Context) error { if err != nil { return splitJobCatalogStatusError(err) } - if job, ok := nextCleanupSplitJob(jobs); ok { - return s.promoteSplitJobTargetAndComplete(ctx, job) + if job, ok := nextRunnableSplitJob(jobs); ok { + return s.runSplitJobPhase(ctx, job) } return nil } @@ -263,10 +285,22 @@ func (s *DistributionServer) verifySplitJobRunnerLeader(ctx context.Context) (bo return true, nil } -func nextCleanupSplitJob(jobs []distribution.SplitJob) (distribution.SplitJob, bool) { +func nextRunnableSplitJob(jobs []distribution.SplitJob) (distribution.SplitJob, bool) { for _, job := range jobs { - if job.Phase == distribution.SplitJobPhaseCleanup { + switch job.Phase { + case distribution.SplitJobPhasePlanned, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseCutover, + distribution.SplitJobPhaseCleanup, + distribution.SplitJobPhaseAbandoning: return job, true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned: + continue } } return distribution.SplitJob{}, false diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 42373f024..b317acdb6 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "io" "testing" "time" @@ -981,6 +982,81 @@ func TestDistributionServerRunSplitJobRunnerOnce_TerminalizesPromotedCleanupJob( require.Equal(t, saved.Version+1, s.engine.Version()) } +func TestDistributionServerRunSplitJobRunnerOnce_CompletesCrossGroupMigration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 1, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(saved)) + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + engine, + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + for range 200 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseDone { + break + } + } + + completed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseDone, completed.Phase) + require.True(t, completed.WriteTrackerArmed) + require.True(t, completed.PostFenceDrainCompleted) + require.NotZero(t, completed.SnapshotTS) + require.NotZero(t, completed.FenceTS) + require.NotZero(t, completed.CutoverVersion) + require.True(t, completed.TargetPromotionDone) + require.NotEmpty(t, source.controls) + require.NotEmpty(t, target.controls) + require.NotEmpty(t, source.exports) + require.Len(t, target.imports, len(source.exports)) + + finalSnapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, finalSnapshot.Routes, 2) + right := distributionRouteByID(t, finalSnapshot.Routes, 3) + require.Equal(t, uint64(2), right.GroupID) + require.Equal(t, distribution.RouteStateActive, right.State) + require.False(t, right.StagedVisibilityActive) + require.Equal(t, completed.FenceTS, right.MinWriteTSExclusive) +} + func TestDistributionServerRunSplitJobRunnerOnce_SkipsFollower(t *testing.T) { t.Parallel() @@ -1866,6 +1942,94 @@ type splitPromotionClientStub struct { calls int } +type splitMigrationClientStub struct { + splitPromotionClientStub + exports []*pb.ExportRangeVersionsRequest + imports []*pb.ImportRangeVersionsRequest + controls []*pb.TargetStagedReadinessRequest +} + +func (s *splitMigrationClientStub) ExportRangeVersions( + _ context.Context, + req *pb.ExportRangeVersionsRequest, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + cloned := &pb.ExportRangeVersionsRequest{ + RangeStart: distribution.CloneBytes(req.GetRangeStart()), + RangeEnd: distribution.CloneBytes(req.GetRangeEnd()), + MaxCommitTs: req.GetMaxCommitTs(), + MinCommitTs: req.GetMinCommitTs(), + Cursor: distribution.CloneBytes(req.GetCursor()), + ChunkBytes: req.GetChunkBytes(), + RouteStart: distribution.CloneBytes(req.GetRouteStart()), + RouteEnd: distribution.CloneBytes(req.GetRouteEnd()), + MaxScannedBytes: req.GetMaxScannedBytes(), + KeyFamily: req.GetKeyFamily(), + ExcludeKnownInternal: req.GetExcludeKnownInternal(), + ExcludePrefixes: req.GetExcludePrefixes(), + } + s.exports = append(s.exports, cloned) + phaseByte := byte(0) + if req.GetMinCommitTs() != 0 { + phaseByte = 1 + } + cursor := []byte{byte(req.GetKeyFamily()), phaseByte} + return &splitMigrationStreamStub{responses: []*pb.ExportRangeVersionsResponse{{ + Versions: []*pb.MVCCVersion{{ + Key: []byte("n"), + CommitTs: 10, + Value: []byte("v"), + KeyFamily: req.GetKeyFamily(), + }}, + NextCursor: cursor, + Done: true, + }}}, nil +} + +func (s *splitMigrationClientStub) ImportRangeVersions( + _ context.Context, + req *pb.ImportRangeVersionsRequest, + _ ...grpc.CallOption, +) (*pb.ImportRangeVersionsResponse, error) { + s.imports = append(s.imports, req) + return &pb.ImportRangeVersionsResponse{AckedCursor: distribution.CloneBytes(req.GetCursor())}, nil +} + +func (s *splitMigrationClientStub) ApplyTargetStagedReadiness( + _ context.Context, + req *pb.TargetStagedReadinessRequest, + _ ...grpc.CallOption, +) (*pb.TargetStagedReadinessResponse, error) { + s.controls = append(s.controls, req) + minAdmittedTS := uint64(0) + if req.GetTrackWrites() { + minAdmittedTS = 10 + } + return &pb.TargetStagedReadinessResponse{MinAdmittedTs: minAdmittedTS}, nil +} + +func (s *splitMigrationClientStub) ProbeMigrationLocks( + context.Context, + *pb.ProbeMigrationLocksRequest, + ...grpc.CallOption, +) (*pb.ProbeMigrationLocksResponse, error) { + return &pb.ProbeMigrationLocksResponse{}, nil +} + +type splitMigrationStreamStub struct { + grpc.ClientStream + responses []*pb.ExportRangeVersionsResponse +} + +func (s *splitMigrationStreamStub) Recv() (*pb.ExportRangeVersionsResponse, error) { + if len(s.responses) == 0 { + return nil, io.EOF + } + resp := s.responses[0] + s.responses = s.responses[1:] + return resp, nil +} + func (s *splitPromotionClientStub) PromoteStagedVersions( _ context.Context, req *pb.PromoteStagedVersionsRequest, diff --git a/adapter/internal.go b/adapter/internal.go index c6ac0eeb0..a7a0d50ae 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -294,17 +294,8 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta } func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) { - if req == nil { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness request is nil")) - } - if req.GetJobId() == 0 { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness job_id is required")) - } - if req.GetMigrationJobId() == 0 { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness migration_job_id is required")) - } - if req.GetArmed() && req.GetMinWriteTsExclusive() == 0 { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness min_write_ts_exclusive is required when armed")) + if err := validateTargetStagedReadinessRequest(req); err != nil { + return nil, err } if i.migrationProposer == nil { return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "target staged readiness proposer is not configured")) @@ -318,7 +309,65 @@ func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.Targe if err := i.proposeTargetStagedReadiness(ctx, req); err != nil { return nil, errors.WithStack(err) } - return &pb.TargetStagedReadinessResponse{}, nil + return &pb.TargetStagedReadinessResponse{MinAdmittedTs: i.migrationMinAdmittedTS(ctx, req.GetJobId())}, nil +} + +func (i *Internal) migrationMinAdmittedTS(ctx context.Context, jobID uint64) uint64 { + reader, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return 0 + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return 0 + } + for _, state := range states { + if state.JobID == jobID { + return state.MinAdmittedTS + } + } + return 0 +} + +func validateTargetStagedReadinessRequest(req *pb.TargetStagedReadinessRequest) error { + switch { + case req == nil: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness request is nil")) + case req.GetJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness job_id is required")) + case req.GetMigrationJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness migration_job_id is required")) + case req.GetArmed() && req.GetMinWriteTsExclusive() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness min_write_ts_exclusive is required when armed")) + } + return validateSourceMigrationControlRequest(req) +} + +func validateSourceMigrationControlRequest(req *pb.TargetStagedReadinessRequest) error { + if req.GetSourceReadFence() && !req.GetSourceWriteFence() { + return errors.WithStack(status.Error(codes.InvalidArgument, "source read fence requires source write fence")) + } + if (req.GetSourceWriteFence() || req.GetSourceReadFence()) && req.GetRetentionPinTs() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "source migration control retention_pin_ts is required")) + } + return nil +} + +func (i *Internal) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest) (*pb.ProbeMigrationLocksResponse, error) { + if req == nil || len(req.GetRouteStart()) == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration lock probe route_start is required")) + } + if i.store == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration lock probe store is not configured")) + } + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + locks, err := kv.PendingTxnLocksInRoute(ctx, i.store, req.GetRouteStart(), req.GetRouteEnd(), ^uint64(0), int(req.GetLimit())) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.ProbeMigrationLocksResponse{PendingCount: uint32(len(locks))}, nil //nolint:gosec // result is bounded by uint32 wire limit } func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { diff --git a/adapter/split_job_runner.go b/adapter/split_job_runner.go new file mode 100644 index 000000000..b794055a2 --- /dev/null +++ b/adapter/split_job_runner.go @@ -0,0 +1,552 @@ +package adapter + +import ( + "bytes" + "context" + "io" + "math" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +const ( + defaultSplitMigrationChunkBytes = 1 << 20 + defaultSplitMigrationMaxScannedBytes = 4 << 20 + defaultSplitMigrationLockProbeLimit = 1024 +) + +func (s *DistributionServer) runSplitJobPhase(ctx context.Context, job distribution.SplitJob) error { + if job.Phase != distribution.SplitJobPhaseCleanup && s.splitMigrationClientFactory == nil { + return errors.New("split migration client factory is not configured") + } + if job.Phase == distribution.SplitJobPhaseBackfill || job.Phase == distribution.SplitJobPhaseDeltaCopy { + return s.runSplitJobCopyPhase(ctx, job) + } + return s.runSplitJobControlPhase(ctx, job) +} + +func (s *DistributionServer) runSplitJobCopyPhase(ctx context.Context, job distribution.SplitJob) error { + if job.Phase == distribution.SplitJobPhaseBackfill { + return s.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseBackfill, 0, job.SnapshotTS) + } + return s.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseDeltaCopy, job.DeltaFloor, job.FenceTS) +} + +func (s *DistributionServer) runSplitJobControlPhase(ctx context.Context, job distribution.SplitJob) error { + switch job.Phase { + case distribution.SplitJobPhasePlanned: + return s.beginSplitJobBackfill(ctx, job) + case distribution.SplitJobPhaseFence: + return s.finalizeSplitJobFence(ctx, job) + case distribution.SplitJobPhaseCutover: + return s.cutoverSplitJob(ctx, job) + case distribution.SplitJobPhaseCleanup: + return s.cleanupSplitJob(ctx, job) + case distribution.SplitJobPhaseAbandoning: + return errors.New("split job abandon cleanup is not configured") + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned: + return nil + } + return errors.New("split job phase is not runnable") +} + +func (s *DistributionServer) cleanupSplitJob(ctx context.Context, job distribution.SplitJob) error { + if job.SourceRetentionPinTS > 0 && job.SourceRetentionPinTS != math.MaxUint64 { + if err := s.releaseSplitJobSourceRetention(ctx, job); err != nil { + return err + } + return nil + } + return s.promoteSplitJobTargetAndComplete(ctx, job) +} + +func (s *DistributionServer) releaseSplitJobSourceRetention(ctx context.Context, job distribution.SplitJob) error { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + sourceGroupID, routeEnd, ok := splitJobSourceSibling(snapshot.Routes, job) + if !ok { + return errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + source, _, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + if err != nil { + return errors.WithStack(err) + } + if _, err := applySplitMigrationControl( + ctx, + source, + job, + routeEnd, + job.CutoverVersion, + job.FenceTS, + true, + true, + false, + math.MaxUint64, + ); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceRetentionPinTS = math.MaxUint64 + current.SourceCutoverAckCursor = []byte("retention-released") + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func splitJobSourceSibling(routes []distribution.RouteDescriptor, job distribution.SplitJob) (uint64, []byte, bool) { + var sourceGroupID uint64 + var routeEnd []byte + for _, route := range routes { + if route.ParentRouteID != job.SourceRouteID { + continue + } + if bytes.Equal(route.Start, job.SplitKey) { + routeEnd = distribution.CloneBytes(route.End) + continue + } + if bytes.Equal(route.End, job.SplitKey) { + sourceGroupID = route.GroupID + } + } + return sourceGroupID, routeEnd, sourceGroupID != 0 +} + +func (s *DistributionServer) beginSplitJobBackfill(ctx context.Context, job distribution.SplitJob) error { + snapshot, parent, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + source, _, err := s.splitMigrationClientFactory(ctx, job, parent.GroupID) + if err != nil { + return errors.WithStack(err) + } + if source == nil { + return errors.New("split migration source client is nil") + } + snapshotTS := s.engine.NextTimestamp() + if snapshotTS == 0 { + return errors.New("split migration snapshot timestamp is zero") + } + // The source guard is applied before BACKFILL opens. This keeps the initial + // implementation conservative: no write can land outside the copied window. + if _, err := applySplitMigrationControl(ctx, source, job, parent.End, snapshot.Version+1, snapshotTS, false, false, true, 1); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhasePlanned { + return current, nil + } + current.Phase = distribution.SplitJobPhaseBackfill + current.SnapshotTS = snapshotTS + current.WriteTrackerArmed = true + current.SourceRetentionPinTS = 1 + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) copySplitJobPhase( + ctx context.Context, + job distribution.SplitJob, + exportPhase distribution.SplitJobExportPhase, + minCommitTS uint64, + maxCommitTS uint64, +) error { + if maxCommitTS == 0 { + return errors.New("split migration copy upper timestamp is zero") + } + _, sourceRoute, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + brackets, err := distribution.PlanExportBrackets(job.SplitKey, sourceRoute.End) + if err != nil { + return errors.WithStack(err) + } + progressIndex, bracket, ok := nextSplitJobBracket(job, brackets, exportPhase) + if !ok { + return s.advanceCompletedCopyPhase(ctx, job, exportPhase) + } + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return errors.WithStack(err) + } + if source == nil || target == nil { + return errors.New("split migration source or target client is nil") + } + progress := job.BracketProgress[progressIndex] + stream, err := source.ExportRangeVersions(ctx, &pb.ExportRangeVersionsRequest{ + RangeStart: bracket.Start, + RangeEnd: bracket.End, + MaxCommitTs: maxCommitTS, + MinCommitTs: minCommitTS, + Cursor: progress.Cursor, + ChunkBytes: defaultSplitMigrationChunkBytes, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + MaxScannedBytes: defaultSplitMigrationMaxScannedBytes, + KeyFamily: bracket.Family, + ExcludeKnownInternal: bracket.ExcludeKnownInternal, + ExcludePrefixes: bracket.ExcludePrefixes, + }) + if err != nil { + return errors.WithStack(err) + } + return s.copySplitJobStream(ctx, job, progressIndex, bracket, progress, stream, target) +} + +func (s *DistributionServer) copySplitJobStream( + ctx context.Context, + job distribution.SplitJob, + progressIndex int, + bracket distribution.MigrationBracket, + progress distribution.SplitJobBracketProgress, + stream grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], + target SplitMigrationClient, +) error { + for { + resp, recvErr := stream.Recv() + if errors.Is(recvErr, io.EOF) { + return nil + } + if recvErr != nil { + return errors.WithStack(recvErr) + } + nextCursor := distribution.CloneBytes(resp.GetNextCursor()) + batchSeq := progress.LastAckedBatchSeq + 1 + importResp, importErr := target.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ + JobId: job.JobID, + Versions: resp.GetVersions(), + Cursor: nextCursor, + BracketId: bracket.BracketID, + BatchSeq: batchSeq, + }) + if importErr != nil { + return errors.WithStack(importErr) + } + if !bytes.Equal(importResp.GetAckedCursor(), nextCursor) { + return errors.New("split migration import acknowledged a different cursor") + } + progress.Cursor = nextCursor + progress.Done = resp.GetDone() + progress.AcceptedRows += uint64(len(resp.GetVersions())) + progress.LastAckedBatchSeq = batchSeq + for _, version := range resp.GetVersions() { + if version.GetCommitTs() > job.MaxImportedTS { + job.MaxImportedTS = version.GetCommitTs() + } + } + job.BracketProgress[progressIndex] = progress + job.Cursor = nextCursor + job.UpdatedAtMs = time.Now().UnixMilli() + if err := s.persistSplitJobCopyProgress(ctx, job); err != nil { + return err + } + if progress.Done { + return nil + } + } +} + +func nextSplitJobBracket( + job distribution.SplitJob, + brackets []distribution.MigrationBracket, + exportPhase distribution.SplitJobExportPhase, +) (int, distribution.MigrationBracket, bool) { + byID := make(map[uint64]distribution.MigrationBracket, len(brackets)) + for _, bracket := range brackets { + byID[bracket.BracketID] = bracket + } + for i, progress := range job.BracketProgress { + if progress.ExportPhase != exportPhase || progress.Done { + continue + } + bracket, found := byID[progress.BracketID] + if found && bracket.Family == progress.Family { + return i, bracket, true + } + } + return 0, distribution.MigrationBracket{}, false +} + +func (s *DistributionServer) persistSplitJobCopyProgress(ctx context.Context, next distribution.SplitJob) error { + return s.updateSplitJobViaCoordinator(ctx, next.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != next.Phase { + return current, nil + } + return distribution.CloneSplitJob(next), nil + }) +} + +func (s *DistributionServer) advanceCompletedCopyPhase(ctx context.Context, job distribution.SplitJob, phase distribution.SplitJobExportPhase) error { + switch phase { + case distribution.SplitJobExportPhaseBackfill: + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseBackfill { + current.Phase = distribution.SplitJobPhaseFence + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + case distribution.SplitJobExportPhaseDeltaCopy: + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseDeltaCopy { + current.Phase = distribution.SplitJobPhaseCutover + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + case distribution.SplitJobExportPhaseNone: + return errors.New("unknown split migration export phase") + } + return errors.New("unknown split migration export phase") +} + +func (s *DistributionServer) finalizeSplitJobFence(ctx context.Context, job distribution.SplitJob) error { + snapshot, sourceRoute, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + snapshot, sourceRoute, err = s.ensureSplitJobCatalogFence(ctx, job, snapshot, sourceRoute) + if err != nil { + return err + } + source, _, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return errors.WithStack(err) + } + minAdmittedTS, err := applySplitMigrationControl( + ctx, + source, + job, + sourceRoute.End, + snapshot.Version, + job.SnapshotTS, + true, + false, + true, + 1, + ) + if err != nil { + return err + } + pendingLocks, err := splitJobPendingLocks(ctx, source, job.SplitKey, sourceRoute.End) + if err != nil { + return err + } + if pendingLocks { + return nil + } + return s.commitSplitJobFenceState(ctx, job, snapshot.Version, minAdmittedTS) +} + +func (s *DistributionServer) commitSplitJobFenceState(ctx context.Context, job distribution.SplitJob, fenceCatalogVersion, minAdmittedTS uint64) error { + fenceTS := s.engine.NextTimestamp() + deltaFloor := job.SnapshotTS + if minAdmittedTS > 0 && minAdmittedTS-1 < deltaFloor { + deltaFloor = minAdmittedTS - 1 + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseFence { + return current, nil + } + current.Phase = distribution.SplitJobPhaseDeltaCopy + current.PostFenceDrainCompleted = true + current.SnapshotMinAdmittedTS = minAdmittedTS + current.DeltaFloor = deltaFloor + current.FenceTS = fenceTS + current.FenceCatalogVersion = fenceCatalogVersion + current.FenceAckCursor = []byte("raft-applied") + current.SourceRetentionPinTS = 1 + for i := range current.BracketProgress { + current.BracketProgress[i].ExportPhase = distribution.SplitJobExportPhaseDeltaCopy + current.BracketProgress[i].Cursor = nil + current.BracketProgress[i].Done = false + current.BracketProgress[i].ScannedBytes = 0 + current.BracketProgress[i].AcceptedRows = 0 + } + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) ensureSplitJobCatalogFence( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + sourceRoute distribution.RouteDescriptor, +) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + if sourceRoute.RouteID != job.SourceRouteID { + return snapshot, sourceRoute, nil + } + leftID, rightID, err := s.allocateChildRouteIDs(ctx, snapshot.ReadTS, snapshot.Routes) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + left, right := splitCatalogRoutes(sourceRoute, job.SplitKey, leftID, rightID) + right.State = distribution.RouteStateWriteFenced + snapshot, err = s.saveSplitResultViaCoordinator(ctx, snapshot.ReadTS, snapshot.Version, sourceRoute.RouteID, nil, left, right) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + if err := s.applyEngineSnapshot(snapshot); err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + return snapshot, right, nil +} + +func splitJobPendingLocks(ctx context.Context, source SplitMigrationClient, routeStart []byte, routeEnd []byte) (bool, error) { + resp, err := source.ProbeMigrationLocks(ctx, &pb.ProbeMigrationLocksRequest{ + RouteStart: routeStart, + RouteEnd: routeEnd, + Limit: defaultSplitMigrationLockProbeLimit, + }) + if err != nil { + return false, errors.WithStack(err) + } + return resp.GetPendingCount() != 0, nil +} + +func (s *DistributionServer) cutoverSplitJob(ctx context.Context, job distribution.SplitJob) error { + snapshot, sourceRoute, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + if snapshot.Version == math.MaxUint64 { + return errors.New("split migration catalog version overflow") + } + expectedVersion := snapshot.Version + 1 + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return errors.WithStack(err) + } + if _, err := applySplitMigrationControl(ctx, target, job, sourceRoute.End, expectedVersion, job.FenceTS, false, false, false, 0); err != nil { + return err + } + if _, err := applySplitMigrationControl(ctx, source, job, sourceRoute.End, expectedVersion, job.FenceTS, true, true, false, 1); err != nil { + return err + } + return s.commitSplitJobCutover(ctx, snapshot, sourceRoute, job, expectedVersion) +} + +func applySplitMigrationControl( + ctx context.Context, + client SplitMigrationClient, + job distribution.SplitJob, + routeEnd []byte, + expectedVersion uint64, + minWriteTS uint64, + sourceWriteFence bool, + sourceReadFence bool, + trackWrites bool, + retentionPinTS uint64, +) (uint64, error) { + if client == nil { + return 0, errors.New("split migration control client is nil") + } + resp, err := client.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: job.JobID, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCutoverVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: minWriteTS, + Armed: true, + SourceWriteFence: sourceWriteFence, + SourceReadFence: sourceReadFence, + RetentionPinTs: retentionPinTS, + TrackWrites: trackWrites, + }) + if err != nil { + return 0, errors.WithStack(err) + } + return resp.GetMinAdmittedTs(), nil +} + +func (s *DistributionServer) splitJobSourceRoute( + ctx context.Context, + job distribution.SplitJob, +) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + if parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + return snapshot, parent, nil + } + for _, route := range snapshot.Routes { + if route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) { + return snapshot, distribution.CloneRouteDescriptor(route), nil + } + } + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, splitJobCatalogStatusError(distribution.ErrMigrationSourceRouteChanged) +} + +func (s *DistributionServer) commitSplitJobCutover( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + right distribution.RouteDescriptor, + job distribution.SplitJob, + nextVersion uint64, +) error { + right.GroupID = job.TargetGroupID + right.State = distribution.RouteStateActive + right.StagedVisibilityActive = true + right.MigrationJobID = job.JobID + right.MinWriteTSExclusive = job.FenceTS + encodedRoute, err := distribution.EncodeRouteDescriptorForCatalogWrite(right, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return errors.WithStack(err) + } + nextJob := distribution.CloneSplitJob(job) + nextJob.Phase = distribution.SplitJobPhaseCleanup + nextJob.CutoverVersion = nextVersion + nextJob.CutoverReadFenceState = distribution.SplitJobBarrierArmed + nextJob.TargetStagedReadinessState = distribution.SplitJobBarrierArmed + nextJob.SourceCutoverReadFenceAckCursor = []byte("raft-applied") + nextJob.TargetStagedReadinessAckCursor = []byte("raft-applied") + nextJob.UpdatedAtMs = time.Now().UnixMilli() + encodedJob, err := distribution.EncodeSplitJob(nextJob) + if err != nil { + return errors.WithStack(err) + } + versionKey := distribution.CatalogVersionKey() + routeKey := distribution.CatalogRouteKey(right.RouteID) + jobKey := distribution.CatalogSplitJobKey(job.JobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: routeKey, Value: encodedRoute}, + {Op: kv.Put, Key: versionKey, Value: distribution.EncodeCatalogVersion(nextVersion)}, + {Op: kv.Put, Key: jobKey, Value: encodedJob}, + }, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: [][]byte{routeKey, versionKey, jobKey}, + }); err != nil { + if errors.Is(err, store.ErrWriteConflict) { + return grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) + } + return errors.WithStack(err) + } + updated, err := s.loadCatalogSnapshotAtLeastVersion(ctx, nextVersion) + if err != nil { + return err + } + return s.applyEngineSnapshot(updated) +} diff --git a/kv/compactor.go b/kv/compactor.go index c13c7d836..8700f92bb 100644 --- a/kv/compactor.go +++ b/kv/compactor.go @@ -234,6 +234,13 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun if !ok { return nil } + safeMinTS, err := migrationRetentionMinTS(ctx, runtime.Store, safeMinTS) + if err != nil { + return errors.Wrapf(err, "read migration retention pins for group %d", runtime.GroupID) + } + if safeMinTS <= retention.MinRetainedTS() { + return nil + } compactCtx, cancel := c.compactContext(ctx, status) defer cancel() @@ -253,6 +260,23 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun return nil } +func migrationRetentionMinTS(ctx context.Context, st store.MVCCStore, safeMinTS uint64) (uint64, error) { + reader, ok := st.(store.MigrationTargetReadinessReader) + if !ok { + return safeMinTS, nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return 0, errors.WithStack(err) + } + for _, state := range states { + if state.Armed && state.RetentionPinTS > 0 && state.RetentionPinTS < safeMinTS { + safeMinTS = state.RetentionPinTS + } + } + return safeMinTS, nil +} + func (c *FSMCompactor) compactionRuntimeReady(runtime FSMCompactRuntime) (store.RetentionController, raftengine.Status, bool) { if runtime.StatusReader == nil || runtime.Store == nil { return nil, raftengine.Status{}, false diff --git a/kv/compactor_test.go b/kv/compactor_test.go index b6417a8e0..c6ae90f0f 100644 --- a/kv/compactor_test.go +++ b/kv/compactor_test.go @@ -56,6 +56,15 @@ type metricsCapturingStore struct { metrics *pebble.Metrics } +type migrationPinnedCompactionStore struct { + deadlineCapturingStore + states []store.TargetStagedReadinessState +} + +func (s *migrationPinnedCompactionStore) MigrationTargetReadinessStates(context.Context) ([]store.TargetStagedReadinessState, error) { + return s.states, nil +} + func (s *metricsCapturingStore) Metrics() *pebble.Metrics { return s.metrics } @@ -146,6 +155,69 @@ func TestFSMCompactorRespectsPinnedTimestamp(t *testing.T) { require.Equal(t, []byte("v20"), val) } +func TestFSMCompactorRespectsMigrationRetentionPin(t *testing.T) { + t.Parallel() + + st := &migrationPinnedCompactionStore{ + deadlineCapturingStore: deadlineCapturingStore{lastCommitTS: ^uint64(0)}, + states: []store.TargetStagedReadinessState{{ + JobID: 1, + MigrationJobID: 1, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 42, + }}, + } + compactor := NewFSMCompactor( + []FSMCompactRuntime{{ + GroupID: 1, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, + AppliedIndex: 1, + CommitIndex: 1, + }}, + Store: st, + }}, + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + ) + + require.NoError(t, compactor.SyncOnce(context.Background())) + require.True(t, st.compactCalled) + require.Equal(t, uint64(42), st.compactMinTS) +} + +func TestFSMCompactorIgnoresDisarmedMigrationRetentionPin(t *testing.T) { + t.Parallel() + + st := &migrationPinnedCompactionStore{ + deadlineCapturingStore: deadlineCapturingStore{lastCommitTS: ^uint64(0)}, + states: []store.TargetStagedReadinessState{{ + JobID: 1, + MigrationJobID: 1, + MinWriteTSExclusive: 1, + RetentionPinTS: 42, + }}, + } + compactor := NewFSMCompactor( + []FSMCompactRuntime{{ + GroupID: 1, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, + AppliedIndex: 1, + CommitIndex: 1, + }}, + Store: st, + }}, + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + ) + + require.NoError(t, compactor.SyncOnce(context.Background())) + require.True(t, st.compactCalled) + require.Greater(t, st.compactMinTS, uint64(42)) +} + func TestFSMCompactorSkipsLaggingRuntime(t *testing.T) { st := store.NewMVCCStore() ctx := context.Background() diff --git a/kv/fsm.go b/kv/fsm.go index 5221a008a..3737899bb 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -536,6 +536,9 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui if err := f.validateRawMutationsForApply(ctx, r, commitTS); err != nil { return err } + if err := f.recordMigrationWrite(ctx, r.Mutations, commitTS); err != nil { + return err + } muts, err := toStoreMutations(r.Mutations) if err != nil { @@ -568,8 +571,11 @@ func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutatio if isTxnInternalKey(mut.Key) { return errors.WithStack(ErrInvalidRequest) } + if err := f.verifySourceWriteFenceForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } if _, bypass := writeFenceBypassKeys[string(mut.Key)]; !bypass { - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { + if err := f.verifyRouteNotFencedForKey(ctx, mut.Key); err != nil { return err } } @@ -599,7 +605,7 @@ func extractDelPrefix(muts []*pb.Mutation) (bool, []byte) { // handleDelPrefix delegates prefix deletion to the store. Transaction-internal // keys are always excluded to preserve transactional integrity. func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uint64) error { - if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { + if err := f.verifyRouteNotFencedForPrefix(ctx, prefix); err != nil { return err } if err := f.verifyTargetReadinessForPrefix(ctx, prefix); err != nil { @@ -608,6 +614,9 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin if err := f.verifyRouteWriteFloorForPrefix(prefix, commitTS); err != nil { return err } + if err := f.recordMigrationWrite(ctx, []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: prefix}}, commitTS); err != nil { + return err + } routes := f.stagedVisibilityRoutesForPrefix(prefix) deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { return f.store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, 0) @@ -641,22 +650,25 @@ func (f *kvFSM) stagedVisibilityRoutesForPrefix(prefix []byte) []distribution.Ro return out } -func (f *kvFSM) verifyRouteNotFencedForMutations(muts []*pb.Mutation, bypassKeys map[string]struct{}) error { +func (f *kvFSM) verifyRouteNotFencedForMutations(ctx context.Context, muts []*pb.Mutation, bypassKeys map[string]struct{}) error { for _, mut := range muts { if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { continue } + if err := f.verifySourceWriteFenceForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } if _, bypass := bypassKeys[string(mut.Key)]; bypass { continue } - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { + if err := f.verifyRouteNotFencedForKey(ctx, mut.Key); err != nil { return err } } return nil } -func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { +func (f *kvFSM) verifyRouteNotFencedForKey(_ context.Context, key []byte) error { if f.routes == nil { return nil } @@ -674,7 +686,11 @@ func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { return nil } -func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { +func (f *kvFSM) verifyRouteNotFencedForPrefix(ctx context.Context, prefix []byte) error { + start, end := routePrefixRange(prefix) + if err := f.verifySourceWriteFenceForRouteRange(ctx, start, end); err != nil { + return err + } if f.routes == nil { return nil } @@ -682,13 +698,34 @@ func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { if !ok { return nil } - start, end := routePrefixRange(prefix) if !snap.WriteFencedIntersects(start, end) { return nil } return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", prefix, start, end) } +func (f *kvFSM) verifySourceWriteFenceForRange(ctx context.Context, start []byte, end []byte) error { + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + return f.verifySourceWriteFenceForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) verifySourceWriteFenceForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + for _, state := range states { + if state.Armed && state.SourceWriteFence && routeRangeIntersects(routeStart, routeEnd, state.RouteStart, state.RouteEnd) { + return errors.Wrapf(ErrRouteWriteFenced, "source migration fence job %d route range [%q,%q)", state.JobID, routeStart, routeEnd) + } + } + return nil +} + func (f *kvFSM) verifyRouteWriteFloorForMutations(muts []*pb.Mutation, commitTS uint64) error { for _, mut := range muts { if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { @@ -804,7 +841,8 @@ func targetReadinessStatesSatisfied( proof bool, ) bool { for _, ready := range states { - if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + if !ready.Armed || ready.SourceWriteFence || ready.SourceReadFence || ready.TrackWrites || + !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { continue } if !proof || !routesSatisfyTargetReadiness(routes, ready, groupID, catalogVersion) { @@ -1469,8 +1507,14 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } + return f.recordAndApplyPrepare(ctx, uniq, storeMuts, r.ReadKeys, floorTS, startTS) +} - if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, r.ReadKeys, startTS, startTS, f.pendingApplyIdx); err != nil { +func (f *kvFSM) recordAndApplyPrepare(ctx context.Context, muts []*pb.Mutation, storeMuts []*store.KVPairMutation, readKeys [][]byte, floorTS, startTS uint64) error { + if err := f.recordMigrationWrite(ctx, muts, floorTS); err != nil { + return err + } + if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, readKeys, startTS, startTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } return nil @@ -1536,6 +1580,9 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com if err != nil { return err } + if err := f.recordMigrationWrite(ctx, uniq, commitTS); err != nil { + return err + } if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, r.ReadKeys, startTS, commitTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } @@ -1575,7 +1622,7 @@ func (f *kvFSM) uniqueMutationsNotFenced( if err != nil { return nil, err } - if err := f.verifyRouteNotFencedForMutations(uniq, writeFenceBypassKeySet(writeFenceBypassKeys)); err != nil { + if err := f.verifyRouteNotFencedForMutations(ctx, uniq, writeFenceBypassKeySet(writeFenceBypassKeys)); err != nil { return nil, err } if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 787ccefce..e23e86d5b 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -98,6 +98,92 @@ func applyTargetReadinessToFSM(t *testing.T, fsm *kvFSM, state store.TargetStage require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) } +func applySourceMigrationControlToFSM(t *testing.T, fsm *kvFSM, writeFence, readFence bool) { + t.Helper() + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: writeFence, + SourceReadFence: readFence, + RetentionPinTS: 40, + }) +} + +func newMigrationWriteTrackerFSM(t *testing.T) *kvFSM { + t.Helper() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 11, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 1, + TrackWrites: true, + }) + return fsm +} + +func migrationTrackerMinimum(t *testing.T, fsm *kvFSM) uint64 { + t.Helper() + reader, ok := fsm.store.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(context.Background()) + require.NoError(t, err) + require.Len(t, states, 1) + return states[0].MinAdmittedTS +} + +func TestFSMMigrationWriteTrackerCoversAllAdmissionPaths(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newMigrationWriteTrackerFSM(t) + require.NoError(t, fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("raw")}}, + }, 80)) + require.Equal(t, uint64(80), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 70)) + require.Equal(t, uint64(70), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Ts: 50, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("o"), CommitTS: 60})}, + {Op: pb.Op_PUT, Key: []byte("o"), Value: []byte("one-phase")}, + }, + }, 60)) + require.Equal(t, uint64(60), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 40, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("p"), CommitTS: 55, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("p"), Value: []byte("prepared")}, + }, + }, 40)) + require.Equal(t, uint64(55), migrationTrackerMinimum(t, fsm)) +} + func newReadinessReadKeyFSM(t *testing.T) *kvFSM { t.Helper() @@ -154,7 +240,7 @@ func TestFSMWriteFenceBypassAllowsMarkedRawPointWrite(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) - key := []byte("z") + key := []byte("n") err := fsm.handleRawRequest(context.Background(), &pb.Request{ WriteFenceBypassKeys: [][]byte{key}, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, @@ -166,6 +252,30 @@ func TestFSMWriteFenceBypassAllowsMarkedRawPointWrite(t *testing.T) { require.Equal(t, []byte("v"), got) } +func TestFSMDurableSourceFenceRejectsRawWriteDespiteCatalogBypass(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, true, false) + key := []byte("n") + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + +func TestFSMDurableSourceFenceRejectsPrefixWrite(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, true, false) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMWriteFenceBypassAllowsPinnedTxnOnNonOwningGroup(t *testing.T) { t.Parallel() @@ -217,6 +327,24 @@ func TestFSMWriteFenceBypassAllowsPinnedOnePhaseTxnOnNonOwningGroup(t *testing.T require.Equal(t, []byte("v"), got) } +func TestFSMDurableSourceFenceRejectsPinnedOnePhaseTxn(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, true, false) + key := []byte("n") + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Ts: 50, + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("v")}, + }, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMWriteFenceBypassDoesNotAllowDelPrefix(t *testing.T) { t.Parallel() diff --git a/kv/fsm_migration_readiness.go b/kv/fsm_migration_readiness.go index 91ea21cd8..211cc409e 100644 --- a/kv/fsm_migration_readiness.go +++ b/kv/fsm_migration_readiness.go @@ -37,12 +37,95 @@ func (f *kvFSM) applyTargetStagedReadiness(ctx context.Context, data []byte) any if !ok { return errors.WithStack(store.ErrNotSupported) } - if err := writer.ApplyTargetStagedReadiness(ctx, targetStagedReadinessStateFromProto(req)); err != nil { + state := targetStagedReadinessStateFromProto(req) + state = f.preserveMigrationTrackerMinimum(ctx, state) + if err := writer.ApplyTargetStagedReadiness(ctx, state); err != nil { return errors.WithStack(err) } return nil } +func (f *kvFSM) preserveMigrationTrackerMinimum(ctx context.Context, state store.TargetStagedReadinessState) store.TargetStagedReadinessState { + if !state.TrackWrites { + return state + } + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return state + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return state + } + for _, current := range states { + if current.JobID == state.JobID && (state.MinAdmittedTS == 0 || current.MinAdmittedTS < state.MinAdmittedTS) { + state.MinAdmittedTS = current.MinAdmittedTS + } + } + return state +} + +func (f *kvFSM) recordMigrationWrite(ctx context.Context, muts []*pb.Mutation, commitTS uint64) error { + if commitTS == 0 { + return nil + } + reader, writer, ok := f.migrationReadinessStore() + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + return recordMigrationWriteInStates(ctx, writer, states, muts, commitTS) +} + +func (f *kvFSM) migrationReadinessStore() (store.MigrationTargetReadinessReader, store.MigrationTargetReadinessWriter, bool) { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, nil, false + } + writer, ok := f.store.(store.MigrationTargetReadinessWriter) + if !ok { + return nil, nil, false + } + return reader, writer, true +} + +func recordMigrationWriteInStates(ctx context.Context, writer store.MigrationTargetReadinessWriter, states []store.TargetStagedReadinessState, muts []*pb.Mutation, commitTS uint64) error { + for _, state := range states { + if !state.Armed || !state.TrackWrites || !migrationMutationsIntersect(muts, state.RouteStart, state.RouteEnd) { + continue + } + if state.MinAdmittedTS != 0 && state.MinAdmittedTS <= commitTS { + continue + } + state.MinAdmittedTS = commitTS + if err := writer.ApplyTargetStagedReadiness(ctx, state); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func migrationMutationsIntersect(muts []*pb.Mutation, routeStart []byte, routeEnd []byte) bool { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + var start, end []byte + if mut.GetOp() == pb.Op_DEL_PREFIX { + start, end = routePrefixRange(mut.Key) + } else { + start, end = readinessRouteRangeForScan(mut.Key, nextScanCursor(mut.Key)) + } + if routeRangeIntersects(start, end, routeStart, routeEnd) { + return true + } + } + return false +} + func targetStagedReadinessStateFromProto(req *pb.TargetStagedReadinessRequest) store.TargetStagedReadinessState { if req == nil { return store.TargetStagedReadinessState{} @@ -55,5 +138,10 @@ func targetStagedReadinessStateFromProto(req *pb.TargetStagedReadinessRequest) s MigrationJobID: req.GetMigrationJobId(), MinWriteTSExclusive: req.GetMinWriteTsExclusive(), Armed: req.GetArmed(), + SourceWriteFence: req.GetSourceWriteFence(), + SourceReadFence: req.GetSourceReadFence(), + RetentionPinTS: req.GetRetentionPinTs(), + TrackWrites: req.GetTrackWrites(), + MinAdmittedTS: req.GetMinAdmittedTs(), } } diff --git a/kv/shard_store.go b/kv/shard_store.go index 19c29c2c0..6d8f54ff4 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -240,6 +240,9 @@ func (s *ShardStore) targetReadyRoutesForRouteRange(ctx context.Context, g *Shar if err != nil { return nil, errors.WithStack(err) } + if sourceReadFenceApplies(states, routeStart, routeEnd) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } applicable := targetReadinessApplicableStates(states, route, routeStart, routeEnd) if len(applicable) == 0 { return []distribution.Route{route}, nil @@ -263,6 +266,9 @@ func targetReadinessApplicableStates( ) []store.TargetStagedReadinessState { applicable := make([]store.TargetStagedReadinessState, 0, len(states)) for _, ready := range states { + if ready.SourceWriteFence || ready.SourceReadFence || ready.TrackWrites { + continue + } if targetReadinessAppliesToRoute(route, routeStart, routeEnd, ready) { applicable = append(applicable, ready) } @@ -270,6 +276,15 @@ func targetReadinessApplicableStates( return applicable } +func sourceReadFenceApplies(states []store.TargetStagedReadinessState, routeStart []byte, routeEnd []byte) bool { + for _, state := range states { + if state.Armed && state.SourceReadFence && routeRangeIntersects(routeStart, routeEnd, state.RouteStart, state.RouteEnd) { + return true + } + } + return false +} + func readinessProofSatisfiesStates( states []store.TargetStagedReadinessState, routes []distribution.Route, diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index f46812e2d..34e33141f 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -199,6 +199,65 @@ func TestShardStoreCommittedVersionAtChecksTargetReadiness(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } +func TestShardStoreSourceReadFenceRejectsPointRead(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 40, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live"), 60, 0)) + + _, err := st.GetAt(ctx, []byte("n"), 60) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, err = st.ScanAt(ctx, []byte("m"), []byte("z"), 60, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreSourceWriteFenceKeepsReadsAvailable(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + RetentionPinTS: 40, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live"), 60, 0)) + + got, err := st.GetAt(ctx, []byte("n"), 60) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) +} + func TestShardStoreCommittedVersionAtRechecksTargetReadinessAfterFence(t *testing.T) { ctx := context.Background() st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ diff --git a/main.go b/main.go index a54a3e535..46a5496c9 100644 --- a/main.go +++ b/main.go @@ -1673,6 +1673,10 @@ func prepareDistributionRuntimeServer( splitPromotionTargetLeaderResolver(in.shardGroups), splitPromotionConnCache, )), + adapter.WithSplitMigrationClientFactory(splitMigrationClientFactory( + splitMigrationGroupLeaderResolver(in.shardGroups), + splitPromotionConnCache, + )), adapter.WithSplitJobRunnerReadinessGate(splitMigrationLocalReadinessGate), adapter.WithSplitJobRunnerReady(), } @@ -1722,6 +1726,7 @@ func startDistributionRuntimeAfterTransport(in distributionRuntimeStartupInput) } type splitPromotionLeaderResolver func(distribution.SplitJob) (string, error) +type splitMigrationLeaderResolver func(uint64) (string, error) func splitPromotionTargetLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) splitPromotionLeaderResolver { return func(job distribution.SplitJob) (string, error) { @@ -1737,6 +1742,20 @@ func splitPromotionTargetLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) s } } +func splitMigrationGroupLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) splitMigrationLeaderResolver { + return func(groupID uint64) (string, error) { + sg := shardGroups[groupID] + if sg == nil || sg.Engine == nil { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + addr := strings.TrimSpace(sg.Engine.Leader().Address) + if addr == "" { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + return addr, nil + } +} + func splitPromotionClientFactory(resolve splitPromotionLeaderResolver, connCache *kv.GRPCConnCache) adapter.SplitPromotionClientFactory { return func(_ context.Context, job distribution.SplitJob) (adapter.SplitPromotionClient, error) { if resolve == nil { @@ -1757,6 +1776,34 @@ func splitPromotionClientFactory(resolve splitPromotionLeaderResolver, connCache } } +func splitMigrationClientFactory(resolve splitMigrationLeaderResolver, connCache *kv.GRPCConnCache) adapter.SplitMigrationClientFactory { + return func(_ context.Context, job distribution.SplitJob, sourceGroupID uint64) (adapter.SplitMigrationClient, adapter.SplitMigrationClient, error) { + if resolve == nil { + return nil, nil, errors.New("split migration leader resolver is not configured") + } + if connCache == nil { + return nil, nil, errors.New("split migration gRPC connection cache is not configured") + } + sourceAddr, err := resolve(sourceGroupID) + if err != nil { + return nil, nil, err + } + targetAddr, err := resolve(job.TargetGroupID) + if err != nil { + return nil, nil, errors.WithStack(err) + } + sourceConn, err := connCache.ConnFor(sourceAddr) + if err != nil { + return nil, nil, errors.WithStack(err) + } + targetConn, err := connCache.ConnFor(targetAddr) + if err != nil { + return nil, nil, errors.WithStack(err) + } + return pb.NewInternalClient(sourceConn), pb.NewInternalClient(targetConn), nil + } +} + func splitMigrationLocalReadinessGate(context.Context) error { if !adapter.MigrationImportOpcodeEnabledFromEnv() { return errors.Errorf("%s is disabled", adapter.MigrationImportOpcodeEnv) diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 044d30cbb..002c29242 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -1097,8 +1097,15 @@ type TargetStagedReadinessRequest struct { MigrationJobId uint64 `protobuf:"varint,5,opt,name=migration_job_id,json=migrationJobId,proto3" json:"migration_job_id,omitempty"` MinWriteTsExclusive uint64 `protobuf:"varint,6,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` Armed bool `protobuf:"varint,7,opt,name=armed,proto3" json:"armed,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-side controls reuse the same Raft-replicated durable record. They + // are ignored by target staged-readiness checks. + SourceWriteFence bool `protobuf:"varint,8,opt,name=source_write_fence,json=sourceWriteFence,proto3" json:"source_write_fence,omitempty"` + SourceReadFence bool `protobuf:"varint,9,opt,name=source_read_fence,json=sourceReadFence,proto3" json:"source_read_fence,omitempty"` + RetentionPinTs uint64 `protobuf:"varint,10,opt,name=retention_pin_ts,json=retentionPinTs,proto3" json:"retention_pin_ts,omitempty"` + TrackWrites bool `protobuf:"varint,11,opt,name=track_writes,json=trackWrites,proto3" json:"track_writes,omitempty"` + MinAdmittedTs uint64 `protobuf:"varint,12,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TargetStagedReadinessRequest) Reset() { @@ -1180,8 +1187,44 @@ func (x *TargetStagedReadinessRequest) GetArmed() bool { return false } +func (x *TargetStagedReadinessRequest) GetSourceWriteFence() bool { + if x != nil { + return x.SourceWriteFence + } + return false +} + +func (x *TargetStagedReadinessRequest) GetSourceReadFence() bool { + if x != nil { + return x.SourceReadFence + } + return false +} + +func (x *TargetStagedReadinessRequest) GetRetentionPinTs() uint64 { + if x != nil { + return x.RetentionPinTs + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetTrackWrites() bool { + if x != nil { + return x.TrackWrites + } + return false +} + +func (x *TargetStagedReadinessRequest) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + type TargetStagedReadinessResponse struct { state protoimpl.MessageState `protogen:"open.v1"` + MinAdmittedTs uint64 `protobuf:"varint,1,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1216,6 +1259,117 @@ func (*TargetStagedReadinessResponse) Descriptor() ([]byte, []int) { return file_internal_proto_rawDescGZIP(), []int{15} } +func (x *TargetStagedReadinessResponse) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +type ProbeMigrationLocksRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RouteStart []byte `protobuf:"bytes,1,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,2,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + Limit uint32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationLocksRequest) Reset() { + *x = ProbeMigrationLocksRequest{} + mi := &file_internal_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationLocksRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationLocksRequest) ProtoMessage() {} + +func (x *ProbeMigrationLocksRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationLocksRequest.ProtoReflect.Descriptor instead. +func (*ProbeMigrationLocksRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{16} +} + +func (x *ProbeMigrationLocksRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *ProbeMigrationLocksRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *ProbeMigrationLocksRequest) GetLimit() uint32 { + if x != nil { + return x.Limit + } + return 0 +} + +type ProbeMigrationLocksResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + PendingCount uint32 `protobuf:"varint,1,opt,name=pending_count,json=pendingCount,proto3" json:"pending_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationLocksResponse) Reset() { + *x = ProbeMigrationLocksResponse{} + mi := &file_internal_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationLocksResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationLocksResponse) ProtoMessage() {} + +func (x *ProbeMigrationLocksResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationLocksResponse.ProtoReflect.Descriptor instead. +func (*ProbeMigrationLocksResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{17} +} + +func (x *ProbeMigrationLocksResponse) GetPendingCount() uint32 { + if x != nil { + return x.PendingCount + } + return 0 +} + var File_internal_proto protoreflect.FileDescriptor const file_internal_proto_rawDesc = "" + @@ -1297,7 +1451,7 @@ const file_internal_proto_rawDesc = "" + "nextCursor\x12\x12\n" + "\x04done\x18\x02 \x01(\bR\x04done\x12#\n" + "\rpromoted_rows\x18\x03 \x01(\x04R\fpromotedRows\x12&\n" + - "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs\"\xa2\x02\n" + + "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs\"\xf1\x03\n" + "\x1cTargetStagedReadinessRequest\x12\x15\n" + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x1f\n" + "\vroute_start\x18\x02 \x01(\fR\n" + @@ -1306,8 +1460,22 @@ const file_internal_proto_rawDesc = "" + "\x18expected_cutover_version\x18\x04 \x01(\x04R\x16expectedCutoverVersion\x12(\n" + "\x10migration_job_id\x18\x05 \x01(\x04R\x0emigrationJobId\x123\n" + "\x16min_write_ts_exclusive\x18\x06 \x01(\x04R\x13minWriteTsExclusive\x12\x14\n" + - "\x05armed\x18\a \x01(\bR\x05armed\"\x1f\n" + - "\x1dTargetStagedReadinessResponse*&\n" + + "\x05armed\x18\a \x01(\bR\x05armed\x12,\n" + + "\x12source_write_fence\x18\b \x01(\bR\x10sourceWriteFence\x12*\n" + + "\x11source_read_fence\x18\t \x01(\bR\x0fsourceReadFence\x12(\n" + + "\x10retention_pin_ts\x18\n" + + " \x01(\x04R\x0eretentionPinTs\x12!\n" + + "\ftrack_writes\x18\v \x01(\bR\vtrackWrites\x12&\n" + + "\x0fmin_admitted_ts\x18\f \x01(\x04R\rminAdmittedTs\"G\n" + + "\x1dTargetStagedReadinessResponse\x12&\n" + + "\x0fmin_admitted_ts\x18\x01 \x01(\x04R\rminAdmittedTs\"p\n" + + "\x1aProbeMigrationLocksRequest\x12\x1f\n" + + "\vroute_start\x18\x01 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x02 \x01(\fR\brouteEnd\x12\x14\n" + + "\x05limit\x18\x03 \x01(\rR\x05limit\"B\n" + + "\x1bProbeMigrationLocksResponse\x12#\n" + + "\rpending_count\x18\x01 \x01(\rR\fpendingCount*&\n" + "\x02Op\x12\a\n" + "\x03PUT\x10\x00\x12\a\n" + "\x03DEL\x10\x01\x12\x0e\n" + @@ -1318,14 +1486,15 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\xdc\x03\n" + + "\x05ABORT\x10\x032\xb0\x04\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00\x12T\n" + "\x13ExportRangeVersions\x12\x1b.ExportRangeVersionsRequest\x1a\x1c.ExportRangeVersionsResponse\"\x000\x01\x12R\n" + "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00\x12X\n" + "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00\x12]\n" + - "\x1aApplyTargetStagedReadiness\x12\x1d.TargetStagedReadinessRequest\x1a\x1e.TargetStagedReadinessResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\x1aApplyTargetStagedReadiness\x12\x1d.TargetStagedReadinessRequest\x1a\x1e.TargetStagedReadinessResponse\"\x00\x12R\n" + + "\x13ProbeMigrationLocks\x12\x1b.ProbeMigrationLocksRequest\x1a\x1c.ProbeMigrationLocksResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_internal_proto_rawDescOnce sync.Once @@ -1340,7 +1509,7 @@ func file_internal_proto_rawDescGZIP() []byte { } var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_internal_proto_goTypes = []any{ (Op)(0), // 0: Op (Phase)(0), // 1: Phase @@ -1360,6 +1529,8 @@ var file_internal_proto_goTypes = []any{ (*PromoteStagedVersionsResponse)(nil), // 15: PromoteStagedVersionsResponse (*TargetStagedReadinessRequest)(nil), // 16: TargetStagedReadinessRequest (*TargetStagedReadinessResponse)(nil), // 17: TargetStagedReadinessResponse + (*ProbeMigrationLocksRequest)(nil), // 18: ProbeMigrationLocksRequest + (*ProbeMigrationLocksResponse)(nil), // 19: ProbeMigrationLocksResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op @@ -1375,14 +1546,16 @@ var file_internal_proto_depIdxs = []int32{ 12, // 10: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest 14, // 11: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest 16, // 12: Internal.ApplyTargetStagedReadiness:input_type -> TargetStagedReadinessRequest - 6, // 13: Internal.Forward:output_type -> ForwardResponse - 8, // 14: Internal.RelayPublish:output_type -> RelayPublishResponse - 10, // 15: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse - 13, // 16: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse - 15, // 17: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse - 17, // 18: Internal.ApplyTargetStagedReadiness:output_type -> TargetStagedReadinessResponse - 13, // [13:19] is the sub-list for method output_type - 7, // [7:13] is the sub-list for method input_type + 18, // 13: Internal.ProbeMigrationLocks:input_type -> ProbeMigrationLocksRequest + 6, // 14: Internal.Forward:output_type -> ForwardResponse + 8, // 15: Internal.RelayPublish:output_type -> RelayPublishResponse + 10, // 16: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse + 13, // 17: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse + 15, // 18: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse + 17, // 19: Internal.ApplyTargetStagedReadiness:output_type -> TargetStagedReadinessResponse + 19, // 20: Internal.ProbeMigrationLocks:output_type -> ProbeMigrationLocksResponse + 14, // [14:21] is the sub-list for method output_type + 7, // [7:14] is the sub-list for method input_type 7, // [7:7] is the sub-list for extension type_name 7, // [7:7] is the sub-list for extension extendee 0, // [0:7] is the sub-list for field type_name @@ -1399,7 +1572,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), NumEnums: 2, - NumMessages: 16, + NumMessages: 18, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index 11c05a5b7..eeb6c65ab 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -11,6 +11,7 @@ service Internal { rpc ImportRangeVersions(ImportRangeVersionsRequest) returns (ImportRangeVersionsResponse) {} rpc PromoteStagedVersions(PromoteStagedVersionsRequest) returns (PromoteStagedVersionsResponse) {} rpc ApplyTargetStagedReadiness(TargetStagedReadinessRequest) returns (TargetStagedReadinessResponse) {} + rpc ProbeMigrationLocks(ProbeMigrationLocksRequest) returns (ProbeMigrationLocksResponse) {} } // internal.proto is node to node communication message in raft replication. @@ -160,6 +161,25 @@ message TargetStagedReadinessRequest { uint64 migration_job_id = 5; uint64 min_write_ts_exclusive = 6; bool armed = 7; + // Source-side controls reuse the same Raft-replicated durable record. They + // are ignored by target staged-readiness checks. + bool source_write_fence = 8; + bool source_read_fence = 9; + uint64 retention_pin_ts = 10; + bool track_writes = 11; + uint64 min_admitted_ts = 12; } -message TargetStagedReadinessResponse {} +message TargetStagedReadinessResponse { + uint64 min_admitted_ts = 1; +} + +message ProbeMigrationLocksRequest { + bytes route_start = 1; + bytes route_end = 2; + uint32 limit = 3; +} + +message ProbeMigrationLocksResponse { + uint32 pending_count = 1; +} diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index 6af68894d..3ca0b1fe1 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -25,6 +25,7 @@ const ( Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" Internal_ApplyTargetStagedReadiness_FullMethodName = "/Internal/ApplyTargetStagedReadiness" + Internal_ProbeMigrationLocks_FullMethodName = "/Internal/ProbeMigrationLocks" ) // InternalClient is the client API for Internal service. @@ -38,6 +39,7 @@ type InternalClient interface { ImportRangeVersions(ctx context.Context, in *ImportRangeVersionsRequest, opts ...grpc.CallOption) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(ctx context.Context, in *PromoteStagedVersionsRequest, opts ...grpc.CallOption) (*PromoteStagedVersionsResponse, error) ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) + ProbeMigrationLocks(ctx context.Context, in *ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*ProbeMigrationLocksResponse, error) } type internalClient struct { @@ -117,6 +119,16 @@ func (c *internalClient) ApplyTargetStagedReadiness(ctx context.Context, in *Tar return out, nil } +func (c *internalClient) ProbeMigrationLocks(ctx context.Context, in *ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*ProbeMigrationLocksResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ProbeMigrationLocksResponse) + err := c.cc.Invoke(ctx, Internal_ProbeMigrationLocks_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // InternalServer is the server API for Internal service. // All implementations must embed UnimplementedInternalServer // for forward compatibility. @@ -128,6 +140,7 @@ type InternalServer interface { ImportRangeVersions(context.Context, *ImportRangeVersionsRequest) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) + ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) mustEmbedUnimplementedInternalServer() } @@ -156,6 +169,9 @@ func (UnimplementedInternalServer) PromoteStagedVersions(context.Context, *Promo func (UnimplementedInternalServer) ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) { return nil, status.Error(codes.Unimplemented, "method ApplyTargetStagedReadiness not implemented") } +func (UnimplementedInternalServer) ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ProbeMigrationLocks not implemented") +} func (UnimplementedInternalServer) mustEmbedUnimplementedInternalServer() {} func (UnimplementedInternalServer) testEmbeddedByValue() {} @@ -278,6 +294,24 @@ func _Internal_ApplyTargetStagedReadiness_Handler(srv interface{}, ctx context.C return interceptor(ctx, in, info, handler) } +func _Internal_ProbeMigrationLocks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeMigrationLocksRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ProbeMigrationLocks(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ProbeMigrationLocks_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ProbeMigrationLocks(ctx, req.(*ProbeMigrationLocksRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Internal_ServiceDesc is the grpc.ServiceDesc for Internal service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -305,6 +339,10 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "ApplyTargetStagedReadiness", Handler: _Internal_ApplyTargetStagedReadiness_Handler, }, + { + MethodName: "ProbeMigrationLocks", + Handler: _Internal_ProbeMigrationLocks_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/store/migration_readiness.go b/store/migration_readiness.go index 2799840a8..b3be4b0dd 100644 --- a/store/migration_readiness.go +++ b/store/migration_readiness.go @@ -11,8 +11,12 @@ import ( ) const ( - targetReadinessCodecVersion byte = 1 - targetReadinessArmedFlag byte = 1 + targetReadinessCodecVersionV1 byte = 1 + targetReadinessCodecVersion byte = 2 + targetReadinessArmedFlag byte = 1 + targetReadinessSourceWrite byte = 1 + targetReadinessSourceRead byte = 2 + targetReadinessTrackWrites byte = 4 ) func validateTargetStagedReadinessState(state TargetStagedReadinessState) error { @@ -25,9 +29,21 @@ func validateTargetStagedReadinessState(state TargetStagedReadinessState) error return errors.New("target staged readiness min_write_ts_exclusive is required when armed") case state.RouteEnd != nil && bytes.Compare(state.RouteStart, state.RouteEnd) >= 0: return errors.New("target staged readiness route range is invalid") - default: - return nil } + return validateSourceMigrationControlState(state) +} + +func validateSourceMigrationControlState(state TargetStagedReadinessState) error { + if state.SourceReadFence && !state.SourceWriteFence { + return errors.New("source read fence requires source write fence") + } + if (state.SourceWriteFence || state.SourceReadFence) && state.RetentionPinTS == 0 { + return errors.New("source migration control retention pin is required") + } + if state.TrackWrites && state.RetentionPinTS == 0 { + return errors.New("migration write tracker retention pin is required") + } + return nil } func (s *mvccStore) ApplyTargetStagedReadiness(_ context.Context, state TargetStagedReadinessState) error { @@ -110,10 +126,23 @@ func encodeTargetStagedReadinessState(state TargetStagedReadinessState) []byte { } else { buf = append(buf, 0) } + var sourceFlags byte + if state.SourceWriteFence { + sourceFlags |= targetReadinessSourceWrite + } + if state.SourceReadFence { + sourceFlags |= targetReadinessSourceRead + } + if state.TrackWrites { + sourceFlags |= targetReadinessTrackWrites + } + buf = append(buf, sourceFlags) buf = binary.BigEndian.AppendUint64(buf, state.JobID) buf = binary.BigEndian.AppendUint64(buf, state.ExpectedCutoverVersion) buf = binary.BigEndian.AppendUint64(buf, state.MigrationJobID) buf = binary.BigEndian.AppendUint64(buf, state.MinWriteTSExclusive) + buf = binary.BigEndian.AppendUint64(buf, state.RetentionPinTS) + buf = binary.BigEndian.AppendUint64(buf, state.MinAdmittedTS) buf = appendVarBytes(buf, state.RouteStart) if state.RouteEnd == nil { buf = append(buf, 0) @@ -138,14 +167,29 @@ func decodeTargetStagedReadinessState(data []byte) (TargetStagedReadinessState, } func decodeTargetReadinessHeader(data []byte) (TargetStagedReadinessState, []byte, bool) { - if len(data) < 2+4*migrationUint64Bytes || data[0] != targetReadinessCodecVersion { + if len(data) == 0 { + return TargetStagedReadinessState{}, nil, false + } + if data[0] == targetReadinessCodecVersionV1 { + return decodeTargetReadinessHeaderV1(data) + } + if len(data) < 3+6*migrationUint64Bytes || data[0] != targetReadinessCodecVersion { return TargetStagedReadinessState{}, nil, false } if data[1] != 0 && data[1] != targetReadinessArmedFlag { return TargetStagedReadinessState{}, nil, false } - state := TargetStagedReadinessState{Armed: data[1] == targetReadinessArmedFlag} - rest := data[2:] + sourceFlags := data[2] + if sourceFlags & ^(targetReadinessSourceWrite|targetReadinessSourceRead|targetReadinessTrackWrites) != 0 { + return TargetStagedReadinessState{}, nil, false + } + state := TargetStagedReadinessState{ + Armed: data[1] == targetReadinessArmedFlag, + SourceWriteFence: sourceFlags&targetReadinessSourceWrite != 0, + SourceReadFence: sourceFlags&targetReadinessSourceRead != 0, + TrackWrites: sourceFlags&targetReadinessTrackWrites != 0, + } + rest := data[3:] state.JobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) rest = rest[migrationUint64Bytes:] state.ExpectedCutoverVersion = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) @@ -154,9 +198,29 @@ func decodeTargetReadinessHeader(data []byte) (TargetStagedReadinessState, []byt rest = rest[migrationUint64Bytes:] state.MinWriteTSExclusive = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) rest = rest[migrationUint64Bytes:] + state.RetentionPinTS = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinAdmittedTS = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] return state, rest, true } +func decodeTargetReadinessHeaderV1(data []byte) (TargetStagedReadinessState, []byte, bool) { + if len(data) < 2+4*migrationUint64Bytes || data[1] != 0 && data[1] != targetReadinessArmedFlag { + return TargetStagedReadinessState{}, nil, false + } + state := TargetStagedReadinessState{Armed: data[1] == targetReadinessArmedFlag} + rest := data[2:] + state.JobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.ExpectedCutoverVersion = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MigrationJobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinWriteTSExclusive = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + return state, rest[migrationUint64Bytes:], true +} + func decodeTargetReadinessRange(state *TargetStagedReadinessState, data []byte) bool { routeStart, rest, ok := readVarBytes(data) if !ok || len(rest) == 0 { @@ -176,7 +240,7 @@ func decodeTargetReadinessRange(state *TargetStagedReadinessState, data []byte) } func targetReadinessEncodedSize(state TargetStagedReadinessState) int { - size := 2 + 4*migrationUint64Bytes + binary.MaxVarintLen64 + len(state.RouteStart) + 1 + size := 3 + 6*migrationUint64Bytes + binary.MaxVarintLen64 + len(state.RouteStart) + 1 if state.RouteEnd != nil { size += binary.MaxVarintLen64 + len(state.RouteEnd) } @@ -209,6 +273,11 @@ func cloneTargetStagedReadinessState(state TargetStagedReadinessState) TargetSta MigrationJobID: state.MigrationJobID, MinWriteTSExclusive: state.MinWriteTSExclusive, Armed: state.Armed, + SourceWriteFence: state.SourceWriteFence, + SourceReadFence: state.SourceReadFence, + RetentionPinTS: state.RetentionPinTS, + TrackWrites: state.TrackWrites, + MinAdmittedTS: state.MinAdmittedTS, } } diff --git a/store/migration_readiness_test.go b/store/migration_readiness_test.go new file mode 100644 index 000000000..5b3db499b --- /dev/null +++ b/store/migration_readiness_test.go @@ -0,0 +1,79 @@ +package store + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTargetStagedReadinessCodecRoundTripSourceControls(t *testing.T) { + t.Parallel() + + want := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 11, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 80, + TrackWrites: true, + MinAdmittedTS: 70, + } + + got, ok := decodeTargetStagedReadinessState(encodeTargetStagedReadinessState(want)) + require.True(t, ok) + require.Equal(t, want, got) +} + +func TestTargetStagedReadinessCodecDecodesV1(t *testing.T) { + t.Parallel() + + data := []byte{targetReadinessCodecVersionV1, targetReadinessArmedFlag} + for _, value := range []uint64{9, 11, 7, 100} { + data = binary.BigEndian.AppendUint64(data, value) + } + data = appendVarBytes(data, []byte("m")) + data = append(data, 1) + data = appendVarBytes(data, []byte("z")) + + got, ok := decodeTargetStagedReadinessState(data) + require.True(t, ok) + require.Equal(t, TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 11, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }, got) +} + +func TestValidateTargetStagedReadinessSourceControls(t *testing.T) { + t.Parallel() + + base := TargetStagedReadinessState{ + JobID: 9, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + } + + readOnly := base + readOnly.SourceReadFence = true + readOnly.RetentionPinTS = 80 + require.ErrorContains(t, validateTargetStagedReadinessState(readOnly), "requires source write fence") + + withoutPin := base + withoutPin.SourceWriteFence = true + require.ErrorContains(t, validateTargetStagedReadinessState(withoutPin), "retention pin is required") + + trackerWithoutPin := base + trackerWithoutPin.TrackWrites = true + require.ErrorContains(t, validateTargetStagedReadinessState(trackerWithoutPin), "tracker retention pin is required") +} diff --git a/store/store.go b/store/store.go index 07f40365f..9661626e7 100644 --- a/store/store.go +++ b/store/store.go @@ -174,6 +174,11 @@ type TargetStagedReadinessState struct { MigrationJobID uint64 MinWriteTSExclusive uint64 Armed bool + SourceWriteFence bool + SourceReadFence bool + RetentionPinTS uint64 + TrackWrites bool + MinAdmittedTS uint64 } // MigrationPromoter is implemented by stores that can promote staged range From 634a75e89ac0954333c52bc8728d6e4f9769ec94 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 01:04:22 +0900 Subject: [PATCH 157/162] Complete hotspot split M2 migration lifecycle --- adapter/distribution_server.go | 66 +- adapter/distribution_server_test.go | 226 +++- adapter/internal.go | 209 ++++ adapter/internal_migration_probe_test.go | 117 +++ adapter/split_job_runner.go | 976 +++++++++++++++++- adapter/split_job_voter_ack_test.go | 67 ++ cmd/elastickv-split/main.go | 60 ++ cmd/elastickv-split/main_test.go | 33 +- distribution/migration_promotion_complete.go | 13 +- .../migration_promotion_complete_test.go | 24 +- distribution/split_job_catalog.go | 8 +- distribution/split_job_catalog_test.go | 1 + jepsen/src/elastickv/db.clj | 15 +- jepsen/src/elastickv/jepsen_test.clj | 8 +- jepsen/src/elastickv/split_workload.clj | 252 +++++ jepsen/test/elastickv/split_workload_test.clj | 40 + kv/fsm.go | 5 + kv/fsm_migration_cleanup.go | 93 ++ kv/fsm_migration_cleanup_test.go | 42 + main.go | 50 + proto/distribution.pb.go | 13 +- proto/distribution.proto | 1 + proto/internal.pb.go | 819 ++++++++++++++- proto/internal.proto | 74 ++ proto/internal_grpc.pb.go | 114 ++ store/migration_cleanup.go | 205 ++++ store/migration_promote.go | 9 +- store/migration_versions_test.go | 73 ++ store/store.go | 33 + 29 files changed, 3497 insertions(+), 149 deletions(-) create mode 100644 adapter/internal_migration_probe_test.go create mode 100644 adapter/split_job_voter_ack_test.go create mode 100644 jepsen/src/elastickv/split_workload.clj create mode 100644 jepsen/test/elastickv/split_workload_test.clj create mode 100644 kv/fsm_migration_cleanup.go create mode 100644 kv/fsm_migration_cleanup_test.go create mode 100644 store/migration_cleanup.go diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 25837a8fd..fe1999dcd 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -35,7 +35,9 @@ type DistributionServer struct { splitJobRunnerReadinessGate SplitMigrationCapabilityGate splitPromotionClientFactory SplitPromotionClientFactory splitMigrationClientFactory SplitMigrationClientFactory + splitMigrationVoterFactory SplitMigrationVoterFactory knownRaftGroups map[uint64]struct{} + splitJobHistoryGCLast time.Time reloadRetry struct { attempts int interval time.Duration @@ -68,11 +70,24 @@ type SplitMigrationClient interface { ImportRangeVersions(context.Context, *pb.ImportRangeVersionsRequest, ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) ApplyTargetStagedReadiness(context.Context, *pb.TargetStagedReadinessRequest, ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) ProbeMigrationLocks(context.Context, *pb.ProbeMigrationLocksRequest, ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) + CleanupMigration(context.Context, *pb.CleanupMigrationRequest, ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) + ProbeMigrationState(context.Context, *pb.ProbeMigrationStateRequest, ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(context.Context, *pb.IssueMigrationTimestampRequest, ...grpc.CallOption) (*pb.IssueMigrationTimestampResponse, error) } // SplitMigrationClientFactory resolves both participating group leaders. type SplitMigrationClientFactory func(context.Context, distribution.SplitJob, uint64) (source SplitMigrationClient, target SplitMigrationClient, err error) +type SplitMigrationVoter struct { + ID string + Address string + Client SplitMigrationClient +} + +// SplitMigrationVoterFactory resolves the current voter set and a direct +// client for each voter. The runner re-resolves it before every barrier. +type SplitMigrationVoterFactory func(context.Context, uint64) ([]SplitMigrationVoter, error) + // WithDistributionCoordinator configures the coordinator used for Raft-backed // catalog mutations in SplitRange. func WithDistributionCoordinator(coordinator kv.Coordinator) DistributionServerOption { @@ -131,6 +146,12 @@ func WithSplitMigrationClientFactory(factory SplitMigrationClientFactory) Distri } } +func WithSplitMigrationVoterFactory(factory SplitMigrationVoterFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitMigrationVoterFactory = factory + } +} + // WithDistributionKnownRaftGroups configures the Raft group IDs this node can // route migration work to. func WithDistributionKnownRaftGroups(groupIDs ...uint64) DistributionServerOption { @@ -266,13 +287,52 @@ func (s *DistributionServer) RunSplitJobRunnerOnce(ctx context.Context) error { return splitJobCatalogStatusError(err) } if job, ok := nextRunnableSplitJob(jobs); ok { + if job.Phase != distribution.SplitJobPhaseAbandoning { + ready, gateErr := s.splitJobCapabilityReady(ctx, job) + if gateErr != nil || !ready { + return gateErr + } + } return s.runSplitJobPhase(ctx, job) } - return nil + return s.gcSplitJobHistory(ctx, jobs, time.Now()) +} + +func (s *DistributionServer) splitJobCapabilityReady(ctx context.Context, job distribution.SplitJob) (bool, error) { + if s.migrationCapabilityGate == nil { + return true, nil + } + gateErr := s.migrationCapabilityGate(ctx) + if gateErr != nil { + if job.CapabilityRegressed { + return false, nil + } + err := s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == job.Phase { + current.CapabilityRegressed = true + current.LastError = "cluster migration capability regressed: " + gateErr.Error() + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return false, err + } + if !job.CapabilityRegressed { + return true, nil + } + err := s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == job.Phase && current.CapabilityRegressed { + current.CapabilityRegressed = false + current.LastError = "" + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return false, err } func (s *DistributionServer) splitJobRunnerConfigured() bool { - return s != nil && s.catalog != nil && s.coordinator != nil && s.splitPromotionClientFactory != nil + return s != nil && s.catalog != nil && s.coordinator != nil && s.splitPromotionClientFactory != nil && s.splitMigrationClientFactory != nil } func (s *DistributionServer) verifySplitJobRunnerLeader(ctx context.Context) (bool, error) { @@ -1146,7 +1206,7 @@ func splitJobPromotionMatchesExpected(expected, current distribution.SplitJob) ( if expected.TargetPromotionDone || !current.TargetPromotionDone || current.PromotionCompletedTS == 0 || - current.Phase != distribution.SplitJobPhaseDone { + (current.Phase != expected.Phase && current.Phase != distribution.SplitJobPhaseDone) { return false, nil } normalized := distribution.CloneSplitJob(current) diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index b317acdb6..355527b15 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -828,10 +828,10 @@ func TestDistributionServerCompleteSplitJobTargetPromotion_UsesCoordinatorCAS(t require.NoError(t, err) require.Equal(t, saved.Version+1, snapshot.Version) require.True(t, completed.TargetPromotionDone) - require.Equal(t, distribution.SplitJobPhaseDone, completed.Phase) + require.Equal(t, distribution.SplitJobPhaseCleanup, completed.Phase) require.Equal(t, before.ReadTS+10, completed.PromotionCompletedTS) require.Equal(t, int64(2000), completed.UpdatedAtMs) - require.Equal(t, int64(2000), completed.TerminalAtMs) + require.Zero(t, completed.TerminalAtMs) require.Equal(t, 1, coordinator.dispatchCalls) require.Equal(t, 1, coordinator.timestampCalls) require.Equal(t, before.ReadTS, coordinator.lastStartTS) @@ -892,6 +892,8 @@ func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T {Done: true}, }, } + sourceMigration := &splitMigrationClientStub{} + targetMigration := &splitMigrationClientStub{} coordinator := newDistributionCoordinatorStub(baseStore, true) s := NewDistributionServer( distribution.NewEngine(), @@ -900,6 +902,9 @@ func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { return client, nil }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return sourceMigration, targetMigration, nil + }), ) require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) @@ -916,9 +921,9 @@ func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T require.NoError(t, err) require.True(t, found) require.True(t, loadedJob.TargetPromotionDone) - require.Equal(t, distribution.SplitJobPhaseDone, loadedJob.Phase) + require.Equal(t, distribution.SplitJobPhaseCleanup, loadedJob.Phase) require.NotZero(t, loadedJob.PromotionCompletedTS) - require.Positive(t, loadedJob.TerminalAtMs) + require.Zero(t, loadedJob.TerminalAtMs) loaded, err := catalog.Snapshot(ctx) require.NoError(t, err) @@ -952,22 +957,31 @@ func TestDistributionServerRunSplitJobRunnerOnce_TerminalizesPromotedCleanupJob( job.UpdatedAtMs = 2000 require.NoError(t, catalog.CreateSplitJob(ctx, job)) - client := &splitPromotionClientStub{ - responses: []*pb.PromoteStagedVersionsResponse{{Done: true}}, - } + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} coordinator := newDistributionCoordinatorStub(baseStore, true) s := NewDistributionServer( distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator), WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { - return client, nil + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil }), ) - require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) - require.Zero(t, client.calls) - require.Equal(t, 1, coordinator.dispatchCalls) + for range 200 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseDone { + break + } + } + require.Zero(t, target.calls) loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) require.NoError(t, err) @@ -978,8 +992,7 @@ func TestDistributionServerRunSplitJobRunnerOnce_TerminalizesPromotedCleanupJob( loaded, err := catalog.Snapshot(ctx) require.NoError(t, err) - require.Equal(t, saved.Version+1, loaded.Version) - require.Equal(t, saved.Version+1, s.engine.Version()) + require.Equal(t, saved.Version, loaded.Version) } func TestDistributionServerRunSplitJobRunnerOnce_CompletesCrossGroupMigration(t *testing.T) { @@ -1046,6 +1059,8 @@ func TestDistributionServerRunSplitJobRunnerOnce_CompletesCrossGroupMigration(t require.NotEmpty(t, target.controls) require.NotEmpty(t, source.exports) require.Len(t, target.imports, len(source.exports)) + require.GreaterOrEqual(t, len(source.events), 2) + require.Equal(t, []string{"control", "timestamp"}, source.events[:2], "the tracker barrier must precede snapshot timestamp selection") finalSnapshot, err := catalog.Snapshot(ctx) require.NoError(t, err) @@ -1057,6 +1072,151 @@ func TestDistributionServerRunSplitJobRunnerOnce_CompletesCrossGroupMigration(t require.Equal(t, completed.FenceTS, right.MinWriteTSExclusive) } +func TestDistributionServerRunSplitJobRunnerOnce_CompletesAbandonCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 7, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + job.Phase = distribution.SplitJobPhaseAbandoning + job.AbandonFromPhase = distribution.SplitJobPhaseBackfill + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + for range 10 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseAbandoned { + break + } + } + completed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoned, completed.Phase) + require.Positive(t, completed.TerminalAtMs) + require.NotEmpty(t, source.cleanups) + require.NotEmpty(t, target.cleanups) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, target.cleanups[0].GetMode()) +} + +func TestDistributionServerRunSplitJobRunnerOnce_PausesOnCapabilityRegression(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 9, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + capabilityErr := errors.New("node capability missing") + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { return capabilityErr }), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + paused, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, paused.CapabilityRegressed) + require.Equal(t, distribution.SplitJobPhasePlanned, paused.Phase) + require.Empty(t, source.controls) + + s.migrationCapabilityGate = func(context.Context) error { return nil } + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + resumed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.False(t, resumed.CapabilityRegressed) + require.Equal(t, distribution.SplitJobPhasePlanned, resumed.Phase) + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + armed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseBackfill, armed.Phase) + require.NotZero(t, armed.SnapshotTS) +} + +func TestSplitJobHistoryGCKeysAppliesTTLAndCountBound(t *testing.T) { + t.Parallel() + + now := time.UnixMilli(10 * splitJobHistoryTTL.Milliseconds()) + jobs := make([]distribution.SplitJob, 0, splitJobHistoryLimit+3) + jobs = append(jobs, distribution.SplitJob{ + JobID: 1, + Phase: distribution.SplitJobPhaseDone, + TerminalAtMs: now.Add(-splitJobHistoryTTL - time.Second).UnixMilli(), + }) + for i := 0; i < splitJobHistoryLimit+2; i++ { + jobs = append(jobs, distribution.SplitJob{ + JobID: uint64(i) + 2, //nolint:gosec // loop bound is splitJobHistoryLimit. + Phase: distribution.SplitJobPhaseDone, + TerminalAtMs: now.Add(-time.Hour).UnixMilli() + int64(i), + }) + } + + keys := splitJobHistoryGCKeys(jobs, now) + require.Len(t, keys, 3) + require.Equal(t, distribution.CatalogSplitJobHistoryKey(jobs[0].TerminalAtMs, jobs[0].JobID), keys[0]) +} + func TestDistributionServerRunSplitJobRunnerOnce_SkipsFollower(t *testing.T) { t.Parallel() @@ -1947,6 +2107,11 @@ type splitMigrationClientStub struct { exports []*pb.ExportRangeVersionsRequest imports []*pb.ImportRangeVersionsRequest controls []*pb.TargetStagedReadinessRequest + cleanups []*pb.CleanupMigrationRequest + probes []*pb.ProbeMigrationStateRequest + probeFn func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) + nextTS uint64 + events []string } func (s *splitMigrationClientStub) ExportRangeVersions( @@ -2001,6 +2166,7 @@ func (s *splitMigrationClientStub) ApplyTargetStagedReadiness( _ ...grpc.CallOption, ) (*pb.TargetStagedReadinessResponse, error) { s.controls = append(s.controls, req) + s.events = append(s.events, "control") minAdmittedTS := uint64(0) if req.GetTrackWrites() { minAdmittedTS = 10 @@ -2016,6 +2182,40 @@ func (s *splitMigrationClientStub) ProbeMigrationLocks( return &pb.ProbeMigrationLocksResponse{}, nil } +func (s *splitMigrationClientStub) CleanupMigration( + _ context.Context, + req *pb.CleanupMigrationRequest, + _ ...grpc.CallOption, +) (*pb.CleanupMigrationResponse, error) { + s.cleanups = append(s.cleanups, req) + return &pb.CleanupMigrationResponse{Done: true, NextCursor: distribution.CloneBytes(req.GetCursor())}, nil +} + +func (s *splitMigrationClientStub) ProbeMigrationState( + _ context.Context, + req *pb.ProbeMigrationStateRequest, + _ ...grpc.CallOption, +) (*pb.ProbeMigrationStateResponse, error) { + s.probes = append(s.probes, req) + if s.probeFn != nil { + return s.probeFn(req) + } + return &pb.ProbeMigrationStateResponse{Ready: true, CatalogVersion: req.GetExpectedCatalogVersion()}, nil +} + +func (s *splitMigrationClientStub) IssueMigrationTimestamp( + context.Context, + *pb.IssueMigrationTimestampRequest, + ...grpc.CallOption, +) (*pb.IssueMigrationTimestampResponse, error) { + s.events = append(s.events, "timestamp") + if s.nextTS == 0 { + s.nextTS = 100 + } + s.nextTS++ + return &pb.IssueMigrationTimestampResponse{Timestamp: s.nextTS, LastCommitTs: s.nextTS - 1}, nil +} + type splitMigrationStreamStub struct { grpc.ClientStream responses []*pb.ExportRangeVersionsResponse diff --git a/adapter/internal.go b/adapter/internal.go index a7a0d50ae..3c47a6c0c 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -5,6 +5,7 @@ import ( "context" "os" "strings" + "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" @@ -64,6 +65,12 @@ func WithInternalRouteEngine(engine *distribution.Engine) InternalOption { } } +func WithInternalActiveTimestampTracker(tracker *kv.ActiveTimestampTracker) InternalOption { + return func(i *Internal) { + i.readTracker = tracker + } +} + func WithInternalMigrationExportRouting(groupID uint64, resolver kv.PartitionResolver) InternalOption { return func(i *Internal) { i.migrationExportGroupID = groupID @@ -97,6 +104,7 @@ type Internal struct { migrationImportGate func(context.Context) error migrationPromoteGate func(context.Context) error routeEngine *distribution.Engine + readTracker *kv.ActiveTimestampTracker migrationExportGroupID uint64 migrationExportResolver kv.PartitionResolver @@ -144,6 +152,139 @@ func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.For }, nil } +func (i *Internal) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + if req == nil || req.GetJobId() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration probe job_id is required")) + } + return i.probeMigrationStateKind(ctx, req) +} + +func (i *Internal) probeMigrationStateKind(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + switch req.GetKind() { + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED: + return i.probeMigrationControl(ctx, req) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED: + return i.probeMigrationRoute(req, true), nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED: + return i.probeMigrationRoute(req, false), nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED: + return &pb.ProbeMigrationStateResponse{ + Ready: time.Now().UnixMilli() >= req.GetReadDrainNotBeforeMs() && + (i.readTracker == nil || i.readTracker.Oldest() == 0), + CatalogVersion: i.migrationCatalogVersion(), + }, nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED: + return i.probeMigrationMetadataCleared(ctx, req.GetJobId()) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED: + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration probe kind is required")) + default: + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "unknown migration probe kind")) + } +} + +func (i *Internal) IssueMigrationTimestamp(ctx context.Context, _ *pb.IssueMigrationTimestampRequest) (*pb.IssueMigrationTimestampResponse, error) { + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + if i.store == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration timestamp store is not configured")) + } + lastCommitTS := i.store.LastCommitTS() + ts, err := i.nextTimestampAfter(ctx, lastCommitTS, "issue migration timestamp") + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.IssueMigrationTimestampResponse{Timestamp: ts, LastCommitTs: lastCommitTS}, nil +} + +func (i *Internal) probeMigrationControl(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + reader, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration readiness reader is not configured")) + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + for _, state := range states { + if state.JobID != req.GetJobId() { + continue + } + ready := migrationControlMatches(state, req) + return &pb.ProbeMigrationStateResponse{Ready: ready, CatalogVersion: i.migrationCatalogVersion(), MinAdmittedTs: state.MinAdmittedTS}, nil + } + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil +} + +func migrationControlMatches(state store.TargetStagedReadinessState, req *pb.ProbeMigrationStateRequest) bool { + return state.Armed && + bytes.Equal(state.RouteStart, req.GetRouteStart()) && + bytes.Equal(state.RouteEnd, normalizeOptionalEnd(req.GetRouteEnd())) && + state.ExpectedCutoverVersion == req.GetExpectedCatalogVersion() && + state.MigrationJobID == req.GetMigrationJobId() && + state.MinWriteTSExclusive == req.GetMinWriteTsExclusive() && + state.SourceWriteFence == req.GetSourceWriteFence() && + state.SourceReadFence == req.GetSourceReadFence() && + state.TrackWrites == req.GetTrackWrites() && + state.RetentionPinTS == req.GetRetentionPinTs() +} + +func (i *Internal) probeMigrationRoute(req *pb.ProbeMigrationStateRequest, requireCleared bool) *pb.ProbeMigrationStateResponse { + version := i.migrationCatalogVersion() + if i.routeEngine == nil || version < req.GetExpectedCatalogVersion() { + return &pb.ProbeMigrationStateResponse{CatalogVersion: version} + } + route, found := i.routeEngine.GetRoute(req.GetRouteStart()) + ready := found && route.GroupID == req.GetExpectedGroupId() && + bytes.Equal(route.Start, req.GetRouteStart()) && + bytes.Equal(route.End, normalizeOptionalEnd(req.GetRouteEnd())) + if requireCleared { + ready = ready && !route.StagedVisibilityActive && route.MigrationJobID == 0 && + route.MinWriteTSExclusive == req.GetMinWriteTsExclusive() + } + return &pb.ProbeMigrationStateResponse{Ready: ready, CatalogVersion: version} +} + +func (i *Internal) probeMigrationMetadataCleared(ctx context.Context, jobID uint64) (*pb.ProbeMigrationStateResponse, error) { + readiness, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration readiness reader is not configured")) + } + states, err := readiness.MigrationTargetReadinessStates(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + for _, state := range states { + if state.JobID == jobID { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + } + if promotion, ok := i.store.(store.MigrationPromotionStateReader); ok { + _, found, err := promotion.MigrationPromotionState(ctx, jobID) + if err != nil { + return nil, errors.WithStack(err) + } + if found { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + } + return &pb.ProbeMigrationStateResponse{Ready: true, CatalogVersion: i.migrationCatalogVersion()}, nil +} + +func (i *Internal) migrationCatalogVersion() uint64 { + if i.routeEngine == nil { + return 0 + } + return i.routeEngine.Version() +} + +func normalizeOptionalEnd(end []byte) []byte { + if len(end) == 0 { + return nil + } + return end +} + func (i *Internal) RelayPublish(_ context.Context, req *pb.RelayPublishRequest) (*pb.RelayPublishResponse, error) { if req == nil || i.relay == nil { return &pb.RelayPublishResponse{}, nil @@ -312,6 +453,47 @@ func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.Targe return &pb.TargetStagedReadinessResponse{MinAdmittedTs: i.migrationMinAdmittedTS(ctx, req.GetJobId())}, nil } +func (i *Internal) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest) (*pb.CleanupMigrationResponse, error) { + if err := validateCleanupMigrationRequest(req); err != nil { + return nil, err + } + if i.migrationProposer == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration cleanup proposer is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { + return nil, err + } + result, err := i.proposeMigrationCleanup(ctx, req) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.CleanupMigrationResponse{ + NextCursor: result.NextCursor, + Done: result.Done, + DeletedRows: result.DeletedRows, + DeletedBytes: result.DeletedBytes, + ScannedBytes: result.ScannedBytes, + }, nil +} + +func validateCleanupMigrationRequest(req *pb.CleanupMigrationRequest) error { + switch { + case req == nil: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup request is nil")) + case req.GetJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup job_id is required")) + case req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && + req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup mode is invalid")) + case req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && req.GetKeyFamily() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup key_family is required")) + } + return nil +} + func (i *Internal) migrationMinAdmittedTS(ctx context.Context, jobID uint64) uint64 { reader, ok := i.store.(store.MigrationTargetReadinessReader) if !ok { @@ -506,6 +688,33 @@ func (i *Internal) proposeMigrationPromote(ctx context.Context, req *pb.PromoteS } } +func (i *Internal) proposeMigrationCleanup(ctx context.Context, req *pb.CleanupMigrationRequest) (store.CleanupVersionsResult, error) { + cmd, err := kv.MarshalMigrationCleanupCommand(req) + if err != nil { + return store.CleanupVersionsResult{}, errors.WithStack(err) + } + resp, err := i.proposeMigrationCommand(ctx, cmd, "migration cleanup") + if err != nil { + return store.CleanupVersionsResult{}, errors.WithStack(err) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA && resp == nil { + return store.CleanupVersionsResult{Done: true}, nil + } + switch resp := resp.(type) { + case store.CleanupVersionsResult: + return resp, nil + case *store.CleanupVersionsResult: + if resp == nil { + return store.CleanupVersionsResult{}, errors.New("migration cleanup apply returned nil result") + } + return *resp, nil + case error: + return store.CleanupVersionsResult{}, errors.WithStack(resp) + default: + return store.CleanupVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration cleanup apply response type %T", resp)) + } +} + func (i *Internal) proposeTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) error { cmd, err := kv.MarshalTargetStagedReadinessCommand(req) if err != nil { diff --git a/adapter/internal_migration_probe_test.go b/adapter/internal_migration_probe_test.go new file mode 100644 index 000000000..d087d4475 --- /dev/null +++ b/adapter/internal_migration_probe_test.go @@ -0,0 +1,117 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestInternalProbeMigrationStateUsesLocalFSMAndCatalog(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 9, + Routes: []distribution.RouteDescriptor{{ + RouteID: 2, + Start: []byte("m"), + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 88, + }}, + })) + tracker := kv.NewActiveTimestampTracker() + internal := NewInternalWithEngine(nil, nil, nil, nil, + WithInternalStore(st), + WithInternalRouteEngine(engine), + WithInternalActiveTimestampTracker(tracker), + ) + + state := store.TargetStagedReadinessState{ + JobID: 7, + RouteStart: []byte("m"), + ExpectedCutoverVersion: 9, + MigrationJobID: 7, + MinWriteTSExclusive: 88, + Armed: true, + } + readinessWriter, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, readinessWriter.ApplyTargetStagedReadiness(ctx, state)) + control, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: []byte("m"), + ExpectedCatalogVersion: 9, + MigrationJobId: 7, + MinWriteTsExclusive: 88, + }) + require.NoError(t, err) + require.True(t, control.Ready) + + cleared, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED, + RouteStart: []byte("m"), + ExpectedCatalogVersion: 9, + ExpectedGroupId: 2, + MinWriteTsExclusive: 88, + }) + require.NoError(t, err) + require.True(t, cleared.Ready) + + pin := tracker.Pin(100) + drained, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(-time.Second).UnixMilli(), + }) + require.NoError(t, err) + require.False(t, drained.Ready) + pin.Release() + drained, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(-time.Second).UnixMilli(), + }) + require.NoError(t, err) + require.True(t, drained.Ready) + + metadata, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.False(t, metadata.Ready) + cleaner, ok := st.(store.MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + metadata, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.True(t, metadata.Ready) +} + +func TestInternalIssueMigrationTimestampFollowsSourceLastCommit(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.ApplyMutations(ctx, []*store.KVPairMutation{{Key: []byte("m"), Value: []byte("value")}}, nil, 50, 50)) + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + + resp, err := internal.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) + require.NoError(t, err) + require.Equal(t, uint64(50), resp.GetLastCommitTs()) + require.Greater(t, resp.GetTimestamp(), resp.GetLastCommitTs()) +} diff --git a/adapter/split_job_runner.go b/adapter/split_job_runner.go index b794055a2..b57bb8225 100644 --- a/adapter/split_job_runner.go +++ b/adapter/split_job_runner.go @@ -3,8 +3,10 @@ package adapter import ( "bytes" "context" + "encoding/binary" "io" "math" + "sort" "time" "github.com/bootjp/elastickv/distribution" @@ -20,10 +22,167 @@ const ( defaultSplitMigrationChunkBytes = 1 << 20 defaultSplitMigrationMaxScannedBytes = 4 << 20 defaultSplitMigrationLockProbeLimit = 1024 + defaultSplitMigrationReadFenceGrace = 30 * time.Second + splitCleanupCursorVersion = byte(1) + splitVoterAckCursorVersion = byte(1) + splitCleanupCursorHeaderBytes = 5 + splitJobHistoryLimit = 1000 + splitJobHistoryDeleteBatchLimit = 1000 + splitJobHistoryTTL = 7 * 24 * time.Hour + splitJobHistoryGCInterval = time.Minute ) +var splitCleanupDoneCursor = []byte{splitCleanupCursorVersion, 0xff, 0xff, 0xff, 0xff} + +type splitVoterAck struct { + id string + address string + acked bool +} + +func (s *DistributionServer) syncSplitMigrationVoterBarrier( + ctx context.Context, + groupID uint64, + existing []byte, + fallback SplitMigrationClient, + req *pb.ProbeMigrationStateRequest, +) ([]byte, bool, error) { + voters, err := s.splitMigrationVoters(ctx, groupID, fallback) + if err != nil { + return nil, false, err + } + prior, err := decodeSplitVoterAckCursor(existing) + if err != nil { + return nil, false, err + } + priorAcked := make(map[string]bool, len(prior)) + for _, voter := range prior { + priorAcked[splitVoterAckKey(voter.id, voter.address)] = voter.acked + } + acks := make([]splitVoterAck, 0, len(voters)) + complete := true + for _, voter := range voters { + key := splitVoterAckKey(voter.ID, voter.Address) + acked := priorAcked[key] + resp, probeErr := voter.Client.ProbeMigrationState(ctx, req) + if probeErr == nil { + acked = resp.GetReady() + } + complete = complete && acked + acks = append(acks, splitVoterAck{id: voter.ID, address: voter.Address, acked: acked}) + } + return encodeSplitVoterAckCursor(acks), complete, nil +} + +func (s *DistributionServer) splitMigrationVoters(ctx context.Context, groupID uint64, fallback SplitMigrationClient) ([]SplitMigrationVoter, error) { + if s.splitMigrationVoterFactory == nil { + if fallback == nil { + return nil, errors.New("split migration voter factory is not configured") + } + return []SplitMigrationVoter{{ID: "leader", Address: "leader", Client: fallback}}, nil + } + voters, err := s.splitMigrationVoterFactory(ctx, groupID) + if err != nil { + return nil, errors.WithStack(err) + } + if len(voters) == 0 { + return nil, errors.New("split migration voter set is empty") + } + if err := validateSplitMigrationVoters(voters); err != nil { + return nil, err + } + return voters, nil +} + +func validateSplitMigrationVoters(voters []SplitMigrationVoter) error { + sort.Slice(voters, func(i, j int) bool { + if voters[i].ID == voters[j].ID { + return voters[i].Address < voters[j].Address + } + return voters[i].ID < voters[j].ID + }) + for index, voter := range voters { + if voter.ID == "" || voter.Address == "" || voter.Client == nil { + return errors.New("split migration voter is incomplete") + } + if index > 0 && voter.ID == voters[index-1].ID { + return errors.New("split migration voter id is duplicated") + } + } + return nil +} + +func splitVoterAckKey(id, address string) string { + return id + "\x00" + address +} + +func encodeSplitVoterAckCursor(acks []splitVoterAck) []byte { + out := []byte{splitVoterAckCursorVersion} + out = binary.AppendUvarint(out, uint64(len(acks))) + for _, ack := range acks { + out = binary.AppendUvarint(out, uint64(len(ack.id))) + out = append(out, ack.id...) + out = binary.AppendUvarint(out, uint64(len(ack.address))) + out = append(out, ack.address...) + if ack.acked { + out = append(out, 1) + } else { + out = append(out, 0) + } + } + return out +} + +func decodeSplitVoterAckCursor(raw []byte) ([]splitVoterAck, error) { + if len(raw) == 0 { + return nil, nil + } + if raw[0] != splitVoterAckCursorVersion { + return nil, errors.New("invalid split voter ack cursor version") + } + raw = raw[1:] + count, rest, err := consumeSplitCursorUvarint(raw) + if err != nil { + return nil, err + } + raw = rest + acks := make([]splitVoterAck, 0, count) + for range count { + id, next, err := consumeSplitCursorString(raw) + if err != nil { + return nil, err + } + address, next, err := consumeSplitCursorString(next) + if err != nil || len(next) == 0 || next[0] > 1 { + return nil, errors.New("invalid split voter ack cursor entry") + } + acks = append(acks, splitVoterAck{id: id, address: address, acked: next[0] == 1}) + raw = next[1:] + } + if len(raw) != 0 { + return nil, errors.New("split voter ack cursor has trailing bytes") + } + return acks, nil +} + +func consumeSplitCursorString(raw []byte) (string, []byte, error) { + size, rest, err := consumeSplitCursorUvarint(raw) + if err != nil || size > uint64(len(rest)) { + return "", nil, errors.New("invalid split voter ack cursor string") + } + return string(rest[:size]), rest[size:], nil +} + +func consumeSplitCursorUvarint(raw []byte) (uint64, []byte, error) { + value, size := binary.Uvarint(raw) + if size <= 0 { + return 0, nil, errors.New("invalid split voter ack cursor integer") + } + return value, raw[size:], nil +} + func (s *DistributionServer) runSplitJobPhase(ctx context.Context, job distribution.SplitJob) error { - if job.Phase != distribution.SplitJobPhaseCleanup && s.splitMigrationClientFactory == nil { + if s.splitMigrationClientFactory == nil { return errors.New("split migration client factory is not configured") } if job.Phase == distribution.SplitJobPhaseBackfill || job.Phase == distribution.SplitJobPhaseDeltaCopy { @@ -50,7 +209,7 @@ func (s *DistributionServer) runSplitJobControlPhase(ctx context.Context, job di case distribution.SplitJobPhaseCleanup: return s.cleanupSplitJob(ctx, job) case distribution.SplitJobPhaseAbandoning: - return errors.New("split job abandon cleanup is not configured") + return s.abandonSplitJob(ctx, job) case distribution.SplitJobPhaseNone, distribution.SplitJobPhaseBackfill, distribution.SplitJobPhaseDeltaCopy, @@ -63,52 +222,586 @@ func (s *DistributionServer) runSplitJobControlPhase(ctx context.Context, job di } func (s *DistributionServer) cleanupSplitJob(ctx context.Context, job distribution.SplitJob) error { - if job.SourceRetentionPinTS > 0 && job.SourceRetentionPinTS != math.MaxUint64 { - if err := s.releaseSplitJobSourceRetention(ctx, job); err != nil { + if !job.TargetPromotionDone { + return s.promoteSplitJobTargetAndComplete(ctx, job) + } + if !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) { + if err := s.cleanupSplitJobTargetProofs(ctx, job); err != nil { return err } return nil } - return s.promoteSplitJobTargetAndComplete(ctx, job) + if !bytes.Equal(job.SourceCutoverAckCursor, splitCleanupDoneCursor) { + if err := s.ackSplitJobSourceRouteRemoval(ctx, job); err != nil { + return err + } + return nil + } + if !bytes.Equal(job.SourceReadDrainCursor, splitCleanupDoneCursor) { + if err := s.ackSplitJobSourceReadDrain(ctx, job); err != nil { + return err + } + return nil + } + if !bytes.Equal(job.Cursor, splitCleanupDoneCursor) { + return s.cleanupSplitJobSourceData(ctx, job) + } + if job.SourceRetentionPinTS != math.MaxUint64 { + return s.cleanupSplitJobSourceProofs(ctx, job) + } + return s.finishSplitJobHistory(ctx, job, distribution.SplitJobPhaseDone) } -func (s *DistributionServer) releaseSplitJobSourceRetention(ctx context.Context, job distribution.SplitJob) error { +func (s *DistributionServer) splitJobMigrationClients(ctx context.Context, job distribution.SplitJob) (SplitMigrationClient, SplitMigrationClient, []byte, error) { + if s.splitMigrationClientFactory == nil { + return nil, nil, nil, errors.New("split migration client factory is not configured") + } snapshot, err := s.loadCatalogSnapshot(ctx) if err != nil { - return err + return nil, nil, nil, err } sourceGroupID, routeEnd, ok := splitJobSourceSibling(snapshot.Routes, job) if !ok { - return errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + if parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + sourceGroupID = parent.GroupID + routeEnd = distribution.CloneBytes(parent.End) + ok = true + } + } + if !ok { + return nil, nil, nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if source == nil || target == nil { + return nil, nil, nil, errors.New("split migration source or target client is nil") } - source, _, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + return source, target, routeEnd, nil +} + +func (s *DistributionServer) cleanupSplitJobTargetProofs(ctx context.Context, job distribution.SplitJob) error { + _, target, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + descriptorCursor, descriptorComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, job.TargetClearedDescriptorAckCursor, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCatalogVersion: job.CutoverVersion, + ExpectedGroupId: job.TargetGroupID, + MinWriteTsExclusive: job.FenceTS, + }) if err != nil { + return err + } + if !descriptorComplete { + return s.persistSplitJobTargetCleanupCursor(ctx, job.JobID, descriptorCursor) + } + if _, err := target.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, + }); err != nil { return errors.WithStack(err) } - if _, err := applySplitMigrationControl( - ctx, - source, - job, - routeEnd, - job.CutoverVersion, - job.FenceTS, - true, - true, - false, - math.MaxUint64, - ); err != nil { + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, nil, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil || !metadataComplete { return err } return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup && current.TargetPromotionDone { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) persistSplitJobTargetCleanupCursor(ctx context.Context, jobID uint64, cursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { if current.Phase == distribution.SplitJobPhaseCleanup { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(cursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) ackSplitJobSourceRouteRemoval(ctx context.Context, job distribution.SplitJob) error { + source, _, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return err + } + cursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, job.SourceCutoverAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCatalogVersion: job.CutoverVersion, + ExpectedGroupId: job.TargetGroupID, + }) + if err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceCutoverAckCursor = distribution.CloneBytes(cursor) + if complete { + current.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + } + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) ackSplitJobSourceReadDrain(ctx context.Context, job distribution.SplitJob) error { + source, _, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return err + } + notBefore := int64(job.PromotionCompletedTS>>kv.HLCLogicalBits) + defaultSplitMigrationReadFenceGrace.Milliseconds() //nolint:gosec // HLC physical millis fit int64. + cursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, job.SourceReadDrainCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: notBefore, + }) + if err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceReadDrainCursor = distribution.CloneBytes(cursor) + if complete { + current.SourceReadDrainCursor = distribution.CloneBytes(splitCleanupDoneCursor) + } + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) splitJobSourceGroupID(ctx context.Context, job distribution.SplitJob) (uint64, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return 0, err + } + groupID, _, ok := splitJobSourceSibling(snapshot.Routes, job) + if !ok { + if parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + return parent.GroupID, nil + } + return 0, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + return groupID, nil +} + +func (s *DistributionServer) cleanupSplitJobSourceData(ctx context.Context, job distribution.SplitJob) error { + source, _, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + brackets, err := distribution.PlanExportBrackets(job.SplitKey, routeEnd) + if err != nil { + return errors.WithStack(err) + } + index, cursor, done, err := decodeSplitCleanupCursor(job.Cursor, len(brackets)) + if err != nil || done { + return err + } + bracket := brackets[index] + resp, err := source.CleanupMigration(ctx, splitMigrationCleanupRequest(job, routeEnd, bracket, cursor, job.FenceTS)) + if err != nil { + return errors.WithStack(err) + } + encoded, err := nextSplitCleanupCursor(resp, cursor, index, len(brackets)) + if err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup && bytes.Equal(current.Cursor, job.Cursor) { + current.Cursor = encoded + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func nextSplitCleanupCursor(resp *pb.CleanupMigrationResponse, cursor []byte, index, bracketCount int) ([]byte, error) { + if !resp.GetDone() { + next := resp.GetNextCursor() + if len(next) == 0 || bytes.Equal(next, cursor) { + return nil, errors.New("split migration source cleanup made no cursor progress") + } + return encodeSplitCleanupCursor(index, next, false), nil + } + nextIndex := index + 1 + return encodeSplitCleanupCursor(nextIndex, nil, nextIndex >= bracketCount), nil +} + +func splitMigrationCleanupRequest(job distribution.SplitJob, routeEnd []byte, bracket distribution.MigrationBracket, cursor []byte, maxCommitTS uint64) *pb.CleanupMigrationRequest { + return &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: bracket.Start, + RangeEnd: bracket.End, + Cursor: cursor, + MaxCommitTs: maxCommitTS, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + KeyFamily: bracket.Family, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExcludeKnownInternal: bracket.ExcludeKnownInternal, + ExcludePrefixes: bracket.ExcludePrefixes, + RequiresRouteKeyCheck: bracket.RequiresRouteKeyCheck, + RequiresDecodedS3: bracket.RequiresDecodedS3, + } +} + +func (s *DistributionServer) cleanupSplitJobSourceProofs(ctx context.Context, job distribution.SplitJob) error { + if err := s.cleanupSplitJobSourceMetadata(ctx, job); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.SourceRetentionPinTS = math.MaxUint64 + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func encodeSplitCleanupCursor(index int, cursor []byte, done bool) []byte { + if done { + return distribution.CloneBytes(splitCleanupDoneCursor) + } + out := make([]byte, splitCleanupCursorHeaderBytes+len(cursor)) + out[0] = splitCleanupCursorVersion + binary.BigEndian.PutUint32(out[1:], uint32(index)) //nolint:gosec // bracket count is bounded. + copy(out[splitCleanupCursorHeaderBytes:], cursor) + return out +} + +func decodeSplitCleanupCursor(raw []byte, bracketCount int) (int, []byte, bool, error) { + if len(raw) == 0 { + return 0, nil, bracketCount == 0, nil + } + if bytes.Equal(raw, splitCleanupDoneCursor) { + return bracketCount, nil, true, nil + } + if len(raw) < splitCleanupCursorHeaderBytes || raw[0] != splitCleanupCursorVersion { + return 0, nil, false, errors.New("invalid split cleanup cursor") + } + index := int(binary.BigEndian.Uint32(raw[1:splitCleanupCursorHeaderBytes])) + if index < 0 || index >= bracketCount { + return 0, nil, false, errors.New("split cleanup cursor bracket is out of range") + } + return index, distribution.CloneBytes(raw[splitCleanupCursorHeaderBytes:]), false, nil +} + +func (s *DistributionServer) abandonSplitJob(ctx context.Context, job distribution.SplitJob) error { + if err := s.rollbackAbandonedSplitJobFence(ctx, job); err != nil { + return err + } + if !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) { + return s.cleanupAbandonedSplitJobTarget(ctx, job) + } + if !bytes.Equal(job.SourceCutoverAckCursor, splitCleanupDoneCursor) { + return s.cleanupAbandonedSplitJobSource(ctx, job) + } + return s.finishSplitJobHistory(ctx, job, distribution.SplitJobPhaseAbandoned) +} + +func (s *DistributionServer) cleanupAbandonedSplitJobTarget(ctx context.Context, job distribution.SplitJob) error { + _, target, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + _, cursor, done, err := decodeSplitCleanupCursor(job.TargetClearedDescriptorAckCursor, 1) + if err != nil { + return err + } + if !done { + complete, err := s.cleanupAbandonedSplitJobTargetVersions(ctx, target, job, cursor) + if err != nil || !complete { + return err + } + } + if _, err := target.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, + }); err != nil { + return errors.WithStack(err) + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, nil, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil || !metadataComplete { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) cleanupAbandonedSplitJobTargetVersions( + ctx context.Context, + target SplitMigrationClient, + job distribution.SplitJob, + cursor []byte, +) (bool, error) { + prefix := distribution.MigrationStagedDataKeyPrefix(job.JobID) + resp, err := target.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: prefix, + RangeEnd: store.PrefixScanEnd(prefix), + Cursor: cursor, + MaxCommitTs: math.MaxUint64, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + KeyFamily: distribution.MigrationFamilyUser, + }) + if err != nil { + return false, errors.WithStack(err) + } + if resp.GetDone() { + return true, nil + } + if len(resp.GetNextCursor()) == 0 || bytes.Equal(resp.GetNextCursor(), cursor) { + return false, errors.New("split migration abandon cleanup made no cursor progress") + } + encoded := encodeSplitCleanupCursor(0, resp.GetNextCursor(), false) + err = s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning && bytes.Equal(current.TargetClearedDescriptorAckCursor, job.TargetClearedDescriptorAckCursor) { + current.TargetClearedDescriptorAckCursor = encoded + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return false, err +} + +func (s *DistributionServer) cleanupAbandonedSplitJobSource(ctx context.Context, job distribution.SplitJob) error { + if err := s.cleanupSplitJobSourceMetadata(ctx, job); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning { + current.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) current.SourceRetentionPinTS = math.MaxUint64 - current.SourceCutoverAckCursor = []byte("retention-released") current.UpdatedAtMs = time.Now().UnixMilli() } return current, nil }) } +func (s *DistributionServer) cleanupSplitJobSourceMetadata(ctx context.Context, job distribution.SplitJob) error { + source, _, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + if _, err := source.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, + }); err != nil { + return errors.WithStack(err) + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return err + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, nil, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil || !metadataComplete { + return err + } + return nil +} + +func (s *DistributionServer) rollbackAbandonedSplitJobFence(ctx context.Context, job distribution.SplitJob) error { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + fenced := abandonedSplitJobFencedRoute(snapshot.Routes, job) + if fenced == nil { + return nil + } + if snapshot.Version == math.MaxUint64 { + return errors.New("split migration catalog version overflow") + } + fenced.State = distribution.RouteStateActive + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(*fenced, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return errors.WithStack(err) + } + nextVersion := snapshot.Version + 1 + if err := s.commitAbandonedSplitJobFenceRollback(ctx, snapshot, *fenced, encoded, nextVersion); err != nil { + return err + } + loaded, err := s.loadCatalogSnapshotAtLeastVersion(ctx, nextVersion) + if err != nil { + return err + } + return s.applyEngineSnapshot(loaded) +} + +func abandonedSplitJobFencedRoute(routes []distribution.RouteDescriptor, job distribution.SplitJob) *distribution.RouteDescriptor { + for i := range routes { + route := &routes[i] + if route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) && route.State == distribution.RouteStateWriteFenced { + return route + } + } + return nil +} + +func (s *DistributionServer) commitAbandonedSplitJobFenceRollback( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + fenced distribution.RouteDescriptor, + encoded []byte, + nextVersion uint64, +) error { + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: distribution.CatalogRouteKey(fenced.RouteID), Value: encoded}, + {Op: kv.Put, Key: distribution.CatalogVersionKey(), Value: distribution.EncodeCatalogVersion(nextVersion)}, + }, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: [][]byte{ + distribution.CatalogRouteKey(fenced.RouteID), + distribution.CatalogVersionKey(), + }, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) finishSplitJobHistory(ctx context.Context, job distribution.SplitJob, terminalPhase distribution.SplitJobPhase) error { + current, readTS, err := s.catalog.LiveSplitJobForUpdate(ctx, job.JobID) + if err != nil { + return splitJobCatalogStatusError(err) + } + if current.Phase != job.Phase { + return nil + } + nowMs := time.Now().UnixMilli() + next := distribution.CloneSplitJob(current) + next.Phase = terminalPhase + next.RetryPhase = distribution.SplitJobPhaseNone + next.AbandonFromPhase = distribution.SplitJobPhaseNone + next.TerminalAtMs = nowMs + next.UpdatedAtMs = nowMs + encoded, err := distribution.EncodeSplitJob(next) + if err != nil { + return splitJobCatalogStatusError(err) + } + liveKey := distribution.CatalogSplitJobKey(job.JobID) + historyKey := distribution.CatalogSplitJobHistoryKey(nowMs, job.JobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: historyKey, Value: encoded}, + {Op: kv.Del, Key: liveKey}, + }, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{liveKey, historyKey}, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) gcSplitJobHistory(ctx context.Context, jobs []distribution.SplitJob, now time.Time) error { + s.mu.Lock() + if !s.splitJobHistoryGCLast.IsZero() && now.Sub(s.splitJobHistoryGCLast) < splitJobHistoryGCInterval { + s.mu.Unlock() + return nil + } + s.mu.Unlock() + + keys := splitJobHistoryGCKeys(jobs, now) + if len(keys) > 0 { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + elems := make([]*kv.Elem[kv.OP], 0, len(keys)) + for _, key := range keys { + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: key}) + } + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: elems, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: keys, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + } + s.mu.Lock() + s.splitJobHistoryGCLast = now + s.mu.Unlock() + return nil +} + +func splitJobHistoryGCKeys(jobs []distribution.SplitJob, now time.Time) [][]byte { + history := make([]distribution.SplitJob, 0, len(jobs)) + for _, job := range jobs { + if (job.Phase == distribution.SplitJobPhaseDone || job.Phase == distribution.SplitJobPhaseAbandoned) && job.TerminalAtMs > 0 { + history = append(history, job) + } + } + sort.Slice(history, func(i, j int) bool { + if history[i].TerminalAtMs == history[j].TerminalAtMs { + return history[i].JobID < history[j].JobID + } + return history[i].TerminalAtMs < history[j].TerminalAtMs + }) + cutoff := now.Add(-splitJobHistoryTTL).UnixMilli() + excess := len(history) - splitJobHistoryLimit + keys := make([][]byte, 0, min(len(history), splitJobHistoryDeleteBatchLimit)) + for i, job := range history { + if job.TerminalAtMs >= cutoff && i >= excess { + continue + } + keys = append(keys, distribution.CatalogSplitJobHistoryKey(job.TerminalAtMs, job.JobID)) + if len(keys) == splitJobHistoryDeleteBatchLimit { + break + } + } + return keys +} + func splitJobSourceSibling(routes []distribution.RouteDescriptor, job distribution.SplitJob) (uint64, []byte, bool) { var sourceGroupID uint64 var routeEnd []byte @@ -139,15 +832,63 @@ func (s *DistributionServer) beginSplitJobBackfill(ctx context.Context, job dist if source == nil { return errors.New("split migration source client is nil") } - snapshotTS := s.engine.NextTimestamp() - if snapshotTS == 0 { - return errors.New("split migration snapshot timestamp is zero") - } - // The source guard is applied before BACKFILL opens. This keeps the initial - // implementation conservative: no write can land outside the copied window. - if _, err := applySplitMigrationControl(ctx, source, job, parent.End, snapshot.Version+1, snapshotTS, false, false, true, 1); err != nil { + ackCursor, complete, err := s.armSplitJobSourceTracker(ctx, job, snapshot, parent, source) + if err != nil { return err } + if !complete { + return s.persistSplitJobBackfillAck(ctx, job.JobID, ackCursor) + } + return s.openSplitJobBackfill(ctx, job, source) +} + +const initialMigrationRetentionPinTS = uint64(1) + +func (s *DistributionServer) armSplitJobSourceTracker( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + parent distribution.RouteDescriptor, + source SplitMigrationClient, +) ([]byte, bool, error) { + // Arm the write tracker and its no-prune retention pin before choosing the + // snapshot boundary. This closes the window where a low-ts write could land + // after timestamp selection without being represented in the delta floor. + if _, err := applySplitMigrationControl(ctx, source, job, parent.End, snapshot.Version+1, initialMigrationRetentionPinTS, false, false, true, initialMigrationRetentionPinTS); err != nil { + return nil, false, err + } + return s.syncSplitMigrationVoterBarrier(ctx, parent.GroupID, job.FenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: parent.End, + ExpectedCatalogVersion: snapshot.Version + 1, + MigrationJobId: job.JobID, + MinWriteTsExclusive: initialMigrationRetentionPinTS, + TrackWrites: true, + RetentionPinTs: initialMigrationRetentionPinTS, + }) +} + +func (s *DistributionServer) persistSplitJobBackfillAck(ctx context.Context, jobID uint64, ackCursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhasePlanned { + current.FenceAckCursor = ackCursor + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) openSplitJobBackfill(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient) error { + issued, err := source.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) + if err != nil { + return errors.WithStack(err) + } + snapshotTS := issued.GetTimestamp() + if snapshotTS == 0 || snapshotTS <= issued.GetLastCommitTs() { + return errors.New("split migration source issued an invalid snapshot timestamp") + } return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { if current.Phase != distribution.SplitJobPhasePlanned { return current, nil @@ -155,7 +896,8 @@ func (s *DistributionServer) beginSplitJobBackfill(ctx context.Context, job dist current.Phase = distribution.SplitJobPhaseBackfill current.SnapshotTS = snapshotTS current.WriteTrackerArmed = true - current.SourceRetentionPinTS = 1 + current.SourceRetentionPinTS = initialMigrationRetentionPinTS + current.FenceAckCursor = nil current.UpdatedAtMs = time.Now().UnixMilli() return current, nil }) @@ -327,10 +1069,40 @@ func (s *DistributionServer) finalizeSplitJobFence(ctx context.Context, job dist if err != nil { return err } - source, _, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + source, minAdmittedTS, ackCursor, complete, err := s.applySplitJobFenceBarrier(ctx, job, snapshot, sourceRoute) + if err != nil { + return err + } + if !complete { + return s.persistSplitJobFenceAck(ctx, job.JobID, ackCursor) + } + pendingLocks, err := splitJobPendingLocks(ctx, source, job.SplitKey, sourceRoute.End) + if err != nil { + return err + } + if pendingLocks { + return nil + } + issued, err := source.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) if err != nil { return errors.WithStack(err) } + if issued.GetTimestamp() == 0 || issued.GetTimestamp() <= issued.GetLastCommitTs() { + return errors.New("split migration source issued an invalid fence timestamp") + } + return s.commitSplitJobFenceState(ctx, job, snapshot.Version, minAdmittedTS, ackCursor, issued.GetTimestamp()) +} + +func (s *DistributionServer) applySplitJobFenceBarrier( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + sourceRoute distribution.RouteDescriptor, +) (SplitMigrationClient, uint64, []byte, bool, error) { + source, _, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return nil, 0, nil, false, errors.WithStack(err) + } minAdmittedTS, err := applySplitMigrationControl( ctx, source, @@ -344,20 +1116,37 @@ func (s *DistributionServer) finalizeSplitJobFence(ctx context.Context, job dist 1, ) if err != nil { - return err + return nil, 0, nil, false, err } - pendingLocks, err := splitJobPendingLocks(ctx, source, job.SplitKey, sourceRoute.End) + ackCursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceRoute.GroupID, job.FenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: snapshot.Version, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.SnapshotTS, + SourceWriteFence: true, + TrackWrites: true, + RetentionPinTs: 1, + }) if err != nil { - return err - } - if pendingLocks { - return nil + return nil, 0, nil, false, err } - return s.commitSplitJobFenceState(ctx, job, snapshot.Version, minAdmittedTS) + return source, minAdmittedTS, ackCursor, complete, nil } -func (s *DistributionServer) commitSplitJobFenceState(ctx context.Context, job distribution.SplitJob, fenceCatalogVersion, minAdmittedTS uint64) error { - fenceTS := s.engine.NextTimestamp() +func (s *DistributionServer) persistSplitJobFenceAck(ctx context.Context, jobID uint64, ackCursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseFence { + current.FenceAckCursor = distribution.CloneBytes(ackCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) commitSplitJobFenceState(ctx context.Context, job distribution.SplitJob, fenceCatalogVersion, minAdmittedTS uint64, ackCursor []byte, fenceTS uint64) error { deltaFloor := job.SnapshotTS if minAdmittedTS > 0 && minAdmittedTS-1 < deltaFloor { deltaFloor = minAdmittedTS - 1 @@ -372,7 +1161,7 @@ func (s *DistributionServer) commitSplitJobFenceState(ctx context.Context, job d current.DeltaFloor = deltaFloor current.FenceTS = fenceTS current.FenceCatalogVersion = fenceCatalogVersion - current.FenceAckCursor = []byte("raft-applied") + current.FenceAckCursor = distribution.CloneBytes(ackCursor) current.SourceRetentionPinTS = 1 for i := range current.BracketProgress { current.BracketProgress[i].ExportPhase = distribution.SplitJobExportPhaseDeltaCopy @@ -432,17 +1221,111 @@ func (s *DistributionServer) cutoverSplitJob(ctx context.Context, job distributi return errors.New("split migration catalog version overflow") } expectedVersion := snapshot.Version + 1 + if job.CutoverVersion == 0 || job.CutoverVersion != expectedVersion { + return s.armSplitJobCutoverWitness(ctx, job, expectedVersion) + } + targetCursor, sourceCursor, targetComplete, sourceComplete, err := s.applySplitJobCutoverBarriers(ctx, job, sourceRoute, expectedVersion) + if err != nil { + return err + } + if splitJobCutoverBarrierPending(job, targetComplete, sourceComplete) { + return s.persistSplitJobCutoverAcks(ctx, job.JobID, targetCursor, sourceCursor, targetComplete, sourceComplete) + } + return s.commitSplitJobCutover(ctx, snapshot, sourceRoute, job, expectedVersion) +} + +func (s *DistributionServer) applySplitJobCutoverBarriers( + ctx context.Context, + job distribution.SplitJob, + sourceRoute distribution.RouteDescriptor, + expectedVersion uint64, +) ([]byte, []byte, bool, bool, error) { source, target, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) if err != nil { - return errors.WithStack(err) + return nil, nil, false, false, errors.WithStack(err) } if _, err := applySplitMigrationControl(ctx, target, job, sourceRoute.End, expectedVersion, job.FenceTS, false, false, false, 0); err != nil { - return err + return nil, nil, false, false, err } if _, err := applySplitMigrationControl(ctx, source, job, sourceRoute.End, expectedVersion, job.FenceTS, true, true, false, 1); err != nil { - return err + return nil, nil, false, false, err } - return s.commitSplitJobCutover(ctx, snapshot, sourceRoute, job, expectedVersion) + targetCursor, targetComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, job.TargetStagedReadinessAckCursor, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.FenceTS, + }) + if err != nil { + return nil, nil, false, false, err + } + sourceCursor, sourceComplete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceRoute.GroupID, job.SourceCutoverReadFenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.FenceTS, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTs: 1, + }) + if err != nil { + return nil, nil, false, false, err + } + return targetCursor, sourceCursor, targetComplete, sourceComplete, nil +} + +func splitJobCutoverBarrierPending(job distribution.SplitJob, targetComplete, sourceComplete bool) bool { + return !targetComplete || !sourceComplete || + job.TargetStagedReadinessState != distribution.SplitJobBarrierArmed || + job.CutoverReadFenceState != distribution.SplitJobBarrierArmed +} + +func (s *DistributionServer) armSplitJobCutoverWitness(ctx context.Context, job distribution.SplitJob, expectedVersion uint64) error { + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseCutover { + return current, nil + } + current.CutoverVersion = expectedVersion + current.CutoverReadFenceState = distribution.SplitJobBarrierArming + current.TargetStagedReadinessState = distribution.SplitJobBarrierArming + current.SourceCutoverReadFenceAckCursor = nil + current.TargetStagedReadinessAckCursor = nil + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) persistSplitJobCutoverAcks( + ctx context.Context, + jobID uint64, + targetCursor []byte, + sourceCursor []byte, + targetComplete bool, + sourceComplete bool, +) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseCutover { + return current, nil + } + current.TargetStagedReadinessAckCursor = distribution.CloneBytes(targetCursor) + current.SourceCutoverReadFenceAckCursor = distribution.CloneBytes(sourceCursor) + current.TargetStagedReadinessState = distribution.SplitJobBarrierArming + if targetComplete { + current.TargetStagedReadinessState = distribution.SplitJobBarrierArmed + } + current.CutoverReadFenceState = distribution.SplitJobBarrierArming + if sourceComplete { + current.CutoverReadFenceState = distribution.SplitJobBarrierArmed + } + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) } func applySplitMigrationControl( @@ -519,8 +1402,7 @@ func (s *DistributionServer) commitSplitJobCutover( nextJob.CutoverVersion = nextVersion nextJob.CutoverReadFenceState = distribution.SplitJobBarrierArmed nextJob.TargetStagedReadinessState = distribution.SplitJobBarrierArmed - nextJob.SourceCutoverReadFenceAckCursor = []byte("raft-applied") - nextJob.TargetStagedReadinessAckCursor = []byte("raft-applied") + nextJob.Cursor = nil nextJob.UpdatedAtMs = time.Now().UnixMilli() encodedJob, err := distribution.EncodeSplitJob(nextJob) if err != nil { diff --git a/adapter/split_job_voter_ack_test.go b/adapter/split_job_voter_ack_test.go new file mode 100644 index 000000000..08a12acc7 --- /dev/null +++ b/adapter/split_job_voter_ack_test.go @@ -0,0 +1,67 @@ +package adapter + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +func TestSplitMigrationVoterBarrierReopensForMembershipChanges(t *testing.T) { + t.Parallel() + + ready := func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{Ready: true}, nil + } + blocked := func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{}, nil + } + n1 := &splitMigrationClientStub{probeFn: ready} + n2 := &splitMigrationClientStub{probeFn: ready} + n3 := &splitMigrationClientStub{probeFn: blocked} + voters := []SplitMigrationVoter{ + {ID: "n1", Address: "n1:50051", Client: n1}, + {ID: "n2", Address: "n2:50051", Client: n2}, + } + server := &DistributionServer{splitMigrationVoterFactory: func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return voters, nil + }} + req := &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED} + + cursor, complete, err := server.syncSplitMigrationVoterBarrier(context.Background(), 1, nil, nil, req) + require.NoError(t, err) + require.True(t, complete) + acks, err := decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 2) + + voters = append(voters, SplitMigrationVoter{ID: "n3", Address: "n3:50051", Client: n3}) + cursor, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.False(t, complete) + acks, err = decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 3) + require.False(t, acks[2].acked) + + n3.probeFn = ready + _, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.True(t, complete) + + n2.probeFn = blocked + voters[1].Address = "n2:50052" + _, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.False(t, complete, "an address change must be treated as a new voter endpoint") +} + +func TestSplitVoterAckCursorRejectsCorruption(t *testing.T) { + t.Parallel() + + _, err := decodeSplitVoterAckCursor([]byte{99}) + require.Error(t, err) + _, err = decodeSplitVoterAckCursor([]byte{splitVoterAckCursorVersion, 1, 4, 'n'}) + require.Error(t, err) +} diff --git a/cmd/elastickv-split/main.go b/cmd/elastickv-split/main.go index 767359c71..787f1e901 100644 --- a/cmd/elastickv-split/main.go +++ b/cmd/elastickv-split/main.go @@ -42,6 +42,9 @@ var ( routeID = flag.Uint64("route-id", 0, "RouteID of the route to split (required)") splitKey = flag.String("split-key", "", "Split key — must lie strictly inside the route's [Start, End) range; rejected if == Start or == End by validateSplitKey (required)") expectedVersion = flag.Uint64("expected-version", 0, "Expected catalog version for OCC; obtain by calling ListRoutes first (required, must be >= 1 — catalog version is 1-based)") + targetGroupID = flag.Uint64("target-group-id", 0, "Target Raft group for an asynchronous cross-group migration; zero uses the existing same-group split") + abandonJobID = flag.Uint64("abandon-job-id", 0, "Abandon a pre-cutover migration job; mutually exclusive with split flags") + getJobID = flag.Uint64("get-job-id", 0, "Print one migration job; mutually exclusive with mutation flags") ) func main() { @@ -76,7 +79,16 @@ func run() error { rpcCtx, rpcCancel := context.WithTimeout(context.Background(), rpcTimeout) defer rpcCancel() + if *getJobID != 0 { + return runGetSplitJob(rpcCtx, client) + } + if *abandonJobID != 0 { + return runAbandonSplitJob(rpcCtx, client) + } + if *targetGroupID != 0 { + return runStartSplitMigration(rpcCtx, client) + } req := &pb.SplitRangeRequest{ ExpectedCatalogVersion: *expectedVersion, RouteId: *routeID, @@ -90,7 +102,51 @@ func run() error { return nil } +func runGetSplitJob(ctx context.Context, client pb.DistributionClient) error { + resp, err := client.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: *getJobID}) + if err != nil { + return errors.Wrap(err, "GetSplitJob") + } + job := resp.GetJob() + fmt.Printf("job_id: %d\nphase: %s\n", job.GetJobId(), job.GetPhase()) + return nil +} + +func runAbandonSplitJob(ctx context.Context, client pb.DistributionClient) error { + if _, err := client.AbandonSplitJob(ctx, &pb.AbandonSplitJobRequest{JobId: *abandonJobID}); err != nil { + return errors.Wrap(err, "AbandonSplitJob") + } + fmt.Printf("abandoned_job_id: %d\n", *abandonJobID) + return nil +} + +func runStartSplitMigration(ctx context.Context, client pb.DistributionClient) error { + resp, err := client.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: *expectedVersion, + RouteId: *routeID, + SplitKey: []byte(*splitKey), + TargetGroupId: *targetGroupID, + }) + if err != nil { + return errors.Wrap(err, "StartSplitMigration") + } + fmt.Printf("catalog_version: %d\njob_id: %d\n", resp.GetCatalogVersion(), resp.GetJobId()) + return nil +} + func validateFlags() error { + if *getJobID != 0 { + if splitMutationFlagsSet() || *abandonJobID != 0 { + return errors.New("--get-job-id is mutually exclusive with mutation flags") + } + return nil + } + if *abandonJobID != 0 { + if splitMutationFlagsSet() { + return errors.New("--abandon-job-id is mutually exclusive with split flags") + } + return nil + } switch { case *routeID == 0: return errors.New("--route-id is required and must be > 0") @@ -102,6 +158,10 @@ func validateFlags() error { return nil } +func splitMutationFlagsSet() bool { + return *routeID != 0 || *splitKey != "" || *expectedVersion != 0 || *targetGroupID != 0 +} + func printResponse(resp *pb.SplitRangeResponse) { fmt.Printf("catalog_version: %d\n", resp.GetCatalogVersion()) printRoute("left: ", resp.GetLeft()) diff --git a/cmd/elastickv-split/main_test.go b/cmd/elastickv-split/main_test.go index bc9d40769..903d7e171 100644 --- a/cmd/elastickv-split/main_test.go +++ b/cmd/elastickv-split/main_test.go @@ -48,6 +48,34 @@ func TestValidateFlags_AcceptsAllPresent(t *testing.T) { require.NoError(t, validateFlags()) } +func TestValidateFlags_AcceptsAbandonOnly(t *testing.T) { + resetFlags(t) + *abandonJobID = 42 + require.NoError(t, validateFlags()) +} + +func TestValidateFlags_RejectsAbandonWithSplitFlags(t *testing.T) { + resetFlags(t) + *abandonJobID = 42 + *routeID = 7 + err := validateFlags() + require.ErrorContains(t, err, "mutually exclusive") +} + +func TestValidateFlags_AcceptsGetOnly(t *testing.T) { + resetFlags(t) + *getJobID = 42 + require.NoError(t, validateFlags()) +} + +func TestValidateFlags_RejectsGetWithMutationFlags(t *testing.T) { + resetFlags(t) + *getJobID = 42 + *abandonJobID = 42 + err := validateFlags() + require.ErrorContains(t, err, "mutually exclusive") +} + // resetFlags reinitialises the package-level flag pointers around // each test so subtests don't leak state through them. We rebind // the globals directly rather than re-parsing argv because the @@ -58,12 +86,15 @@ func resetFlags(t *testing.T) { // init). errors.New rather than t.Fatalf so a future // refactor that drops the package-level flags surfaces here // rather than silently no-oping the resets. - if routeID == nil || splitKey == nil || expectedVersion == nil || address == nil { + if routeID == nil || splitKey == nil || expectedVersion == nil || targetGroupID == nil || abandonJobID == nil || getJobID == nil || address == nil { t.Fatal(errors.New("package-level flag pointers were not initialised")) } *routeID = 0 *splitKey = "" *expectedVersion = 0 + *targetGroupID = 0 + *abandonJobID = 0 + *getJobID = 0 *address = "127.0.0.1:50051" _ = flag.CommandLine // touch to avoid unused import on future trims } diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go index a0ffe801a..aa4487a9c 100644 --- a/distribution/migration_promotion_complete.go +++ b/distribution/migration_promotion_complete.go @@ -52,8 +52,6 @@ func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs out.Changed = true out.ClearedRouteIDs = cleared out.Job.TargetPromotionDone = true - out.Job.Phase = SplitJobPhaseDone - out.Job.TerminalAtMs = nowMs out.Job.UpdatedAtMs = nowMs return out, nil } @@ -62,11 +60,10 @@ func completeAlreadyPromotedTarget(out TargetPromotionCompletion, nowMs int64) ( if !targetClearedDescriptorPresent(out.Job, out.Routes) { return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) } - if out.Job.Phase != SplitJobPhaseDone { - out.Changed = true - out.Job.Phase = SplitJobPhaseDone - } - if out.Job.TerminalAtMs <= 0 { + // Preserve compatibility for jobs terminalized by older binaries, but do + // not move a current CLEANUP job to DONE until source and target cleanup + // proofs have been removed by the runner. + if out.Job.Phase == SplitJobPhaseDone && out.Job.TerminalAtMs <= 0 { out.Changed = true out.Job.TerminalAtMs = nowMs } @@ -250,7 +247,7 @@ func promotionCompleteJobMatchesExpected(expected SplitJob, expectedRaw []byte, if expected.TargetPromotionDone || !current.TargetPromotionDone || current.PromotionCompletedTS == 0 || - current.Phase != SplitJobPhaseDone { + (current.Phase != expected.Phase && current.Phase != SplitJobPhaseDone) { return false, nil } normalized := CloneSplitJob(current) diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go index d68cd7c00..96bfc7e57 100644 --- a/distribution/migration_promotion_complete_test.go +++ b/distribution/migration_promotion_complete_test.go @@ -22,8 +22,8 @@ func TestCompleteTargetPromotionStateClearsStagedFieldsAndRetainsFloor(t *testin require.True(t, result.Job.TargetPromotionDone) require.Zero(t, result.Job.PromotionCompletedTS) require.Equal(t, int64(1000), result.Job.UpdatedAtMs) - require.Equal(t, int64(1000), result.Job.TerminalAtMs) - require.Equal(t, SplitJobPhaseDone, result.Job.Phase) + require.Zero(t, result.Job.TerminalAtMs) + require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) target := routeByID(t, result.Routes, 3) require.False(t, target.StagedVisibilityActive) @@ -50,11 +50,11 @@ func TestCompleteTargetPromotionStateAcceptsAlreadyClearedDescriptor(t *testing. result, err := CompleteTargetPromotionState(job, routes, 1100) require.NoError(t, err) - require.True(t, result.Changed) - require.Equal(t, SplitJobPhaseDone, result.Job.Phase) + require.False(t, result.Changed) + require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) require.Equal(t, uint64(900), result.Job.PromotionCompletedTS) - require.Equal(t, int64(1100), result.Job.UpdatedAtMs) - require.Equal(t, int64(1100), result.Job.TerminalAtMs) + require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Zero(t, result.Job.TerminalAtMs) target := routeByID(t, result.Routes, 3) require.False(t, target.StagedVisibilityActive) @@ -78,7 +78,7 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionCommitsRouteAndJobTogether(t require.NoError(t, err) require.Equal(t, saved.Version+1, snapshot.Version) require.True(t, completed.TargetPromotionDone) - require.Equal(t, SplitJobPhaseDone, completed.Phase) + require.Equal(t, SplitJobPhaseCleanup, completed.Phase) require.Greater(t, completed.PromotionCompletedTS, before.ReadTS) require.Equal(t, int64(1000), completed.UpdatedAtMs) @@ -138,15 +138,15 @@ func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDesc snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1100) require.NoError(t, err) - require.Equal(t, saved.Version+1, snapshot.Version) - require.Equal(t, SplitJobPhaseDone, completed.Phase) + require.Equal(t, saved.Version, snapshot.Version) + require.Equal(t, SplitJobPhaseCleanup, completed.Phase) require.Equal(t, uint64(900), completed.PromotionCompletedTS) - require.Equal(t, int64(1100), completed.UpdatedAtMs) - require.Equal(t, int64(1100), completed.TerminalAtMs) + require.Equal(t, int64(1000), completed.UpdatedAtMs) + require.Zero(t, completed.TerminalAtMs) loaded, err := cs.Snapshot(ctx) require.NoError(t, err) - require.Equal(t, saved.Version+1, loaded.Version) + require.Equal(t, saved.Version, loaded.Version) } func TestCompleteTargetPromotionStateBackfillsMissingTerminalTime(t *testing.T) { diff --git a/distribution/split_job_catalog.go b/distribution/split_job_catalog.go index d127940e8..b09fb26c9 100644 --- a/distribution/split_job_catalog.go +++ b/distribution/split_job_catalog.go @@ -168,6 +168,7 @@ type SplitJob struct { StartedAtMs int64 UpdatedAtMs int64 TerminalAtMs int64 + CapabilityRegressed bool } // CatalogNextSplitJobIDKey returns the reserved key used for split-job ID allocation. @@ -589,12 +590,12 @@ func (p SplitJobPhase) retryable() bool { func (p SplitJobPhase) abandonable() bool { switch p { - case SplitJobPhaseBackfill, + case SplitJobPhasePlanned, + SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: return true case SplitJobPhaseNone, - SplitJobPhasePlanned, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, @@ -906,6 +907,7 @@ func CloneSplitJob(job SplitJob) SplitJob { StartedAtMs: job.StartedAtMs, UpdatedAtMs: job.UpdatedAtMs, TerminalAtMs: job.TerminalAtMs, + CapabilityRegressed: job.CapabilityRegressed, } } @@ -950,6 +952,7 @@ func splitJobToProto(job SplitJob) *pb.SplitJob { StartedAtMs: job.StartedAtMs, UpdatedAtMs: job.UpdatedAtMs, TerminalAtMs: job.TerminalAtMs, + CapabilityRegressed: job.CapabilityRegressed, } } @@ -1015,6 +1018,7 @@ func splitJobFromProto(msg *pb.SplitJob) (SplitJob, error) { StartedAtMs: msg.GetStartedAtMs(), UpdatedAtMs: msg.GetUpdatedAtMs(), TerminalAtMs: msg.GetTerminalAtMs(), + CapabilityRegressed: msg.GetCapabilityRegressed(), }, nil } diff --git a/distribution/split_job_catalog_test.go b/distribution/split_job_catalog_test.go index 9e161c22e..d8c836368 100644 --- a/distribution/split_job_catalog_test.go +++ b/distribution/split_job_catalog_test.go @@ -119,6 +119,7 @@ func TestSplitJobCodecValidatesRestartPhases(t *testing.T) { } for _, phase := range []SplitJobPhase{ + SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index 33130023d..6b1105808 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -15,6 +15,8 @@ (def ^:private pid-file "/var/run/elastickv.pid") (def ^:private server-bin (str bin-dir "/elastickv")) (def ^:private raftadmin-bin (str bin-dir "/raftadmin")) +(def ^:private split-bin (str bin-dir "/elastickv-split")) +(def ^:private list-routes-bin (str bin-dir "/elastickv-list-routes")) (def ^:private build-dir ;; local (control node) directory for built binaries @@ -37,7 +39,9 @@ "GOPATH" "/home/vagrant/go" "GOCACHE" "/home/vagrant/.cache/go-build"})] (doseq [[out-cmd args] [["elastickv" ["go" "build" "-o" (str build-dir "/elastickv") "./cmd/server"]] - ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]]]] + ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]] + ["elastickv-split" ["go" "build" "-o" (str build-dir "/elastickv-split") "./cmd/elastickv-split"]] + ["elastickv-list-routes" ["go" "build" "-o" (str build-dir "/elastickv-list-routes") "./cmd/elastickv-list-routes"]]]] (let [{:keys [exit err]} (apply sh/sh (concat args [:env env :dir root]))] (when-not (zero? exit) (throw (ex-info (str "failed to build " out-cmd) {:err err}))))))) @@ -57,7 +61,7 @@ (c/on node (c/su (c/exec :mkdir :-p bin-dir) - (doseq [bin ["elastickv" "raftadmin"]] + (doseq [bin ["elastickv" "raftadmin" "elastickv-split" "elastickv-list-routes"]] (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) @@ -97,7 +101,7 @@ (clojure.string/join ",")))) (defn- start-node! - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine]}] + [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine migration-enabled]}] (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -134,7 +138,10 @@ (apply cu/start-daemon! {:chdir bin-dir :logfile log-file :pidfile pid-file - :background? true} + :background? true + :env (when migration-enabled + {:ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE "true" + :ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE "true"})} args))))) (defn- stop-node! diff --git a/jepsen/src/elastickv/jepsen_test.clj b/jepsen/src/elastickv/jepsen_test.clj index 9de017df0..39d2c3c5f 100644 --- a/jepsen/src/elastickv/jepsen_test.clj +++ b/jepsen/src/elastickv/jepsen_test.clj @@ -6,6 +6,7 @@ [elastickv.dynamodb-types-workload :as dynamodb-types-workload] [elastickv.s3-workload :as s3-workload] [elastickv.sqs-htfifo-workload :as sqs-htfifo-workload] + [elastickv.split-workload :as split-workload] [jepsen.cli :as cli])) (defn elastickv-test @@ -28,6 +29,10 @@ ([] (elastickv-zset-safety-test {})) ([opts] (zset-safety-workload/elastickv-zset-safety-test opts))) +(defn elastickv-split-test + ([] (elastickv-split-test {})) + ([opts] (split-workload/elastickv-split-test opts))) + (def ^:private test-fns "Map of user-facing test names to their constructor fns. The first positional CLI arg selects which workload runs; if absent or unknown, @@ -36,7 +41,8 @@ {"elastickv-test" elastickv-test "elastickv-zset-safety-test" elastickv-zset-safety-test "elastickv-dynamodb-test" elastickv-dynamodb-test - "elastickv-s3-test" elastickv-s3-test}) + "elastickv-s3-test" elastickv-s3-test + "elastickv-split-test" elastickv-split-test}) (defn elastickv-sqs-htfifo-test "HT-FIFO Jepsen test (PR 7b). Run via the workload's own -main: diff --git a/jepsen/src/elastickv/split_workload.clj b/jepsen/src/elastickv/split_workload.clj new file mode 100644 index 000000000..b52ba9ac5 --- /dev/null +++ b/jepsen/src/elastickv/split_workload.clj @@ -0,0 +1,252 @@ +(ns elastickv.split-workload + "Deterministic M2 cross-group split workload over two Redis registers." + (:gen-class) + (:require [clojure.java.shell :as shell] + [clojure.tools.logging :refer [info warn]] + [elastickv.cli :as cli] + [elastickv.composed1-nemesis :as routes] + [elastickv.db :as ekdb] + [jepsen [checker :as checker] + [client :as client] + [generator :as gen] + [independent :as independent] + [net :as net]] + [jepsen.checker.timeline :as timeline] + [jepsen.control :as control] + [jepsen.db :as jdb] + [jepsen.nemesis :as nemesis] + [jepsen.nemesis.combined :as combined] + [jepsen.os :as os] + [jepsen.os.debian :as debian] + [knossos.model :as model] + [taoensso.carmine :as car])) + +(def default-nodes ["n1" "n2" "n3" "n4" "n5"]) +(def ^:private split-key "m2-split") +(def ^:private register-keys ["m2-left" "m2-right"]) + +(defn- helper-path [name] + (str (System/getProperty "user.dir") "/target/elastickv-jepsen/" name)) + +(defrecord SplitRegisterClient [node->port conn-spec] + client/Client + (open! [this test node] + (assoc this :conn-spec + {:pool {} + :spec {:host (or (:redis-host test) (name node)) + :port (get node->port node 6379) + :timeout-ms 10000}})) + (close! [this _test] (assoc this :conn-spec nil)) + (setup! [this _test] + (doseq [k register-keys] + (car/wcar conn-spec (car/del k))) + this) + (teardown! [this _test] this) + (invoke! [_this _test op] + (try + (let [[k value] (:value op) + redis-key (nth register-keys k)] + (case (:f op) + :write (do (car/wcar conn-spec (car/set redis-key (str value))) + (assoc op :type :ok)) + :read (let [raw (car/wcar conn-spec (car/get redis-key)) + value (when raw (Long/parseLong (str raw)))] + (assoc op :type :ok :value (independent/tuple k value))))) + (catch java.net.ConnectException _ + (assoc op :type :info :error :connection-refused)) + (catch java.net.SocketTimeoutException _ + (assoc op :type :info :error :socket-timeout)) + (catch Throwable t + (assoc op :type :info :error (or (.getMessage t) (str t))))))) + +(defn split-register-workload [opts] + (let [client (->SplitRegisterClient + (or (:node->port opts) (zipmap default-nodes (repeat 6379))) + nil) + max-writes (or (:max-writes-per-key opts) 100)] + {:client client + :generator (independent/concurrent-generator + 2 + (range (count register-keys)) + (fn [_] + (gen/mix [(map (fn [v] {:f :write :value v}) (range max-writes)) + (repeat max-writes {:f :read})]))) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable {:model (model/register) + :algorithm :competition}) + :timeline (timeline/html)}))})) + +(defn- grpc-address [opts test] + (or (:grpc-host-port opts) + (:grpc-host-port test) + (let [node (name (first (:nodes test))) + groups (:raft-groups test) + port (if (seq groups) + (get groups (first (sort (keys groups)))) + (or (:grpc-port test) 50051))] + (str node ":" port)))) + +(defn- run-helper! [bin & args] + (let [result (apply shell/sh bin args)] + (when-not (zero? (:exit result)) + (throw (ex-info (str bin " failed") result))) + (:out result))) + +(defn- current-plan [opts test] + (let [address (grpc-address opts test) + snapshot (routes/parse-routes-json + (run-helper! (or (:list-routes-bin opts) (helper-path "elastickv-list-routes")) + "--address" address)) + source (routes/route-containing-key (:routes snapshot) split-key) + groups (sort (keys (:raft-groups test))) + target (or (:target-group-id opts) + (first (remove #{(:raft-group-id source)} groups)))] + (when-not (and source target) + (throw (ex-info "split workload requires an active source route and a different target group" + {:source source :groups groups}))) + {:address address + :catalog-version (:catalog-version snapshot) + :route-id (:route-id source) + :target-group-id target})) + +(defn- start-split! [opts test] + (let [{:keys [address catalog-version route-id target-group-id] :as plan} + (current-plan opts test) + output (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" address + "--route-id" (str route-id) + "--split-key" split-key + "--expected-version" (str catalog-version) + "--target-group-id" (str target-group-id)) + job-id (some->> (re-find #"job_id:\s*(\d+)" output) second Long/parseLong)] + (when-not job-id + (throw (ex-info "split helper did not return job_id" {:output output :plan plan}))) + (assoc plan :job-id job-id :output output))) + +(defn- job-phase [opts address job-id] + (let [output (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" address "--get-job-id" (str job-id))] + (some->> (re-find #"phase:\s*(\S+)" output) second))) + +(defn- await-phase! [opts address job-id wanted timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (let [phase (job-phase opts address job-id)] + (cond + (contains? wanted phase) phase + (> (System/currentTimeMillis) deadline) + (throw (ex-info "timed out waiting for split job phase" + {:job-id job-id :phase phase :wanted wanted})) + :else (do (Thread/sleep 100) (recur))))))) + +(defn split-nemesis [opts] + (let [active (atom nil)] + (reify + nemesis/Reflection + (fs [_] #{:start-cross-group-split :verify-cross-group-split}) + nemesis/Nemesis + (setup! [this _test] this) + (invoke! [_ test op] + (try + (case (:f op) + :start-cross-group-split + (let [plan (start-split! opts test)] + (reset! active plan) + (if (:abandon-at-fence opts) + (do + (await-phase! opts (:address plan) (:job-id plan) + #{"SPLIT_JOB_PHASE_FENCE"} 60000) + (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" (:address plan) + "--abandon-job-id" (str (:job-id plan)))) + (info "started M2 split job" (:job-id plan))) + (assoc op :value plan)) + + :verify-cross-group-split + (let [{:keys [address job-id] :as plan} @active + wanted (if (:abandon-at-fence opts) + #{"SPLIT_JOB_PHASE_ABANDONED"} + #{"SPLIT_JOB_PHASE_DONE"}) + phase (await-phase! opts address job-id wanted 120000)] + (assoc op :value (assoc plan :phase phase))) + + op) + (catch Throwable t + (warn t "M2 split nemesis failed") + (assoc op :type :fail :error (or (.getMessage t) (str t)))))) + (teardown! [this _test] this)))) + +(defn split-package [opts] + {:generator (gen/phases + (gen/sleep (or (:split-at-seconds opts) 5)) + (gen/once {:type :info :f :start-cross-group-split}) + (gen/sleep (or (:split-verify-after-seconds opts) 20)) + (gen/once {:type :info :f :verify-cross-group-split})) + :final-generator nil + :nemesis (split-nemesis opts) + :perf #{{:name "cross-group-split" + :fs #{:start-cross-group-split :verify-cross-group-split} + :start #{:start-cross-group-split} + :stop #{:verify-cross-group-split} + :color "#4A90E2"}}}) + +(defn elastickv-split-test + ([] (elastickv-split-test {})) + ([opts] + (let [nodes (or (:nodes opts) default-nodes) + ports (or (:redis-ports opts) (repeat (count nodes) (or (:redis-port opts) 6379))) + node->port (or (:node->port opts) (cli/ports->node-map ports nodes)) + local? (:local opts) + db (if local? jdb/noop + (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) + :redis-port node->port + :raft-groups (:raft-groups opts) + :shard-ranges (:shard-ranges opts) + :migration-enabled true})) + fault-package (when-not local? + (combined/nemesis-package + {:db db + :faults (cli/normalize-faults (or (:faults opts) [:partition :kill])) + :interval (or (:fault-interval opts) 10)})) + package (combined/compose-packages + (cond-> [(split-package opts)] fault-package (conj fault-package))) + workload (split-register-workload (assoc opts :node->port node->port))] + (merge workload + {:name (or (:name opts) "elastickv-m2-cross-group-split") + :nodes nodes + :db db + :raft-groups (:raft-groups opts) + :grpc-port (:grpc-port opts) + :grpc-host-port (:grpc-host-port opts) + :redis-host (:redis-host opts) + :os (if local? os/noop debian/os) + :net (if local? net/noop net/iptables) + :ssh (merge {:username "vagrant" + :private-key-path "/home/vagrant/.ssh/id_rsa" + :strict-host-key-checking false} + (when local? {:dummy true}) (:ssh opts)) + :remote control/ssh + :nemesis (:nemesis package) + :final-generator nil + :concurrency (or (:concurrency opts) 6) + :generator (->> (:generator workload) + (gen/nemesis (:generator package)) + (gen/stagger (/ (double (or (:rate opts) 10)))) + (gen/time-limit (or (:time-limit opts) 60)))})))) + +(def split-cli-opts + [[nil "--redis-port PORT" "Redis port." :default 6379 :parse-fn #(Integer/parseInt %)] + [nil "--target-group-id ID" "Target Raft group." :parse-fn #(Long/parseLong %)] + [nil "--grpc-host-port HOST:PORT" "Distribution gRPC address."] + [nil "--split-bin PATH" "Path to elastickv-split."] + [nil "--list-routes-bin PATH" "Path to elastickv-list-routes."] + [nil "--split-at-seconds SECONDS" "Workload time to start migration." :default 5 :parse-fn #(Integer/parseInt %)] + [nil "--split-verify-after-seconds SECONDS" "Delay before verifying terminal state." :default 20 :parse-fn #(Integer/parseInt %)] + [nil "--abandon-at-fence" "Abandon after observing FENCE." :default false]]) + +(defn -main [& args] + (cli/run-workload! args + (into cli/common-cli-opts split-cli-opts) + #(cli/parse-common-opts % nil) + elastickv-split-test)) diff --git a/jepsen/test/elastickv/split_workload_test.clj b/jepsen/test/elastickv/split_workload_test.clj new file mode 100644 index 000000000..d3ebf82b7 --- /dev/null +++ b/jepsen/test/elastickv/split_workload_test.clj @@ -0,0 +1,40 @@ +(ns elastickv.split-workload-test + (:require [clojure.java.shell :as shell] + [clojure.test :refer :all] + [elastickv.split-workload :as split] + [jepsen.nemesis :as nemesis])) + +(defn- routes-json [] + (str "{\"catalog_version\":7,\"routes\":[" + "{\"route_id\":100,\"raft_group_id\":1,\"start\":\"\",\"end\":\"\",\"state\":\"ROUTE_STATE_ACTIVE\"}" + "]}")) + +(deftest split-package-is-deterministic + (let [package (split/split-package {:split-at-seconds 1 + :split-verify-after-seconds 2})] + (is (some? (:generator package))) + (is (some? (:nemesis package))) + (is (= #{:start-cross-group-split :verify-cross-group-split} + (nemesis/fs (:nemesis package)))))) + +(deftest split-nemesis-starts-cross-group-job + (let [calls (atom [])] + (with-redefs [shell/sh (fn [& args] + (swap! calls conj args) + (if (.contains (first args) "list-routes") + {:exit 0 :out (routes-json) :err ""} + {:exit 0 :out "catalog_version: 8\njob_id: 44\n" :err ""}))] + (let [result (nemesis/invoke! + (split/split-nemesis + {:list-routes-bin "elastickv-list-routes" + :split-bin "elastickv-split" + :target-group-id 2 + :grpc-host-port "n1:50051"}) + {:nodes ["n1"] :raft-groups {1 50051, 2 50052}} + {:type :info :f :start-cross-group-split})] + (is (= 44 (get-in result [:value :job-id]))) + (is (= [["elastickv-list-routes" "--address" "n1:50051"] + ["elastickv-split" "--address" "n1:50051" + "--route-id" "100" "--split-key" "m2-split" + "--expected-version" "7" "--target-group-id" "2"]] + @calls)))))) diff --git a/kv/fsm.go b/kv/fsm.go index 3737899bb..29fcc71b8 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -392,6 +392,8 @@ func (f *kvFSM) applyReservedOpcode(ctx context.Context, data []byte) (any, bool return f.applyMigrationPromote(ctx, data[1:]), true case data[0] == raftEncodeTargetReadiness: return f.applyTargetStagedReadiness(ctx, data[1:]), true + case data[0] == raftEncodeMigrationCleanup: + return f.applyMigrationCleanup(ctx, data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true default: @@ -435,6 +437,9 @@ const ( // guard. It must be replicated through the target Raft group before the // migration controller can treat a target as fail-closed for cutover. raftEncodeTargetReadiness byte = 0x0c + // raftEncodeMigrationCleanup carries one bounded source/target physical + // cleanup chunk or a per-job proof-record cleanup. + raftEncodeMigrationCleanup byte = 0x0d ) func decodeRaftRequests(data []byte) ([]*pb.Request, error) { diff --git a/kv/fsm_migration_cleanup.go b/kv/fsm_migration_cleanup.go new file mode 100644 index 000000000..9444fb39d --- /dev/null +++ b/kv/fsm_migration_cleanup.go @@ -0,0 +1,93 @@ +package kv + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// MarshalMigrationCleanupCommand encodes a bounded cleanup operation for Raft. +func MarshalMigrationCleanupCommand(req *pb.CleanupMigrationRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled migration cleanup request too large") + } + return prependByte(raftEncodeMigrationCleanup, b), nil +} + +func (f *kvFSM) applyMigrationCleanup(ctx context.Context, data []byte) any { + req := &pb.CleanupMigrationRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + cleaner, ok := f.store.(store.MigrationCleaner) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA { + return errors.WithStack(cleaner.ClearMigrationState(ctx, req.GetJobId(), f.pendingApplyIdx)) + } + result, err := cleaner.CleanupVersions(ctx, migrationCleanupOptionsFromProto(req, f.pendingApplyIdx)) + if err != nil { + return errors.WithStack(err) + } + return result +} + +func migrationCleanupOptionsFromProto(req *pb.CleanupMigrationRequest, appliedIndex uint64) store.CleanupVersionsOptions { + maxVersions := int(req.GetMaxVersions()) + if maxVersions <= 0 { + maxVersions = defaultMigrationPromoteMaxVersions + } + maxBytes := req.GetMaxBytes() + if maxBytes == 0 { + maxBytes = defaultMigrationPromoteMaxBytes + } + maxScannedBytes := req.GetMaxScannedBytes() + if maxScannedBytes == 0 { + maxScannedBytes = defaultMigrationPromoteMaxScannedBytes + } + bracket := distribution.MigrationBracket{ + Family: req.GetKeyFamily(), + Start: bytes.Clone(req.GetRangeStart()), + End: bytes.Clone(req.GetRangeEnd()), + ExcludePrefixes: cloneMigrationByteSlices(req.GetExcludePrefixes()), + ExcludeKnownInternal: req.GetExcludeKnownInternal(), + RequiresRouteKeyCheck: req.GetRequiresRouteKeyCheck(), + RequiresDecodedS3: req.GetRequiresDecodedS3(), + } + return store.CleanupVersionsOptions{ + JobID: req.GetJobId(), + AppliedIndex: appliedIndex, + StartKey: bytes.Clone(req.GetRangeStart()), + EndKey: bytes.Clone(req.GetRangeEnd()), + Cursor: bytes.Clone(req.GetCursor()), + MaxCommitTS: req.GetMaxCommitTs(), + MaxVersions: maxVersions, + MaxBytes: maxBytes, + MaxScannedBytes: maxScannedBytes, + KeyFamily: req.GetKeyFamily(), + AcceptVersion: func(key, value []byte) bool { + return bracket.ContainsRoutedVersion(key, value, req.GetRouteStart(), req.GetRouteEnd(), routeKey) + }, + } +} + +func cloneMigrationByteSlices(in [][]byte) [][]byte { + out := make([][]byte, len(in)) + for i := range in { + out[i] = bytes.Clone(in[i]) + } + return out +} diff --git a/kv/fsm_migration_cleanup_test.go b/kv/fsm_migration_cleanup_test.go new file mode 100644 index 000000000..70aa9ee74 --- /dev/null +++ b/kv/fsm_migration_cleanup_test.go @@ -0,0 +1,42 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestApplyMigrationCleanupCommandDeletesBoundedVersions(t *testing.T) { + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("old"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("new"), 20, 0)) + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + + cmd, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{ + JobId: 1, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: []byte("a"), + RangeEnd: []byte("b"), + MaxCommitTs: 10, + MaxVersions: 16, + MaxBytes: 1 << 20, + MaxScannedBytes: 1 << 20, + KeyFamily: 1, + }) + require.NoError(t, err) + result, ok := fsm.applyMigrationCleanup(ctx, cmd[1:]).(store.CleanupVersionsResult) + require.True(t, ok) + require.True(t, result.Done) + require.Equal(t, uint64(1), result.DeletedRows) + + value, err := st.GetAt(ctx, []byte("a"), 20) + require.NoError(t, err) + require.Equal(t, []byte("new"), value) + _, err = st.GetAt(ctx, []byte("a"), 10) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} diff --git a/main.go b/main.go index 46a5496c9..2631a53db 100644 --- a/main.go +++ b/main.go @@ -1677,6 +1677,10 @@ func prepareDistributionRuntimeServer( splitMigrationGroupLeaderResolver(in.shardGroups), splitPromotionConnCache, )), + adapter.WithSplitMigrationVoterFactory(splitMigrationVoterFactory( + in.shardGroups, + splitPromotionConnCache, + )), adapter.WithSplitJobRunnerReadinessGate(splitMigrationLocalReadinessGate), adapter.WithSplitJobRunnerReady(), } @@ -1804,6 +1808,49 @@ func splitMigrationClientFactory(resolve splitMigrationLeaderResolver, connCache } } +func splitMigrationVoterFactory(shardGroups map[uint64]*kv.ShardGroup, connCache *kv.GRPCConnCache) adapter.SplitMigrationVoterFactory { + return func(ctx context.Context, groupID uint64) ([]adapter.SplitMigrationVoter, error) { + sg := shardGroups[groupID] + if sg == nil || sg.Engine == nil { + return nil, errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + if connCache == nil { + return nil, errors.New("split migration voter gRPC connection cache is not configured") + } + cfg, err := sg.Engine.Configuration(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + voters := make([]adapter.SplitMigrationVoter, 0, len(cfg.Servers)) + for _, member := range cfg.Servers { + if member.Suffrage != "voter" { + continue + } + voter, err := splitMigrationVoter(groupID, member, connCache) + if err != nil { + return nil, err + } + voters = append(voters, voter) + } + if len(voters) == 0 { + return nil, errors.Errorf("split migration group %d has no voters", groupID) + } + return voters, nil + } +} + +func splitMigrationVoter(groupID uint64, member raftengine.Server, connCache *kv.GRPCConnCache) (adapter.SplitMigrationVoter, error) { + addr := strings.TrimSpace(member.Address) + if member.ID == "" || addr == "" { + return adapter.SplitMigrationVoter{}, errors.Errorf("split migration group %d has voter with missing id/address", groupID) + } + conn, err := connCache.ConnFor(addr) + if err != nil { + return adapter.SplitMigrationVoter{}, errors.WithStack(err) + } + return adapter.SplitMigrationVoter{ID: member.ID, Address: addr, Client: pb.NewInternalClient(conn)}, nil +} + func splitMigrationLocalReadinessGate(context.Context) error { if !adapter.MigrationImportOpcodeEnabledFromEnv() { return errors.Errorf("%s is disabled", adapter.MigrationImportOpcodeEnv) @@ -2483,6 +2530,7 @@ func startRaftServers( confChangeInterceptor internalraftadmin.MembershipChangeInterceptor, encWiring encryptionWriteWiring, sqsPartitionResolver kv.PartitionResolver, + readTracker *kv.ActiveTimestampTracker, ) error { forwardLogger := slog.Default().With(slog.String("component", "admin")) // extraOptsCap reserves slots for the unary + stream admin interceptor @@ -2521,6 +2569,7 @@ func startRaftServers( adapter.WithInternalStore(rt.store), adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), adapter.WithInternalRouteEngine(routeEngine), + adapter.WithInternalActiveTimestampTracker(readTracker), adapter.WithInternalMigrationExportRouting(rt.spec.id, sqsPartitionResolver), )..., )) @@ -2990,6 +3039,7 @@ func (r *runtimeServerRunner) startRaftTransport() error { r.encryptionConfChangeInterceptor, r.encWiring, sqsPartitionResolver, + r.readTracker, ); err != nil { return r.startupFailure(err) } diff --git a/proto/distribution.pb.go b/proto/distribution.pb.go index a4a586aa9..6aea9f134 100644 --- a/proto/distribution.pb.go +++ b/proto/distribution.pb.go @@ -679,6 +679,7 @@ type SplitJob struct { StartedAtMs int64 `protobuf:"varint,31,opt,name=started_at_ms,json=startedAtMs,proto3" json:"started_at_ms,omitempty"` UpdatedAtMs int64 `protobuf:"varint,32,opt,name=updated_at_ms,json=updatedAtMs,proto3" json:"updated_at_ms,omitempty"` TerminalAtMs int64 `protobuf:"varint,33,opt,name=terminal_at_ms,json=terminalAtMs,proto3" json:"terminal_at_ms,omitempty"` + CapabilityRegressed bool `protobuf:"varint,34,opt,name=capability_regressed,json=capabilityRegressed,proto3" json:"capability_regressed,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -944,6 +945,13 @@ func (x *SplitJob) GetTerminalAtMs() int64 { return 0 } +func (x *SplitJob) GetCapabilityRegressed() bool { + if x != nil { + return x.CapabilityRegressed + } + return false +} + type ListRoutesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1985,7 +1993,7 @@ const file_distribution_proto_rawDesc = "" + "\x04done\x18\x05 \x01(\bR\x04done\x12#\n" + "\rscanned_bytes\x18\x06 \x01(\x04R\fscannedBytes\x12#\n" + "\raccepted_rows\x18\a \x01(\x04R\facceptedRows\x12/\n" + - "\x14last_acked_batch_seq\x18\b \x01(\x04R\x11lastAckedBatchSeq\"\xe9\f\n" + + "\x14last_acked_batch_seq\x18\b \x01(\x04R\x11lastAckedBatchSeq\"\x9c\r\n" + "\bSplitJob\x12\x15\n" + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12&\n" + "\x0fsource_route_id\x18\x02 \x01(\x04R\rsourceRouteId\x12\x1b\n" + @@ -2024,7 +2032,8 @@ const file_distribution_proto_rawDesc = "" + "last_error\x18\x1e \x01(\tR\tlastError\x12\"\n" + "\rstarted_at_ms\x18\x1f \x01(\x03R\vstartedAtMs\x12\"\n" + "\rupdated_at_ms\x18 \x01(\x03R\vupdatedAtMs\x12$\n" + - "\x0eterminal_at_ms\x18! \x01(\x03R\fterminalAtMs\"\x13\n" + + "\x0eterminal_at_ms\x18! \x01(\x03R\fterminalAtMs\x121\n" + + "\x14capability_regressed\x18\" \x01(\bR\x13capabilityRegressed\"\x13\n" + "\x11ListRoutesRequest\"g\n" + "\x12ListRoutesResponse\x12'\n" + "\x0fcatalog_version\x18\x01 \x01(\x04R\x0ecatalogVersion\x12(\n" + diff --git a/proto/distribution.proto b/proto/distribution.proto index 4f65163b0..9cc9e9743 100644 --- a/proto/distribution.proto +++ b/proto/distribution.proto @@ -127,6 +127,7 @@ message SplitJob { int64 started_at_ms = 31; int64 updated_at_ms = 32; int64 terminal_at_ms = 33; + bool capability_regressed = 34; } message ListRoutesRequest {} diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 002c29242..4629b6349 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -128,6 +128,113 @@ func (Phase) EnumDescriptor() ([]byte, []int) { return file_internal_proto_rawDescGZIP(), []int{1} } +type MigrationCleanupMode int32 + +const ( + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED MigrationCleanupMode = 0 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS MigrationCleanupMode = 1 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA MigrationCleanupMode = 2 +) + +// Enum value maps for MigrationCleanupMode. +var ( + MigrationCleanupMode_name = map[int32]string{ + 0: "MIGRATION_CLEANUP_MODE_UNSPECIFIED", + 1: "MIGRATION_CLEANUP_MODE_VERSIONS", + 2: "MIGRATION_CLEANUP_MODE_METADATA", + } + MigrationCleanupMode_value = map[string]int32{ + "MIGRATION_CLEANUP_MODE_UNSPECIFIED": 0, + "MIGRATION_CLEANUP_MODE_VERSIONS": 1, + "MIGRATION_CLEANUP_MODE_METADATA": 2, + } +) + +func (x MigrationCleanupMode) Enum() *MigrationCleanupMode { + p := new(MigrationCleanupMode) + *p = x + return p +} + +func (x MigrationCleanupMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationCleanupMode) Descriptor() protoreflect.EnumDescriptor { + return file_internal_proto_enumTypes[2].Descriptor() +} + +func (MigrationCleanupMode) Type() protoreflect.EnumType { + return &file_internal_proto_enumTypes[2] +} + +func (x MigrationCleanupMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationCleanupMode.Descriptor instead. +func (MigrationCleanupMode) EnumDescriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{2} +} + +type MigrationStateProbeKind int32 + +const ( + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED MigrationStateProbeKind = 0 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED MigrationStateProbeKind = 1 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED MigrationStateProbeKind = 2 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED MigrationStateProbeKind = 3 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED MigrationStateProbeKind = 4 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED MigrationStateProbeKind = 5 +) + +// Enum value maps for MigrationStateProbeKind. +var ( + MigrationStateProbeKind_name = map[int32]string{ + 0: "MIGRATION_STATE_PROBE_KIND_UNSPECIFIED", + 1: "MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED", + 2: "MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED", + 3: "MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED", + 4: "MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED", + 5: "MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED", + } + MigrationStateProbeKind_value = map[string]int32{ + "MIGRATION_STATE_PROBE_KIND_UNSPECIFIED": 0, + "MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED": 1, + "MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED": 2, + "MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED": 3, + "MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED": 4, + "MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED": 5, + } +) + +func (x MigrationStateProbeKind) Enum() *MigrationStateProbeKind { + p := new(MigrationStateProbeKind) + *p = x + return p +} + +func (x MigrationStateProbeKind) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationStateProbeKind) Descriptor() protoreflect.EnumDescriptor { + return file_internal_proto_enumTypes[3].Descriptor() +} + +func (MigrationStateProbeKind) Type() protoreflect.EnumType { + return &file_internal_proto_enumTypes[3] +} + +func (x MigrationStateProbeKind) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationStateProbeKind.Descriptor instead. +func (MigrationStateProbeKind) EnumDescriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{3} +} + type Mutation struct { state protoimpl.MessageState `protogen:"open.v1"` Op Op `protobuf:"varint,1,opt,name=op,proto3,enum=Op" json:"op,omitempty"` @@ -1370,6 +1477,534 @@ func (x *ProbeMigrationLocksResponse) GetPendingCount() uint32 { return 0 } +type CleanupMigrationRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Mode MigrationCleanupMode `protobuf:"varint,2,opt,name=mode,proto3,enum=MigrationCleanupMode" json:"mode,omitempty"` + RangeStart []byte `protobuf:"bytes,3,opt,name=range_start,json=rangeStart,proto3" json:"range_start,omitempty"` + RangeEnd []byte `protobuf:"bytes,4,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` + Cursor []byte `protobuf:"bytes,5,opt,name=cursor,proto3" json:"cursor,omitempty"` + MaxCommitTs uint64 `protobuf:"varint,6,opt,name=max_commit_ts,json=maxCommitTs,proto3" json:"max_commit_ts,omitempty"` + MaxVersions uint32 `protobuf:"varint,7,opt,name=max_versions,json=maxVersions,proto3" json:"max_versions,omitempty"` + MaxBytes uint64 `protobuf:"varint,8,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + MaxScannedBytes uint64 `protobuf:"varint,9,opt,name=max_scanned_bytes,json=maxScannedBytes,proto3" json:"max_scanned_bytes,omitempty"` + KeyFamily uint32 `protobuf:"varint,10,opt,name=key_family,json=keyFamily,proto3" json:"key_family,omitempty"` + RouteStart []byte `protobuf:"bytes,11,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,12,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExcludeKnownInternal bool `protobuf:"varint,13,opt,name=exclude_known_internal,json=excludeKnownInternal,proto3" json:"exclude_known_internal,omitempty"` + ExcludePrefixes [][]byte `protobuf:"bytes,14,rep,name=exclude_prefixes,json=excludePrefixes,proto3" json:"exclude_prefixes,omitempty"` + RequiresRouteKeyCheck bool `protobuf:"varint,15,opt,name=requires_route_key_check,json=requiresRouteKeyCheck,proto3" json:"requires_route_key_check,omitempty"` + RequiresDecodedS3 bool `protobuf:"varint,16,opt,name=requires_decoded_s3,json=requiresDecodedS3,proto3" json:"requires_decoded_s3,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CleanupMigrationRequest) Reset() { + *x = CleanupMigrationRequest{} + mi := &file_internal_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CleanupMigrationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupMigrationRequest) ProtoMessage() {} + +func (x *CleanupMigrationRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupMigrationRequest.ProtoReflect.Descriptor instead. +func (*CleanupMigrationRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{18} +} + +func (x *CleanupMigrationRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMode() MigrationCleanupMode { + if x != nil { + return x.Mode + } + return MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED +} + +func (x *CleanupMigrationRequest) GetRangeStart() []byte { + if x != nil { + return x.RangeStart + } + return nil +} + +func (x *CleanupMigrationRequest) GetRangeEnd() []byte { + if x != nil { + return x.RangeEnd + } + return nil +} + +func (x *CleanupMigrationRequest) GetCursor() []byte { + if x != nil { + return x.Cursor + } + return nil +} + +func (x *CleanupMigrationRequest) GetMaxCommitTs() uint64 { + if x != nil { + return x.MaxCommitTs + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxVersions() uint32 { + if x != nil { + return x.MaxVersions + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxBytes() uint64 { + if x != nil { + return x.MaxBytes + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxScannedBytes() uint64 { + if x != nil { + return x.MaxScannedBytes + } + return 0 +} + +func (x *CleanupMigrationRequest) GetKeyFamily() uint32 { + if x != nil { + return x.KeyFamily + } + return 0 +} + +func (x *CleanupMigrationRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *CleanupMigrationRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *CleanupMigrationRequest) GetExcludeKnownInternal() bool { + if x != nil { + return x.ExcludeKnownInternal + } + return false +} + +func (x *CleanupMigrationRequest) GetExcludePrefixes() [][]byte { + if x != nil { + return x.ExcludePrefixes + } + return nil +} + +func (x *CleanupMigrationRequest) GetRequiresRouteKeyCheck() bool { + if x != nil { + return x.RequiresRouteKeyCheck + } + return false +} + +func (x *CleanupMigrationRequest) GetRequiresDecodedS3() bool { + if x != nil { + return x.RequiresDecodedS3 + } + return false +} + +type CleanupMigrationResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + NextCursor []byte `protobuf:"bytes,1,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` + Done bool `protobuf:"varint,2,opt,name=done,proto3" json:"done,omitempty"` + DeletedRows uint64 `protobuf:"varint,3,opt,name=deleted_rows,json=deletedRows,proto3" json:"deleted_rows,omitempty"` + DeletedBytes uint64 `protobuf:"varint,4,opt,name=deleted_bytes,json=deletedBytes,proto3" json:"deleted_bytes,omitempty"` + ScannedBytes uint64 `protobuf:"varint,5,opt,name=scanned_bytes,json=scannedBytes,proto3" json:"scanned_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CleanupMigrationResponse) Reset() { + *x = CleanupMigrationResponse{} + mi := &file_internal_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CleanupMigrationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupMigrationResponse) ProtoMessage() {} + +func (x *CleanupMigrationResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupMigrationResponse.ProtoReflect.Descriptor instead. +func (*CleanupMigrationResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{19} +} + +func (x *CleanupMigrationResponse) GetNextCursor() []byte { + if x != nil { + return x.NextCursor + } + return nil +} + +func (x *CleanupMigrationResponse) GetDone() bool { + if x != nil { + return x.Done + } + return false +} + +func (x *CleanupMigrationResponse) GetDeletedRows() uint64 { + if x != nil { + return x.DeletedRows + } + return 0 +} + +func (x *CleanupMigrationResponse) GetDeletedBytes() uint64 { + if x != nil { + return x.DeletedBytes + } + return 0 +} + +func (x *CleanupMigrationResponse) GetScannedBytes() uint64 { + if x != nil { + return x.ScannedBytes + } + return 0 +} + +type ProbeMigrationStateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Kind MigrationStateProbeKind `protobuf:"varint,2,opt,name=kind,proto3,enum=MigrationStateProbeKind" json:"kind,omitempty"` + RouteStart []byte `protobuf:"bytes,3,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,4,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExpectedCatalogVersion uint64 `protobuf:"varint,5,opt,name=expected_catalog_version,json=expectedCatalogVersion,proto3" json:"expected_catalog_version,omitempty"` + ExpectedGroupId uint64 `protobuf:"varint,6,opt,name=expected_group_id,json=expectedGroupId,proto3" json:"expected_group_id,omitempty"` + MigrationJobId uint64 `protobuf:"varint,7,opt,name=migration_job_id,json=migrationJobId,proto3" json:"migration_job_id,omitempty"` + MinWriteTsExclusive uint64 `protobuf:"varint,8,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` + SourceWriteFence bool `protobuf:"varint,9,opt,name=source_write_fence,json=sourceWriteFence,proto3" json:"source_write_fence,omitempty"` + SourceReadFence bool `protobuf:"varint,10,opt,name=source_read_fence,json=sourceReadFence,proto3" json:"source_read_fence,omitempty"` + TrackWrites bool `protobuf:"varint,11,opt,name=track_writes,json=trackWrites,proto3" json:"track_writes,omitempty"` + RetentionPinTs uint64 `protobuf:"varint,12,opt,name=retention_pin_ts,json=retentionPinTs,proto3" json:"retention_pin_ts,omitempty"` + ReadDrainNotBeforeMs int64 `protobuf:"varint,13,opt,name=read_drain_not_before_ms,json=readDrainNotBeforeMs,proto3" json:"read_drain_not_before_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationStateRequest) Reset() { + *x = ProbeMigrationStateRequest{} + mi := &file_internal_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationStateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationStateRequest) ProtoMessage() {} + +func (x *ProbeMigrationStateRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationStateRequest.ProtoReflect.Descriptor instead. +func (*ProbeMigrationStateRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{20} +} + +func (x *ProbeMigrationStateRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetKind() MigrationStateProbeKind { + if x != nil { + return x.Kind + } + return MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED +} + +func (x *ProbeMigrationStateRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *ProbeMigrationStateRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *ProbeMigrationStateRequest) GetExpectedCatalogVersion() uint64 { + if x != nil { + return x.ExpectedCatalogVersion + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetExpectedGroupId() uint64 { + if x != nil { + return x.ExpectedGroupId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetMigrationJobId() uint64 { + if x != nil { + return x.MigrationJobId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetMinWriteTsExclusive() uint64 { + if x != nil { + return x.MinWriteTsExclusive + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetSourceWriteFence() bool { + if x != nil { + return x.SourceWriteFence + } + return false +} + +func (x *ProbeMigrationStateRequest) GetSourceReadFence() bool { + if x != nil { + return x.SourceReadFence + } + return false +} + +func (x *ProbeMigrationStateRequest) GetTrackWrites() bool { + if x != nil { + return x.TrackWrites + } + return false +} + +func (x *ProbeMigrationStateRequest) GetRetentionPinTs() uint64 { + if x != nil { + return x.RetentionPinTs + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetReadDrainNotBeforeMs() int64 { + if x != nil { + return x.ReadDrainNotBeforeMs + } + return 0 +} + +type ProbeMigrationStateResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` + CatalogVersion uint64 `protobuf:"varint,2,opt,name=catalog_version,json=catalogVersion,proto3" json:"catalog_version,omitempty"` + MinAdmittedTs uint64 `protobuf:"varint,3,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationStateResponse) Reset() { + *x = ProbeMigrationStateResponse{} + mi := &file_internal_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationStateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationStateResponse) ProtoMessage() {} + +func (x *ProbeMigrationStateResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationStateResponse.ProtoReflect.Descriptor instead. +func (*ProbeMigrationStateResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{21} +} + +func (x *ProbeMigrationStateResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *ProbeMigrationStateResponse) GetCatalogVersion() uint64 { + if x != nil { + return x.CatalogVersion + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +type IssueMigrationTimestampRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IssueMigrationTimestampRequest) Reset() { + *x = IssueMigrationTimestampRequest{} + mi := &file_internal_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IssueMigrationTimestampRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IssueMigrationTimestampRequest) ProtoMessage() {} + +func (x *IssueMigrationTimestampRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IssueMigrationTimestampRequest.ProtoReflect.Descriptor instead. +func (*IssueMigrationTimestampRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{22} +} + +type IssueMigrationTimestampResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Timestamp uint64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + LastCommitTs uint64 `protobuf:"varint,2,opt,name=last_commit_ts,json=lastCommitTs,proto3" json:"last_commit_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IssueMigrationTimestampResponse) Reset() { + *x = IssueMigrationTimestampResponse{} + mi := &file_internal_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IssueMigrationTimestampResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IssueMigrationTimestampResponse) ProtoMessage() {} + +func (x *IssueMigrationTimestampResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IssueMigrationTimestampResponse.ProtoReflect.Descriptor instead. +func (*IssueMigrationTimestampResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{23} +} + +func (x *IssueMigrationTimestampResponse) GetTimestamp() uint64 { + if x != nil { + return x.Timestamp + } + return 0 +} + +func (x *IssueMigrationTimestampResponse) GetLastCommitTs() uint64 { + if x != nil { + return x.LastCommitTs + } + return 0 +} + var File_internal_proto protoreflect.FileDescriptor const file_internal_proto_rawDesc = "" + @@ -1475,7 +2110,59 @@ const file_internal_proto_rawDesc = "" + "\troute_end\x18\x02 \x01(\fR\brouteEnd\x12\x14\n" + "\x05limit\x18\x03 \x01(\rR\x05limit\"B\n" + "\x1bProbeMigrationLocksResponse\x12#\n" + - "\rpending_count\x18\x01 \x01(\rR\fpendingCount*&\n" + + "\rpending_count\x18\x01 \x01(\rR\fpendingCount\"\xe8\x04\n" + + "\x17CleanupMigrationRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12)\n" + + "\x04mode\x18\x02 \x01(\x0e2\x15.MigrationCleanupModeR\x04mode\x12\x1f\n" + + "\vrange_start\x18\x03 \x01(\fR\n" + + "rangeStart\x12\x1b\n" + + "\trange_end\x18\x04 \x01(\fR\brangeEnd\x12\x16\n" + + "\x06cursor\x18\x05 \x01(\fR\x06cursor\x12\"\n" + + "\rmax_commit_ts\x18\x06 \x01(\x04R\vmaxCommitTs\x12!\n" + + "\fmax_versions\x18\a \x01(\rR\vmaxVersions\x12\x1b\n" + + "\tmax_bytes\x18\b \x01(\x04R\bmaxBytes\x12*\n" + + "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + + "\n" + + "key_family\x18\n" + + " \x01(\rR\tkeyFamily\x12\x1f\n" + + "\vroute_start\x18\v \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\f \x01(\fR\brouteEnd\x124\n" + + "\x16exclude_known_internal\x18\r \x01(\bR\x14excludeKnownInternal\x12)\n" + + "\x10exclude_prefixes\x18\x0e \x03(\fR\x0fexcludePrefixes\x127\n" + + "\x18requires_route_key_check\x18\x0f \x01(\bR\x15requiresRouteKeyCheck\x12.\n" + + "\x13requires_decoded_s3\x18\x10 \x01(\bR\x11requiresDecodedS3\"\xbc\x01\n" + + "\x18CleanupMigrationResponse\x12\x1f\n" + + "\vnext_cursor\x18\x01 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x02 \x01(\bR\x04done\x12!\n" + + "\fdeleted_rows\x18\x03 \x01(\x04R\vdeletedRows\x12#\n" + + "\rdeleted_bytes\x18\x04 \x01(\x04R\fdeletedBytes\x12#\n" + + "\rscanned_bytes\x18\x05 \x01(\x04R\fscannedBytes\"\xc3\x04\n" + + "\x1aProbeMigrationStateRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12,\n" + + "\x04kind\x18\x02 \x01(\x0e2\x18.MigrationStateProbeKindR\x04kind\x12\x1f\n" + + "\vroute_start\x18\x03 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x04 \x01(\fR\brouteEnd\x128\n" + + "\x18expected_catalog_version\x18\x05 \x01(\x04R\x16expectedCatalogVersion\x12*\n" + + "\x11expected_group_id\x18\x06 \x01(\x04R\x0fexpectedGroupId\x12(\n" + + "\x10migration_job_id\x18\a \x01(\x04R\x0emigrationJobId\x123\n" + + "\x16min_write_ts_exclusive\x18\b \x01(\x04R\x13minWriteTsExclusive\x12,\n" + + "\x12source_write_fence\x18\t \x01(\bR\x10sourceWriteFence\x12*\n" + + "\x11source_read_fence\x18\n" + + " \x01(\bR\x0fsourceReadFence\x12!\n" + + "\ftrack_writes\x18\v \x01(\bR\vtrackWrites\x12(\n" + + "\x10retention_pin_ts\x18\f \x01(\x04R\x0eretentionPinTs\x126\n" + + "\x18read_drain_not_before_ms\x18\r \x01(\x03R\x14readDrainNotBeforeMs\"\x84\x01\n" + + "\x1bProbeMigrationStateResponse\x12\x14\n" + + "\x05ready\x18\x01 \x01(\bR\x05ready\x12'\n" + + "\x0fcatalog_version\x18\x02 \x01(\x04R\x0ecatalogVersion\x12&\n" + + "\x0fmin_admitted_ts\x18\x03 \x01(\x04R\rminAdmittedTs\" \n" + + "\x1eIssueMigrationTimestampRequest\"e\n" + + "\x1fIssueMigrationTimestampResponse\x12\x1c\n" + + "\ttimestamp\x18\x01 \x01(\x04R\ttimestamp\x12$\n" + + "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs*&\n" + "\x02Op\x12\a\n" + "\x03PUT\x10\x00\x12\a\n" + "\x03DEL\x10\x01\x12\x0e\n" + @@ -1486,7 +2173,18 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\xb0\x04\n" + + "\x05ABORT\x10\x03*\x88\x01\n" + + "\x14MigrationCleanupMode\x12&\n" + + "\"MIGRATION_CLEANUP_MODE_UNSPECIFIED\x10\x00\x12#\n" + + "\x1fMIGRATION_CLEANUP_MODE_VERSIONS\x10\x01\x12#\n" + + "\x1fMIGRATION_CLEANUP_MODE_METADATA\x10\x02*\xc9\x02\n" + + "\x17MigrationStateProbeKind\x12*\n" + + "&MIGRATION_STATE_PROBE_KIND_UNSPECIFIED\x10\x00\x12.\n" + + "*MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED\x10\x01\x128\n" + + "4MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED\x10\x02\x123\n" + + "/MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED\x10\x03\x122\n" + + ".MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED\x10\x04\x12/\n" + + "+MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED\x10\x052\xaf\x06\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00\x12T\n" + @@ -1494,7 +2192,10 @@ const file_internal_proto_rawDesc = "" + "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00\x12X\n" + "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00\x12]\n" + "\x1aApplyTargetStagedReadiness\x12\x1d.TargetStagedReadinessRequest\x1a\x1e.TargetStagedReadinessResponse\"\x00\x12R\n" + - "\x13ProbeMigrationLocks\x12\x1b.ProbeMigrationLocksRequest\x1a\x1c.ProbeMigrationLocksResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\x13ProbeMigrationLocks\x12\x1b.ProbeMigrationLocksRequest\x1a\x1c.ProbeMigrationLocksResponse\"\x00\x12I\n" + + "\x10CleanupMigration\x12\x18.CleanupMigrationRequest\x1a\x19.CleanupMigrationResponse\"\x00\x12R\n" + + "\x13ProbeMigrationState\x12\x1b.ProbeMigrationStateRequest\x1a\x1c.ProbeMigrationStateResponse\"\x00\x12^\n" + + "\x17IssueMigrationTimestamp\x12\x1f.IssueMigrationTimestampRequest\x1a .IssueMigrationTimestampResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_internal_proto_rawDescOnce sync.Once @@ -1508,57 +2209,73 @@ func file_internal_proto_rawDescGZIP() []byte { return file_internal_proto_rawDescData } -var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_internal_proto_goTypes = []any{ - (Op)(0), // 0: Op - (Phase)(0), // 1: Phase - (*Mutation)(nil), // 2: Mutation - (*Request)(nil), // 3: Request - (*RaftCommand)(nil), // 4: RaftCommand - (*ForwardRequest)(nil), // 5: ForwardRequest - (*ForwardResponse)(nil), // 6: ForwardResponse - (*RelayPublishRequest)(nil), // 7: RelayPublishRequest - (*RelayPublishResponse)(nil), // 8: RelayPublishResponse - (*ExportRangeVersionsRequest)(nil), // 9: ExportRangeVersionsRequest - (*ExportRangeVersionsResponse)(nil), // 10: ExportRangeVersionsResponse - (*MVCCVersion)(nil), // 11: MVCCVersion - (*ImportRangeVersionsRequest)(nil), // 12: ImportRangeVersionsRequest - (*ImportRangeVersionsResponse)(nil), // 13: ImportRangeVersionsResponse - (*PromoteStagedVersionsRequest)(nil), // 14: PromoteStagedVersionsRequest - (*PromoteStagedVersionsResponse)(nil), // 15: PromoteStagedVersionsResponse - (*TargetStagedReadinessRequest)(nil), // 16: TargetStagedReadinessRequest - (*TargetStagedReadinessResponse)(nil), // 17: TargetStagedReadinessResponse - (*ProbeMigrationLocksRequest)(nil), // 18: ProbeMigrationLocksRequest - (*ProbeMigrationLocksResponse)(nil), // 19: ProbeMigrationLocksResponse + (Op)(0), // 0: Op + (Phase)(0), // 1: Phase + (MigrationCleanupMode)(0), // 2: MigrationCleanupMode + (MigrationStateProbeKind)(0), // 3: MigrationStateProbeKind + (*Mutation)(nil), // 4: Mutation + (*Request)(nil), // 5: Request + (*RaftCommand)(nil), // 6: RaftCommand + (*ForwardRequest)(nil), // 7: ForwardRequest + (*ForwardResponse)(nil), // 8: ForwardResponse + (*RelayPublishRequest)(nil), // 9: RelayPublishRequest + (*RelayPublishResponse)(nil), // 10: RelayPublishResponse + (*ExportRangeVersionsRequest)(nil), // 11: ExportRangeVersionsRequest + (*ExportRangeVersionsResponse)(nil), // 12: ExportRangeVersionsResponse + (*MVCCVersion)(nil), // 13: MVCCVersion + (*ImportRangeVersionsRequest)(nil), // 14: ImportRangeVersionsRequest + (*ImportRangeVersionsResponse)(nil), // 15: ImportRangeVersionsResponse + (*PromoteStagedVersionsRequest)(nil), // 16: PromoteStagedVersionsRequest + (*PromoteStagedVersionsResponse)(nil), // 17: PromoteStagedVersionsResponse + (*TargetStagedReadinessRequest)(nil), // 18: TargetStagedReadinessRequest + (*TargetStagedReadinessResponse)(nil), // 19: TargetStagedReadinessResponse + (*ProbeMigrationLocksRequest)(nil), // 20: ProbeMigrationLocksRequest + (*ProbeMigrationLocksResponse)(nil), // 21: ProbeMigrationLocksResponse + (*CleanupMigrationRequest)(nil), // 22: CleanupMigrationRequest + (*CleanupMigrationResponse)(nil), // 23: CleanupMigrationResponse + (*ProbeMigrationStateRequest)(nil), // 24: ProbeMigrationStateRequest + (*ProbeMigrationStateResponse)(nil), // 25: ProbeMigrationStateResponse + (*IssueMigrationTimestampRequest)(nil), // 26: IssueMigrationTimestampRequest + (*IssueMigrationTimestampResponse)(nil), // 27: IssueMigrationTimestampResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op 1, // 1: Request.phase:type_name -> Phase - 2, // 2: Request.mutations:type_name -> Mutation - 3, // 3: RaftCommand.requests:type_name -> Request - 3, // 4: ForwardRequest.requests:type_name -> Request - 11, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion - 11, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion - 5, // 7: Internal.Forward:input_type -> ForwardRequest - 7, // 8: Internal.RelayPublish:input_type -> RelayPublishRequest - 9, // 9: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest - 12, // 10: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest - 14, // 11: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest - 16, // 12: Internal.ApplyTargetStagedReadiness:input_type -> TargetStagedReadinessRequest - 18, // 13: Internal.ProbeMigrationLocks:input_type -> ProbeMigrationLocksRequest - 6, // 14: Internal.Forward:output_type -> ForwardResponse - 8, // 15: Internal.RelayPublish:output_type -> RelayPublishResponse - 10, // 16: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse - 13, // 17: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse - 15, // 18: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse - 17, // 19: Internal.ApplyTargetStagedReadiness:output_type -> TargetStagedReadinessResponse - 19, // 20: Internal.ProbeMigrationLocks:output_type -> ProbeMigrationLocksResponse - 14, // [14:21] is the sub-list for method output_type - 7, // [7:14] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 4, // 2: Request.mutations:type_name -> Mutation + 5, // 3: RaftCommand.requests:type_name -> Request + 5, // 4: ForwardRequest.requests:type_name -> Request + 13, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion + 13, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion + 2, // 7: CleanupMigrationRequest.mode:type_name -> MigrationCleanupMode + 3, // 8: ProbeMigrationStateRequest.kind:type_name -> MigrationStateProbeKind + 7, // 9: Internal.Forward:input_type -> ForwardRequest + 9, // 10: Internal.RelayPublish:input_type -> RelayPublishRequest + 11, // 11: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest + 14, // 12: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest + 16, // 13: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest + 18, // 14: Internal.ApplyTargetStagedReadiness:input_type -> TargetStagedReadinessRequest + 20, // 15: Internal.ProbeMigrationLocks:input_type -> ProbeMigrationLocksRequest + 22, // 16: Internal.CleanupMigration:input_type -> CleanupMigrationRequest + 24, // 17: Internal.ProbeMigrationState:input_type -> ProbeMigrationStateRequest + 26, // 18: Internal.IssueMigrationTimestamp:input_type -> IssueMigrationTimestampRequest + 8, // 19: Internal.Forward:output_type -> ForwardResponse + 10, // 20: Internal.RelayPublish:output_type -> RelayPublishResponse + 12, // 21: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse + 15, // 22: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse + 17, // 23: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse + 19, // 24: Internal.ApplyTargetStagedReadiness:output_type -> TargetStagedReadinessResponse + 21, // 25: Internal.ProbeMigrationLocks:output_type -> ProbeMigrationLocksResponse + 23, // 26: Internal.CleanupMigration:output_type -> CleanupMigrationResponse + 25, // 27: Internal.ProbeMigrationState:output_type -> ProbeMigrationStateResponse + 27, // 28: Internal.IssueMigrationTimestamp:output_type -> IssueMigrationTimestampResponse + 19, // [19:29] is the sub-list for method output_type + 9, // [9:19] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_internal_proto_init() } @@ -1571,8 +2288,8 @@ func file_internal_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), - NumEnums: 2, - NumMessages: 18, + NumEnums: 4, + NumMessages: 24, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index eeb6c65ab..31c79528b 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -12,6 +12,9 @@ service Internal { rpc PromoteStagedVersions(PromoteStagedVersionsRequest) returns (PromoteStagedVersionsResponse) {} rpc ApplyTargetStagedReadiness(TargetStagedReadinessRequest) returns (TargetStagedReadinessResponse) {} rpc ProbeMigrationLocks(ProbeMigrationLocksRequest) returns (ProbeMigrationLocksResponse) {} + rpc CleanupMigration(CleanupMigrationRequest) returns (CleanupMigrationResponse) {} + rpc ProbeMigrationState(ProbeMigrationStateRequest) returns (ProbeMigrationStateResponse) {} + rpc IssueMigrationTimestamp(IssueMigrationTimestampRequest) returns (IssueMigrationTimestampResponse) {} } // internal.proto is node to node communication message in raft replication. @@ -183,3 +186,74 @@ message ProbeMigrationLocksRequest { message ProbeMigrationLocksResponse { uint32 pending_count = 1; } + +enum MigrationCleanupMode { + MIGRATION_CLEANUP_MODE_UNSPECIFIED = 0; + MIGRATION_CLEANUP_MODE_VERSIONS = 1; + MIGRATION_CLEANUP_MODE_METADATA = 2; +} + +message CleanupMigrationRequest { + uint64 job_id = 1; + MigrationCleanupMode mode = 2; + bytes range_start = 3; + bytes range_end = 4; + bytes cursor = 5; + uint64 max_commit_ts = 6; + uint32 max_versions = 7; + uint64 max_bytes = 8; + uint64 max_scanned_bytes = 9; + uint32 key_family = 10; + bytes route_start = 11; + bytes route_end = 12; + bool exclude_known_internal = 13; + repeated bytes exclude_prefixes = 14; + bool requires_route_key_check = 15; + bool requires_decoded_s3 = 16; +} + +message CleanupMigrationResponse { + bytes next_cursor = 1; + bool done = 2; + uint64 deleted_rows = 3; + uint64 deleted_bytes = 4; + uint64 scanned_bytes = 5; +} + +enum MigrationStateProbeKind { + MIGRATION_STATE_PROBE_KIND_UNSPECIFIED = 0; + MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED = 1; + MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED = 2; + MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED = 3; + MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED = 4; + MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED = 5; +} + +message ProbeMigrationStateRequest { + uint64 job_id = 1; + MigrationStateProbeKind kind = 2; + bytes route_start = 3; + bytes route_end = 4; + uint64 expected_catalog_version = 5; + uint64 expected_group_id = 6; + uint64 migration_job_id = 7; + uint64 min_write_ts_exclusive = 8; + bool source_write_fence = 9; + bool source_read_fence = 10; + bool track_writes = 11; + uint64 retention_pin_ts = 12; + int64 read_drain_not_before_ms = 13; +} + +message ProbeMigrationStateResponse { + bool ready = 1; + uint64 catalog_version = 2; + uint64 min_admitted_ts = 3; +} + +message IssueMigrationTimestampRequest {} + +message IssueMigrationTimestampResponse { + uint64 timestamp = 1; + uint64 last_commit_ts = 2; +} diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index 3ca0b1fe1..18ed111d8 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -26,6 +26,9 @@ const ( Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" Internal_ApplyTargetStagedReadiness_FullMethodName = "/Internal/ApplyTargetStagedReadiness" Internal_ProbeMigrationLocks_FullMethodName = "/Internal/ProbeMigrationLocks" + Internal_CleanupMigration_FullMethodName = "/Internal/CleanupMigration" + Internal_ProbeMigrationState_FullMethodName = "/Internal/ProbeMigrationState" + Internal_IssueMigrationTimestamp_FullMethodName = "/Internal/IssueMigrationTimestamp" ) // InternalClient is the client API for Internal service. @@ -40,6 +43,9 @@ type InternalClient interface { PromoteStagedVersions(ctx context.Context, in *PromoteStagedVersionsRequest, opts ...grpc.CallOption) (*PromoteStagedVersionsResponse, error) ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) ProbeMigrationLocks(ctx context.Context, in *ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*ProbeMigrationLocksResponse, error) + CleanupMigration(ctx context.Context, in *CleanupMigrationRequest, opts ...grpc.CallOption) (*CleanupMigrationResponse, error) + ProbeMigrationState(ctx context.Context, in *ProbeMigrationStateRequest, opts ...grpc.CallOption) (*ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(ctx context.Context, in *IssueMigrationTimestampRequest, opts ...grpc.CallOption) (*IssueMigrationTimestampResponse, error) } type internalClient struct { @@ -129,6 +135,36 @@ func (c *internalClient) ProbeMigrationLocks(ctx context.Context, in *ProbeMigra return out, nil } +func (c *internalClient) CleanupMigration(ctx context.Context, in *CleanupMigrationRequest, opts ...grpc.CallOption) (*CleanupMigrationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CleanupMigrationResponse) + err := c.cc.Invoke(ctx, Internal_CleanupMigration_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) ProbeMigrationState(ctx context.Context, in *ProbeMigrationStateRequest, opts ...grpc.CallOption) (*ProbeMigrationStateResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ProbeMigrationStateResponse) + err := c.cc.Invoke(ctx, Internal_ProbeMigrationState_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) IssueMigrationTimestamp(ctx context.Context, in *IssueMigrationTimestampRequest, opts ...grpc.CallOption) (*IssueMigrationTimestampResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(IssueMigrationTimestampResponse) + err := c.cc.Invoke(ctx, Internal_IssueMigrationTimestamp_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // InternalServer is the server API for Internal service. // All implementations must embed UnimplementedInternalServer // for forward compatibility. @@ -141,6 +177,9 @@ type InternalServer interface { PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) + CleanupMigration(context.Context, *CleanupMigrationRequest) (*CleanupMigrationResponse, error) + ProbeMigrationState(context.Context, *ProbeMigrationStateRequest) (*ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(context.Context, *IssueMigrationTimestampRequest) (*IssueMigrationTimestampResponse, error) mustEmbedUnimplementedInternalServer() } @@ -172,6 +211,15 @@ func (UnimplementedInternalServer) ApplyTargetStagedReadiness(context.Context, * func (UnimplementedInternalServer) ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) { return nil, status.Error(codes.Unimplemented, "method ProbeMigrationLocks not implemented") } +func (UnimplementedInternalServer) CleanupMigration(context.Context, *CleanupMigrationRequest) (*CleanupMigrationResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CleanupMigration not implemented") +} +func (UnimplementedInternalServer) ProbeMigrationState(context.Context, *ProbeMigrationStateRequest) (*ProbeMigrationStateResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ProbeMigrationState not implemented") +} +func (UnimplementedInternalServer) IssueMigrationTimestamp(context.Context, *IssueMigrationTimestampRequest) (*IssueMigrationTimestampResponse, error) { + return nil, status.Error(codes.Unimplemented, "method IssueMigrationTimestamp not implemented") +} func (UnimplementedInternalServer) mustEmbedUnimplementedInternalServer() {} func (UnimplementedInternalServer) testEmbeddedByValue() {} @@ -312,6 +360,60 @@ func _Internal_ProbeMigrationLocks_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _Internal_CleanupMigration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CleanupMigrationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).CleanupMigration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_CleanupMigration_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).CleanupMigration(ctx, req.(*CleanupMigrationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_ProbeMigrationState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeMigrationStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ProbeMigrationState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ProbeMigrationState_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ProbeMigrationState(ctx, req.(*ProbeMigrationStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_IssueMigrationTimestamp_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IssueMigrationTimestampRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).IssueMigrationTimestamp(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_IssueMigrationTimestamp_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).IssueMigrationTimestamp(ctx, req.(*IssueMigrationTimestampRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Internal_ServiceDesc is the grpc.ServiceDesc for Internal service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -343,6 +445,18 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "ProbeMigrationLocks", Handler: _Internal_ProbeMigrationLocks_Handler, }, + { + MethodName: "CleanupMigration", + Handler: _Internal_CleanupMigration_Handler, + }, + { + MethodName: "ProbeMigrationState", + Handler: _Internal_ProbeMigrationState_Handler, + }, + { + MethodName: "IssueMigrationTimestamp", + Handler: _Internal_IssueMigrationTimestamp_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/store/migration_cleanup.go b/store/migration_cleanup.go new file mode 100644 index 000000000..78198224c --- /dev/null +++ b/store/migration_cleanup.go @@ -0,0 +1,205 @@ +package store + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +func cleanupExportOptions(opts CleanupVersionsOptions) ExportVersionsOptions { + return ExportVersionsOptions{ + StartKey: opts.StartKey, + EndKey: opts.EndKey, + Cursor: opts.Cursor, + MaxCommitTSInclusive: opts.MaxCommitTS, + MaxVersions: opts.MaxVersions, + MaxBytes: opts.MaxBytes, + MaxScannedBytes: opts.MaxScannedBytes, + KeyFamily: opts.KeyFamily, + AcceptVersion: opts.AcceptVersion, + } +} + +func cleanupResult(exported ExportVersionsResult) CleanupVersionsResult { + var deletedBytes uint64 + for _, version := range exported.Versions { + deletedBytes += versionExportSize(version.Key, len(version.Value)) + } + return CleanupVersionsResult{ + NextCursor: bytes.Clone(exported.NextCursor), + Done: exported.Done, + DeletedRows: uint64(len(exported.Versions)), //nolint:gosec // bounded by MaxVersions. + DeletedBytes: deletedBytes, + ScannedBytes: exported.ScannedBytes, + } +} + +func (s *mvccStore) CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) { + if opts.MaxVersions <= 0 { + return CleanupVersionsResult{Done: true}, nil + } + s.mtx.Lock() + defer s.mtx.Unlock() + + exportOpts := normalizeExportVersionsOptions(cleanupExportOptions(opts)) + pos, err := decodeExportCursorForOptions(exportOpts) + if err != nil { + return CleanupVersionsResult{}, err + } + exported, err := s.exportVersionsLocked(ctx, exportOpts, pos) + if err != nil { + return CleanupVersionsResult{}, err + } + for _, version := range exported.Versions { + s.removeVersionLocked(version.Key, version.CommitTS) + } + return cleanupResult(exported), nil +} + +func (s *mvccStore) ClearMigrationState(_ context.Context, jobID, _ uint64) error { + if jobID == 0 { + return errors.New("migration cleanup job id is required") + } + s.mtx.Lock() + defer s.mtx.Unlock() + + for id := range s.migrationAcks { + if id.jobID == jobID { + delete(s.migrationAcks, id) + } + } + delete(s.migrationHLCFloors, jobID) + delete(s.migrationPromotions, jobID) + delete(s.migrationReadiness, jobID) + s.migrationReadinessCache = upsertReadinessCacheWithoutJob(s.migrationReadinessCache, jobID) + return nil +} + +func upsertReadinessCacheWithoutJob(states []TargetStagedReadinessState, jobID uint64) []TargetStagedReadinessState { + out := states[:0] + for _, state := range states { + if state.JobID != jobID { + out = append(out, state) + } + } + return out +} + +func (s *pebbleStore) CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) { + if opts.MaxVersions <= 0 { + return CleanupVersionsResult{Done: true}, nil + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + s.applyMu.Lock() + defer s.applyMu.Unlock() + + exported, err := s.exportVersionsLocked(ctx, cleanupExportOptions(opts)) + if err != nil { + return CleanupVersionsResult{}, err + } + batch := s.db.NewBatch() + defer batch.Close() + for _, version := range exported.Versions { + if err := batch.Delete(encodeKey(version.Key, version.CommitTS), nil); err != nil { + return CleanupVersionsResult{}, errors.WithStack(err) + } + } + if opts.AppliedIndex > 0 { + if err := setPebbleUint64InBatch(batch, metaAppliedIndexBytes, opts.AppliedIndex); err != nil { + return CleanupVersionsResult{}, err + } + } + if err := batch.Commit(s.cleanupWriteOpts(opts.AppliedIndex)); err != nil { + return CleanupVersionsResult{}, errors.WithStack(err) + } + return cleanupResult(exported), nil +} + +func (s *pebbleStore) ClearMigrationState(_ context.Context, jobID, appliedIndex uint64) error { + if jobID == 0 { + return errors.New("migration cleanup job id is required") + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + s.applyMu.Lock() + defer s.applyMu.Unlock() + + metadata, err := s.migrationMetadataWithoutJob(jobID) + if err != nil { + return err + } + + batch := s.db.NewBatch() + defer batch.Close() + if err := stageMigrationMetadataCleanup(batch, jobID, appliedIndex, metadata); err != nil { + return err + } + if err := batch.Commit(s.cleanupWriteOpts(appliedIndex)); err != nil { + return errors.WithStack(err) + } + s.mtx.Lock() + s.migrationReadinessCache = upsertReadinessCacheWithoutJob(s.migrationReadinessCache, jobID) + s.mtx.Unlock() + return nil +} + +type migrationMetadataState struct { + acks map[migrationAckID]migrationImportAck + floors map[uint64]uint64 + promotions map[uint64]PromotionState +} + +func (s *pebbleStore) migrationMetadataWithoutJob(jobID uint64) (migrationMetadataState, error) { + acks, err := s.readMigrationImportAcks() + if err != nil { + return migrationMetadataState{}, err + } + for id := range acks { + if id.jobID == jobID { + delete(acks, id) + } + } + floors, err := s.readMigrationHLCFloors() + if err != nil { + return migrationMetadataState{}, err + } + delete(floors, jobID) + promotions, err := s.readPebblePromotionStates() + if err != nil { + return migrationMetadataState{}, err + } + delete(promotions, jobID) + return migrationMetadataState{acks: acks, floors: floors, promotions: promotions}, nil +} + +func stageMigrationMetadataCleanup(batch *pebble.Batch, jobID, appliedIndex uint64, metadata migrationMetadataState) error { + if err := batch.Set(migrationAckMetaKeyBytes, encodeMigrationImportAcks(metadata.acks), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Set(migrationHLCFloorMetaKeyBytes, encodeMigrationHLCFloors(metadata.floors), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Set(migrationPromoteMetaKeyBytes, encodeMigrationPromotionStates(metadata.promotions), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Delete(migrationReadyKey(jobID), nil); err != nil { + return errors.WithStack(err) + } + if appliedIndex == 0 { + return nil + } + return setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex) +} + +func (s *pebbleStore) cleanupWriteOpts(appliedIndex uint64) *pebble.WriteOptions { + if appliedIndex > 0 { + return s.raftApplyWriteOpts() + } + return s.directApplyWriteOpts() +} + +var _ MigrationCleaner = (*mvccStore)(nil) +var _ MigrationCleaner = (*pebbleStore)(nil) diff --git a/store/migration_promote.go b/store/migration_promote.go index fc81eb350..c79bac7e5 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -142,25 +142,24 @@ func (s *mvccStore) MigrationPromotionState(_ context.Context, jobID uint64) (Pr return clonePromotionState(state), ok, nil } -func (s *mvccStore) removeVersionLocked(key []byte, commitTS uint64) bool { +func (s *mvccStore) removeVersionLocked(key []byte, commitTS uint64) { existing, ok := s.tree.Get(key) if !ok { - return false + return } versions, _ := existing.([]VersionedValue) idx := findVersionIndex(versions, commitTS) if idx < 0 { - return false + return } next := make([]VersionedValue, len(versions)-1) copy(next, versions[:idx]) copy(next[idx:], versions[idx+1:]) if len(next) == 0 { s.tree.Remove(key) - return true + return } s.tree.Put(bytes.Clone(key), next) - return true } func findVersionIndex(versions []VersionedValue, commitTS uint64) int { diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 680a84368..540320d94 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -1424,3 +1424,76 @@ func TestPebbleSnapshotPreservesMigrationMetadata(t *testing.T) { _, err = dst.GetAt(ctx, []byte("fresh"), 60) require.ErrorIs(t, err, ErrKeyNotFound) } + +func TestCleanupVersionsRemovesOnlySelectedCommittedVersions(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("v1"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("v2"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("keep"), 10, 0)) + + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + first, err := cleaner.CleanupVersions(ctx, CleanupVersionsOptions{ + JobID: 1, + StartKey: []byte("a"), + EndKey: []byte("b"), + MaxCommitTS: 10, + MaxVersions: 1, + MaxBytes: 1 << 20, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), first.DeletedRows) + + value, err := st.GetAt(ctx, []byte("a"), 20) + require.NoError(t, err) + require.Equal(t, []byte("v2"), value) + _, err = st.GetAt(ctx, []byte("a"), 10) + require.ErrorIs(t, err, ErrKeyNotFound) + value, err = st.GetAt(ctx, []byte("b"), 20) + require.NoError(t, err) + require.Equal(t, []byte("keep"), value) + }) +} + +func TestClearMigrationStateRemovesPerJobProofs(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + _, err := st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 1, + BatchSeq: 1, + Versions: []MVCCVersion{{ + Key: []byte("!dist|migstage|row"), + CommitTS: 50, + Value: []byte("v"), + }}, + }) + require.NoError(t, err) + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 7, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + MinWriteTSExclusive: 50, + Armed: true, + })) + + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + floor, err := st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Zero(t, floor) + reader, ok := st.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + for _, state := range states { + require.NotEqual(t, uint64(7), state.JobID) + } + }) +} diff --git a/store/store.go b/store/store.go index 9661626e7..8b01564f6 100644 --- a/store/store.go +++ b/store/store.go @@ -163,6 +163,32 @@ type PromotionState struct { LastError string } +// CleanupVersionsOptions physically removes exact committed versions selected +// by the migration bracket filter. It is cursor-resumable and is used only +// after ownership has moved or while abandoning an unpublished migration. +type CleanupVersionsOptions struct { + JobID uint64 + AppliedIndex uint64 + StartKey []byte + EndKey []byte + Cursor []byte + MaxCommitTS uint64 + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + KeyFamily uint32 + AcceptVersion func(key, value []byte) bool +} + +// CleanupVersionsResult reports one bounded physical-delete chunk. +type CleanupVersionsResult struct { + NextCursor []byte + Done bool + DeletedRows uint64 + DeletedBytes uint64 + ScannedBytes uint64 +} + // TargetStagedReadinessState is a target-local guard that makes a target voter // fail closed for a moving route until it has either the staged descriptor or // the cleared descriptor with the retained write timestamp floor. @@ -187,6 +213,13 @@ type MigrationPromoter interface { PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) } +// MigrationCleaner removes source or abandoned target versions and the +// per-job migration proof records through the Raft apply path. +type MigrationCleaner interface { + CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) + ClearMigrationState(ctx context.Context, jobID, appliedIndex uint64) error +} + // MigrationPromotionStateReader reads target-local staged promotion state. type MigrationPromotionStateReader interface { MigrationPromotionState(ctx context.Context, jobID uint64) (PromotionState, bool, error) From 1cbee6a7cc0b5eec1714ba914cf8c8d28b84c4e6 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 01:09:15 +0900 Subject: [PATCH 158/162] Track post-arm two-phase commits during split --- kv/fsm.go | 7 +++++ kv/fsm_migration_fence_test.go | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/kv/fsm.go b/kv/fsm.go index 29fcc71b8..9ca0af63c 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -1736,9 +1736,16 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } + return f.recordAndApplyCommit(ctx, storeMuts, uniq, applyStartTS, commitTS) +} + +func (f *kvFSM) recordAndApplyCommit(ctx context.Context, storeMuts []*store.KVPairMutation, uniq []*pb.Mutation, applyStartTS, commitTS uint64) error { if len(storeMuts) == 0 { return nil } + if err := f.recordMigrationWrite(ctx, uniq, commitTS); err != nil { + return err + } if err := f.applyCommitWithIdempotencyFallback(ctx, storeMuts, uniq, applyStartTS, commitTS); err != nil { return err } diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index e23e86d5b..4853a2758 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -184,6 +184,56 @@ func TestFSMMigrationWriteTrackerCoversAllAdmissionPaths(t *testing.T) { require.Equal(t, uint64(55), migrationTrackerMinimum(t, fsm)) } +func TestFSMMigrationWriteTrackerRecordsCommitPreparedBeforeArm(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + startTS, commitTS := uint64(40), uint64(60) + meta := &pb.Mutation{ + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("n"), + CommitTS: commitTS, + LockTTLms: defaultTxnLockTTLms, + }), + } + mutation := &pb.Mutation{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("value")} + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: startTS, + Mutations: []*pb.Mutation{meta, mutation}, + }, startTS)) + + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 12, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 12, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 1, + TrackWrites: true, + }) + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: startTS, + Mutations: []*pb.Mutation{meta, mutation}, + }, commitTS)) + require.Equal(t, commitTS, migrationTrackerMinimum(t, fsm)) +} + func newReadinessReadKeyFSM(t *testing.T) *kvFSM { t.Helper() From 8264ea3835b3c8fd447278fb674f083eb92e3c46 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 01:15:06 +0900 Subject: [PATCH 159/162] Close split migration startup and retry gaps --- adapter/distribution_server.go | 20 ++++++------- adapter/distribution_server_test.go | 34 +++++++++++++++++++++++ main.go | 5 ++++ main_encryption_rotate_on_startup_test.go | 5 ++++ 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index fe1999dcd..af29a0a3f 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -1028,7 +1028,7 @@ func (s *DistributionServer) completeSplitJobTargetPromotionViaCoordinator( if alreadyDone { return snapshot, current, nil } - completion, err := distribution.CompleteTargetPromotionState(expected, snapshot.Routes, nowMs) + completion, err := distribution.CompleteTargetPromotionState(current, snapshot.Routes, nowMs) if err != nil { return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) } @@ -1065,12 +1065,12 @@ func (s *DistributionServer) loadPromotionCompletionCandidate( if snapshot.Version == expectedVersion && distribution.SplitJobsEquivalent(current, expected) { return snapshot, current, false, nil } - job, err := completedPromotionRetryJob(snapshot.Routes, expected, current, nowMs) + completion, err := completedPromotionRetry(snapshot.Routes, expected, current, nowMs) if err != nil { return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(err) } - if job.TargetPromotionDone { - return snapshot, job, true, nil + if completion.Job.TargetPromotionDone { + return snapshot, current, !completion.Changed, nil } if snapshot.Version != expectedVersion { return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogVersionMismatch) @@ -1078,24 +1078,24 @@ func (s *DistributionServer) loadPromotionCompletionCandidate( return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) } -func completedPromotionRetryJob( +func completedPromotionRetry( routes []distribution.RouteDescriptor, expected distribution.SplitJob, current distribution.SplitJob, nowMs int64, -) (distribution.SplitJob, error) { +) (distribution.TargetPromotionCompletion, error) { matches, err := splitJobPromotionMatchesExpected(expected, current) if err != nil { - return distribution.SplitJob{}, errors.WithStack(err) + return distribution.TargetPromotionCompletion{}, errors.WithStack(err) } if !matches { - return distribution.SplitJob{}, nil + return distribution.TargetPromotionCompletion{}, nil } completion, err := distribution.CompleteTargetPromotionState(current, routes, nowMs) if err != nil { - return distribution.SplitJob{}, errors.WithStack(err) + return distribution.TargetPromotionCompletion{}, errors.WithStack(err) } - return completion.Job, nil + return completion, nil } func (s *DistributionServer) dispatchPromotionCompletion( diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 355527b15..da4a97940 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -875,6 +875,40 @@ func TestDistributionServerCompleteSplitJobTargetPromotion_RetryAfterCommitIsIde require.Equal(t, 1, coordinator.dispatchCalls) } +func TestDistributionServerCompleteSplitJobTargetPromotion_PersistsLegacyTerminalTime(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + expected := promotionCompleteDistributionJob() + legacy := distribution.CloneSplitJob(expected) + legacy.Phase = distribution.SplitJobPhaseDone + legacy.TargetPromotionDone = true + legacy.PromotionCompletedTS = saved.ReadTS + 1 + require.NoError(t, catalog.CreateSplitJob(ctx, legacy)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + snapshot, completed, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, expected, 3000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.Equal(t, int64(3000), completed.TerminalAtMs) + require.Equal(t, int64(3000), completed.UpdatedAtMs) + require.Equal(t, 1, coordinator.dispatchCalls) + + loaded, found, err := catalog.SplitJob(ctx, legacy.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, completed, loaded) +} + func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 2631a53db..5bf9aee0d 100644 --- a/main.go +++ b/main.go @@ -2292,9 +2292,14 @@ func startupRotationGatedMethod(fullMethod string) bool { switch fullMethod { case pb.Internal_Forward_FullMethodName, pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_GetSplitMigrationCapability_FullMethodName, pb.Distribution_AbandonSplitJob_FullMethodName, pb.Distribution_RetrySplitJob_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index fa53d4a68..76b998160 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -420,9 +420,14 @@ func TestStartupPublicKVGate_BlocksMutatorsUntilReady(t *testing.T) { pb.TransactionalKV_Get_FullMethodName, pb.Internal_Forward_FullMethodName, pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_GetSplitMigrationCapability_FullMethodName, pb.Distribution_AbandonSplitJob_FullMethodName, pb.Distribution_RetrySplitJob_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, From db933c0f65894e2580d3bdf23b0d68cf3b619087 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 18:18:58 +0900 Subject: [PATCH 160/162] docs: mark hotspot split M2 implemented --- .../2026_02_18_partial_hotspot_shard_split.md | 13 +++++++------ ...emented_hotspot_split_milestone2_migration.md} | 15 +++++++++------ .../design/2026_06_12_proposed_scaling_roadmap.md | 8 ++++---- .../design/2026_06_23_proposed_scaling_roadmap.md | 8 ++++---- 4 files changed, 24 insertions(+), 20 deletions(-) rename docs/design/{2026_06_11_partial_hotspot_split_milestone2_migration.md => 2026_06_11_implemented_hotspot_split_milestone2_migration.md} (99%) diff --git a/docs/design/2026_02_18_partial_hotspot_shard_split.md b/docs/design/2026_02_18_partial_hotspot_shard_split.md index 55ecfe1b0..ee8a0b5b8 100644 --- a/docs/design/2026_02_18_partial_hotspot_shard_split.md +++ b/docs/design/2026_02_18_partial_hotspot_shard_split.md @@ -293,12 +293,13 @@ Add RPCs: 2. Job phases: BACKFILL/FENCE/DELTA/CUTOVER 3. Manual split with target-group relocation -Status: partial. SplitJob catalog/codec, MVCC export/import primitives, staged -visibility, write-fence checks, target readiness, and target-promotion catalog -components exist in the M2 stack, but no production runner advances cross-group -jobs to `DONE` yet. M2 remains open in -[`2026_06_11_partial_hotspot_split_milestone2_migration.md`](2026_06_11_partial_hotspot_split_milestone2_migration.md) -until the cross-group acceptance criteria and Jepsen workload pass. +Status: implemented. The production runner advances durable cross-group jobs +through BACKFILL, FENCE, DELTA_COPY, CUTOVER, CLEANUP, and `DONE`; current-voter +readiness and cleanup barriers protect leadership changes; and the deterministic +Jepsen workload covers the split alongside leader-kill and partition packages. +The completed M2 contract is recorded in +[`2026_06_11_implemented_hotspot_split_milestone2_migration.md`](2026_06_11_implemented_hotspot_split_milestone2_migration.md) +and its final runner/lifecycle slice landed in PR #1096. ### Milestone 3: Automation diff --git a/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md similarity index 99% rename from docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md rename to docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md index dc13a3044..80b7716dd 100644 --- a/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md +++ b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md @@ -1,13 +1,13 @@ # Hotspot Shard Split — Milestone 2: Migration Plane -Status: Partial +Status: Implemented Author: bootjp Date: 2026-06-11 Parent: [2026_02_18_partial_hotspot_shard_split.md](2026_02_18_partial_hotspot_shard_split.md). M1 (control plane) is as-built in [2026_02_18_implemented_hotspot_split_milestone1_pr.md](2026_02_18_implemented_hotspot_split_milestone1_pr.md). -Current implementation status: partial. The SplitJob catalog/codec, versioned route descriptor storage, range export/import primitives, staged-read visibility, write-fence checks, target readiness, and target-promotion catalog pieces have landed or are covered by the active M2 stack. The feature is not implemented end to end until a production runner advances cross-group jobs to `DONE`, `ListRoutes` exposes the moved child on its target group after cutover, and the Jepsen cross-group split workload passes the acceptance criteria in §13. +Current implementation status: implemented. The M2 stack provides the durable SplitJob catalog, versioned route descriptors, resumable range export/import, staged/live visibility, source write and read fences, current-voter readiness and cleanup barriers, constant-time route cutover, incremental target promotion, bounded source/target cleanup, terminal `DONE` history, and the deterministic cross-group Jepsen workload. PR #1096 completed the production runner and lifecycle wiring. Automatic hotspot detection and split scheduling remain M3 scope; the broader route-shuffle and fault campaign remains M4 scope. ## 1. Background @@ -1243,7 +1243,7 @@ Phased into reviewable PRs, each lands behind its own doc-or-test gate: | M2-PR5 | Coordinator + FSM FENCE rejection (`ErrRouteWriteFenced`) with point and `DEL_PREFIX` range-footprint checks + route-faithful txn-lock drain (§3.2a.0a) + same-group `SplitRange` overlap rejection while a SplitJob is live | fsm + coordinator unit + `migrator_lock_drain_test` + `catalog_test` overlap red controls | | M2-PR6 | Cross-group end-to-end: ExportRangeVersions / ImportVersions server-side handlers + migrator BACKFILL/DELTA_COPY + raw-candidate staged/live merge read path in both scan directions and `LatestCommitTS` + §7.2.2e source-side cutover read-fence arm with every-current-source-voter ACK, membership-epoch re-ACK, catalog-version waiter, route-key-normalized scan ownership checks, and server-stamped RawKV read versions | integration incl. delete/TTL DELTA_COPY, Redis list/hash/set/zset/stream migration, HLC restart/fence-floor cases + `kv/fsm_cutover_read_fence_test.go` | | M2-PR7 | Target-local `PromotionState` + background promoter + ordered default-group promotion-complete CAS retaining `min_write_ts_exclusive` + source/target cleanup before DONE history move; readiness guard accepts matching cleared descriptors while retained; `AbandonSplitJob` with durable `ABANDONING` cleanup witness + CLEANUP GC + Jepsen split workload | `kv/fsm_promote_staged_test.go` raw hidden-version/LatestCommitTS merge + jepsen suite | -| M2-PR8 | Rename `*_proposed_*` → `*_partial_*` after PR1 ships; update parent partial doc M2 status; rename to `*_implemented_*` after PR7 | | +| M2-PR8 | Rename `*_proposed_*` → `*_partial_*` after PR1 ships; update parent partial doc M2 status; rename to `*_implemented_*` after PR7 | Completed after the runner, voter barriers, cleanup lifecycle, route publication, and Jepsen workload landed in PR #1096. | Each PR follows the five-lens self-review and is gated by its tests + `make lint`. @@ -1283,9 +1283,12 @@ This is independent of the existing rolling-upgrade protocol for unrelated subsy ## 14. Lifecycle -This document is `*_partial_*` because M2-PR1 and later component slices have landed, but the cross-group migration plane is not complete end to end. +This document is `*_implemented_*` because the M2 cross-group migration plane is complete as a central subsystem. The production runner now resumes durable jobs across leadership changes, drives every phase through `CLEANUP`, waits for current source and target voter proofs, publishes the target route, removes bounded migration state, and moves the job to `DONE` history. The deterministic Jepsen split workload exercises the cross-group operation alongside leader-kill and partition fault packages. -- Track per-PR landing under §11. -- Rename to `*_implemented_*` only after the acceptance criteria in §13 pass: cross-group `StartSplitMigration` reaches `phase=DONE`, `ListRoutes` shows the target-group child after cutover, leader-kill recovery is automatic, and the Jepsen split workload is green. +The remaining work is intentionally outside M2's central scope: + +- M3 owns automatic hotspot detection, target selection, and split scheduling. +- M4 owns the broader route-shuffle nemesis matrix and production-scale fault campaigns. +- Reverse migration after CUTOVER and concurrent migration jobs remain future extensions. `git mv` is used so history follows. diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 6a7968cc5..a29a201c0 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -336,9 +336,9 @@ control-plane (`*_proposed_*` doc TBD).** - M1 standalone but doesn't enable cross-region writes — it just makes Raft survive cross-WAN partition. -- M2 depends on the M2 hotspot-split migration contract - (`2026_06_11_partial_hotspot_split_milestone2_migration.md`) - being implemented so the monotone-merge primitive exists. +- M2's hotspot-split migration dependency is implemented in + `2026_06_11_implemented_hotspot_split_milestone2_migration.md`, including + the monotone-merge primitive. - M3 depends on M1's region-aware membership and M2's per-region ceiling. - M4 depends on M2 and M3. @@ -531,7 +531,7 @@ the ceiling shape: Composability invariant: **every monotone-merge happens via the same `SetPhysicalCeiling` + `Observe` primitive**. The M2 hotspot-split contract (§6.2.1 of -`2026_06_11_partial_hotspot_split_milestone2_migration.md`) is the +`2026_06_11_implemented_hotspot_split_milestone2_migration.md`) is the reference implementation; per-region and per-group merges reuse it. ### 7.2 Capability bits diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index 6e2d11628..d27574331 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -98,10 +98,10 @@ memory each group's private cache/memtable pins. - **Range split — distribute a range across groups.** Same-group split shipped in M1 (`distribution/`). Cross-group migration (the part that - actually relocates data and reduces per-node volume) is tracked by **PR #945** - and the active M2 stack - (`docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md`, - branch `docs/hotspot-split-m2-proposal`): a resumable `SplitJob` with + actually relocates data and reduces per-node volume) is implemented by the + M2 stack recorded in + `docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md`: + a resumable `SplitJob` with `PLANNED → BACKFILL → FENCE → DELTA_COPY → CUTOVER → CLEANUP → DONE` phases driven by a migrator on the default-group leader. M2 is the required ownership-migration mechanism, but it reduces per-node bytes only when the From f270d8d54f964d0a7a0fadefc00663253ebc8fe2 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 18:22:03 +0900 Subject: [PATCH 161/162] docs: link implemented hotspot split design --- docs/design/2026_06_12_proposed_scaling_roadmap.md | 4 ++-- docs/design/2026_06_23_proposed_scaling_roadmap.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index a29a201c0..2ee3e458c 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -337,7 +337,7 @@ control-plane (`*_proposed_*` doc TBD).** - M1 standalone but doesn't enable cross-region writes — it just makes Raft survive cross-WAN partition. - M2's hotspot-split migration dependency is implemented in - `2026_06_11_implemented_hotspot_split_milestone2_migration.md`, including + [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md), including the monotone-merge primitive. - M3 depends on M1's region-aware membership and M2's per-region ceiling. @@ -531,7 +531,7 @@ the ceiling shape: Composability invariant: **every monotone-merge happens via the same `SetPhysicalCeiling` + `Observe` primitive**. The M2 hotspot-split contract (§6.2.1 of -`2026_06_11_implemented_hotspot_split_milestone2_migration.md`) is the +[2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md)) is the reference implementation; per-region and per-group merges reuse it. ### 7.2 Capability bits diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index d27574331..210d52c14 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -100,7 +100,7 @@ memory each group's private cache/memtable pins. shipped in M1 (`distribution/`). Cross-group migration (the part that actually relocates data and reduces per-node volume) is implemented by the M2 stack recorded in - `docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md`: + [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md): a resumable `SplitJob` with `PLANNED → BACKFILL → FENCE → DELTA_COPY → CUTOVER → CLEANUP → DONE` phases driven by a migrator on the default-group leader. M2 is the required From 6b54f22ddc482ddf5af1befb3b08a248e7123aef Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 20 Jul 2026 04:45:42 +0000 Subject: [PATCH 162/162] docs: resolve merge conflicts with base branch --- adapter/distribution_server.go | 62 - adapter/distribution_server_test.go | 57 - adapter/internal.go | 610 -------- adapter/internal_migration_test.go | 6 - adapter/internal_test.go | 158 -- adapter/redis_delta_compactor.go | 4 - adapter/redis_delta_compactor_test.go | 13 - adapter/redis_list_dedup_test.go | 13 - adapter/redis_retry.go | 4 - adapter/split_job_runner.go | 4 - distribution/engine.go | 17 - distribution/engine_test.go | 3 - .../2026_02_18_partial_hotspot_shard_split.md | 9 - ...nted_hotspot_split_milestone2_migration.md | 15 - ...tial_hotspot_split_milestone2_migration.md | 1309 ----------------- .../2026_06_12_proposed_scaling_roadmap.md | 10 - .../2026_06_23_proposed_scaling_roadmap.md | 7 - .../etcd/engine_applied_index_test.go | 6 - internal/raftengine/etcd/grpc_transport.go | 3 - jepsen/src/elastickv/db.clj | 15 - jepsen/src/elastickv/split_workload.clj | 4 - jepsen/test/elastickv/split_workload_test.clj | 3 - kv/fsm.go | 18 - kv/fsm_migration_fence_test.go | 35 - kv/shard_key.go | 15 - kv/shard_key_test.go | 3 - kv/shard_store.go | 433 ------ kv/shard_store_test.go | 15 - kv/sharded_coordinator.go | 65 - kv/sharded_coordinator_del_prefix_test.go | 3 - kv/sharded_coordinator_txn_test.go | 3 - kv/transcoder.go | 3 - main.go | 254 +--- main_catalog_test.go | 9 - monitoring/hotpath.go | 3 - proto/internal.pb.go | 4 - store/lsm_migration.go | 3 - store/migration_versions.go | 7 - store/migration_versions_test.go | 3 - store/mvcc_store.go | 5 - store/mvcc_store_snapshot_test.go | 3 - 41 files changed, 29 insertions(+), 3187 deletions(-) delete mode 100644 docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 0634a4af6..07c044d65 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -12,10 +12,7 @@ import ( "time" "github.com/bootjp/elastickv/distribution" -<<<<<<< HEAD -======= "github.com/bootjp/elastickv/internal/fskeys" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" @@ -33,10 +30,7 @@ type DistributionServer struct { catalog *distribution.CatalogStore coordinator kv.Coordinator readTracker *kv.ActiveTimestampTracker -<<<<<<< HEAD -======= fsObserver DistributionFilesystemObserver ->>>>>>> origin/design/hotspot-split-m2-promotion-complete readBlocked func() bool migrationCapabilityGate SplitMigrationCapabilityGate splitJobRunnerReady bool @@ -56,15 +50,12 @@ type DistributionServer struct { // DistributionServerOption configures DistributionServer behavior. type DistributionServerOption func(*DistributionServer) -<<<<<<< HEAD -======= type DistributionFilesystemObserver interface { ObserveFilePinnedHotspot(reason string) } const DistributionFilePinnedHotspotSplitBoundary = "split_boundary" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete // SplitMigrationCapabilityGate reports whether this node can safely create // migration-only side effects. A nil gate keeps StartSplitMigration fail-closed. type SplitMigrationCapabilityGate func(context.Context) error @@ -119,15 +110,12 @@ func WithDistributionActiveTimestampTracker(tracker *kv.ActiveTimestampTracker) } } -<<<<<<< HEAD -======= func WithDistributionFilesystemObserver(observer DistributionFilesystemObserver) DistributionServerOption { return func(s *DistributionServer) { s.fsObserver = observer } } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func WithDistributionReadGate(blocked func() bool) DistributionServerOption { return func(s *DistributionServer) { s.readBlocked = blocked @@ -477,11 +465,7 @@ func (s *DistributionServer) GetRoute(ctx context.Context, req *pb.GetRouteReque if err := s.requireReadReady(); err != nil { return nil, err } -<<<<<<< HEAD - r, ok := s.engine.GetRoute(req.Key) -======= r, ok := s.engine.GetRoute(kv.RouteKey(req.Key)) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if !ok { return &pb.GetRouteResponse{}, nil } @@ -795,34 +779,12 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR return nil, err } -<<<<<<< HEAD - parent, found := findRouteByID(snapshot.Routes, req.GetRouteId()) - if !found { - return nil, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error()) - } - - splitKey := distribution.CloneBytes(req.GetSplitKey()) - if err := validateSplitKey(parent, splitKey); err != nil { - return nil, err - } - splitJobReadKeys, err := s.splitJobOverlapReadKeys(ctx, snapshot, parent) - if err != nil { - return nil, err - } - - leftID, rightID, err := s.allocateChildRouteIDs(ctx, snapshot.ReadTS, snapshot.Routes) -======= plan, err := s.planSplitRange(ctx, snapshot, req) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if err != nil { return nil, err } -<<<<<<< HEAD - saved, err := s.saveSplitResultViaCoordinator(ctx, snapshot.ReadTS, req.GetExpectedCatalogVersion(), parent.RouteID, splitJobReadKeys, left, right) -======= saved, err := s.saveSplitResultViaCoordinator(ctx, snapshot.ReadTS, req.GetExpectedCatalogVersion(), plan.parentID, plan.readKeys, plan.left, plan.right) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if err != nil { return nil, err } @@ -879,8 +841,6 @@ func (s *DistributionServer) pinReadTS(ts uint64) *kv.ActiveTimestampToken { return s.readTracker.Pin(ts) } -<<<<<<< HEAD -======= func (s *DistributionServer) observeFilePinnedHotspotIfNeeded(rawSplitKey []byte, splitKey []byte, err error) { if s == nil || s.fsObserver == nil || bytes.Equal(rawSplitKey, splitKey) || !isSplitBoundaryError(err) { return @@ -903,7 +863,6 @@ func isSplitBoundaryError(err error) bool { strings.Contains(err.Error(), errDistributionSplitKeyAtBoundary.Error()) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func releaseReadPin(token *kv.ActiveTimestampToken) { if token == nil { return @@ -947,21 +906,13 @@ func (s *DistributionServer) saveSplitResultViaCoordinator( if err != nil { return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "build split mutations: %v", err) } -<<<<<<< HEAD - if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ -======= resp, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ ->>>>>>> origin/design/hotspot-split-m2-promotion-complete Elems: ops, IsTxn: true, StartTS: readTS, ReadKeys: readKeys, -<<<<<<< HEAD - }); err != nil { -======= }) if err != nil { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if errors.Is(err, store.ErrWriteConflict) { return distribution.CatalogSnapshot{}, grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) } @@ -1893,19 +1844,6 @@ func toProtoRoute(route distribution.Route) *pb.RouteDescriptor { } } -func toProtoRoute(route distribution.Route) *pb.RouteDescriptor { - return &pb.RouteDescriptor{ - RouteId: route.RouteID, - Start: distribution.CloneBytes(route.Start), - End: distribution.CloneBytes(route.End), - RaftGroupId: route.GroupID, - State: toProtoRouteState(route.State), - StagedVisibilityActive: route.StagedVisibilityActive, - MigrationJobId: route.MigrationJobID, - MinWriteTsExclusive: route.MinWriteTSExclusive, - } -} - func toProtoRouteState(state distribution.RouteState) pb.RouteState { switch state { case distribution.RouteStateActive: diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index e87b7e41f..f266e8047 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -3,20 +3,14 @@ package adapter import ( "bytes" "context" -<<<<<<< HEAD -======= "encoding/binary" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete "errors" "io" "testing" "time" "github.com/bootjp/elastickv/distribution" -<<<<<<< HEAD -======= "github.com/bootjp/elastickv/internal/fskeys" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" @@ -51,8 +45,6 @@ func TestDistributionServerGetRoute_HitAndMiss(t *testing.T) { require.Nil(t, miss.End) } -<<<<<<< HEAD -======= func TestDistributionServerGetRoute_NormalizesFilesystemChunkKeys(t *testing.T) { t.Parallel() @@ -72,7 +64,6 @@ func TestDistributionServerGetRoute_NormalizesFilesystemChunkKeys(t *testing.T) require.Equal(t, uint64(2), resp.RaftGroupId) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestDistributionServerRouteReadsHonorStartupGate(t *testing.T) { t.Parallel() @@ -2519,23 +2510,6 @@ func (s *splitPromotionClientStub) PromoteStagedVersions( } type distributionCoordinatorStub struct { -<<<<<<< HEAD - store store.MVCCStore - leader bool - dispatchErr error - nextTS uint64 - timestampNext uint64 - timestampErr error - timestampCalls int - lastStartTS uint64 - lastCommitTS uint64 - lastReadKeys [][]byte - beforeApply func(context.Context, store.MVCCStore) error - afterDispatch func(context.Context, store.MVCCStore, uint64) error - asyncApplyDone chan error - asyncApplyDelay time.Duration - dispatchCalls int -======= store store.MVCCStore leader bool clock *kv.HLC @@ -2553,7 +2527,6 @@ type distributionCoordinatorStub struct { asyncApplyDone chan error asyncApplyDelay time.Duration dispatchCalls int ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } func newDistributionCoordinatorStub(st store.MVCCStore, leader bool) *distributionCoordinatorStub { @@ -2569,10 +2542,7 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope return nil, err } s.dispatchCalls++ -<<<<<<< HEAD -======= s.lastRequestedCommitTS = reqs.CommitTS ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if s.dispatchErr != nil { return nil, s.dispatchErr } @@ -2619,12 +2589,9 @@ func (s *distributionCoordinatorStub) validateDispatch(reqs *kv.OperationGroup[k func (s *distributionCoordinatorStub) nextTimestamps(startTS uint64, requestedCommitTS uint64) (uint64, uint64) { if requestedCommitTS != 0 { -<<<<<<< HEAD -======= if s.clock != nil { s.clock.Observe(requestedCommitTS) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if s.nextTS <= requestedCommitTS { s.nextTS = requestedCommitTS + 1 } @@ -2694,11 +2661,7 @@ func requireReadKeysNotContain(t *testing.T, readKeys [][]byte, want []byte) { } } -<<<<<<< HEAD -func coordinatorStubMutations(elems []*kv.Elem[kv.OP]) ([]*store.KVPairMutation, error) { -======= func coordinatorStubMutations(elems []*kv.Elem[kv.OP], commitTS uint64) ([]*store.KVPairMutation, error) { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete mutations := make([]*store.KVPairMutation, 0, len(elems)) for _, elem := range elems { mutation, err := coordinatorStubMutation(elem, commitTS) @@ -2791,26 +2754,6 @@ func (s *distributionCoordinatorStub) NextAfter(_ context.Context, min uint64) ( return next, nil } -func (s *distributionCoordinatorStub) Next(ctx context.Context) (uint64, error) { - return s.NextAfter(ctx, 0) -} - -func (s *distributionCoordinatorStub) NextAfter(_ context.Context, min uint64) (uint64, error) { - s.timestampCalls++ - if s.timestampErr != nil { - return 0, s.timestampErr - } - next := s.timestampNext - if next == 0 { - next = s.store.LastCommitTS() + 1 - } - if next <= min { - next = min + 1 - } - s.timestampNext = next + 1 - return next, nil -} - func (s *distributionCoordinatorStub) LinearizableRead(_ context.Context) (uint64, error) { return 0, nil } diff --git a/adapter/internal.go b/adapter/internal.go index 6be6fdf9c..01a74dc43 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -25,12 +25,9 @@ const ( // MigrationPromoteOpcodeEnv enables the internal staged promotion opcode // after every voter is running a compatible build. MigrationPromoteOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE" -<<<<<<< HEAD -======= // MigrationCleanupOpcodeEnv enables the internal migration cleanup opcode // after every voter is running a compatible build. MigrationCleanupOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_CLEANUP_OPCODE" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete ) type InternalOption func(*Internal) @@ -65,15 +62,12 @@ func WithInternalMigrationPromoteGate(gate func(context.Context) error) Internal } } -<<<<<<< HEAD -======= func WithInternalMigrationCleanupGate(gate func(context.Context) error) InternalOption { return func(i *Internal) { i.migrationCleanupGate = gate } } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func WithInternalRouteEngine(engine *distribution.Engine) InternalOption { return func(i *Internal) { i.routeEngine = engine @@ -101,10 +95,7 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c relay: relay, migrationImportGate: defaultMigrationImportGate, migrationPromoteGate: defaultMigrationPromoteGate, -<<<<<<< HEAD -======= migrationCleanupGate: defaultMigrationCleanupGate, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } for _, opt := range opts { opt(i) @@ -122,10 +113,7 @@ type Internal struct { migrationProposer raftengine.Proposer migrationImportGate func(context.Context) error migrationPromoteGate func(context.Context) error -<<<<<<< HEAD -======= migrationCleanupGate func(context.Context) error ->>>>>>> origin/design/hotspot-split-m2-promotion-complete routeEngine *distribution.Engine readTracker *kv.ActiveTimestampTracker migrationExportGroupID uint64 @@ -330,574 +318,6 @@ func (i *Internal) ExportRangeVersions(req *pb.ExportRangeVersionsRequest, strea } func (i *Internal) validateExportRangeVersionsRequest(req *pb.ExportRangeVersionsRequest) error { -<<<<<<< HEAD - if req == nil { - return errors.WithStack(status.Error(codes.InvalidArgument, "export range versions request is nil")) - } - if i.store == nil { - return errors.WithStack(status.Error(codes.FailedPrecondition, "migration export store is not configured")) - } - if req.GetMaxCommitTs() == 0 { - return errors.WithStack(status.Error(codes.InvalidArgument, "migration export max_commit_ts is required")) - } - if req.GetKeyFamily() == 0 { - return errors.WithStack(status.Error(codes.InvalidArgument, "migration export key_family is required")) - } - if exportRangeVersionsRequestFullyUnbounded(req) { - return errors.WithStack(status.Error(codes.InvalidArgument, "migration export requires a raw or route bound")) - } - return nil -} - -func exportRangeVersionsRequestFullyUnbounded(req *pb.ExportRangeVersionsRequest) bool { - return len(req.GetRangeStart()) == 0 && - len(req.GetRangeEnd()) == 0 && - len(req.GetRouteStart()) == 0 && - len(req.GetRouteEnd()) == 0 -} - -func (i *Internal) streamExportRangeVersions(req *pb.ExportRangeVersionsRequest, stream pb.Internal_ExportRangeVersionsServer) error { - opts := i.exportRangeVersionsOptions(req) - for { - result, err := i.store.ExportVersions(stream.Context(), opts) - if err != nil { - return errors.WithStack(err) - } - if !result.Done && bytes.Equal(opts.Cursor, result.NextCursor) { - return errors.WithStack(status.Error(codes.Internal, "migration export cursor did not progress")) - } - if err := stream.Send(&pb.ExportRangeVersionsResponse{ - Versions: protoMVCCVersionsFromStore(result.Versions), - NextCursor: result.NextCursor, - Done: result.Done, - }); err != nil { - return errors.WithStack(err) - } - if result.Done { - return nil - } - opts.Cursor = result.NextCursor - } -} - -func (i *Internal) ImportRangeVersions(ctx context.Context, req *pb.ImportRangeVersionsRequest) (*pb.ImportRangeVersionsResponse, error) { - if req == nil { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "import range versions request is nil")) - } - if err := validateImportRangeVersionsRequest(req); err != nil { - return nil, err - } - if i.migrationProposer == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration import proposer is not configured")) - } - if err := i.verifyInternalLeader(ctx); err != nil { - return nil, err - } - if err := i.verifyMigrationImportEnabled(ctx); err != nil { - return nil, err - } - result, err := i.proposeMigrationImport(ctx, req) - if err != nil { - return nil, errors.WithStack(err) - } - if i.clock != nil && result.MaxImportedTS > 0 { - i.clock.Observe(result.MaxImportedTS) - } - return &pb.ImportRangeVersionsResponse{AckedCursor: result.AckedCursor}, nil -} - -func validateImportRangeVersionsRequest(req *pb.ImportRangeVersionsRequest) error { - if req.GetJobId() == 0 { - return errors.WithStack(status.Error(codes.InvalidArgument, "import range versions job_id is required")) - } - if req.GetBracketId() == 0 { - return errors.WithStack(status.Error(codes.InvalidArgument, "import range versions bracket_id is required")) - } - return nil -} - -func (i *Internal) verifyMigrationImportEnabled(ctx context.Context) error { - if i.migrationImportGate == nil { - return nil - } - if err := i.migrationImportGate(ctx); err != nil { - return errors.WithStack(err) - } - return nil -} - -func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteStagedVersionsRequest) (*pb.PromoteStagedVersionsResponse, error) { - if req == nil { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "promote staged versions request is nil")) - } - if req.GetJobId() == 0 { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "promote staged versions job_id is required")) - } - if i.migrationProposer == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote proposer is not configured")) - } - if err := i.verifyInternalLeader(ctx); err != nil { - return nil, err - } - if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { - return nil, err - } - if err := validatePromoteStagedVersionsRequest(req); err != nil { - return nil, errors.WithStack(err) - } - result, err := i.proposeMigrationPromote(ctx, req) - if err != nil { - return nil, errors.WithStack(err) - } - if i.clock != nil && result.MaxPromotedTS > 0 { - i.clock.Observe(result.MaxPromotedTS) - } - return &pb.PromoteStagedVersionsResponse{ - NextCursor: result.NextCursor, - Done: result.Done, - PromotedRows: result.PromotedRows, - MaxPromotedTs: result.MaxPromotedTS, - }, nil -} - -func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) { - if err := validateTargetStagedReadinessRequest(req); err != nil { - return nil, err - } - if i.migrationProposer == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "target staged readiness proposer is not configured")) - } - if err := i.verifyInternalLeader(ctx); err != nil { - return nil, err - } - if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { - return nil, err - } - if err := i.proposeTargetStagedReadiness(ctx, req); err != nil { - return nil, errors.WithStack(err) - } - return &pb.TargetStagedReadinessResponse{MinAdmittedTs: i.migrationMinAdmittedTS(ctx, req.GetJobId())}, nil -} - -func (i *Internal) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest) (*pb.CleanupMigrationResponse, error) { - if err := validateCleanupMigrationRequest(req); err != nil { - return nil, err - } - if i.migrationProposer == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration cleanup proposer is not configured")) - } - if err := i.verifyInternalLeader(ctx); err != nil { - return nil, err - } - if err := i.verifyMigrationPromoteEnabled(ctx); err != nil { - return nil, err - } - result, err := i.proposeMigrationCleanup(ctx, req) - if err != nil { - return nil, errors.WithStack(err) - } - return &pb.CleanupMigrationResponse{ - NextCursor: result.NextCursor, - Done: result.Done, - DeletedRows: result.DeletedRows, - DeletedBytes: result.DeletedBytes, - ScannedBytes: result.ScannedBytes, - }, nil -} - -func validateCleanupMigrationRequest(req *pb.CleanupMigrationRequest) error { - switch { - case req == nil: - return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup request is nil")) - case req.GetJobId() == 0: - return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup job_id is required")) - case req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && - req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA: - return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup mode is invalid")) - case req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && req.GetKeyFamily() == 0: - return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup key_family is required")) - } - return nil -} - -func (i *Internal) migrationMinAdmittedTS(ctx context.Context, jobID uint64) uint64 { - reader, ok := i.store.(store.MigrationTargetReadinessReader) - if !ok { - return 0 - } - states, err := reader.MigrationTargetReadinessStates(ctx) - if err != nil { - return 0 - } - for _, state := range states { - if state.JobID == jobID { - return state.MinAdmittedTS - } - } - return 0 -} - -func validateTargetStagedReadinessRequest(req *pb.TargetStagedReadinessRequest) error { - switch { - case req == nil: - return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness request is nil")) - case req.GetJobId() == 0: - return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness job_id is required")) - case req.GetMigrationJobId() == 0: - return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness migration_job_id is required")) - case req.GetArmed() && req.GetMinWriteTsExclusive() == 0: - return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness min_write_ts_exclusive is required when armed")) - } - return validateSourceMigrationControlRequest(req) -} - -func validateSourceMigrationControlRequest(req *pb.TargetStagedReadinessRequest) error { - if req.GetSourceReadFence() && !req.GetSourceWriteFence() { - return errors.WithStack(status.Error(codes.InvalidArgument, "source read fence requires source write fence")) - } - if (req.GetSourceWriteFence() || req.GetSourceReadFence()) && req.GetRetentionPinTs() == 0 { - return errors.WithStack(status.Error(codes.InvalidArgument, "source migration control retention_pin_ts is required")) - } - return nil -} - -func (i *Internal) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest) (*pb.ProbeMigrationLocksResponse, error) { - if req == nil || len(req.GetRouteStart()) == 0 { - return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration lock probe route_start is required")) - } - if i.store == nil { - return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration lock probe store is not configured")) - } - if err := i.verifyInternalLeaderApplied(ctx); err != nil { - return nil, err - } - locks, err := kv.PendingTxnLocksInRoute(ctx, i.store, req.GetRouteStart(), req.GetRouteEnd(), ^uint64(0), int(req.GetLimit())) - if err != nil { - return nil, errors.WithStack(err) - } - return &pb.ProbeMigrationLocksResponse{PendingCount: uint32(len(locks))}, nil //nolint:gosec // result is bounded by uint32 wire limit -} - -func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { - if i.migrationPromoteGate == nil { - return nil - } - if err := i.migrationPromoteGate(ctx); err != nil { - return errors.WithStack(err) - } - return nil -} - -func validatePromoteStagedVersionsRequest(req *pb.PromoteStagedVersionsRequest) error { - prefix := distribution.MigrationStagedDataKeyPrefix(req.GetJobId()) - if err := store.ValidatePromotionCursorForRange(req.GetCursor(), prefix, store.PrefixScanEnd(prefix)); err != nil { - if errors.Is(err, store.ErrInvalidExportCursor) { - return errors.WithStack(status.Error(codes.InvalidArgument, store.ErrInvalidExportCursor.Error())) - } - return errors.WithStack(status.Errorf(codes.Internal, "validate promote cursor: %v", err)) - } - return nil -} - -func defaultMigrationImportGate(context.Context) error { - if migrationImportOpcodeEnabledFromEnv() { - return nil - } - return errors.WithStack(status.Error(codes.FailedPrecondition, "migration import opcode is disabled; enable after every voter is running a build that supports migration import")) -} - -func migrationImportOpcodeEnabledFromEnv() bool { - return MigrationImportOpcodeEnabledFromEnv() -} - -func defaultMigrationPromoteGate(context.Context) error { - if migrationPromoteOpcodeEnabledFromEnv() { - return nil - } - return errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote opcode is disabled; enable after every voter is running a build that supports migration promotion")) -} - -func migrationPromoteOpcodeEnabledFromEnv() bool { - return MigrationPromoteOpcodeEnabledFromEnv() -} - -// MigrationImportOpcodeEnabledFromEnv reports whether the staged import opcode -// is enabled for this process. -func MigrationImportOpcodeEnabledFromEnv() bool { - return envFlagEnabled(MigrationImportOpcodeEnv) -} - -// MigrationPromoteOpcodeEnabledFromEnv reports whether the staged promotion -// opcode is enabled for this process. -func MigrationPromoteOpcodeEnabledFromEnv() bool { - return envFlagEnabled(MigrationPromoteOpcodeEnv) -} - -func envFlagEnabled(name string) bool { - switch strings.ToLower(strings.TrimSpace(os.Getenv(name))) { - case "1", "true", "yes", "on": - return true - default: - return false - } -} - -func (i *Internal) verifyInternalLeader(ctx context.Context) error { - if i.leader == nil { - return errors.WithStack(ErrNotLeader) - } - if i.leader.State() != raftengine.StateLeader { - return errors.WithStack(ErrNotLeader) - } - if err := i.leader.VerifyLeader(ctx); err != nil { - return errors.WithStack(ErrNotLeader) - } - return nil -} - -func (i *Internal) verifyInternalLeaderApplied(ctx context.Context) error { - if i.leader == nil { - return errors.WithStack(ErrNotLeader) - } - if i.leader.State() != raftengine.StateLeader { - return errors.WithStack(ErrNotLeader) - } - _, err := i.leader.LinearizableRead(ctx) - return errors.WithStack(err) -} - -func (i *Internal) proposeMigrationImport(ctx context.Context, req *pb.ImportRangeVersionsRequest) (store.ImportVersionsResult, error) { - cmd, err := kv.MarshalMigrationImportCommand(req) - if err != nil { - return store.ImportVersionsResult{}, errors.WithStack(err) - } - resp, err := i.proposeMigrationCommand(ctx, cmd, "migration import") - if err != nil { - return store.ImportVersionsResult{}, errors.WithStack(err) - } - switch resp := resp.(type) { - case store.ImportVersionsResult: - return resp, nil - case *store.ImportVersionsResult: - if resp == nil { - return store.ImportVersionsResult{}, errors.New("migration import apply returned nil result") - } - return *resp, nil - case error: - return store.ImportVersionsResult{}, errors.WithStack(resp) - default: - return store.ImportVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration import apply response type %T", resp)) - } -} - -func (i *Internal) proposeMigrationPromote(ctx context.Context, req *pb.PromoteStagedVersionsRequest) (store.PromoteVersionsResult, error) { - cmd, err := kv.MarshalMigrationPromoteCommand(req) - if err != nil { - return store.PromoteVersionsResult{}, errors.WithStack(err) - } - resp, err := i.proposeMigrationCommand(ctx, cmd, "migration promote") - if err != nil { - return store.PromoteVersionsResult{}, errors.WithStack(err) - } - switch resp := resp.(type) { - case store.PromoteVersionsResult: - return resp, nil - case *store.PromoteVersionsResult: - if resp == nil { - return store.PromoteVersionsResult{}, errors.New("migration promote apply returned nil result") - } - return *resp, nil - case error: - return store.PromoteVersionsResult{}, errors.WithStack(resp) - default: - return store.PromoteVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration promote apply response type %T", resp)) - } -} - -func (i *Internal) proposeMigrationCleanup(ctx context.Context, req *pb.CleanupMigrationRequest) (store.CleanupVersionsResult, error) { - cmd, err := kv.MarshalMigrationCleanupCommand(req) - if err != nil { - return store.CleanupVersionsResult{}, errors.WithStack(err) - } - resp, err := i.proposeMigrationCommand(ctx, cmd, "migration cleanup") - if err != nil { - return store.CleanupVersionsResult{}, errors.WithStack(err) - } - if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA && resp == nil { - return store.CleanupVersionsResult{Done: true}, nil - } - switch resp := resp.(type) { - case store.CleanupVersionsResult: - return resp, nil - case *store.CleanupVersionsResult: - if resp == nil { - return store.CleanupVersionsResult{}, errors.New("migration cleanup apply returned nil result") - } - return *resp, nil - case error: - return store.CleanupVersionsResult{}, errors.WithStack(resp) - default: - return store.CleanupVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration cleanup apply response type %T", resp)) - } -} - -func (i *Internal) proposeTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) error { - cmd, err := kv.MarshalTargetStagedReadinessCommand(req) - if err != nil { - return errors.WithStack(err) - } - resp, err := i.proposeMigrationCommand(ctx, cmd, "target staged readiness") - if err != nil { - return errors.WithStack(err) - } - if resp == nil { - return nil - } - if err, ok := resp.(error); ok { - return errors.WithStack(err) - } - return errors.WithStack(errors.Newf("unexpected target readiness apply response type %T", resp)) -} - -func (i *Internal) proposeMigrationCommand(ctx context.Context, cmd []byte, label string) (any, error) { - result, err := i.migrationProposer.Propose(ctx, cmd) - if err != nil { - return nil, errors.WithStack(err) - } - if result == nil { - return nil, errors.WithStack(errors.Newf("%s proposal returned nil result", label)) - } - return result.Response, nil -} - -func (i *Internal) exportRangeVersionsOptions(req *pb.ExportRangeVersionsRequest) store.ExportVersionsOptions { - chunkBytes := uint64(req.GetChunkBytes()) - if chunkBytes == 0 { - chunkBytes = defaultMigrationExportChunkBytes - } - maxScannedBytes := req.GetMaxScannedBytes() - if maxScannedBytes == 0 { - maxScannedBytes = chunkBytes * defaultMigrationExportScanFactor - } - opts := store.ExportVersionsOptions{ - StartKey: req.GetRangeStart(), - EndKey: req.GetRangeEnd(), - MinCommitTSExclusive: req.GetMinCommitTs(), - MaxCommitTSInclusive: req.GetMaxCommitTs(), - Cursor: req.GetCursor(), - MaxVersions: defaultMigrationExportMaxVersions, - MaxBytes: chunkBytes, - MaxScannedBytes: maxScannedBytes, - KeyFamily: req.GetKeyFamily(), - AcceptKey: i.migrationExportFilter(req), - } - return opts -} - -func (i *Internal) migrationExportFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool { - routeFilter := i.migrationExportRouteFilter(req) - if migrationFamilyRequiresDecodedS3(req.GetKeyFamily()) { - routeFilter = decodedS3BucketRouteFilter(req.GetKeyFamily(), req.GetRouteStart(), req.GetRouteEnd()) - } - excludeKnownInternal := req.GetExcludeKnownInternal() || req.GetKeyFamily() == distribution.MigrationFamilyUser - bracket := distribution.MigrationBracket{ - Family: req.GetKeyFamily(), - Start: bytes.Clone(req.GetRangeStart()), - End: bytes.Clone(req.GetRangeEnd()), - ExcludeKnownInternal: excludeKnownInternal, - ExcludePrefixes: cloneByteSlices(req.GetExcludePrefixes()), - } - return func(rawKey []byte) bool { - return bracket.ContainsRawKey(rawKey) && routeFilter(rawKey) - } -} - -func (i *Internal) migrationExportRouteFilter(req *pb.ExportRangeVersionsRequest) func([]byte) bool { - if i != nil && i.migrationExportGroupID != 0 && i.migrationExportResolver != nil { - return kv.RouteKeyFilterForGroup(req.GetRouteStart(), req.GetRouteEnd(), i.migrationExportGroupID, i.migrationExportResolver) - } - return kv.RouteKeyFilter(req.GetRouteStart(), req.GetRouteEnd()) -} - -func migrationFamilyRequiresDecodedS3(family uint32) bool { - return family == distribution.MigrationFamilyS3BucketMeta || - family == distribution.MigrationFamilyS3BucketGeneration -} - -func decodedS3BucketRouteFilter(family uint32, routeStart, routeEnd []byte) func([]byte) bool { - allowRawRouteMatch := !s3BucketRouteBounds(routeStart, routeEnd) - return func(rawKey []byte) bool { - bucket, ok := decodedS3BucketName(family, rawKey) - if !ok { - return false - } - if allowRawRouteMatch && kv.RouteKeyFilter(routeStart, routeEnd)(rawKey) { - return true - } - return decodedS3BucketRouteIntersects(bucket, routeStart, routeEnd) - } -} - -func s3BucketRouteBounds(routeStart, routeEnd []byte) bool { - return bytes.HasPrefix(routeStart, []byte(s3keys.RoutePrefix)) || - bytes.HasPrefix(routeEnd, []byte(s3keys.RoutePrefix)) -} - -func decodedS3BucketRouteIntersects(bucket string, routeStart, routeEnd []byte) bool { - bucketRouteStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) - return rangesIntersect(routeStart, routeEnd, bucketRouteStart, prefixScanEnd(bucketRouteStart)) -} - -func rangesIntersect(aStart, aEnd, bStart, bEnd []byte) bool { - if len(aEnd) > 0 && bytes.Compare(aEnd, bStart) <= 0 { - return false - } - if len(bEnd) > 0 && bytes.Compare(bEnd, aStart) <= 0 { - return false - } - return true -} - -func decodedS3BucketName(family uint32, rawKey []byte) (string, bool) { - switch family { - case distribution.MigrationFamilyS3BucketMeta: - return s3keys.ParseBucketMetaKey(rawKey) - case distribution.MigrationFamilyS3BucketGeneration: - return s3keys.ParseBucketGenerationKey(rawKey) - default: - return "", false - } -} - -func cloneByteSlices(in [][]byte) [][]byte { - if len(in) == 0 { - return nil - } - out := make([][]byte, len(in)) - for i := range in { - out[i] = bytes.Clone(in[i]) - } - return out -} - -func protoMVCCVersionsFromStore(in []store.MVCCVersion) []*pb.MVCCVersion { - out := make([]*pb.MVCCVersion, 0, len(in)) - for _, version := range in { - out = append(out, &pb.MVCCVersion{ - Key: bytes.Clone(version.Key), - CommitTs: version.CommitTS, - Tombstone: version.Tombstone, - Value: bytes.Clone(version.Value), - KeyFamily: version.KeyFamily, - ExpireAt: version.ExpireAt, - }) - } - return out -} - -func (i *Internal) stampTimestamps(ctx context.Context, req *pb.ForwardRequest) error { -======= ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if req == nil { return errors.WithStack(status.Error(codes.InvalidArgument, "export range versions request is nil")) } @@ -1629,12 +1049,6 @@ func routeWriteTimestampFloorApplies(route distribution.Route, mut *pb.Mutation, start, end := kv.RoutePrefixRange(mut.GetKey()) return rangesIntersect(route.Start, route.End, start, end) } -<<<<<<< HEAD - return kv.RouteKeyFilter(route.Start, route.End)(mut.GetKey()) -} - -func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) error { -======= if start, end, ok := forwardedS3BucketAuxiliaryRouteRange(mut.GetKey()); ok { return rangesIntersect(route.Start, route.End, start, end) } @@ -1654,7 +1068,6 @@ func forwardedS3BucketAuxiliaryRouteRange(key []byte) ([]byte, []byte, bool) { } func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) (uint64, error) { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete startTS := forwardedTxnStartTS(reqs) if startTS == 0 { ts, err := i.nextTimestamp(ctx, "stampTxnTimestamps startTS") @@ -1674,12 +1087,6 @@ func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) ( } } -<<<<<<< HEAD - if err := i.fillForwardedTxnCommitTS(ctx, reqs, startTS); err != nil { - return err - } - return i.rejectWriteTimestampFloorTxnRequests(reqs) -======= commitTS, err := i.fillForwardedTxnCommitTS(ctx, reqs, startTS) if err != nil { return 0, err @@ -1688,7 +1095,6 @@ func (i *Internal) stampTxnTimestamps(ctx context.Context, reqs []*pb.Request) ( return 0, err } return commitTS, nil ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } func forwardedTxnStartTS(reqs []*pb.Request) uint64 { @@ -1794,22 +1200,6 @@ func (i *Internal) rejectWriteTimestampFloorTxnRequests(reqs []*pb.Request) erro return nil } -<<<<<<< HEAD -func (i *Internal) rejectWriteTimestampFloorTxnRequests(reqs []*pb.Request) error { - for _, r := range reqs { - commitTS, ok := forwardedTxnRequestCommitTS(r) - if !ok { - continue - } - if err := i.rejectWriteTimestampFloorMutations(r.Mutations, commitTS); err != nil { - return err - } - } - return nil -} - -======= ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func forwardedTxnRequestCommitTS(r *pb.Request) (uint64, bool) { if r == nil || (r.Phase != pb.Phase_COMMIT && r.Phase != pb.Phase_NONE) { return 0, false diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index 1bea12210..2f3d495c3 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -147,8 +147,6 @@ func TestInternalExportRangeVersionsUsesStoreAndRouteFilter(t *testing.T) { }, stream.responses[0].GetVersions()) } -<<<<<<< HEAD -======= func TestInternalExportRangeVersionsUsesValueAwareLegacyListDeltaRouteFilter(t *testing.T) { t.Parallel() @@ -177,7 +175,6 @@ func TestInternalExportRangeVersionsUsesValueAwareLegacyListDeltaRouteFilter(t * }, stream.responses[0].GetVersions()) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestInternalExportRangeVersionsUsesAppliedReadFence(t *testing.T) { t.Parallel() @@ -762,8 +759,6 @@ func TestInternalApplyTargetStagedReadinessRejectsArmedZeroMinWriteTS(t *testing require.Equal(t, codes.InvalidArgument, status.Code(err)) require.Equal(t, uint64(0), proposer.calls) } -<<<<<<< HEAD -======= func TestInternalCleanupMigrationRejectsWhenOpcodeGateClosed(t *testing.T) { t.Parallel() @@ -785,4 +780,3 @@ func TestInternalCleanupMigrationRejectsWhenOpcodeGateClosed(t *testing.T) { require.Equal(t, codes.FailedPrecondition, status.Code(err)) require.Zero(t, proposer.calls) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete diff --git a/adapter/internal_test.go b/adapter/internal_test.go index ab8cd99b0..ec2b9eb8b 100644 --- a/adapter/internal_test.go +++ b/adapter/internal_test.go @@ -6,10 +6,7 @@ import ( "testing" "github.com/bootjp/elastickv/distribution" -<<<<<<< HEAD -======= "github.com/bootjp/elastickv/internal/s3keys" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" @@ -433,158 +430,3 @@ func TestStampTxnTimestampsIgnoresMetadataForRouteWriteFloor(t *testing.T) { _, err := i.stampTxnTimestamps(context.Background(), reqs) require.NoError(t, err) } - -func TestStampRawTimestampsRejectsRouteWriteFloor(t *testing.T) { - t.Parallel() - - engine := distribution.NewEngine() - require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, - Routes: []distribution.RouteDescriptor{{ - RouteID: 1, - Start: []byte(""), - End: nil, - GroupID: 1, - State: distribution.RouteStateActive, - MinWriteTSExclusive: 100, - }}, - })) - i := &Internal{ - tsAllocator: fixedInternalTimestampAllocator(100), - routeEngine: engine, - } - reqs := []*pb.Request{{ - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}}, - }} - - err := i.stampRawTimestamps(context.Background(), reqs) - require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) -} - -func TestStampRawTimestampsRejectsPreStampedRouteWriteFloor(t *testing.T) { - t.Parallel() - - engine := distribution.NewEngine() - require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, - Routes: []distribution.RouteDescriptor{{ - RouteID: 1, - Start: []byte(""), - End: nil, - GroupID: 1, - State: distribution.RouteStateActive, - MinWriteTSExclusive: 100, - }}, - })) - i := &Internal{routeEngine: engine} - reqs := []*pb.Request{{ - Ts: 100, - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}}, - }} - - err := i.stampRawTimestamps(context.Background(), reqs) - require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) -} - -func TestStampRawTimestampsRejectsPreStampedDelPrefixRouteWriteFloor(t *testing.T) { - t.Parallel() - - engine := distribution.NewEngine() - require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, - Routes: []distribution.RouteDescriptor{ - { - RouteID: 1, - Start: []byte(""), - End: []byte("m"), - GroupID: 1, - State: distribution.RouteStateActive, - MinWriteTSExclusive: 0, - }, - { - RouteID: 2, - Start: []byte("m"), - End: nil, - GroupID: 2, - State: distribution.RouteStateActive, - MinWriteTSExclusive: 100, - }, - }, - })) - i := &Internal{routeEngine: engine} - reqs := []*pb.Request{{ - Ts: 100, - Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: nil}}, - }} - - err := i.stampRawTimestamps(context.Background(), reqs) - require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) -} - -func TestStampTxnTimestampsRejectsRouteWriteFloor(t *testing.T) { - t.Parallel() - - engine := distribution.NewEngine() - require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, - Routes: []distribution.RouteDescriptor{{ - RouteID: 1, - Start: []byte(""), - End: nil, - GroupID: 1, - State: distribution.RouteStateActive, - MinWriteTSExclusive: 100, - }}, - })) - i := &Internal{routeEngine: engine} - reqs := []*pb.Request{{ - IsTxn: true, - Phase: pb.Phase_NONE, - Ts: 50, - Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("k"), CommitTS: 100})}, - {Op: pb.Op_PUT, Key: []byte("k"), Value: []byte("v")}, - }, - }} - - err := i.stampTxnTimestamps(context.Background(), reqs) - require.ErrorIs(t, err, kv.ErrRouteWriteTimestampTooLow) -} - -func TestStampTxnTimestampsIgnoresMetadataForRouteWriteFloor(t *testing.T) { - t.Parallel() - - engine := distribution.NewEngine() - require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, - Routes: []distribution.RouteDescriptor{ - { - RouteID: 1, - Start: []byte(""), - End: []byte("m"), - GroupID: 1, - State: distribution.RouteStateActive, - MinWriteTSExclusive: 100, - }, - { - RouteID: 2, - Start: []byte("m"), - End: nil, - GroupID: 2, - State: distribution.RouteStateActive, - }, - }, - })) - i := &Internal{routeEngine: engine} - reqs := []*pb.Request{{ - IsTxn: true, - Phase: pb.Phase_NONE, - Ts: 50, - Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(kv.TxnMetaPrefix), Value: kv.EncodeTxnMeta(kv.TxnMeta{PrimaryKey: []byte("z"), CommitTS: 100})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, - }, - }} - - require.NoError(t, i.stampTxnTimestamps(context.Background(), reqs)) -} diff --git a/adapter/redis_delta_compactor.go b/adapter/redis_delta_compactor.go index 0a601a16f..0e01d55b7 100644 --- a/adapter/redis_delta_compactor.go +++ b/adapter/redis_delta_compactor.go @@ -837,11 +837,7 @@ func (c *DeltaCompactor) legacyListHandler() collectionDeltaHandler { extractUserKey: store.ExtractLegacyListUserKeyFromDelta, acceptDeltaKV: isListMetaDeltaKV, deltaKeyPrefixFn: store.LegacyListMetaDeltaScanPrefix, -<<<<<<< HEAD - buildElems: c.buildListCompactElems, -======= buildElems: c.buildLegacyListCompactElems, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } } diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index be2f073c9..17600a4d7 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -1690,18 +1690,6 @@ func TestDeltaCompactor_UrgentCompactionPagination(t *testing.T) { // update before all delete elems have been applied under the race detector. c.compactUrgentKey(ctx, urgentCompactionRequest{typeName: "hash", userKey: userKey}) -<<<<<<< HEAD - readTS := st.LastCommitTS() - raw, err := st.GetAt(ctx, store.HashMetaKey(userKey), readTS) - require.NoError(t, err) - got, err := store.UnmarshalHashMeta(raw) - require.NoError(t, err) - require.Equal(t, int64(totalDeltasU64), got.Len, "all %d delta keys should be compacted into base meta", totalDeltasU64) - - // No delta keys should remain after pagination compaction. - prefix := store.HashMetaDeltaScanPrefix(userKey) - end := store.PrefixScanEnd(prefix) -======= runCtx, cancel := context.WithCancel(ctx) defer cancel() go func() { _ = c.Run(runCtx) }() @@ -1729,7 +1717,6 @@ func TestDeltaCompactor_UrgentCompactionPagination(t *testing.T) { // No delta keys should remain after pagination compaction. readTS := st.LastCommitTS() ->>>>>>> origin/design/hotspot-split-m2-promotion-complete remaining, err := st.ScanAt(ctx, prefix, end, int(totalDeltasU64)+1, readTS) require.NoError(t, err) require.Empty(t, remaining, "all delta keys must be deleted after urgent compaction") diff --git a/adapter/redis_list_dedup_test.go b/adapter/redis_list_dedup_test.go index 06ea2f6a9..93ab303d0 100644 --- a/adapter/redis_list_dedup_test.go +++ b/adapter/redis_list_dedup_test.go @@ -4,10 +4,7 @@ import ( "bytes" "context" "encoding/binary" -<<<<<<< HEAD -======= "errors" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete "testing" "github.com/bootjp/elastickv/kv" @@ -45,14 +42,11 @@ type dedupTestCoordinator struct { // WITHOUT applying — an ambiguous retryable error distinct from // WriteConflict, exercising the "advance pending.commitTS and retry" branch. txnLockedAtDispatch int -<<<<<<< HEAD -======= // wireWriteConflicts converts every typed write conflict this test // coordinator returns into the keyed gRPC status produced by leader // forwarding. It exercises adapter-side type restoration without changing // the underlying landed-vs-not-landed scenario. wireWriteConflicts bool ->>>>>>> origin/design/hotspot-split-m2-promotion-complete // routeFenceAtDispatch makes the named dispatch return ErrRouteWriteFenced // before the FSM dedup probe or apply. It is retryable but cannot be // treated as an ambiguous landing. @@ -301,11 +295,7 @@ func (c *dedupTestCoordinator) Dispatch(ctx context.Context, req *kv.OperationGr return resp, err } if err := c.preApplyError(n); err != nil { -<<<<<<< HEAD - return nil, err -======= return nil, c.maybeWireWriteConflict(req, err) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } resp, err := c.occAdapterCoordinator.Dispatch(ctx, req) if err != nil { @@ -325,8 +315,6 @@ func (c *dedupTestCoordinator) Dispatch(ctx context.Context, req *kv.OperationGr return resp, nil } -<<<<<<< HEAD -======= func (c *dedupTestCoordinator) maybeWireWriteConflict(req *kv.OperationGroup[kv.OP], err error) error { if !c.wireWriteConflicts || !errors.Is(err, store.ErrWriteConflict) { return err @@ -338,7 +326,6 @@ func (c *dedupTestCoordinator) maybeWireWriteConflict(req *kv.OperationGroup[kv. return status.Error(codes.Unknown, store.NewWriteConflictError(key).Error()) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func (c *dedupTestCoordinator) shouldRouteFence(dispatch int) bool { return dispatch == c.routeFenceAtDispatch } diff --git a/adapter/redis_retry.go b/adapter/redis_retry.go index df4b240e4..72289d4ea 100644 --- a/adapter/redis_retry.go +++ b/adapter/redis_retry.go @@ -44,14 +44,10 @@ var ( ) func isRetryableRedisTxnErr(err error) bool { -<<<<<<< HEAD - return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || errors.Is(err, kv.ErrRouteWriteFenced) -======= return errors.Is(err, store.ErrWriteConflict) || errors.Is(err, kv.ErrTxnLocked) || wireRedisTxnErrKind(err) == redisTxnWireErrLocked || errors.Is(err, kv.ErrRouteWriteFenced) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } func shouldPreserveRedisTxnAttempt(err error) bool { diff --git a/adapter/split_job_runner.go b/adapter/split_job_runner.go index 880c3da9f..17ed16d39 100644 --- a/adapter/split_job_runner.go +++ b/adapter/split_job_runner.go @@ -1188,11 +1188,7 @@ func (s *DistributionServer) ensureSplitJobCatalogFence( if err != nil { return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err } -<<<<<<< HEAD - left, right := splitCatalogRoutes(sourceRoute, job.SplitKey, leftID, rightID) -======= left, right := splitCatalogRoutes(sourceRoute, job.SplitKey, leftID, rightID, job.FenceTS) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete right.State = distribution.RouteStateWriteFenced snapshot, err = s.saveSplitResultViaCoordinator(ctx, snapshot.ReadTS, snapshot.Version, sourceRoute.RouteID, nil, left, right) if err != nil { diff --git a/distribution/engine.go b/distribution/engine.go index 61cfb50fa..e55b6dc46 100644 --- a/distribution/engine.go +++ b/distribution/engine.go @@ -421,27 +421,10 @@ func (e *Engine) GetIntersectingRoutesWithVersion(start, end []byte) ([]Route, u } // Route intersects with scan range result = append(result, cloneRoute(*r)) -<<<<<<< HEAD -======= } return result, e.catalogVersion } -func cloneRoute(r Route) Route { - return Route{ - RouteID: r.RouteID, - Start: CloneBytes(r.Start), - End: CloneBytes(r.End), - GroupID: r.GroupID, - State: r.State, - StagedVisibilityActive: r.StagedVisibilityActive, - MigrationJobID: r.MigrationJobID, - MinWriteTSExclusive: r.MinWriteTSExclusive, - Load: r.Load, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete - } -} - func cloneRoute(r Route) Route { return Route{ RouteID: r.RouteID, diff --git a/distribution/engine_test.go b/distribution/engine_test.go index 21ccd6648..836303b93 100644 --- a/distribution/engine_test.go +++ b/distribution/engine_test.go @@ -282,8 +282,6 @@ func assertStagedRouteMetadata(t *testing.T, route Route) { if !route.StagedVisibilityActive || route.MigrationJobID != 42 || route.MinWriteTSExclusive != 99 { t.Fatalf("staged route metadata was not preserved: %+v", route) } -<<<<<<< HEAD -======= } func TestEngineRouteLookupsReturnMatchingCatalogVersion(t *testing.T) { @@ -309,7 +307,6 @@ func TestEngineRouteLookupsReturnMatchingCatalogVersion(t *testing.T) { if len(routes) != 2 || version != 12 { t.Fatalf("unexpected atomic range lookup: routes=%+v version=%d", routes, version) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } func TestEngineApplySnapshot_RejectsOldVersion(t *testing.T) { diff --git a/docs/design/2026_02_18_partial_hotspot_shard_split.md b/docs/design/2026_02_18_partial_hotspot_shard_split.md index bd0bfca8e..ee8a0b5b8 100644 --- a/docs/design/2026_02_18_partial_hotspot_shard_split.md +++ b/docs/design/2026_02_18_partial_hotspot_shard_split.md @@ -293,7 +293,6 @@ Add RPCs: 2. Job phases: BACKFILL/FENCE/DELTA/CUTOVER 3. Manual split with target-group relocation -<<<<<<< HEAD Status: implemented. The production runner advances durable cross-group jobs through BACKFILL, FENCE, DELTA_COPY, CUTOVER, CLEANUP, and `DONE`; current-voter readiness and cleanup barriers protect leadership changes; and the deterministic @@ -301,14 +300,6 @@ Jepsen workload covers the split alongside leader-kill and partition packages. The completed M2 contract is recorded in [`2026_06_11_implemented_hotspot_split_milestone2_migration.md`](2026_06_11_implemented_hotspot_split_milestone2_migration.md) and its final runner/lifecycle slice landed in PR #1096. -======= -Status: partial. SplitJob catalog/codec, MVCC export/import primitives, staged -visibility, write-fence checks, target readiness, and target-promotion catalog -components exist in the M2 stack, but no production runner advances cross-group -jobs to `DONE` yet. M2 remains open in -[`2026_06_11_partial_hotspot_split_milestone2_migration.md`](2026_06_11_partial_hotspot_split_milestone2_migration.md) -until the cross-group acceptance criteria and Jepsen workload pass. ->>>>>>> origin/design/hotspot-split-m2-promotion-complete ### Milestone 3: Automation diff --git a/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md index e52f0da08..80b7716dd 100644 --- a/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md +++ b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md @@ -1,21 +1,13 @@ # Hotspot Shard Split — Milestone 2: Migration Plane -<<<<<<<< HEAD:docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md Status: Implemented -======== -Status: Partial ->>>>>>>> origin/design/hotspot-split-m2-promotion-complete:docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md Author: bootjp Date: 2026-06-11 Parent: [2026_02_18_partial_hotspot_shard_split.md](2026_02_18_partial_hotspot_shard_split.md). M1 (control plane) is as-built in [2026_02_18_implemented_hotspot_split_milestone1_pr.md](2026_02_18_implemented_hotspot_split_milestone1_pr.md). -<<<<<<<< HEAD:docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md Current implementation status: implemented. The M2 stack provides the durable SplitJob catalog, versioned route descriptors, resumable range export/import, staged/live visibility, source write and read fences, current-voter readiness and cleanup barriers, constant-time route cutover, incremental target promotion, bounded source/target cleanup, terminal `DONE` history, and the deterministic cross-group Jepsen workload. PR #1096 completed the production runner and lifecycle wiring. Automatic hotspot detection and split scheduling remain M3 scope; the broader route-shuffle and fault campaign remains M4 scope. -======== -Current implementation status: partial. The SplitJob catalog/codec, versioned route descriptor storage, range export/import primitives, staged-read visibility, write-fence checks, target readiness, and target-promotion catalog pieces have landed or are covered by the active M2 stack. The feature is not implemented end to end until a production runner advances cross-group jobs to `DONE`, `ListRoutes` exposes the moved child on its target group after cutover, and the Jepsen cross-group split workload passes the acceptance criteria in §13. ->>>>>>>> origin/design/hotspot-split-m2-promotion-complete:docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md ## 1. Background @@ -1291,7 +1283,6 @@ This is independent of the existing rolling-upgrade protocol for unrelated subsy ## 14. Lifecycle -<<<<<<<< HEAD:docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md This document is `*_implemented_*` because the M2 cross-group migration plane is complete as a central subsystem. The production runner now resumes durable jobs across leadership changes, drives every phase through `CLEANUP`, waits for current source and target voter proofs, publishes the target route, removes bounded migration state, and moves the job to `DONE` history. The deterministic Jepsen split workload exercises the cross-group operation alongside leader-kill and partition fault packages. The remaining work is intentionally outside M2's central scope: @@ -1299,11 +1290,5 @@ The remaining work is intentionally outside M2's central scope: - M3 owns automatic hotspot detection, target selection, and split scheduling. - M4 owns the broader route-shuffle nemesis matrix and production-scale fault campaigns. - Reverse migration after CUTOVER and concurrent migration jobs remain future extensions. -======== -This document is `*_partial_*` because M2-PR1 and later component slices have landed, but the cross-group migration plane is not complete end to end. - -- Track per-PR landing under §11. -- Rename to `*_implemented_*` only after the acceptance criteria in §13 pass: cross-group `StartSplitMigration` reaches `phase=DONE`, `ListRoutes` shows the target-group child after cutover, leader-kill recovery is automatic, and the Jepsen split workload is green. ->>>>>>>> origin/design/hotspot-split-m2-promotion-complete:docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md `git mv` is used so history follows. diff --git a/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md b/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md deleted file mode 100644 index e52f0da08..000000000 --- a/docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md +++ /dev/null @@ -1,1309 +0,0 @@ -# Hotspot Shard Split — Milestone 2: Migration Plane - -<<<<<<<< HEAD:docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md -Status: Implemented -======== -Status: Partial ->>>>>>>> origin/design/hotspot-split-m2-promotion-complete:docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md -Author: bootjp -Date: 2026-06-11 - -Parent: [2026_02_18_partial_hotspot_shard_split.md](2026_02_18_partial_hotspot_shard_split.md). -M1 (control plane) is as-built in [2026_02_18_implemented_hotspot_split_milestone1_pr.md](2026_02_18_implemented_hotspot_split_milestone1_pr.md). - -<<<<<<<< HEAD:docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md -Current implementation status: implemented. The M2 stack provides the durable SplitJob catalog, versioned route descriptors, resumable range export/import, staged/live visibility, source write and read fences, current-voter readiness and cleanup barriers, constant-time route cutover, incremental target promotion, bounded source/target cleanup, terminal `DONE` history, and the deterministic cross-group Jepsen workload. PR #1096 completed the production runner and lifecycle wiring. Automatic hotspot detection and split scheduling remain M3 scope; the broader route-shuffle and fault campaign remains M4 scope. -======== -Current implementation status: partial. The SplitJob catalog/codec, versioned route descriptor storage, range export/import primitives, staged-read visibility, write-fence checks, target readiness, and target-promotion catalog pieces have landed or are covered by the active M2 stack. The feature is not implemented end to end until a production runner advances cross-group jobs to `DONE`, `ListRoutes` exposes the moved child on its target group after cutover, and the Jepsen cross-group split workload passes the acceptance criteria in §13. ->>>>>>>> origin/design/hotspot-split-m2-promotion-complete:docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md - -## 1. Background - -M1 landed the control plane: - -- `distribution/catalog.go` — durable `RouteDescriptor` records in the default Raft group's MVCC state. -- `distribution/engine.go` — in-memory route cache + history ring (consumed by Composed-1 in `kv/fsm.go`). -- `distribution/watcher.go` — applies versioned catalog snapshots into the engine. -- `proto/distribution.proto` — `SplitRange` RPC; M1 implementation splits a range **inside the same Raft group** only (no data movement, no target field on the wire). -- `RouteState` already reserves `WRITE_FENCED`, `MIGRATING_SOURCE`, `MIGRATING_TARGET` for M2 (`distribution/catalog.go:51-58`). - -Composed-1 cross-group commit guard (PR #927) and the routeKey normalization fix (PR #932) close the apply-time hazards that an unsafe M2 cutover would trip. M2 builds on top of both. - -M2 does **not** ship hotspot detection or auto-split scheduling (M3). - -## 2. Goals and Non-Goals - -### 2.1 Goals - -1. `SplitRange` can move the right child of a split to a different Raft group, atomically with respect to readers and writers. -2. The migration is **resumable**: a leader change during BACKFILL / DELTA_COPY / CUTOVER recovers without operator intervention or data loss. -3. The FENCE phase rejects writes on the moving range with a retryable error code (so adapter callers retry transparently). -4. No write loss, no MVCC visibility regression, no Composed-1 violation at the cutover boundary. -5. The same-group `SplitRange` path from M1 keeps working byte-for-byte; cross-group migration uses the new `StartSplitMigration` RPC. - -### 2.2 Non-Goals - -1. Automatic hotspot detection / scheduler — M3. -2. Online range merge (inverse of split) — out of scope for M2. -3. Adding or removing Raft groups dynamically. Target group must already exist in `--raftGroups`. -4. Cross-region migration. M2 assumes all groups are reachable on the cluster gRPC mesh. -5. Per-tenant migration policy, throttling, or QoS. - -## 3. Data Model Additions - -### 3.1 SplitJob - -A split job is durable state for the migration. Persist under reserved keys in the default group, alongside the route catalog. - -Key layout: - -- `!dist|meta|next_job_id` — uint64, allocator for `job_id`. -- `!dist|job|` — encoded `SplitJob` record (in-flight + terminal until GC). -- `!dist|jobhist||` — moved here at GC time for short-term audit; periodically truncated. - -`SplitJob` fields: - -| Field | Type | Notes | -|---|---|---| -| `job_id` | uint64 | Allocated from `next_job_id`. | -| `source_route_id` | uint64 | Route being split. | -| `split_key` | bytes | Logical (routing-key) boundary. | -| `target_group_id` | uint64 | Where the right child lands; equals source group for same-group split. | -| `phase` | enum | `PLANNED` / `BACKFILL` / `FENCE` / `DELTA_COPY` / `CUTOVER` / `CLEANUP` / `DONE` / `FAILED` / `ABANDONING` / `ABANDONED`. | -| `retry_phase` | enum | Durable restart target used only while `phase == FAILED`. Set atomically when the migrator moves a job to `FAILED`, choosing the last safe idempotent phase to replay (`BACKFILL`, `FENCE`, `DELTA_COPY`, `CUTOVER`, or `CLEANUP`) from the phase that failed and the side effects already acknowledged. `CUTOVER` is used when target readiness / source read-fence guards are already `ARMED` but the catalog CAS has not durably committed; retry resumes the same CUTOVER attempt and must not clear those guards first. `CLEANUP` is used for post-CUTOVER promotion/source/target cleanup failures; retry resumes idempotent cleanup because `AbandonSplitJob` is rejected after CUTOVER. `NONE` outside `FAILED`. `RetrySplitJob` must CAS back to this field's value; it must not infer a phase from cursors, `last_error`, or wall-clock diagnostics after a restart. | -| `abandon_from_phase` | enum | Set atomically with `phase=ABANDONING` before any abandon cleanup side effect is removed. Records the live pre-CUTOVER phase (`BACKFILL`, `FENCE`, or `DELTA_COPY`) being abandoned for audit and for bounded cleanup selection; `NONE` otherwise. A restart that sees `ABANDONING` resumes abandon cleanup and must not retry `abandon_from_phase`. | -| `snapshot_ts` | uint64 | **Provisional** HLC commit-ts pinned at BACKFILL entry — the ts BACKFILL iterates at, not the DELTA_COPY lower bound. Selected by §6.1.0 step 1 after the source-group migration write tracker is armed: `min(HLC.Next(), snapshot_min_admitted_ts - 1)` over current in-flight prepares / tracker rows. The authoritative DELTA_COPY lower bound is `delta_floor`, finalised only after the fence (codex round-14 P1 line 405; codex round-15 P1 — fence before boundary). | -| `snapshot_min_admitted_ts` | uint64 | **Running** minimum `commit_ts` of migration-window writes admitted for the moving range — keyed on any point mutation / locked key whose `routeKey()` falls in `[routeStart, routeEnd)`, and on any range mutation whose route-key footprint intersects it. It covers raw requests (`handleRawRequest` at `r.Ts`, including `Op_DEL_PREFIX` range tombstones), one-phase txns (`handleOnePhaseTxnRequest` at `TxnMeta.CommitTS`), and prepared txns on **any** locked key (primary or secondary; codex round-16 P1). Sampled from the source-group migration write tracker armed before `snapshot_ts` selection (§6.1.0 step 1) plus the live §3.2a.0a `!txn|lock|` drain for backward-compat locks that predate the tracker. It is **not** computed from every historical committed MVCC version in the range; old retained history that BACKFILL already copied must not lower the floor. Monotone-down; persisted on each advance. Final once `post_fence_drain_completed = true` — no new raw/one-phase write, `DEL_PREFIX`, or prepare can land on any key in the moving range after the fence, and every pre-fence prepare has either resolved or aborted. `0` means no migration-window write was observed. | -| `write_tracker_armed` | bool | Default-group witness that the source group has durably applied `ArmMigrationWriteTracker(job_id, route_start, route_end)` before BACKFILL opens. The tracker records every later write whose point or range footprint routes into the moving range: raw writes and one-phase txns at apply time, and prepares at admission time so same-tick land-and-resolve cases that a polling drain could miss are still represented. A restart must not pick `snapshot_ts` or start BACKFILL until this is true. | -| `delta_floor` | uint64 | Authoritative DELTA_COPY lower bound, fixed at the §3.2a post-fence-drain → step-2 boundary as `min(snapshot_ts, snapshot_min_admitted_ts - 1)` (§6.1.0 step 3). DELTA_COPY exports `(delta_floor, fence_ts]`. Ensures every migration-window low-ts write with `commit_ts ≤ snapshot_ts` (raw point write, `DEL_PREFIX` range tombstone, one-phase, or prepared; caller-supplied / lagging-clock ts accepted while the source was still `Active`) satisfies `commit_ts > delta_floor` and is captured. Persisted so a DELTA_COPY resume uses the same lower bound. | -| `post_fence_drain_completed` | bool | True once §3.2a step 0b's post-fence intent-lock drain has returned empty (after step 1's voter-ACK barrier). Restart-safe: a migrator restart between step 0b and step 2 (`fence_ts` pin) knows whether to re-run the drain (closes codex round-14 P1 line 171). | -| `fence_ts` | uint64 | HLC commit-ts pinned at FENCE → DELTA_COPY entry. | -| `cutover_version` | uint64 | Expected catalog version that the CUTOVER write will produce (`current_catalog_version + 1` under the CAS input). Fixed and persisted before `cutover_read_fence_state = ARMING`, then verified as the produced version when the CUTOVER CAS lands. Used by §7.2 read-fence and §4.2 GC. | -| `cutover_read_fence_state` | enum | `NONE` / `ARMING` / `ARMED` / `CLEARING` restart witness for §7.2.2e. The migrator first persists `ARMING` on the default-group SplitJob **before** proposing/probing `ArmCutoverReadFence` on every current source voter, with the expected `cutover_version` already fixed. It CASes the state to `ARMED` only after `source_cutover_read_fence_ack_cursor` shows every current source voter has applied the pending read-fence barrier for the moving range and expected version. A restart at `ARMING` resumes the per-voter probes and then records `ARMED`; a restart at `ARMED` either completes the CUTOVER CAS with the same expected version or clears the source fence on abort. `CLEARING` is persisted before `ClearCutoverReadFence` so abandon/rollback also resumes idempotently. | -| `target_staged_readiness_state` | enum | `NONE` / `ARMING` / `ARMED` / `CLEARING` restart witness for §6.4 step 0. The migrator persists `ARMING` before proposing/probing `ApplyTargetStagedReadiness(job_id, route_start, route_end, expected_cutover_version, migration_job_id, min_write_ts_exclusive)` on every current target voter, records `ARMED` only after `target_staged_readiness_ack_cursor` shows every current target voter has applied the guard, and clears through `CLEARING` after the default-group promotion-complete witness or on pre-CUTOVER rollback. This makes any target voter that can later become leader fail closed instead of serving live-only reads while its catalog watcher is behind the CUTOVER descriptor. | -| `source_cutover_read_fence_ack_cursor` | bytes | Opaque per-voter ACK progress for the pre-CUTOVER source read-fence guard in §7.2.2e. Each ACK means that source voter has applied a source-group Raft entry carrying `ArmCutoverReadFence(job_id, route_start, route_end, expected_cutover_version, target_group_id)` and will fail closed for intersecting stale reads before touching MVCC. A single leader ACK, heartbeat watermark, or watcher-applied catalog version does not count. | -| `target_staged_readiness_ack_cursor` | bytes | Opaque per-voter ACK progress for the pre-CUTOVER target readiness guard in §6.4 step 0. Each ACK means that target voter has applied a target-group Raft entry carrying `ApplyTargetStagedReadiness` plus the route's durable `min_write_ts_exclusive` and will fail closed for intersecting target reads/writes until the matching descriptor is locally available. A single target leader ACK does not count. | -| `cursor` | bytes | Opaque (raw_key, commit_ts) export position for BACKFILL / DELTA_COPY resume — addresses an MVCC version, not a raw key, so a hot key with many committed versions can be chunked safely across batches (see §6.1.1). | -| `max_imported_ts` | uint64 | Default-group witness for the largest `commit_ts` ever included in a non-empty ack'd import batch for this job. Empty scan-progress chunks (§6.2) advance bracket cursors but do not change this value because they contain no timestamp. The target group also persists the same full HLC value locally under `!migstage|hlc_floor|` (§6.2.1) so restart/snapshot restore can re-observe the logical half before serving post-CUTOVER writes. | -| `target_promotion_done` | bool | Default-group witness that the target group has durably completed §6.4 promotion using its target-local `PromotionState`. This is **not** a cursor; promotion progress is owned by the target Raft group so a default-group crash cannot skip target-local staged rows. | -| `promotion_completed_ts` | uint64 | HLC commit-ts of the default-group CAS that cleared `staged_visibility_active` after the target reported promotion complete. Diagnostic / audit only; the job remains in CLEANUP until source/target cleanup ACKs are durable and the final CLEANUP → DONE history move lands. | -| `fence_catalog_version` | uint64 | Catalog version produced by the FENCE catalog write. It labels the `ApplyMigrationWriteFence(job_id, route_start, route_end, fence_catalog_version)` tuple and lets the source-group durable fence record prove which catalog transition it protects; it is **not** itself the ACK condition. Set at FENCE entry. | -| `fence_ack_cursor` | bytes | Opaque encoding of the per-voter ACK progress for §3.2a's fence-apply barrier. Each ACK means that voter has applied the source-group Raft entry that persists `!migfence|` for the exact `(route_start, route_end, fence_catalog_version)` tuple, optionally carrying the source Raft apply index/term for diagnostics. A watcher-only `catalog_version_applied >= fence_catalog_version` heartbeat is insufficient and must not advance this cursor. Restart-safe: a migrator restart resumes from the pending voter list and re-proposes/probes the durable fence record rather than re-polling completed voters. | -| `source_cutover_ack_cursor` | bytes | Opaque per-voter ACK progress for the successful CUTOVER route-removal barrier. Each ACK means that source voter has applied a source-group Raft entry such as `AcknowledgeCutoverRouteRemoval(job_id, cutover_version)` whose apply handler first proves the voter's local route snapshot has applied `cutover_version` and no longer contains the moved source route. Watcher-only heartbeat watermarks do not count. Source write fences, pending read fences, and retention pins cannot be disarmed until every current source voter has ACKed this proof. | -| `source_read_drain_cursor` | bytes | Opaque per-voter ACK progress for §7.2.4's active-read drain. Each ACK means the source voter has applied the cutover read-fence epoch for `cutover_version`, has stopped admitting new source reads for the moved interval, and has no active read pin whose normalized point/range footprint intersects the moved route, regardless of `read_ts`. The pin's `read_ts` remains an audit/logging field, not the drain predicate. Source data cleanup cannot delete versions until every current source voter has ACKed this proof. | -| `target_cleared_descriptor_ack_cursor` | bytes | Opaque per-voter ACK progress for §6.4 step 5. Each ACK means a target voter has applied a target-group Raft entry proving its local serving route snapshot has observed the default-group descriptor with `staged_visibility_active=false` / `migration_job_id=0` and the retained `min_write_ts_exclusive` at `promotion_completed_ts`. Target-local import ACKs, per-job HLC floor, readiness, and promotion proof records cannot be deleted until every current target voter has ACKed this cleared-descriptor proof. | -| `bracket_progress` | repeated `BracketProgress` | Per-family export bracket state for §6.3.1's parallel exports. Each entry carries `(bracket_id uint64, family uint32, export_phase enum, cursor bytes, done bool, scanned_bytes uint64, accepted_rows uint64, last_acked_batch_seq uint64)`. `bracket_id` is stable for the life of the SplitJob and forms part of the §6.1.1 importer idempotency key `(job_id, bracket_id, batch_seq)`. `last_acked_batch_seq` is monotone across the whole job, not reset per phase. When BACKFILL completes, the `BACKFILL -> DELTA_COPY` transition rewrites every bracket to `export_phase=DELTA_COPY`, clears `cursor`, `done`, `scanned_bytes`, and `accepted_rows`, and preserves `last_acked_batch_seq`; the first DELTA_COPY import uses `batch_seq = last_acked_batch_seq + 1`. This explicit phase boundary reopens exhausted BACKFILL brackets without reusing importer idempotency keys. A leader flap resumes the current phase of each bracket independently. `phase` advances from BACKFILL/DELTA_COPY only when every entry for that phase has `done = true`. | -| `source_retention_pin_ts` | uint64 | Source-group MVCC retention floor witness (§6.1.0a). The source group installs a route-scoped no-prune pin (`min_ts=0`, the BACKFILL lower bound) before BACKFILL opens and keeps it until every BACKFILL bracket is done, because BACKFILL exports `(0, snapshot_ts]`. After `delta_floor` is finalised, the same source-local pin is relaxed to `delta_floor` for DELTA_COPY's `(delta_floor, fence_ts]` window. Released only after successful source cleanup or the abandon/FAILED cleanup path disarms the job. | -| `last_error` | string | Most recent transient failure (for operator diagnosis). | -| `started_at`, `updated_at`, `terminal_at` | int64 | Wall-clock ms; **diagnostic only**, must not be read by any ordering-sensitive logic (CLAUDE.md HLC rule). `terminal_at` is set when phase enters DONE or ABANDONED; FAILED keeps the job live and does not set a terminal history key. | - -Encoding mirrors `RouteDescriptor` (a single-byte codec version prefix + protobuf body). Reserve a new codec constant `catalogJobCodecVersion = 1`. - -Target-local promotion/readiness state is **not** stored in this default-group `SplitJob`. The target group owns small Raft-applied records under `!migstage|ready|` with `(route_start, route_end, expected_cutover_version, migration_job_id, min_write_ts_exclusive, armed bool)` and under `!migstage|promote|` with `(promote_cursor bytes, done bool, promoted_rows uint64, last_error string)`. `ApplyTargetStagedReadiness` creates the former before CUTOVER, and `PromoteStaged` updates the latter in the same target-group apply that copies/deletes staged rows (§6.4). The default group only records restart witnesses and the final promotion witness (`target_promotion_done = true`) after the target reports `done=true`. - -All per-voter ACK cursors are keyed by the Raft configuration epoch / voter-set fingerprint observed when the barrier starts. Before marking a barrier `ARMED`, before issuing CUTOVER, and before deleting guard/proof records during CLEANUP, the migrator re-reads `engine.Configuration(ctx)` for the source/target group. If the current voter set differs from the cursor's epoch, the cursor is not complete: existing ACKs are retained for voters that remain, newly added/promoted voters are added as pending, and the migrator re-proposes/probes the same guard on those voters before phase advancement. A membership change that would add a voter or promote a learner for a group participating in a live SplitJob must either wait until the relevant guarded phase is disarmed or drive this re-ACK path before the new voter can serve the moving range. Treating "all voters at the time we first sampled the config" as permanent is unsafe; any voter that can later become leader must have applied the same readiness/read-fence/cleanup proof or fail closed until it catches up. - -#### 3.1.1 Job retention (bounded storage) - -Without a retention policy the `!dist|job|*` namespace grows unboundedly across a long-lived cluster (gemini medium on PR #945). M2 ships these bounds: - -- **Live cap and same-group split exclusion**: M2 v1 allows at most `maxLiveJobs = 1` non-terminal SplitJob in the catalog at any time. `StartSplitMigration` rejects a second cross-group request with `ErrTooManyInFlightJobs` while any job is live, including retryable `FAILED` jobs. The existing M1 `SplitRange` same-group path remains the same wire shape, but it must preflight the SplitJob catalog and reject with `ErrSplitJobOverlap` if the requested parent route / resulting child interval intersects any live SplitJob's `(route_start, route_end)` on either source or target side. A disjoint same-group split can proceed; an overlapping one cannot rewrite route IDs/bounds while BACKFILL/FENCE/DELTA_COPY/CLEANUP still rely on the job's original route interval, fence records, staged prefixes, and cleanup cursors. This matches §8's single migrator goroutine and avoids under-specified interactions between overlapping source/target fences, HLC floors, staged-readiness barriers, and cleanup. M3 may raise the cap only after adding explicit conflict detection for overlapping route intervals, target readiness records, and cleanup cursors. -- **Audit retention**: terminal jobs (`DONE` and `ABANDONED`) are moved by the migrator from `!dist|job|` to `!dist|jobhist||` after their cleanup obligations complete. `FAILED` is intentionally kept in the live namespace so `RetrySplitJob` / `AbandonSplitJob` can act on the full job body; it moves to history only after a successful retry reaches `DONE` or an abandon reaches `ABANDONED`. The history key sorts by terminal time so a single bounded prefix-scan trims oldest entries. -- **History bound**: `maxJobHistory = 1_000` total entries OR retention age `historyTTL = 7d`, whichever is tighter. The migrator runs a GC sweep at most once per minute on the default-group leader; it batch-deletes excess entries with a single Raft proposal. -- **Listing**: `ListSplitJobs` returns live + history (default cap 200 newest), with `since_terminal_at_ms` / `phase` filters; operators page through history via cursor. - -Bounded both by count (worst case ≤ `1 + maxJobHistory ≈ 1k` records ≈ low-MB storage) and by time. The cap on live jobs also prevents `ListSplitJobs` enumeration from blowing the response. - -### 3.2 RouteState transitions during a job - -Same-group split (M1): `Active → (atomic CAS) → Active+Active`. - -Cross-group split (M2) — **catalog stays single-source-of-truth, never holds an overlapping route**: - -```text -catalog snapshot evolution migrator action -============================ ==================== -[ source.Active(full range) ] - | - | PLANNED → BACKFILL SplitJob created on default group; - | source-group write tracker armed - | before snapshot_ts is chosen; no - | catalog mutation. Target accepts - | imports into shadow keyspace. - v -[ source.Active(full range) ] BACKFILL: chunked export from - source @ snapshot_ts; idempotent - import into target's shadow space. - | - | BACKFILL → FENCE Atomic catalog write happens first: - | source's - v right child split out as a NEW -[ source.Active(left) ] RouteDescriptor with state = -[ source.WriteFenced(right) ] WriteFenced, raft_group_id = - source. Catalog snapshot remains - non-overlapping. Coordinator/FSM - reject new writes/prepares on the - fenced route; prepared-txn - resolution is allowed only for - existing tracked intents. - | - | FENCE → DELTA_COPY Migrator waits for source-side - | fence ACK (§3.2a) — every source- - | group voter has applied the - | WriteFenced route — and the post- - | fence drain returns empty, then - | finalises delta_floor (§6.1.0) and - | pins fence_ts. THEN copies - | (delta_floor, fence_ts] from - | source to target shadow space. - v -[ source.Active(left) ] -[ source.WriteFenced(right) ] - | - | DELTA_COPY → CUTOVER Migrator first proposes a final - v target-local full HLC floor -[ source.Active(left) ] (`!migstage|hlc_floor|`, -[ source.WriteFenced(right) ] SetPhysicalCeiling(ms)+Observe(ts)) - so the target HLC > every import - (§6.2.1), applies the target-side - staged-readiness barrier - (§6.4 step 0), then arms the - source-side cutover read fence - (§7.2.2e). Only after the target - can fail closed without staged - metadata and the source can reject - stale reads does the default group - publish the atomic catalog write under - CAS(expected_catalog_version): - (a) source.WriteFenced(right) - → removed - (b) target.Active(right) inserted - with raft_group_id=target - Catalog snapshot still - non-overlapping; version+=1 - atomically. `cutover_version` - recorded in the SplitJob. -[ source.Active(left) ] -[ target.Active(right) ] - | - | CUTOVER → CLEANUP Source group deletes moved data only - v after the read-fence grace window, -[ source.Active(left) ] the active read-drain ACK barrier, -[ target.Active(right) ] and the post-cutover current-owner - gate are all in force (§7.2.4). - Cleanup walks the same disjoint - bracket plan and routeKey filter as - export; control prefixes are never - migrated or cleaned as data. - | - | CLEANUP → DONE Job moved to !dist|jobhist|<...>. - v -[ source.Active(left) ] -[ target.Active(right) ] -``` - -Key invariant: **catalog snapshots never contain overlapping ranges**. The target shadow keyspace lives outside the catalog (BACKFILL / DELTA_COPY land into private `!dist|migstage||` keys on the target group's MVCC store; CUTOVER **does not** bulk-rename them — promotion is incremental and runs as a target-side background job after CUTOVER, see §6.4). This keeps `routesFromCatalog` + `validateRouteOrder` (`distribution/engine.go:496-527`) green throughout the migration — addresses codex P1 on overlapping routes (PR #945 review). RouteState `MigratingSource` / `MigratingTarget` are reserved but unused in M2; they remain available for a future merge / multi-stage migration design. - -Reserved-control invariant: M2 cross-group migration rejects any moving interval that intersects a migration/control namespace: the default-group `!dist|` catalog namespace (including `!dist|meta|`, `!dist|route|`, `!dist|job|`, `!dist|jobhist|`, and staged data under `!dist|migstage|`), target-local `!migstage|` records (`!migstage|ack|`, `!migstage|hlc_floor|`, `!migstage|ready|`, `!migstage|promote|`), and source-local migration state (`!migwrite|`, `!migfence|`). Those keys are recovery metadata, not user data; if a split exported or cleaned them as ordinary rows it could corrupt another migration's cursor, HLC floor, target readiness proof, promotion state, or fence. `SplitRange` therefore returns `ErrReservedRange` before creating the SplitJob when the requested right child intersects any of these prefixes. This is a hard validation rule, not a best-effort cleanup filter. - -The same reservation is enforced on the **data-plane mutation path**, not only at `SplitRange`. Before any user-visible `PUT`, `DEL`, raw `Op_DEL_PREFIX`, one-phase txn, or prepare can create an MVCC version or intent, the FSM runs a migration-control-prefix pre-gate over the mutation footprint. Point mutations and explicitly targeted prefixes such as `!migstage|` / `!dist|migstage|` are rejected with `ErrReservedRange` before they reach MVCC. Broadcast-style prefix deletes (`Op_DEL_PREFIX ""` and adapter safety-net deletes whose raw interval spans more than one owner) are handled in two stages: the coordinator first proves the full normalized interval does not intersect a `WriteFenced` route, subtracts registered control intervals, and sends each owner only the clipped local user-data subranges; each recipient then verifies that clipped plan before `handleDelPrefix` runs. An empty prefix is therefore allowed to flush user data without deleting catalog or migration-control rows, while a request that truly targets a control prefix is still rejected. Internal maintenance code that needs to mutate those records uses typed catalog/migration APIs, not RawKV user-plane requests. This gate is intentionally independent of route ownership and `WriteFenced`: it runs even when no migration is active, so control-prefix safety is not conditional on a particular SplitJob. - -The Composed-1 gate rejects an apply on the wrong group at the observed catalog version; CUTOVER bumps `catalog_version` so any straggler write from an unaware coordinator fails closed at apply time on either side. - -### 3.2a Source-side fence-apply barrier — fence_ts must follow the source's actual write-reject point (closes codex round-6 P1 line 115) - -Codex round-6 P1 on PR #945 (line 115): the FENCE catalog write commits on the default group, but the source FSM only learns about the new `WriteFenced` state through the watcher's asynchronous catalog snapshot apply (`distribution/watcher.go:11`, `:87-98`). If the migrator records `fence_ts = source.LastCommitTS()` and starts DELTA_COPY immediately after the FENCE catalog write, the source leader may still accept writes for the moved keys for one watcher-tick worth of time after the catalog commit — those writes get a `commit_ts > fence_ts`, fall outside DELTA_COPY's `(delta_floor, fence_ts]` window, and are silently left on the source at CUTOVER. Result: write loss after CUTOVER hands the routing to the target. - -M2 fixes this with a **fence-first drain gate** on the FENCE → DELTA_COPY transition (closes codex round-18 P1 — new prepares must be gated before waiting for an empty drain): - -0. **Publish `WriteFenced` before waiting for the drain to become empty.** The migrator does **not** wait for an empty prepared-txn drain while the source route is still `Active`; that can starve forever under steady transactional load because every resolved intent can be replaced by a new prepare before the next drain tick. Instead, `BACKFILL → FENCE` first performs the atomic default-group catalog write that splits out `source.WriteFenced(right)` and records `fence_catalog_version`. From this point onward, once the source-group durable fence barrier in step 1 has applied, new writes and new prepares for the moving range are rejected synchronously by `verifyRouteNotFenced`, including raw `DEL_PREFIX` requests whose prefix range intersects the moving route. Low-ts writes that land during the catalog/write-fence gap are still safe: the source-group migration write tracker from §6.1.0 step 1 records raw point writes, `DEL_PREFIX` range tombstones, and one-phase writes at apply time, and records prepares at admission time; the post-fence drain below waits for prepared txns to commit or abort before `fence_ts` is picked. - - Existing prepared txns are allowed to resolve during FENCE, but only through the narrow resolution lane in §7.1: `Phase_ABORT` is always allowed, and `Phase_COMMIT` is allowed before the post-fence drain completes only when the matching pre-existing `!txn|lock|` row is present for that `(primary, start_ts, commit_ts)`. New prepares are not allowed through that lane. This avoids stranding a prepared txn, while still guaranteeing that after the drain returns empty there is no unresolved intent left to resolve on either source or target. After the drain, a response-lost COMMIT retry may still return idempotent success only when durable identity-bearing proof for the exact transaction already exists; a plain committed MVCC version is not enough. A durable rollback marker is also probed, but it proves the prior result was abort/rollback and returns that idempotent result instead of COMMIT success. No post-drain retry can create new state or resolve a new/unprepared commit. - - **0a. The lock drain must be route-faithful, not raw-key-bracketed (closes codex P1 on PR #945 — txn-lock drain bounds).** A txn intent lock is stored at `txnLockKey(userKey) = !txn|lock| || userKey` (`kv/txn_keys.go`), i.e. the lock sorts by its **raw user key**, but a route's `[routeStart, routeEnd)` bounds live in the **routing-key** namespace. For families whose raw storage key differs from their routing key these two namespaces diverge, so a drain bracketed by `[txnLockKey(routeStart), txnLockKey(routeEnd))` silently misses locks that belong to the moving range. Concretely (verified against `routeKey` / `dynamoRouteFromTablePrefixedKey` in `kv/shard_key.go`): a DynamoDB item lock is stored at `!txn|lock|!ddb|item||...` while its routing key is `!ddb|route|table|
` (table-segment routing); SQS locks store at `!txn|lock|!sqs|...` but route to `!sqs|route|global`; Redis/list/S3 internal families have the same raw-vs-routing split. Because `!ddb|item|...` sorts well before `!ddb|route|table|...` (`i` < `r`), a DynamoDB item lock falls **outside** any `[txnLockKey(routeStart), txnLockKey(routeEnd))` bracket whose bounds are `!ddb|route|table|...` — the drain reports empty while a prepared lock is still live, and FENCE proceeds with an unresolved prepare in the moving range (stranded `Phase_COMMIT`, or a snapshot boundary in §6.1.0 that drops the eventual committed version). This is the same routing-vs-raw-key split §6.3.1 already had to handle for **data** export: a raw-range scan never visits internal-family keys that sort outside `[routeStart, routeEnd)`. The lock drain therefore reuses §6.3.1's mechanism rather than a raw bracket. The drain is specified as: - - **Scan the full `!txn|lock|` drain-only family bracket** `[txnLockPrefix(), txnLockPrefixEnd())`, AND - - **filter each row by `routeKey(lockedKey) ∈ [routeStart, routeEnd)`**, where `lockedKey = key[len("!txn|lock|"):]` is the embedded raw user key and `routeKey()` (`kv/shard_key.go`) normalizes it to its routing key — exactly the §6.3 `RouteKeyFilter` predicate the data export already uses. A lock whose `routeKey(lockedKey)` falls inside the moving range counts toward the drain regardless of where its raw bytes sort. - - Where a per-family `EncodeBracketStart(routeStart)` hook exists (§6.3.1), the migrator MAY tighten the family-wide `!txn|lock|` bracket to the moving slice as an optimization, but the `routeKey()` filter remains the correctness contract. The cost is bounded: the txn-lock keyspace is small and transient (one row per in-flight prepare, all resolving within a txn-TTL), so a full `!txn|lock|` family scan per drain tick is cheap — this is not the full data scan §6.3.1's scan-budget pacing guards against. The same route-faithful drain definition is used by step 0b (post-fence drain) and by §6.1.0's pre-BACKFILL live-lock scan; both are the identical "`!txn|lock|` family scan + per-row `routeKey` filter" operation, never a raw `[txnLockKey(routeStart), txnLockKey(routeEnd))` bracket. - - **0b. Post-fence drain (closes codex round-14 P1 line 171 — the catalog-write race).** After step 1's unanimous voter-ACK returns and before picking `fence_ts`, the migrator runs the route-faithful intent-lock drain — the same `!txn|lock|` family scan filtered by `routeKey(lockedKey) ∈ [routeStart, routeEnd)` from §3.2a.0a (not a raw `[txnLockKey(routeStart), txnLockKey(routeEnd))` bracket, which would miss DynamoDB/SQS/Redis/list/S3 locks per §3.2a.0a). By the time this drain starts, every source voter rejects new prepares synchronously, so the drain contains only intents that existed before the last voter applied `WriteFenced`; each either commits through the narrow §7.1 resolution lane or is aborted by the lock resolver (`kv/lock_resolver.go`) within its TTL. The wait is therefore bounded by at most one txn-TTL and cannot starve under a steady workload. The migrator logs `last_error = "post-fence drain pending: intent locks in moving range"` while this drain runs. The SplitJob records `post_fence_drain_completed` so a migrator restart between the drain and step 2 resumes correctly. - - **Boundary finalisation is gated on this drain (codex round-15 P1 — fence before the snapshot boundary).** §6.1.0's migration write tracker records a *running minimum* (§6.1.0 step 2) from before `snapshot_ts` is selected through this drain — it is not a single pre-BACKFILL snapshot. The migrator does **not** finalise the DELTA_COPY lower bound `delta_floor` until step 0b's drain has returned empty, because only then is it true that no new raw point write, `DEL_PREFIX`, one-phase txn, or prepare can land in the moving range (step 1 made every voter reject via `verifyRouteNotFenced`, and the drain confirmed no residual in-flight prepare remains). Finalising `delta_floor` before this point would re-create the lost-low-ts-write hazard: a coordinator can submit a raw request, one-phase txn, or prepare with a low caller-supplied / lagging-clock `commit_ts ≤ snapshot_ts` while the source is still `Active`, and that commit would escape both BACKFILL and a `(snapshot_ts, fence_ts]` window. The tracker-backed running minimum + post-drain finalisation (§6.1.0 step 3) is what pushes `delta_floor` below every such migration-window write without scanning old retained MVCC history. - - -1. **Source-group durable write-fence barrier, ACKed by every source-group voter.** After the FENCE catalog write commits, the migrator does **not** immediately pick `fence_ts` and does **not** rely on an in-memory watcher ACK alone. It proposes `ApplyMigrationWriteFence(job_id, route_start, route_end, fence_catalog_version)` through the source group's Raft log. The source FSM persists a local fence entry such as `!migfence|` in source-group state/snapshots and makes `verifyRouteNotFenced` consult this table before the watcher-populated route snapshot. The entry is idempotent for the same tuple and rejects conflicting bounds/version for the same job. Startup, snapshot restore, and leadership acquisition must replay/load these entries before serving writes; a node that cannot load them fails closed for the affected range until it catches up. - - The migrator advances only after the source-group proposal is committed and **every current voter has applied it** (the same voter list comes from `engine.Configuration(ctx)`, already read by the migrator for §3.4 routing). A heartbeat field such as `source_write_fence_applied_version` may report progress, but the progress being reported is the source-group Raft-applied fence record, not just `catalog_version_applied` in `distribution/watcher.go`. This guarantees every node that could be elected source leader already has a durable local rejection rule for the moving range, so any write that reaches its FSM apply path after this point is rejected by §7.1's pre-gate even if the catalog watcher is stale or the process restarts. -2. **`delta_floor` finalised, then `fence_ts` pinned, after the durable barrier and post-fence drain.** After step 1 returns **and step 0b's post-fence drain has returned empty**, the migrator finalises `delta_floor = min(snapshot_ts, snapshot_min_admitted_ts - 1)` (§6.1.0 step 3) and then queries the source group leader for `LastCommitTS()` and pins that as `fence_ts`. Because the durable write-fence entry is now applied on every voter, no in-flight write with `commit_ts > fence_ts` can land for the moved range — the apply path rejects them with `ErrRouteWriteFenced` at the FSM gate. DELTA_COPY's `(delta_floor, fence_ts]` window therefore captures every committed write whose `commit_ts > delta_floor` — including any post-scan raw, one-phase, or prepared write that committed with `commit_ts ≤ snapshot_ts` while the source was still `Active` (`delta_floor < its commit_ts`) — completing the BACKFILL+DELTA_COPY semantic with no lost low-ts write. - -Timing budget: the migrator's heartbeat-driven loop already runs at one tick per `hlcRenewalInterval` (1s), and `cap_migration_v2` heartbeats carry `catalog_version_applied` already (§11.1 reuses the same field). The fence-apply ACK therefore typically completes in one tick. A non-responsive voter blocks the phase — the migrator logs `last_error = "fence ack pending: nodes [, ]"` and either (a) the operator intervenes, or (b) Raft removes the unresponsive voter via the existing leader-loss path. **Importantly the migrator does NOT advance phase on a quorum ACK** — every voter must ACK, because any voter could later become the leader and would re-serve writes for the moved range with the pre-FENCE state if its catalog hadn't applied. Quorum is necessary for catalog-write durability; **unanimous voter-ACK is necessary for fence safety**. - -Crash safety: the fence-ack-pending state is durable in the SplitJob (`fence_ack_cursor` tracks which voters have ACK'd). A migrator restart re-proposes/probes `ApplyMigrationWriteFence` with the same tuple and resumes from the pending voter list. No data is at risk during this wait — the catalog FENCE state and the source-group local fence record are both durable once applied; the migrator just doesn't commit DELTA_COPY's `fence_ts` until every possible source leader has the local fence. - -CUTOVER uses a separate source-side read-fence arm, not a post-hoc watcher wait: before the default group publishes `target.Active`, §7.2.2e installs a pending read fence in the source group's Raft log for the exact moving range and expected `cutover_version`. The source can then reject any stale read that still reaches it even if its catalog watcher has not applied the final CUTOVER snapshot yet. The fence-apply ACK above is specifically about **writes during DELTA_COPY**, where the data move happens; the CUTOVER read barrier is the symmetric reader-side protection. - -## 4. State Machine and Recovery - -The migrator is a goroutine on the **default-group leader**. State lives in the catalog (durable); the goroutine is a thin reconciler that picks up wherever the last leader stopped. - -Phase transitions are themselves applied through the default Raft group so every node sees the same job state. A transition writes the new phase + cursor + ts to `!dist|job|` via a CAS on the prior phase — concurrent migrators on a leadership flap can't race. - -Failure semantics per phase: - -| Phase | Crash safety | -|---|---| -| `PLANNED` | Job record durable; no side effects yet. | -| `BACKFILL` | Source-group migration write tracker is armed before `snapshot_ts` is picked; cursor persisted after each chunk batch (configurable, default every 256 keys); on resume the migrator re-reads cursor and continues. Idempotent ImportRangeVersions (see §6) means re-sending a batch is harmless. | -| `FENCE` | Catalog state is `WriteFenced`; coordinator rejects new writes/prepares until phase advances, while existing prepared txns may resolve through the narrow §7.1 lane until the post-fence drain is empty. After the drain, only idempotent COMMIT retries with durable identity-bearing proof of the prior success may return success; rollback proof returns the prior abort/rollback result, and new/unprepared commits stay rejected. A leader flap here keeps the fence (state is durable). The §6.1.0 step-2 running minimum (`snapshot_min_admitted_ts`), `post_fence_drain_completed`, and `delta_floor` are persisted as they advance, so a flap before `delta_floor` is finalised re-reads the migration write tracker + route-faithful drain and recomputes the same bound (the running minimum is monotone-down and durable). | -| `DELTA_COPY` | Same as BACKFILL — chunked + cursor-persisted. `delta_floor` (lower bound) and `fence_ts` (upper bound) are recorded at the FENCE → DELTA_COPY transition, so a resumed DELTA_COPY uses the same `(delta_floor, fence_ts]` window. | -| `CUTOVER` | Single atomic catalog write under CAS on `expected_catalog_version`. Either fully visible or fully not — no partial split. | -| `CLEANUP` | Source data deletion waits for `readFenceGrace` (default **30 s** per §7.2.4), every source voter's active-read drain ACK, and target promotion durability. Source fence disarm then waits for every source voter to ACK route removal/current-owner gates; target proof deletion waits for every target voter to ACK the cleared descriptor. GC is idempotent (delete-if-version-≤-fence_ts) and operates only on user data found by the disjoint bracket plan (§6.3.1); cross-group jobs intersecting migration/control namespaces are rejected before SplitJob creation (§3.2). Transient cleanup errors should keep the job in `CLEANUP` with `last_error` and backoff whenever the migrator can retry automatically. If operator intervention is required, the job may enter `FAILED` only with `retry_phase=CLEANUP`; `AbandonSplitJob` remains rejected because CUTOVER is already durable. | -| `FAILED` | Operator-visible **live** state. The job remains under `!dist|job|` with all cursors/fences/pins intact until a manual `RetrySplitJob` or, for pre-CUTOVER retry phases only, `AbandonSplitJob` RPC. `RetrySplitJob` CASes the job back to the durable `retry_phase` (`BACKFILL`, `FENCE`, `DELTA_COPY`, `CUTOVER`, or `CLEANUP`) without deleting side effects; idempotent imports/fences/promoters/cleanup make replay safe. A FAILED job with `retry_phase=NONE` is corrupt and must reject retry rather than guessing. `AbandonSplitJob` is allowed only for `retry_phase ∈ {BACKFILL,FENCE,DELTA_COPY}` and first CASes the live job to durable `ABANDONING` before any cleanup side effect is deleted. | -| `ABANDONING` | Operator-visible **live** cleanup state for a pre-CUTOVER abandon. The default-group job has already recorded `abandon_from_phase`, so recovery resumes the abandon cleanup protocol below instead of retrying BACKFILL/FENCE/DELTA_COPY with partially removed trackers/fences/staged rows. `RetrySplitJob` is rejected while `ABANDONING`; repeated `AbandonSplitJob` calls are idempotent cleanup resumes. | -| `ABANDONED` | Terminal audit state after a pre-CUTOVER job is explicitly abandoned and the source/target per-job cleanup protocol has completed. No retry is allowed. | - -`PLANNED → BACKFILL` can be rolled back by `AbandonSplitJob` (no data moved yet). `BACKFILL` / `FENCE` / `DELTA_COPY` can be abandoned only through the cleanup protocol below. After CUTOVER the job is one-way; rollback would require a reverse migration, which is M2-out-of-scope. - -### 4.2 Job garbage collection - -The retention policy from §3.1.1 runs as a CLEANUP-tail sweep on the default-group leader. Concretely: - -1. On a phase transition `CUTOVER → CLEANUP`, the migrator records `terminal_at` candidate, waits for the read-fence grace window (§7.2.4), then runs §6.4 step 5's target cleanup and §6.5's source cleanup/disarm. The job stays live under `!dist|job|` until target cleanup, source cleanup, and the source CUTOVER route-removal ACK barrier are durable. -2. On `CLEANUP → DONE`, the migrator issues one Raft proposal that (a) deletes `!dist|job|`, (b) writes `!dist|jobhist||` with the final body. This proposal is allowed only after target-local ack / HLC-floor / readiness / promotion records are gone and the source write tracker / write fence / cutover read fence / retention pin are disarmed. -3. On `* → FAILED`, the job stays live under `!dist|job|` and is **not** counted against `maxJobHistory`. The transition atomically records `retry_phase`. A failure before the catalog CUTOVER CAS may use `retry_phase=BACKFILL/FENCE/DELTA_COPY/CUTOVER` depending on which side effects are already armed; a failure after CUTOVER uses `retry_phase=CLEANUP`. The live cap in §3.1.1 includes retryable FAILED jobs so a cluster cannot accumulate infinite failed jobs without operator action. -4. On `AbandonSplitJob` before CUTOVER, the default-group leader first CASes the live job body to `phase=ABANDONING` with `abandon_from_phase` set to the current phase (or the FAILED job's `retry_phase`) and `terminal_at` still empty. This CAS happens **before** deleting any source tracker/fence/pin, target staged/ack/readiness state, or catalog fence side effect. From this point on, recovery and duplicate abandon requests resume the bounded abandon-cleanup protocol below; they must not replay `abandon_from_phase`. Only after every cleanup proof below is durable may the leader move the final `ABANDONED` body to `!dist|jobhist||`: - - source group: disarm `ArmMigrationWriteTracker`, delete `!migwrite||*`, clear any `!migfence|` durable write fence / pending cutover read fence, and release `source_retention_pin_ts`; - - target group: delete staged rows under `!dist|migstage||*`, import ack records under `!migstage|ack||*`, `!migstage|hlc_floor|`, `!migstage|ready|`, and `!migstage|promote|` in cursor-bounded batches; - - default group: if FENCE had landed, first CAS the catalog route descriptor from `WriteFenced` back to `Active` at the current catalog version (or prove it is already `Active` for this job and CUTOVER has not published `target.Active`), then only after the source and target cleanup ACKs and the catalog rollback proof are durable CAS the `ABANDONING` live job to `ABANDONED` history. The source group cannot "roll back" `WriteFenced` by deleting `!migfence|` alone: `WriteFenced` is default-group catalog state observed by coordinators/watchers, so failing to restore it in the default group would abandon the job while leaving the route fenced forever. A crash after any cleanup bullet but before the final history CAS is safe because the live `ABANDONING` job suppresses retry and drives the remaining idempotent cleanup again. -5. Once per minute the leader scans `!dist|jobhist|` for entries older than `historyTTL` (default 7d) OR exceeding `maxJobHistory` (1k) and deletes the oldest excess in one proposal. - -GC failure modes are bounded: the sweep is idempotent (delete-if-exists), and a stale leader's proposal lands as a no-op on the new leader's apply because the keys are already gone. - -## 5. Wire / RPC Changes - -### 5.1 `proto/distribution.proto` - -Extend `RouteDescriptor` to carry the staged-visibility flag and migration job ID that §6.4 step 2's MVCC merge read needs (closes claude round-11 P1 on PR #945: the target FSM cannot identify which `!dist|migstage||*` prefix to merge without `migration_job_id` on the route snapshot): - -```proto -message RouteDescriptor { - // ... existing M1 fields (raft_group_id, route_id, start/end, state) ... - - // Milestone 2: set on CUTOVER, cleared by the default-group - // promotion-complete CAS during CLEANUP after target-local promotion - // has proven done and before target-local proof cleanup. While true, - // the target FSM's read path runs the - // staged+live MVCC merge of §6.4 step 2 against the - // !dist|migstage||* prefix; once cleared the route - // is indistinguishable from any other Active route. - bool staged_visibility_active = N; - // Identifies the !dist|migstage||* prefix the §6.4 step 2 merge - // reads. Non-zero iff staged_visibility_active is true; cleared by - // the same promotion-complete CAS that clears staged_visibility_active. - // The target FSM needs this on the - // RouteSnapshot so per-request reads can find the staged prefix - // without an extra lookup against the SplitJob catalog. - uint64 migration_job_id = N+1; - // Exclusive lower bound for any live write/prepare commit timestamp - // on this route. Set on cross-group CUTOVER to - // max(max_imported_ts, fence_ts) and retained after staged visibility - // is cleared, so caller-supplied or replayed commit_ts values cannot - // fall behind imported/source-fenced history. - uint64 min_write_ts_exclusive = N+2; -} -``` - -These fields are persisted on the default-group `RouteDescriptor` record itself, propagated through the normal catalog watcher snapshot, and included in the same CUTOVER CAS that publishes `target.Active(right)`. They are not transient SplitJob-only metadata: a target leader that only has its route snapshot must be able to decide whether staged reads are active, which `!dist|migstage||*` prefix to merge, and the minimum live commit timestamp it may admit. Because a freshly refreshed coordinator can route to the target before the target watcher has applied that descriptor locally, CUTOVER is also preceded by the target-group staged-readiness barrier in §6.4 step 0; a target with the readiness record but without the matching route descriptor fails closed instead of serving live-only data or accepting a write below `min_write_ts_exclusive`. Existing descriptors decode as `staged_visibility_active=false`, `migration_job_id=0`, and `min_write_ts_exclusive=0`; the default-group promotion-complete CAS clears only the staged fields after the target-local promotion completion proof in §6.4 step 5. It **does not** clear `min_write_ts_exclusive`: the floor is a durable route invariant after cross-group migration, later child descriptors copy the parent floor on split and descriptor rewrites preserve `max(existing_floor, new_floor)`, and it continues to reject caller-supplied / replayed `commit_ts <= floor` even after the route is otherwise ordinary MVCC. The final CLEANUP→DONE move happens after the per-job source/target cleanup ACKs are durable. - -**Durable RouteDescriptor codec migration.** The durable catalog route record is **not** the protobuf `RouteDescriptor`; it is the custom `catalogRouteCodecVersion` binary record in `distribution/catalog.go`. M2 therefore includes an explicit route-codec v2 migration for these fields: - -- Add `catalogRouteCodecVersionMin = 1` and a version-keyed decoder. Version 1 keeps the existing layout exactly and requires `r.Len() == 0` after `End`; decoded M1 records get `staged_visibility_active=false`, `migration_job_id=0`, and `min_write_ts_exclusive=0`. Version 2 reads the existing v1 fields plus a fixed tail: one byte `staged_visibility_active` (`0`/`1` only), `uint64 migration_job_id`, and `uint64 min_write_ts_exclusive`, then requires no trailing bytes. Unknown versions are rejected rather than drained because the M2 rollout gate below prevents a newer writer from appearing before every participant can decode v2. -- Make `EncodeRouteDescriptor` conditional: while all three M2 fields are zero, it continues to emit v1 records so ordinary M1 same-group catalog rewrites remain readable during rolling upgrade. It emits v2 only when at least one M2 field is non-zero, which can happen only after `StartSplitMigration` has passed the cluster-wide `cap_migration_v2` gate (§11.1). This avoids an upgraded default-group leader writing a v2 record that a not-yet-upgraded watcher would reject during rollout. The v2 tail must be appended on **both** `End == nil` and `End != nil` paths; the current encoder's early return for nil `End` must be converted to a single fallthrough append path. -- `CloneRouteDescriptor`, `routeDescriptorEqual`, catalog save normalization, and watcher/list conversion must copy/compare the new fields. Tests hand-build v1 and v2 records for both finite-end and `End == nil` routes, assert v1 defaults, v2 round-trip of non-zero staged/floor fields, rejection of truncated or extra v2 tails, and that a zero-field route still encodes as v1 during rollout. - -Do **not** overload the existing M1 `SplitRange` RPC with a `target_group_id` field. In a mixed cluster, an M2 client that sent an extended `SplitRangeRequest` to an old default-group leader would have the unknown field ignored and could accidentally execute the same-group M1 split path. M2 keeps `SplitRange` byte-for-byte same-group only and adds a new cross-group RPC whose absence on old binaries fails safely with `UNIMPLEMENTED`. The new RPC is handled only by a default-group leader that advertises `cap_migration_v2` locally and has observed the cluster-wide capability gate in §11.1; otherwise it returns `ErrClusterNotReadyForMigration` before creating a job. - -Add the cross-group migration RPC, versioned ownership RPCs used by §7.2, and job-introspection RPCs: - -```proto -message StartSplitMigrationRequest { - uint64 expected_catalog_version = 1; - uint64 route_id = 2; - bytes split_key = 3; - // Mandatory for the M2 RPC. Must name a different existing voter group. - // Same-group callers continue to use SplitRange. - uint64 target_group_id = 4; - // Per-job pacing knobs (§8). Reserved string-map so we can add new - // knobs without a proto break. Known keys today: - // "chunk_bytes" — BACKFILL / DELTA_COPY chunk size (bytes) - // "inter_chunk_pacing_ms" — sleep between chunks (ms) - // "fence_grace_ms" — readFenceGrace override (ms) for the job - // Unknown keys are accepted (forward-compat) and logged at - // SplitJob create time, but do not affect behaviour. - map options = 5; -} - -message StartSplitMigrationResponse { - uint64 catalog_version = 1; - // The cross-group split is asynchronous; left/right become observable - // via ListRoutes after CUTOVER completes. - uint64 job_id = 2; -} - -message GetRouteOwnershipRequest { - bytes key = 1; // routeKey-normalized point key - uint64 catalog_version = 2; // exact version to answer from -} - -message GetRouteOwnershipResponse { - RouteDescriptor route = 1; - uint64 catalog_version = 2; - bool found = 3; -} - -message GetIntersectingRoutesRequest { - bytes start = 1; // routeKey-normalized inclusive start - bytes end = 2; // exclusive; empty means +infinity - uint64 catalog_version = 3; // exact version to answer from -} - -message GetIntersectingRoutesResponse { - repeated RouteDescriptor routes = 1; - uint64 catalog_version = 2; -} - -message GetSplitJobRequest { - uint64 job_id = 1; -} - -message GetSplitJobResponse { - SplitJob job = 1; -} - -message ListSplitJobsRequest { - // Inclusive lower bound on terminal_at (ms). 0 = no lower bound. - // §3.1.1's history retention orders entries by terminal_at_ms_be - // so paging through the audit history just bumps this on each call. - uint64 since_terminal_at_ms = 1; - // Optional phase filter; empty = no filter. Matches a SplitJob's - // `phase` field exactly. Supports per-phase audit queries. - string phase = 2; - // Pagination cursor. Empty = first page; opaque bytes from a prior - // response's next_page_cursor for subsequent pages. The default cap - // (§3.1.1) is 200 newest entries per response. - bytes page_cursor = 3; -} - -message ListSplitJobsResponse { - repeated SplitJob jobs = 1; - // Opaque pagination cursor. Empty = caller has reached the last page; - // non-empty = pass back as ListSplitJobsRequest.page_cursor to fetch - // the next page. The encoding is internal to the catalog (typically - // the terminal_at_ms_be of the last entry returned). - bytes next_page_cursor = 2; -} - -message AbandonSplitJobRequest { - uint64 job_id = 1; - // Operator may abandon only PLANNED / BACKFILL / FENCE / DELTA_COPY, - // or FAILED when retry_phase is one of those pre-CUTOVER phases. - // CUTOVER + later is rejected (data is partially landed in target). -} - -message AbandonSplitJobResponse {} - -message RetrySplitJobRequest { - uint64 job_id = 1; - // Allowed only while phase == FAILED. The default-group CAS moves the job - // back to the recorded retry_phase (BACKFILL/FENCE/DELTA_COPY/CUTOVER/ - // CLEANUP); source/target side effects are reused rather than cleaned first. -} - -message RetrySplitJobResponse {} - -service Distribution { - // ... existing M1 RPCs ... - rpc StartSplitMigration (StartSplitMigrationRequest) - returns (StartSplitMigrationResponse) {} - rpc GetRouteOwnership (GetRouteOwnershipRequest) - returns (GetRouteOwnershipResponse) {} - rpc GetIntersectingRoutes (GetIntersectingRoutesRequest) - returns (GetIntersectingRoutesResponse) {} - rpc GetSplitJob (GetSplitJobRequest) returns (GetSplitJobResponse) {} - rpc ListSplitJobs (ListSplitJobsRequest) returns (ListSplitJobsResponse) {} - rpc AbandonSplitJob (AbandonSplitJobRequest) returns (AbandonSplitJobResponse) {} - rpc RetrySplitJob (RetrySplitJobRequest) returns (RetrySplitJobResponse) {} -} -``` - -`SplitJob` is a separate message mirroring the persistent fields in §3.1. - -### 5.1.1 `proto/service.proto` read-path fields - -The read fence in §7.2 is wired through the concrete user-visible read RPCs, not an abstract helper message. M2 extends the existing RawKV service messages as follows: - -```proto -message RawGetRequest { - bytes key = 1; - uint64 ts = 3; - // Catalog version observed by the server-side ShardStore/read adapter - // when it routed this read. User clients leave this unset. - uint64 read_route_version = 4; -} - -message RawScanAtRequest { - bytes start_key = 1; - bytes end_key = 2; - int64 limit = 3; - uint64 ts = 4; - bool reverse = 5; - // Catalog version observed by the server-side ShardStore/read adapter when - // it chose the scan route(s). User clients leave this unset. - uint64 read_route_version = 6; - // Optional route-key-normalized interval supplied by callers that already - // decoded an internal-family raw scan. Empty means the receiver normalizes - // start_key/end_key itself, failing closed if it cannot. - bytes route_start = 7; - bytes route_end = 8; -} - -message RawLatestCommitTSRequest { - bytes key = 1; - // Same ownership-fence stamp as RawGetRequest. User clients leave this unset; - // server-side RawKV / ShardStore stamps it before serving or proxying. - uint64 read_route_version = 2; -} -``` - -External RawKV clients do **not** know distribution catalog versions and are not required to send these fields. The server-side RawKV adapter / `ShardStore` entrypoint first resolves the current route ownership for the requested point or scan interval, stamps `read_route_version` and the normalized route interval, and only then calls the shared read-fence helper or forwards to another node. `TransactionalKV.Get` / `Scan` enter through `ShardStore` and therefore stamp the same value before they proxy to `RawGet` / `RawScanAt`. `RawLatestCommitTS` / `ShardStore.LatestCommitTS` is a point read for fencing purposes and carries the same stamp; after CUTOVER the target answers it from the same staged+live raw-candidate view as §6.4 step 2, returning the greatest committed timestamp found in either live MVCC or `!dist|migstage||*` for that logical key. A source that no longer owns the key returns `ErrRouteMoved` / `ErrRouteOwnershipUnknown` before touching source MVCC, just like `RawGet`. Internal forwarding must preserve `read_route_version` and the normalized interval fields; a proxy target that receives zero after `cap_migration_v2` treats the request as a broken internal legacy hop and returns `ErrRouteOwnershipUnknown` rather than reading source MVCC. Zero-version rejection is **not** applied to user-visible local-leader RawKV requests before the server has had a chance to stamp them. This is intentionally narrower than adding a generic `pb.ReadRequest`: the concrete wire types are what stale coordinators actually send today. - -### 5.2 `proto/internal.proto` - -Add the data-movement RPCs (leader-to-leader, server-streaming for BACKFILL): - -```proto -service Internal { - // ... existing RPCs ... - rpc ExportRangeVersions (ExportRangeVersionsRequest) - returns (stream ExportRangeVersionsResponse) {} - rpc ImportRangeVersions (ImportRangeVersionsRequest) - returns (ImportRangeVersionsResponse) {} -} - -message ExportRangeVersionsRequest { - // Raw scan bounds — what the server iterator's [start, end) range - // is in the MVCC storage key space. For a §6.3.1 internal-family - // bracket these are the family-wide bounds (e.g. txnLockPrefix() → - // txnLockPrefixEnd()), which scan ALL of the family cluster-wide. - bytes range_start = 1; - bytes range_end = 2; - uint64 max_commit_ts = 3; // snapshot_ts (BACKFILL) or fence_ts (DELTA_COPY) - // DELTA_COPY's lower bound is delta_floor (§3.1 / §6.1.0 step 3), NOT - // snapshot_ts: a post-scan raw/one-phase/prepared write with commit_ts ≤ - // snapshot_ts must still land in (delta_floor, fence_ts], and delta_floor - // = min(snapshot_ts, snapshot_min_admitted_ts - 1) is finalised after the - // fence so that delta_floor < every such commit_ts. Sending snapshot_ts - // here would re-exclude exactly the versions delta_floor was computed to - // capture (codex round-16 P1 — use delta_floor as DELTA_COPY lower bound). - uint64 min_commit_ts = 4; // 0 for BACKFILL; delta_floor for DELTA_COPY - bytes cursor = 5; // resume token; empty on first call - uint32 chunk_bytes = 6; // soft cap, server may exceed by one row - // Logical moving range bounds — what the server's KeyFilter (§6.3) - // narrows the raw iterator to (closes claude round-12 P1 on PR #945: - // without these, a family-wide raw bracket cannot be narrowed to the - // moving slice on the server side and would export every cluster- - // wide txn lock / list / redis / etc. to the target). The server - // constructs RouteKeyFilter(route_start, route_end) from these and - // applies it before adding a row to the chunk. Required on every - // call; the migrator already has them on its SplitJob. - bytes route_start = 7; - // Empty == +infinity (open-ended right edge). The internal RPC server - // normalizes len(route_end)==0 to nil before building RouteKeyFilter; the - // filter also checks len(rangeEnd)>0 defensively (§6.3) so the rightmost - // route cannot reject every non-empty routing key by comparing against []. - bytes route_end = 8; - // §6.3.1 scan budget: hard cap on the bytes the server iterator may - // scan (accepted AND rejected rows both count) before it must return - // whatever it has accepted plus next_cursor over the last-scanned - // position. Bounds the per-call wall-clock cost of a family-wide - // sparse bracket whose moving slice is empty or tiny — without it a - // single call could scan an arbitrarily large unrelated internal- - // family prefix (e.g. a cluster-wide 10 GB !sqs| range) to prove - // done=true. 0 means "use the server default" (4 × chunk_bytes, §6.3.1). - uint64 max_scanned_bytes = 9; -} - -message ExportRangeVersionsResponse { - repeated MVCCVersion versions = 1; - bytes next_cursor = 2; // empty when exhausted - bool done = 3; -} - -message MVCCVersion { - bytes key = 1; // raw storage key (NOT routeKey-normalized) - uint64 commit_ts = 2; - bool tombstone = 3; - bytes value = 4; // logical value; omitted when tombstone == true - // Internal-key family (txn/list/redis) tag, so the importer can - // dispatch into the matching store helper instead of inferring - // from the byte prefix. - uint32 key_family = 5; - // Store-level TTL deadline in HLC timestamp units; 0 means no TTL. - // Preserved from VersionedValue.ExpireAt / Pebble's value header so - // PutWithTTLAt / ExpireAt values keep expiring at the original time. - // Tombstones must carry expire_at=0. - uint64 expire_at = 6; -} - -message ImportRangeVersionsRequest { - uint64 job_id = 1; - // May be empty for a scan-budget progress chunk (§6.2): the importer - // still acks bracket_id/batch_seq/cursor but performs no MVCC writes or - // HLC advancement. - repeated MVCCVersion versions = 2; - // Exporter resume token (§6.1.1): carried so the migrator can persist - // it back on ack and resume the bracket after a restart. NOT the - // idempotency key — see bracket_id / batch_seq below. - bytes cursor = 3; - // §6.1.1 importer idempotency key. Because §6.3.1 runs export brackets - // in parallel, dedup keys on (job_id, bracket_id, batch_seq), not on - // (job_id, cursor) which parallel brackets would clobber. - // bracket_id — the §6.3.1 per-bracket id (stable for the SplitJob's - // life; recorded in bracket_progress). - // batch_seq — monotonic per-bracket counter the migrator advances on - // every successful ack. The importer accepts only - // stored+1; a retry of batch_seq ≤ stored is a no-op, and - // batch_seq > stored+1 returns ErrImportBatchGap before - // mutating MVCC/HLC/cursor state. - uint64 bracket_id = 4; - uint64 batch_seq = 5; -} - -message ImportRangeVersionsResponse { - // Cursor durably acked by the target for (job_id, bracket_id, batch_seq). - // On duplicate batch_seq the target returns the stored value. - bytes acked_cursor = 1; -} -``` - -Streaming the export keeps the migrator memory-bounded; a single `ImportRangeVersions` RPC per chunk lets the importer ack the cursor (the migrator persists it in the SplitJob on ack). - -## 6. MVCC Range Export / Import - -### 6.1 Source side (`store/mvcc_store.go`, `store/lsm_store.go`) - -The `store` package is generic and **must not** import `kv` or know about routing keys (gemini medium on PR #945). Filtering is injected as a delegate from the caller. - -#### 6.1.0 `snapshot_ts` selection and DELTA_COPY lower bound — must dominate every low-ts write admitted before the fence (closes codex round-14 P1 line 405; closes codex round-15 P1 — fence-before-boundary; closes codex round-16 P1 — track prepares for moved keys, not only moved primaries; closes codex round-20 P1 — raw/one-phase low-ts writes) - -The corner case: 2PC's `commit_ts` is allocated by the coordinator at **prepare time** (via `HLC.Next()` / `NextFenced()` on the prepare path), not at commit-apply time. Concretely, a transaction `T` can sequence as: - -1. `t = t_prepare`: coordinator picks `commit_ts(T)` and writes intent locks on the participant groups. -2. `t = t_snapshot`: migrator picks a snapshot ts for BACKFILL, with `t_prepare < t_snapshot` so `commit_ts(T) < snapshot_ts` is possible. -3. `t = t_commit_apply`: coordinator's `Phase_COMMIT` lands on the source FSM. The MVCC version `(K, commit_ts(T))` is written **after** `t_snapshot`. - -If the migrator starts BACKFILL at `snapshot_ts` and the export reads the snapshot at exactly `maxCommitTS = snapshot_ts`, the committed value lands at `commit_ts(T) ≤ snapshot_ts` but is **not yet present in the store** when BACKFILL iterates. A naïve DELTA_COPY `(snapshot_ts, fence_ts]` window then excludes it (because `commit_ts(T) ≤ snapshot_ts`), and the FENCE drain alone doesn't help because the drain only waits for intents to disappear, not for past-committed-but-not-yet-applied versions to surface. Net result: a committed write the client has already observed is silently dropped from the target. - -**Why a single pre-BACKFILL scan is insufficient (codex round-15 P1 — fence before fixing the boundary).** An earlier revision picked `snapshot_ts = min(HLC.Next(), min_admitted_commit_ts - 1)` from **one scan taken before BACKFILL** and trusted it as the DELTA_COPY lower bound. That scan only sees writes known at scan time, but **the source route stays `Active` until the much-later FENCE transition (§3.2a)** — so a coordinator can apply a *new* raw request, one-phase txn, or prepare after the scan with `commit_ts ≤ snapshot_ts` and still have it accepted by the source. Two verified mechanisms let a post-scan write carry such a low `commit_ts`: - -- **Caller-supplied commit_ts.** `resolveDispatchCommitTS` uses a caller-supplied `commitTS` directly (it only `Observe`s it; `kv/coordinator.go:872-876`), and `HLC.Observe` is monotone-up-only — it never raises a caller's ts, so a caller-supplied `commit_ts ≤ snapshot_ts` is accepted and used as-is. The caller-supplied-StartTS/CommitTS path is live for the S3/SQS/DynamoDB adapters today (`kv/sharded_coordinator.go:638-665`). -- **Lagging-clock / low logical half.** Even on the coordinator-allocated path, `NextFenced` only floors the **physical** half at the Raft-agreed ceiling (`kv/hlc.go:171-176`); the logical lower 16 bits are a free in-memory counter (`kv/hlc.go:181-190`), and `snapshot_ts = min_admitted_commit_ts - 1` can sit below the leader's own current `HLC.Next()`, so a fresh allocation within the same physical millisecond can be `≤ snapshot_ts`. - -Such a post-scan write can commit after the BACKFILL iterator has passed its key (so it is invisible to BACKFILL) and be `≤ snapshot_ts` (so a `(snapshot_ts, fence_ts]` DELTA_COPY excludes it) → lost write. Prepared txns are only one instance of the problem: `handleRawRequest` writes at `r.Ts` (including `Op_DEL_PREFIX`, which writes tombstones for every visible key under a prefix), and `handleOnePhaseTxnRequest` writes at `TxnMeta.CommitTS`, with no intent lock or other durable marker unless the migration write tracker records them explicitly. The boundary must therefore be **fixed only after writes for the moving range are fenced**, i.e. after no new raw point write, `DEL_PREFIX`, one-phase txn, or prepare can land — which is exactly §3.2a step 1's unanimous `verifyRouteNotFenced` apply plus step 0b's post-fence drain for already-prepared txns. - -**The fix is to record migration-window writes before choosing `snapshot_ts`, then fence new writes before fixing the boundary.** The single pre-BACKFILL scan becomes a *provisional* snapshot for the early bulk copy; the boundary that DELTA_COPY trusts is finalised after the fence. Concretely: - -1. **Arm the migration write tracker and a conservative source retention pin, then choose the provisional snapshot for BACKFILL.** Before opening BACKFILL, the migrator proposes `ArmMigrationWriteTracker(job_id, route_start, route_end)` to the source group and waits for it to apply; the same source-group Raft entry also installs an initial route-scoped migration retention pin in the safest state (`min_ts=0` / "do not compact accepted rows for this moving route" until the migrator tightens it). The source FSM must not admit a tracked raw/one-phase/prepare write before both the tracker and this conservative pin are active. Only then may the migrator set `write_tracker_armed=true` on the default-group SplitJob and choose `snapshot_ts`. The tracker and the initial pin are source-group Raft state. From that apply onward: - - - `handleRawRequest` appends a tracker row for each point mutation whose `routeKey(mut.Key) ∈ [routeStart, routeEnd)`, using the request's `commit_ts = r.Ts`, in the same apply that writes the MVCC version. For `Op_DEL_PREFIX`, the tracker computes the same prefix footprint as §7.1: an empty prefix intersects every route, and a non-empty prefix covers the half-open raw prefix interval `[prefix, prefixScanEnd(prefix))` mapped through the route-key-normalized prefix planner. If that footprint intersects the moving range, or if the prefix is an internal/family-wide prefix that cannot prove disjoint from the moving range, the tracker appends a row using the request's `commit_ts = r.Ts` in the same apply that writes the prefix tombstones. It does not need one row per deleted key; the row records that the DELTA_COPY lower bound must include this range tombstone. - - `handleOnePhaseTxnRequest` appends a tracker row for each written mutation whose `routeKey(mut.Key)` is in the moving range, using `commit_ts = TxnMeta.CommitTS`, in the same apply that writes the MVCC version. A dedup no-op for `PrevCommitTS` does not append a row because no new version is written. - - The prepare apply path appends a tracker row such as `!migwrite||||prepare||` in the same apply that writes the txn lock when `routeKey(lockedKey) ∈ [routeStart, routeEnd)`. This records **admission**, not later polling, so a prepare that lands and resolves between two migrator ticks is still represented. - - M2-PR0 must make the last bullet true on the live prepare path once migration side effects are enabled, not only in the lock-value codec. Today the multi-shard prepare plumbing can reach `handlePrepare` with only `start_ts` and a zero/absent `commit_ts` in the txn metadata. PR0 introduces the new contract behind the cluster capability gate: before every active voter advertises `cap_migration_v2`, normal transaction traffic remains M1-compatible, emits legacy lock values, and does **not** write `!txn|ok|` markers; the migration write tracker cannot be armed because `StartSplitMigration` is rejected. After the PR7 capability gate is open, the coordinator allocates and fences `commit_ts` before emitting PREPARE, the prepare request and txn metadata carry that exact value to every participant, `handlePrepare` writes it into each `txnLock`, and the migration write tracker records the same non-zero value in the same Raft apply before the lock becomes visible. The later COMMIT phase must reuse that already-admitted `commit_ts`; it must not allocate a replacement timestamp. Once the gate is open, a newly written prepare lock whose metadata lacks a non-zero `commit_ts` is rejected as a protocol error; only pre-upgrade lock values may take the legacy direct-commit-record lookup / pending-drain fallback. - - After the tracker is armed, run the **route-faithful lock scan** of §3.2a.0a — the `!txn|lock|` family bracket `[txnLockPrefix(), txnLockPrefixEnd())` filtered by `routeKey(lockedKey) ∈ [routeStart, routeEnd)`, **not** a raw `[txnLockKey(routeStart), txnLockKey(routeEnd))` bracket (which would miss DynamoDB/SQS/Redis/list/S3 item locks whose raw storage key sorts outside the routing-key bounds — see §3.2a.0a). For each accepted pre-tracker lock, decode its recorded `commit_ts` from the M2-PR0 lock-value field. If the field is absent or zero because the lock was written before M2-PR0, fall back to the direct commit-record point lookup described in step 2; if neither source yields a timestamp because the legacy lock is still only prepared, keep the drain pending rather than guessing a floor. Take `min_admitted_commit_ts = min(commit_ts for all in-flight pre-tracker prepares accepted by the filter and all tracker rows so far)`; `∞` if none. Pick the provisional `snapshot_ts = min(HLC.Next(), min_admitted_commit_ts - 1)` and start BACKFILL at it. Over-copy is harmless — a low `snapshot_ts` just hands more work to DELTA_COPY. -2. **Track the running minimum until the fence blocks new writes — over every *key or range footprint* in the moving range, not only moved primaries (closes codex round-16 P1; closes codex round-18 P1 on historical-version over-scan; closes codex round-20 P1 on raw/one-phase low-ts writes; closes codex round-21 P1 on `DEL_PREFIX` range writes).** From BACKFILL through §3.2a step 0b's post-fence drain, the migrator maintains `snapshot_min_admitted_ts` = the smallest `commit_ts` in the source-group migration write tracker plus any still-live pre-tracker locks whose locked key resolves into the moving range (monotone-down, persisted on each advance). Tracker rows update this future DELTA_COPY lower-bound witness; they do **not** relax the active source retention pin while BACKFILL is still copying `(0, snapshot_ts]` (§6.1.0a). The earlier wording — "scan committed MVCC versions in the moving range with `commit_ts ≤ snapshot_ts`" — was too broad: it includes every old retained version that BACKFILL already copied, so a years-old historical version could lower `delta_floor` near zero and turn DELTA_COPY into another full backfill. The tracker is the correctness backstop for same-tick land-and-resolve prepares, for raw point / one-phase writes that create no intent lock, and for `DEL_PREFIX` range tombstones whose prefix intersects the moving route; ordinary committed MVCC history is copied by BACKFILL/DELTA_COPY but is **not** allowed to lower the boundary. - - The tracker is keyed by the **locked key**, not the primary, so a secondary lock in the moving range counts even when the txn's primary routes elsewhere. Verified against `kv/fsm.go:1136-1142` and `kv/txn_keys.go:50-58`: the durable `!txn|cmt||` commit record is written **only when committing the primary** (`if committingPrimary`) and is keyed by `primaryKey`, so for such a txn the commit record's `routeKey()` falls **outside** `[routeStart, routeEnd)` — a commit-record family scan filtered by `routeKey()` never sees it. The migrator may still use the exact `PrimaryKey` + `StartTS` decoded from a live lock to do a **direct point lookup** for backward-compat locks where the M2-PR0 `commit_ts` field is absent or zero, but it never relies on a route-faithful commit-record-family scan for moved-secondaries. - - M2-PR0 also adds a route-local **per-key transaction-success marker** for committed participants. The key is self-delimiting, not separator-delimited: `!txn|ok|||||||`. The commit apply path writes this marker in the same source/target Raft apply that installs the committed MVCC version for each participant key; its value redundantly encodes `(locked_key, primary_key, start_ts, commit_ts)` and the same marker version. Because the decoder can find the end of `locked_key` from the length prefix before reading the fixed timestamps and trailing `primary_key`, `routeKey(txnSuccessKey)` equals `routeKey(locked_key)` even when user keys contain separator bytes or timestamp-like suffixes. A secondary key that moves without its primary therefore carries local identity-bearing proof through BACKFILL/DELTA_COPY. The export bracket planner treats `!txn|ok|` as a concrete txn family bracket, the `routeKey()` decoder understands this length-prefixed shape, and cleanup/txn-GC removes it with the same retention horizon as the primary commit/rollback markers after no idempotent retry can need it. Post-drain COMMIT idempotency may return success from this marker (or a primary commit record / identity-bearing commit marker); it must not fall back to an ordinary committed MVCC version that lacks `(primary_key,start_ts)`. - - The minimum is only **final** once §3.2a step 1's unanimous voter-ACK has made every source voter reject new writes/prepares (`verifyRouteNotFenced`) **and** step 0b's post-fence drain has returned empty — at that point no future raw point write, `DEL_PREFIX`, one-phase write, or prepare can land on any key in the moving range, and every migration-window write was recorded by the tracker or was a pre-tracker prepare lock sampled by the drain path. Tracker rows are retained until `delta_floor` is finalised, then deleted by the same cleanup that clears the import ack state for the job. -3. **Finalise the boundary after the post-fence drain.** When step 0b's drain returns empty (`post_fence_drain_completed = true`), the migrator reads the migration write tracker and live-lock scan **one last time**; the lock side returns empty by construction (no in-flight prepare survives the drain), and the tracker yields the final running minimum over every migration-window write — raw point write, `DEL_PREFIX` range tombstone, one-phase write, prepared primary, or prepared secondary — that resolved into the moving range. The DELTA_COPY lower bound is then fixed as `delta_floor = min(snapshot_ts, snapshot_min_admitted_ts - 1)` and DELTA_COPY exports the window **`(delta_floor, fence_ts]`** (not `(snapshot_ts, fence_ts]`). Any post-scan write that committed with `commit_ts ≤ snapshot_ts` was recorded by the step-2 tracker, so `delta_floor ≤ snapshot_min_admitted_ts - 1 < commit_ts`, i.e. `commit_ts > delta_floor` and the version falls **inside** DELTA_COPY's window — it is no longer dropped. `delta_floor` is persisted on the SplitJob alongside `snapshot_min_admitted_ts` so a DELTA_COPY resume uses the same lower bound. - -This is the reviewer's "block/drain before choosing this boundary," expressed as: keep tracking every admitted write's `commit_ts` until the fence blocks new ones, then fix the DELTA_COPY floor below all low-ts migration-window writes. The alternative — "before BACKFILL, wait for all currently-prepared transactions to resolve, then `snapshot_ts = HLC.Next()`" — does **not** suffice on its own here precisely because the source stays `Active` after that wait and fresh low-ts raw/one-phase/prepared writes can keep arriving until the fence; the fence-then-floor ordering is what makes the boundary safe. - -Edge case: if a migration-window write has an unusually low caller-supplied `commit_ts`, `delta_floor` may push the DELTA_COPY window below recent committed history. That is acceptable — `(delta_floor, fence_ts]` is a proper superset of the BACKFILL-missed window and re-exports the same versions idempotently (§6.2 dedups on `(job_id, bracket_id, batch_seq)`). The bound is now tied to the number and timestamps of writes admitted during this migration, not to arbitrary old retained MVCC versions, so it cannot be lowered merely because the range has long history. The target HLC floor in §6.2.1 uses `max(max_imported_ts, fence_ts)`: `max_imported_ts` covers imported DELTA_COPY versions, while `fence_ts` covers sparse/empty ranges whose next target write still must occur after the source-side fence boundary. - -#### 6.1.0a Source MVCC retention pin — export windows must remain readable - -Before BACKFILL opens, the source already has the conservative route-scoped pin installed by `ArmMigrationWriteTracker` in §6.1.0 step 1. That initial pin is the actual BACKFILL lower bound: `min_ts=0` / "retain every committed version and tombstone in the moving route accepted by the §6.3.1 `RouteKeyFilter`." It is route-scoped and bracket-aware, so compaction may still discard unrelated keys, but it must keep the moving route's `(0, snapshot_ts]` history readable until every BACKFILL bracket has reached `done=true`. There is no window where tracked writes are admitted while the moving route has no source-local retention pin. - -The migrator MUST NOT replace that no-prune pin with `min(snapshot_ts, snapshot_min_admitted_ts - 1)` while BACKFILL is still running. BACKFILL exports the full `(0, snapshot_ts]` window; relaxing the pin early would let compaction discard older committed versions or tombstones that a not-yet-reached bracket still needs. During BACKFILL and FENCE, tracker rows only lower `snapshot_min_admitted_ts` / the future `delta_floor` witness. The source-local active pin stays at `0` even if the next DELTA_COPY lower bound is already known to be higher. - -Only after every BACKFILL bracket is done and §3.2a finalises `delta_floor` does the migrator relax the same source-group pin to `min_ts = delta_floor` before DELTA_COPY starts. At that point the BACKFILL window has been copied, and DELTA_COPY only needs `(delta_floor, fence_ts]`; a low caller-supplied migration-window write may keep `delta_floor` near zero, but it never raises the pin above a version DELTA_COPY must still read. The default-group SplitJob records `source_retention_pin_ts` first as `0` and later as the applied `delta_floor` value for audit/restart, but the source-local pin is the safety mechanism. - -The pin is released only after successful source cleanup (§6.5) or after the abandon/FAILED cleanup protocol in §4.2 disarms the job. A source restart or snapshot restore must reload active migration retention pins before running compaction. This is a correctness fence, not only an optimization: if `MinRetainedTS` or tombstone compaction prunes a version before the bracket cursor reaches it, the target can miss historical visibility or resurrect deleted data after CUTOVER. - -New method on `MVCCStore`: - -```go -// KeyFilter is the in-package contract: the caller decides which raw -// keys belong to the slice of MVCC space being exported. nil means -// "accept every key in [start, end)". The decision MUST be deterministic -// and pure — the export streams it on the iteration goroutine. -type KeyFilter func(rawKey []byte) bool - -// ExportVersions iterates raw committed MVCC versions in the half-open -// range [start, end) whose commit_ts is in (minCommitTS, maxCommitTS], -// resuming from cursor. It includes delete tombstones and versions whose -// ExpireAt would make them invisible to a reader at maxCommitTS; it does -// NOT apply read-visible filtering. It excludes only in-flight intent -// locks (!txn|lock|...) because those are not committed MVCC versions the -// importer should materialise. Returns a slice of versions (capped roughly -// at chunkBytes) and the next cursor, plus a sentinel when exhausted. -// The cursor is opaque (see §6.1.1) — it addresses an MVCC version -// (raw key + commit_ts), not just a raw key, so a single hot key's -// history is allowed to span batches without losing or duplicating -// versions. -// `accept` (nil = no filtering) is the §6.3 decoupling seam. -// `maxScannedBytes` is the §6.3.1 scan budget: a hard cap on the bytes -// the iterator may scan — counting BOTH accepted and rejected rows — -// before it must return whatever it has accepted plus the next cursor -// over the last-scanned position (done=false). It bounds the per-call -// cost of a sparse family-wide bracket whose moving slice is empty or -// tiny, where `accept` rejects most rows and they do not count against -// chunkBytes. The server handler normalizes wire max_scanned_bytes=0 -// to the default 4 × chunkBytes before calling this method, and the -// store layer repeats that normalization for maxScannedBytes <= 0 as a -// defensive default; zero is not interpreted as "unbounded" on export -// RPC paths. chunkBytes still caps the accepted-row payload. -ExportVersions(ctx context.Context, start, end []byte, minCommitTS, maxCommitTS uint64, - cursor []byte, chunkBytes, maxScannedBytes int, accept KeyFilter) ([]MVCCVersion, []byte, bool, error) -``` - -The function uses the existing MVCC storage order and value decoder, but deliberately **does not** use the read-visible snapshot filter as its row source. A read at `maxCommitTS` hides tombstones and values whose `ExpireAt <= maxCommitTS`; migration must still copy those committed versions so the target preserves deletes, TTL expiry, and historical reads across CUTOVER. `accept` is consulted before a row is added to the chunk; rejected rows do not advance the chunk-bytes budget so the caller sees deterministic per-call shapes — but they **do** advance the `maxScannedBytes` budget, so a sparse bracket cannot scan an unbounded prefix to prove `done=true` (§6.3.1). - -#### 6.1.1 Cursor granularity — addresses a scanned MVCC position, not just a raw key (closes codex P2; refined for sparse brackets per coderabbit round-12 Major / codex round-14 P2) - -Codex P2 on PR #945: a "next-key" cursor breaks for hot keys with version histories larger than `chunkBytes` — the migrator would either have to send the whole version chain in one unbounded chunk (blowing the memory bound) or skip/duplicate the tail after a restart. The cursor must therefore address an **MVCC position**, not a raw key. - -Concretely, the cursor is opaque on the wire but encodes the tuple `(raw_key, commit_ts, scanned_position_tag)` of the **last scanned MVCC position** — the position the iterator advanced past in this chunk's tail, regardless of whether the version at that position was emitted or rejected (closes coderabbit round-12 Major line 429 and codex round-14 P2 line 540 on PR #945). The earlier "last emitted" wording assumed every iterated row was emitted; that is true for the dense default path but is incorrect for §6.3.1's sparse-bracket / scan-budget-bounded path where the `accept KeyFilter` rejects most rows before `maxScannedBytes` is hit. The corrected definition is: - -- **`raw_key, commit_ts`** — the MVCC position the iterator advanced past in this call's tail. If the last action was an emit, this is the emitted version's `(raw_key, commit_ts)`. If the last action was a reject (skipped by `accept` or by an internal-family filter), this is the rejected row's `(raw_key, commit_ts)`. In either case the iterator's "where I have looked so far" boundary is captured exactly. -- **`scanned_position_tag`** — a 1-bit tag (`0 = last-emitted, 1 = last-scanned-but-not-emitted`) distinguishing the two cases above. The importer's idempotency check (§6.2) keys on `(job_id, bracket_id, batch_seq)` rather than on this cursor, so the tag is informational for the importer — but the **exporter** needs it to know whether to start the next iteration "strictly past the last-emitted version" (tag=0, the original semantic — guarantees no duplicate) or "strictly past the last-scanned position even though nothing was emitted" (tag=1 — guarantees no infinite rescan of a sparse window). - -Resume is **exclusive on the recorded position**: the next call iterates strictly past `(raw_key, commit_ts)` under the export's `(raw_key ASC, commit_ts DESC)` order (newest-first within a key, matching the MVCC iterator already used by snapshot reads in `store/mvcc_store.go`), so the version the cursor names is **not** re-iterated (avoids duplicate work) and the very next version in iteration order **is** iterated (no skip, no infinite rescan of rejected windows). The earlier wording — "encoded as the exclusive `(raw_key, commit_ts)` of the *next-to-emit* row" — was internally inconsistent on two axes: "next-to-emit" + "exclusive resume" reads as "skip the version the cursor names" (would silently drop one MVCC version per chunk boundary for a hot key whose history spans chunks); and "last emitted" alone gave no resume point for a sparse bracket that accepted zero rows before `maxScannedBytes`, leaving the cursor unable to advance at all (would infinitely rescan the same rejected window). The "last-scanned position + tag" form is the only one that preserves no-duplicate AND no-skip AND no-rescan simultaneously. Three properties follow: - -- **Hot-key safe.** A chunk may contain only versions of a single raw key, and the next chunk picks up the next-older version of the same key without any all-or-nothing constraint on the key's history. -- **Resume-safe across restarts.** Restarting from a persisted opaque cursor lands on the exact same MVCC version as a continuous run — no skipped versions (the cursor is exclusive on the resume side) and no duplicates (the importer's idempotency check, §6.2, dedups any one resume that overlaps an ack window). -- **Importer doesn't care about key boundaries.** Versions arrive in `(raw_key, commit_ts DESC)` order; the importer just applies them. No "this batch is the rest of key K" handshake is needed. - -The opaque encoding is `varint(len(raw_key)) || raw_key || uvarint(commit_ts) || byte(scanned_position_tag)`; an empty cursor signals "start from the beginning." Future evolution can add fields after the tag byte because the consumer never parses the cursor by hand — only the exporter reads its own encoding. - -**Importer idempotency keying (closes codex round-14 P2 line 439 — bracket-parallel cursor collision).** Because §6.3.1 runs export brackets in parallel, the importer's idempotency key cannot be `(job_id, cursor)` alone: bracket A can ack a batch with cursor `cA`, bracket B can then overwrite the stored cursor with `cB`, and a retry of A's already-acked batch no longer matches the stored cursor (depending on the importer's matching rule, that either re-applies side effects or drops the retry while ack state diverges). The corrected idempotency key is `(job_id, bracket_id, batch_seq)` where: - -- `bracket_id` is the per-bracket identifier the migrator assigns when it slices the export space in §6.3.1 (`Stable for the life of the SplitJob` — recorded in `bracket_progress`). -- `batch_seq` is a monotonic per-bracket counter the migrator advances on every successful ImportVersions ack. It is contiguous within a bracket: the target accepts `stored+1`, treats `≤ stored` as a duplicate, and rejects `> stored+1` with retryable `ErrImportBatchGap` before writing versions, cursor, HLC floor, or `metaLastCommitTS`. - -The importer persists `(job_id, bracket_id) → (max_contiguous_batch_seq, acked_cursor)` under `!migstage|ack||`. A retry of `(job_id, bracket_id, batch_seq ≤ stored)` is a no-op that returns the stored `acked_cursor`; `batch_seq == stored+1` applies atomically and advances the stored cursor; `batch_seq > stored+1` is rejected as a gap and must not advance the high-water mark. The migrator may pipeline different brackets, but for a single bracket it either sends imports sequentially or retries a rejected later batch after the missing earlier batch is acked. The cursor is still carried in the ImportVersions request — it is used by the **exporter** to resume after a restart — but it is no longer the dedup key. This separation means cursor semantics can be refined (last-emitted vs last-scanned + tag, above) without touching importer idempotency. - -Zero-version progress chunks are first-class batches. When §6.3.1's scan budget is hit after accepting zero rows, the exporter returns `versions=[]`, `done=false`, and a cursor tagged as last-scanned (`scanned_position_tag = 1`). The migrator still sends that chunk through `ImportVersions` with the next per-bracket `batch_seq`; the importer records `(max_contiguous_batch_seq, acked_cursor)` and returns success without writing MVCC versions or advancing clocks. The migrator then CASes the default-group `SplitJob.bracket_progress` entry to the same `(cursor, last_acked_batch_seq)`. A leader flap may re-export the same sparse window once, but the target-side ack makes the retry idempotent and lets the new migrator advance the durable SplitJob cursor instead of resending the rejected window forever. - -The store-side type stays `[]byte` and the exporter alone owns the codec, preserving the §6.1 decoupling seam: the migrator hands the cursor back as opaque bytes on every `ExportVersions` call. - -### 6.2 Target side - -New method: - -```go -// ImportVersions writes the given versions idempotently. The dedup key -// is (jobID, bracketID, batchSeq) — NOT (jobID, cursor) — because §6.3.1 -// runs export brackets in parallel and a single cursor slot would be -// clobbered across brackets (§6.1.1). For each bracket the importer keeps -// a contiguous high-water mark plus acked cursor under -// !migstage|ack||: batchSeq ≤ stored is a duplicate -// no-op, batchSeq == stored+1 applies, and batchSeq > stored+1 returns -// ErrImportBatchGap before MVCC/HLC/cursor state changes. The opaque -// `cursor` is the exporter's resume token (§6.1.1) — carried so the -// migrator can persist it back on ack, but it is NOT the dedup key. -// versions may be empty when a sparse bracket hits maxScannedBytes before -// accepting a row; that still advances (bracketID, batchSeq, cursor) but -// performs no MVCC writes and no HLC/metaLastCommitTS advancement -// (§6.2.1). Non-empty imports apply tombstones via DeleteAt and -// non-tombstones via PutAt(value, commit_ts, expire_at), then atomically -// advance metaLastCommitTS, the target-local full-HLC floor, and the -// node-local physical ceiling to at least max(commit_ts) of the batch -// (§6.2.1). -ImportVersions(ctx context.Context, jobID, bracketID, batchSeq uint64, - versions []MVCCVersion, cursor []byte) ([]byte, error) -``` - -Idempotency: persist `(jobID, bracketID) → (max_contiguous_batch_seq, acked_cursor)` under `!migstage|ack||` after each batch's apply (the bracket dimension is required — a single `!migstage|cursor|` record would be clobbered by parallel brackets per §6.1.1). On a duplicate request drop any batch whose `batchSeq` is `≤` the recorded high-water mark while returning the stored `acked_cursor`; on `batchSeq > max_contiguous_batch_seq + 1`, return retryable `ErrImportBatchGap` without writing versions, clocks, or cursor. The `cursor` is also persisted on the SplitJob's `bracket_progress` entry for exporter resume, separate from the dedup record. Empty batches follow the same ack path; they are not skipped. Non-empty batches preserve the complete MVCC value header relevant to visibility: `tombstone` and `expire_at` are part of the imported version, not recomputed on the target. - -#### 6.2.1 Target HLC advancement — preventing post-CUTOVER time travel (closes codex P1) - -Codex P1 on PR #945: when the source range contains commit timestamps higher than the target group's current `LastCommitTS` / HLC, an import that only writes versions + cursor leaves the target's clock below the staged data. After CUTOVER the target's first new write at `Next() = max(wall, ceiling)` can therefore receive a `commit_ts` **smaller** than imported values — MVCC visibility then resurrects the pre-cutover value on a snapshot read at the new commit_ts, an unambiguous time-travel hazard. - -The fix runs inside `ImportVersions`. The later `PromoteStaged` apply only copies already-imported versions from staged keys into live keys with their original `commit_ts`, `tombstone`, and `expire_at` header; it must not mint or observe a new timestamp. - -1. **Per-batch advance for non-empty batches.** On every apply of an `ImportVersions` batch with at least one version, the target FSM computes `batchMax = max(versions[i].commit_ts)` and **atomically** (under the same FSM apply lock that mutates the MVCC store): - - sets `metaLastCommitTS = max(metaLastCommitTS, batchMax)`, - - sets target-local `!migstage|hlc_floor| = max(current_floor, batchMax)` — this record stores the **full** HLC value, including the logical lower 16 bits, and lives in the target group's Raft state / snapshots rather than only in the default-group SplitJob, - - calls `hlc.SetPhysicalCeiling(hlcPhysicalMs(batchMax))` — the **physical-ms component only**, not the full HLC `commit_ts` (closes codex round-5 P1 on PR #945 line 367). `SetPhysicalCeiling`'s signature is `int64 ms` (`kv/hlc.go:222`) and is interpreted as Unix milliseconds; passing the full encoded ts `commit_ts = ms<<16 | logical` would advance the ceiling to ~`commit_ts<<16` worth of milliseconds — `2^16 ≈ 65 536` × further in the future than intended — and weaken the lease-ceiling fence until wall time catches up. The conversion is `hlcPhysicalMs(ts) = int64(ts >> 16)` (the same shift `HLC.Next()` uses to extract the physical half, `kv/hlc.go:128-145`), AND - - calls `hlc.Observe(batchMax)` so the current leader's in-memory `last` also tracks the full high-water mark — keeps `Next()`'s logical-half advancement above the imported versions while this process stays alive. - - `SetPhysicalCeiling` is monotone — a lower argument is a no-op — so duplicate batches are safe. A later per-bracket batch arriving before an earlier one is not applied: the `ErrImportBatchGap` rule in §6.2 rejects it before clock or MVCC state changes, preserving the contiguous ack invariant. `Observe` is similarly monotone for duplicate accepted batches. - - Empty `versions=[]` progress chunks have no `batchMax`. They still advance the per-bracket `(batch_seq, cursor)` ack described in §6.1.1 / §6.2, but they MUST NOT change `metaLastCommitTS`, target-local `!migstage|hlc_floor|`, call `SetPhysicalCeiling`, call `Observe`, or update `SplitJob.max_imported_ts`. This keeps sparse-bracket progress durable without inventing a timestamp for a batch that imported no committed version. -2. **Two monotone witnesses with different owners.** The default-group SplitJob carries `max_imported_ts` (§3.1) — the high-water mark of all non-empty ack'd import batches for this job. The migrator updates it whenever it advances a bracket's `cursor` / `last_acked_batch_seq` in `bracket_progress` on ack of a non-empty batch (§6.1.1), so the coordinator can reason about CUTOVER. The target group owns the enforcement copy under `!migstage|hlc_floor|`; target startup and snapshot restore must be correct even if the default-group leader is unavailable. -3. **CUTOVER precondition.** Before the migrator issues the CUTOVER catalog write, it computes `cutoverFloor = max(max_imported_ts, fence_ts)` and ensures the target group's durable full-HLC floor is at least that value by issuing a final target-group proposal (not default-group) that updates `!migstage|hlc_floor| = max(current_floor, cutoverFloor)`, calls `SetPhysicalCeiling(hlcPhysicalMs(cutoverFloor))`, and calls `Observe(cutoverFloor)`. The proposal is gated by `cap_migration_v2` (§11.1) and is the **last** target-side write before CUTOVER. If the target HLC is already above this value (e.g. due to per-batch advancement), the proposal is a no-op except for confirming the durable floor; if a follower flap or snapshot restore dropped in-memory `HLC.last`, this proposal closes the live-leader gap before the catalog is published. The default-group CUTOVER descriptor for `target.Active(right)` carries the same value in `RouteDescriptor.min_write_ts_exclusive`, so the timestamp floor survives after staged metadata is later deleted. `fence_ts` is load-bearing even when the moved range is empty or only contains old versions: a client may have observed a source timestamp near the fence via `RawLatestCommitTS` / `RawGet(ts)`, and the first post-CUTOVER target write must not receive a timestamp `<= fence_ts`. -4. **Restart/snapshot invariant.** `metaLastCommitTS` is not enough: current `HLC.Next()` reads in-memory `HLC.last` and the physical ceiling, but does **not** consult `metaLastCommitTS`. A restored node that only replays `SetPhysicalCeiling(hlcPhysicalMs(max_imported_ts))` can issue `(ms<<16)|0` even when an imported version was `(ms<<16)|logical>0`, and a sparse migration can issue a timestamp below `fence_ts` if it never imported a high version. Therefore, on target FSM startup, snapshot restore, and leadership acquisition before serving writes for any route whose migration high-water mark is non-zero, the target computes `fullFloor = max(store.LastCommitTS(), all !migstage|hlc_floor|*, all local RouteDescriptor.min_write_ts_exclusive values)`, then calls `hlc.SetPhysicalCeiling(hlcPhysicalMs(fullFloor))` and `hlc.Observe(fullFloor)`. Re-observing the **full** HLC value is the preferred design because it preserves logical-half monotonicity without artificially jumping the physical ceiling by one millisecond; if an implementation cannot load the full value, the fail-safe alternative is `SetPhysicalCeiling(hlcPhysicalMs(fullFloor)+1)` before accepting writes. -5. **Post-CUTOVER write-side invariant, including caller-supplied timestamps.** A target-group `Next()` after CUTOVER is strictly greater than both every imported `commit_ts` and the source `fence_ts` because the live HLC has observed `cutoverFloor` before writes are admitted. Reads at any post-CUTOVER `Next()` therefore see imported/source-fenced versions as historical and a new live write as the most-recent committed version — no resurrection, no time travel. - - This `Next()` invariant is not enough for adapters that pass a caller-supplied `commit_ts` through `resolveDispatchCommitTS` unchanged. The target FSM must therefore enforce a durable write-side floor before any live MVCC write or prepare lock is created: compute `floor = max(route.min_write_ts_exclusive, !migstage|hlc_floor| when staged visibility/readiness is still active)` and reject any raw write, one-phase txn, or prepared txn whose supplied / already-allocated `commit_ts <= floor` with `ErrMigrationTimestampTooLow` (mapped to a retryable coordinator error that refreshes and reallocates unless the caller explicitly requires that timestamp). This check runs on the target apply path next to the staged/live merge flag, not only in the coordinator, so a stale or direct RawKV caller cannot insert a low live version below an already-imported staged version or below the source fence boundary. Coordinator-allocated writes pass naturally because `Next()` has observed the full floor; the rejection exists only for externally supplied or replayed low timestamps. Promotion and cleanup may clear `staged_visibility_active` and delete `!migstage|hlc_floor|` after the cleared-descriptor ACK barrier (§6.4 step 5), but they must retain `route.min_write_ts_exclusive` on the target descriptor. The route is ordinary MVCC after promotion except for this permanent lower-bound check on caller-supplied/replayed timestamps. -6. **Same-group split (M1) unaffected.** Same-group split never crosses the FSM apply boundary, so its `Next()` is already bounded by the source's HLC; the new advance path is a no-op when `target_group_id == source_group_id`. - -This reuses the same Raft-agreed monotone physical-ceiling primitive that PR #927 / Composed-1 already relies on, plus a small target-local full-HLC floor for the logical half and fence boundary and a durable route descriptor floor for caller-supplied timestamps after promotion. Test coverage: §10.1 unit gains an explicit `kv/migrator_hlc_advance_test.go` that asserts (a) `metaLastCommitTS`, target-local `!migstage|hlc_floor|`, `RouteDescriptor.min_write_ts_exclusive`, and HLC ceiling are advanced/fixed by `max(batch.commit_ts)` / `max(max_imported_ts, fence_ts)`, (b) the CUTOVER pre-write proposal closes any gap left by a missing batch and floors at `max(max_imported_ts, fence_ts)`, including empty/sparse moved ranges, (c) restart/snapshot restore re-observes a full `max_imported_ts = ms<<16|logical` so the first post-restore `Next()` is `> max_imported_ts`, (d) a control that restores only the physical ceiling can emit `ms<<16|0` and fails, (e) a caller-supplied target write with `commit_ts <= fence_ts` is rejected even when `max_imported_ts < fence_ts`, and (f) the same low timestamp remains rejected after promotion clears `staged_visibility_active` and target cleanup deletes the per-job `!migstage|hlc_floor|` record. Property test extends the §10.1 export-chunks/import-acks/leader-flap sequence with a `Next()` invocation after CUTOVER and after snapshot restore, asserting strict monotonicity vs. every imported ts and `fence_ts`. - -### 6.3 Internal-key coverage and the routeKey delegate - -§9 of the parent doc enumerates the key families. The migrator builds a `KeyFilter` closure in `kv/` (where `routeKey()` lives) and passes it to `store.ExportVersions`: - -```go -// kv/migrator_filter.go (sketch) -func RouteKeyFilter(rangeStart, rangeEnd []byte) store.KeyFilter { - return func(rawKey []byte) bool { - rKey := routeKey(rawKey) - // half-open [rangeStart, rangeEnd) in the routing-key namespace. - if bytes.Compare(rKey, rangeStart) < 0 { return false } - if len(rangeEnd) > 0 && bytes.Compare(rKey, rangeEnd) >= 0 { return false } - return true - } -} -``` - -The closure ensures internal keys (the concrete txn subprefixes such as `!txn|lock|...`, `!txn|cmt|...`, plus `!lst|...`, Redis collection families, etc.) land in the same shard as their logical owner without `store` ever importing `kv`. `rangeEnd` uses the wire convention from §5.2: nil or empty means `+infinity`. The `len(rangeEnd) > 0` check is therefore part of the correctness contract, not just a convenience — an empty-but-non-nil slice must not make every non-empty `routeKey(rawKey)` compare `>= rangeEnd` and reject the entire rightmost route. The importer dispatches by `key_family` into the matching store helper (in a future kv-level `Import` wrapper) to preserve per-family invariants (e.g., list head pointer updates). - -#### 6.3.1 Internal-family brackets — raw range alone misses internal state (closes codex P1 line 310) - -The above closure correctly *rejects* a raw key that does not belong to the moving range, but it cannot *find* internal-family keys that the caller never iterates over in the first place. Codex P1 round-4 on PR #945 (line 310): for a moving user-key interval `[foo, bar)`, an iteration over the raw range `[foo, bar)` never visits `!txn|lock|foo`, `!lst|meta|foo`, `!redis|str|foo`, `!ddb|item||foo`, `!ddb|meta|table|`, `!ddb|meta|gen|`, `!sqs|...`, `!s3|...`, because each of those prefixes sorts **outside** `[foo, bar)` in the raw MVCC key space — the routeKey filter rejects nothing because those bytes were never iterated. Without the fix below, CUTOVER would leave intent locks, list metadata, DynamoDB table schema / generation, and adapter-private state behind on the source and the target would serve reads against a half-populated MVCC space — silent data loss. - -M2 therefore runs **multiple parallel exports per migration**, one per relevant raw-key family, each driven by the same `KeyFilter` so `routeKey()` remains the single source of truth for "does this key belong to the moving range?". The migrator computes the bracket list from the moving range's logical scope: - -```go -// kv/migrator_export_plan.go (sketch) -// -// Each bracket is one (start, end) raw-key band the migrator -// ExportVersions over. The KeyFilter rejects any key whose -// routeKey() falls outside the moving range — necessary because -// each bracket below is FAMILY-WIDE (every txn lock cluster-wide, -// not just the moving range's), and the filter narrows it back to -// the moving slice. -type ExportBracket struct { - Start, End []byte // raw key bounds for ExportVersions - Family uint32 // MVCCVersion.key_family tag - ExcludeKnownInternal bool // true only for familyUser -} - -func PlanExportBrackets(routeStart, routeEnd []byte) []ExportBracket { - return []ExportBracket{ - // User-key bracket itself, in the routing-key namespace. This bracket - // rejects any raw key with a known internal prefix; those rows are - // exported exactly once by the explicit family brackets below. - {Start: routeStart, End: routeEnd, Family: familyUser, ExcludeKnownInternal: true}, - // Each internal-family raw-key band. routeKey() decodes the - // family-prefixed raw key back to its routing key, so the - // filter rejects any entry whose routing key falls outside - // [routeStart, routeEnd). - // Intent locks are drain-only control state (§3.2a.0a/0b), not - // migrated data. ExportVersions excludes in-flight locks, and the - // target's empty lock space after the fence drain is intentional. - {Start: txnCmtPrefix(), End: txnCmtPrefixEnd(), Family: familyTxnCommit}, - {Start: txnOkPrefix(), End: txnOkPrefixEnd(), Family: familyTxnSuccess}, - {Start: txnRbPrefix(), End: txnRbPrefixEnd(), Family: familyTxnRollback}, - {Start: txnMetaPrefix(), End: txnMetaPrefixEnd(), Family: familyTxnMeta}, - {Start: txnIntPrefix(), End: txnIntPrefixEnd(), Family: familyTxnInternal}, - {Start: lstMetaPrefix(), End: lstMetaPrefixEnd(), Family: familyListMeta}, - {Start: lstMetaDeltaPrefix(), End: lstMetaDeltaPrefixEnd(), Family: familyListMetaDelta}, - {Start: lstItmPrefix(), End: lstItmPrefixEnd(), Family: familyListItem}, - {Start: lstClaimPrefix(), End: lstClaimPrefixEnd(), Family: familyListClaim}, - {Start: redisPrefix(), End: redisPrefixEnd(), Family: familyRedisLegacy}, // !redis|... - {Start: redisHashWidePrefix(), End: redisHashWidePrefixEnd(), Family: familyRedisHash}, // !hs|... - {Start: redisSetWidePrefix(), End: redisSetWidePrefixEnd(), Family: familyRedisSet}, // !st|... - {Start: redisZSetWidePrefix(), End: redisZSetWidePrefixEnd(), Family: familyRedisZSet}, // !zs|... - {Start: streamMetaPrefix(), End: streamMetaPrefixEnd(), Family: familyRedisStreamMeta}, - {Start: streamEntryPrefix(), End: streamEntryPrefixEnd(), Family: familyRedisStreamEntry}, - {Start: ddbTableMetaPrefix(), End: ddbTableMetaPrefixEnd(), Family: familyDdbTableMeta}, - {Start: ddbTableGenerationPrefix(), End: ddbTableGenerationPrefixEnd(), Family: familyDdbTableGeneration}, - {Start: ddbItemPrefix(), End: ddbItemPrefixEnd(), Family: familyDdbItem}, - {Start: ddbGSIPrefix(), End: ddbGSIPrefixEnd(), Family: familyDdbGSI}, - // Concrete adapter storage prefixes only, not the whole !sqs| / - // !s3| umbrellas. M2 must narrow sqsRouteKey/s3RouteKey to those - // concrete prefixes too; raw user keys such as !sqs|foo migrate - // through familyUser unless the data plane explicitly reserves them. - sqsConcreteStorageBrackets(routeStart, routeEnd)..., - s3ConcreteStorageBrackets(routeStart, routeEnd)..., - // A future adapter MUST add its bracket here at the same PR - // that teaches routeKey() about its family — both are - // required to land internal-family migration end-to-end. - } -} -``` - -The bracket plan is **disjoint by construction**. `familyUser` is not "whatever bytes fall in `[routeStart, routeEnd)`"; it is user MVCC rows after excluding `knownInternalPrefixes`. The exclusions are **concrete internal subprefixes only** unless the data plane already rejects the entire umbrella prefix. For txn, exclude `!txn|lock|`, `!txn|int|`, `!txn|cmt|`, `!txn|rb|`, `!txn|ok|`, and `!txn|meta|`, not the entire `!txn|` umbrella: a legal user key such as `!txn|foo` stays in `familyUser` and migrates with ordinary raw user rows. `!txn|lock|` is the one txn exclusion that intentionally has no export bracket: intent locks are drained by the route-faithful lock scan (§3.2a.0a/0b), never imported as migrated data, and the target's empty lock space after the drain is correct. The same rule applies to adapter-looking raw keys: DDB excludes `!ddb|meta|table|`, `!ddb|meta|gen|`, `!ddb|item|`, and `!ddb|gsi|`, not every `!ddb|*`; SQS/S3 exclude only their concrete persisted storage subprefixes returned by the adapter bracket helpers; stream excludes only `!stream|meta|` and `!stream|entry|`. A raw user key such as `!ddb|foo`, `!sqs|foo`, `!s3|foo`, or `!stream|foo` remains in `familyUser` unless the corresponding adapter explicitly reserves/rejects that exact prefix on the write path. This requires the route-key decoder to match the bracket plan: txn success markers route by their decoded length-prefixed `locked_key`, and `sqsRouteKey` / `s3RouteKey` must recognize only concrete adapter storage prefixes and otherwise fall through to the raw user route (or the write path must reserve the whole umbrella and this paragraph changes accordingly). A raw `!sqs|foo` must not be mapped to `!sqs|route|global` unless it is a real SQS storage key; otherwise `RouteKeyFilter` would drop it from `familyUser` while no SQS bracket owns it. Store-owned Redis/list/hash/set/zset umbrellas (`!redis|`, `!lst|`, `!hs|`, `!st|`, `!zs|`) remain excluded because those prefixes are treated as internal storage families in the data plane. Reserved migration/control prefixes (`!dist|`, `!migstage|`, `!migwrite|`, `!migfence|`, etc.) are rejected by `StartSplitMigration` up front rather than exported as data. Every excluded prefix must either have an explicit family bracket below, be the drain-only `!txn|lock|` family, or be a reserved control prefix. This prevents the user bracket from exporting the same `(raw_key, commit_ts)` that an internal-family bracket exports; importer idempotency is per `(job_id, bracket_id, batch_seq)` and is not allowed to mask cross-bracket duplicates. - -M2 also extends `routeKey()` for every store-owned collection prefix that persists live user-visible state. Today `routeKey()` handles the `!redis|...` family and base list meta/item keys, but list reads and POP protection also consume `store.ListMetaDeltaPrefix` (`!lst|meta|d|`) and `store.ListClaimPrefix` (`!lst|claim|`). The decoder must therefore check `store.ExtractListUserKeyFromDelta` and `store.ExtractListUserKeyFromClaim` before the broader base-meta `store.ExtractListUserKey` path; `!lst|meta|d|` has `!lst|meta|` as a byte prefix, so the order is load-bearing. Current hash/set/zset/stream state is stored under `store.HashMetaPrefix`, `store.HashFieldPrefix`, `store.HashMetaDeltaPrefix`, `store.SetMetaPrefix`, `store.SetMemberPrefix`, `store.SetMetaDeltaPrefix`, `store.ZSetMetaPrefix`, `store.ZSetMemberPrefix`, `store.ZSetScorePrefix`, `store.ZSetMetaDeltaPrefix`, `store.StreamMetaPrefix`, and `store.StreamEntryPrefix`. Those layouts carry the logical Redis user key after a length-prefixed segment, so the decoder must call the matching `store.ExtractHashUserKeyFrom{Meta,Field,Delta}`, `store.ExtractSetUserKeyFrom{Meta,Member,Delta}`, `store.ExtractZSetUserKeyFrom{Meta,Member,Score,Delta}`, and `store.ExtractStreamUserKeyFrom{Meta,Entry}` helpers. SQS/S3 route decoders follow the same rule: decode only concrete persisted storage prefixes, and leave adapter-looking but unreserved raw keys on their raw route. The `KeyFilter` then accepts an internal-family raw key whose routing key falls inside the moving range and rejects others. Adding a bracket without a matching `routeKey()` decoder, or broadening a route decoder beyond the bracketed concrete prefixes, is a correctness bug, and the export-plan tests below cover both halves together. - -List export includes base meta, meta-delta, item, and claim families because all four can affect the post-CUTOVER value or OCC behavior of a Redis list. Hash/set/zset use non-overlapping umbrella brackets (`!hs|`, `!st|`, `!zs|`) rather than separate meta/delta brackets because their delta prefixes are nested under the meta prefix (`!hs|meta|d|` under `!hs|meta|`, etc.). The import dispatch still inspects the raw subprefix inside the bracket to preserve per-family invariants. Stream uses its two concrete prefixes (`!stream|meta|`, `!stream|entry|`) instead of an umbrella `!stream|` bracket, matching `knownInternalPrefixes`: a user key such as `!stream|foo` is legal and must remain a user-key row, while only the concrete meta/entry prefixes are internal stream storage. - -The Redis collection brackets are mandatory, not an optimization. Redis strings, TTL rows, HLL rows, and legacy/blob collection state use `!redis|...`; list state spans `!lst|meta|`, `!lst|meta|d|`, `!lst|itm|`, and `!lst|claim|`; modern hash/set/zset state lives under the store-owned `!hs|`, `!st|`, and `!zs|` prefixes, while stream state uses only the concrete `!stream|meta|` and `!stream|entry|` prefixes above. Copying only `!redis|...` during a cross-group split would make the target serve empty or partial `LRANGE`, `HGETALL`, `SMEMBERS`, `ZRANGE`, and `XRANGE` results after CUTOVER, and omitting list claims can lose POP claim protection. Stream metadata and stream entries both have to be copied: `KEYS` may expose a stream through metadata, but migration must preserve every entry for `XRANGE` / `XREAD`. A user key named `!stream|foo` remains in `familyUser` and must not be captured by an umbrella stream bracket. - -The DynamoDB metadata brackets are mandatory, not an optimization. `routeKey()` maps `!ddb|meta|table|
` and `!ddb|meta|gen|
` to the same `!ddb|route|table|
` route as `!ddb|item|
|...` and `!ddb|gsi|
|...` (`kv/shard_key.go`). The adapter reads those keys directly in `loadTableSchemaAt` and `loadTableGenerationAt` (`adapter/dynamodb.go`), so copying item/GSI rows without schema/generation leaves the target with data that DynamoDB treats as a missing table or generation 0. A future DynamoDB storage family must therefore add both a `routeKey()` decoder and a bracket in the same PR. - -The migrator runs the brackets **in parallel up to `--migrationExportFanout` (default 4)** within a single phase (BACKFILL or DELTA_COPY): each bracket gets its own opaque cursor (the §6.1.1 codec is extended to key on `(family, raw_key, commit_ts)`), so a bracket can be paused/resumed independently and the §9 resumability matrix applies per bracket. The migrator records per-bracket `cursor` and `done` flags on the SplitJob; the phase advances only when **every bracket** reports `done = true`. Cost is at most `fanout`× the gRPC frame budget and is bounded by the static bracket list, not by data volume. - -**Bounded scan for sparse / out-of-range brackets (closes codex P2 round-5 line 445).** "Returns its first chunk empty and `done=true` immediately" is only true when the iterator can short-circuit; under the §6.1 contract, `accept` rejections do **not** count against `chunkBytes` (so the caller sees deterministic chunk shapes), which would otherwise let a family-wide raw range scan an arbitrarily large prefix to prove `done=true` for a moving range with no intersection. The fix has two layers: - -- **Scan-budget pacing.** Each `ExportVersions` call carries a separate scan budget — the `maxScannedBytes` parameter on the store-layer signature above, carried on the wire as `ExportRangeVersionsRequest.max_scanned_bytes` (default 4 × `chunkBytes`; `0` on the wire means "use the server default", mirroring how the migrator leaves other tuning fields unset) — that **counts both accepted and rejected rows** as iteration cost, where `chunkBytes` alone counts only accepted rows. The internal RPC server translates wire `0` to `4 × chunkBytes` before calling `store.ExportVersions`, and the store repeats the same `<= 0` normalization defensively so a sparse family bracket can never become unbounded by accidentally passing the wire sentinel through. When the iterator hits that budget without filling the accepted chunk, it returns whatever it has accepted so far + the next cursor (over the *rejected* tail position — the §6.1.1 `scanned_position_tag = 1` case, so the next call resumes strictly past the already-rejected window). `done = false`, so the next chunk picks up where the rejection scan left off. Without this field on the contract, a Redis migration where the SQS family has a cluster-wide 10 GB raw prefix would block on a single `ExportVersions` call for the entire prefix duration. -- **Route-key sub-prefix indexing where the family layout allows it.** Each exported internal-family bracket's raw key starts with a known fixed prefix (e.g. `!txn|cmt|`, `!lst|meta|`) followed by the routing-key bytes (mostly — adapters like SQS encode queue ID first; per-family `EncodeBracketStart(routeStart)` is the per-family hook). When such a hook is available, the bracket's `Start` / `End` are tightened to `(familyPrefix || EncodeBracketStart(routeStart), familyPrefix || EncodeBracketStart(routeEnd))` instead of the family-wide bounds, reducing the scan budget consumption from "full family" to "moving slice within family." Families without an encoder hook (initially `!s3|...`, until the design records its routing-key encoding) keep the family-wide bracket + the scan-budget pacing above; the scan-budget alone is sufficient for correctness, the encoder hook is a per-family optimization that lands in the matching adapter's PR. The `!txn|lock|` family uses the same route-key filtering idea only for drain scans, not for data export. - -These two layers together preserve the §9 resumability matrix per bracket (cursor still advances through both accepted and rejected rows; a leader flap resumes at the persisted scan cursor) and bound the per-chunk wall-clock cost regardless of how sparse the bracket happens to be. The `keys per bracket` distribution AND `rejected_rows per bracket` are §7.3 / §11.x metrics so operators can spot a runaway family scan and prioritize adding an encoder hook for that adapter. - -A simple safety check: every exported key in the migrator's send buffer must map back to the moving range under `routeKey()`. The migrator asserts this on every row before the gRPC send; an assertion failure aborts the job and surfaces in `last_error`. The assertion now guards two failure modes: (i) a future internal-family being added that someone forgot to teach `routeKey()`, and (ii) a bracket being added without a matching `routeKey()` decoder. Reuses the same routeKey assertion §6.3 already specifies — no new code surface beyond the bracket list. - -### 6.4 Incremental staged-to-live promotion — CUTOVER is constant-time (closes codex P2 on PR #945) - -Codex P2 round-3 on PR #945 (line 147): "staging every imported row under `!dist|migstage|...` and then promoting the whole range via a single FSM apply makes CUTOVER proportional to the entire migrated range. For any split larger than the Raft proposal/apply budget, this defeats the chunked BACKFILL/DELTA_COPY design and can block or fail exactly when the catalog must switch atomically." The proposal must therefore avoid a per-key bulk move at cutover. M2 ships an incremental promotion path that keeps CUTOVER a constant-time catalog write while preserving the visibility fence: - -0. **Target staged-readiness barrier before the target can become Active.** After DELTA_COPY completes and after the final target full-HLC floor proposal in §6.2.1, but before the default-group CUTOVER CAS can publish `target.Active(right)`, the migrator persists `target_staged_readiness_state = ARMING` on the default-group SplitJob and proposes/probes `ApplyTargetStagedReadiness(job_id, route_start, route_end, expected_cutover_version, migration_job_id, min_write_ts_exclusive=max(max_imported_ts, fence_ts))` until every current target voter has applied it and advanced `target_staged_readiness_ack_cursor`. The target FSM persists `!migstage|ready|` in its own Raft state/snapshots and treats that record as a fail-closed guard for the moving route interval. - - While the readiness record is armed, any target read or write whose route-key-normalized point/range intersects `[route_start, route_end)` must prove one of three things before touching live MVCC: either (a) the target's local route snapshot contains `staged_visibility_active=true` with the matching `migration_job_id` and catalog version, so reads use the staged/live merge and writes enforce the floor; (b) the local route snapshot contains the **matching cleared descriptor** for the same route (`staged_visibility_active=false`, `migration_job_id=0`, and retained `min_write_ts_exclusive >= ready.min_write_ts_exclusive`) after promotion-complete, so the route is ordinary MVCC plus the durable timestamp floor; or (c) the default-group ownership lookup says the CUTOVER has not committed yet and the request should retry. If none of those proofs is available (watcher lag, ownership RPC timeout, snapshot restore before route replay), the target returns `ErrRouteCutoverPending` / `ErrRouteOwnershipUnknown` and **must not** read live-only data or accept a live write for the interval. This closes the target-side mirror of §7.2.2e: a fresh coordinator may learn the new target route from the default group before the target FSM has applied the same descriptor locally, so the target must fail closed until it can run the staged/live merge and timestamp floor. The cleared-descriptor allowance prevents the later cleanup window from making the route unavailable merely because `!migstage|ready|` is intentionally retained until every target voter ACKs §6.4 step 5. - - After every current target voter ACKs the applied guard, the migrator CASes the SplitJob to `target_staged_readiness_state = ARMED`; a restart at `ARMING` probes/re-proposes the same target record idempotently for non-ACKed voters. A pre-CUTOVER rollback transitions through `CLEARING` and removes the target readiness record before returning to DELTA_COPY/FAILED. On the successful path the record remains until after the default-group promotion-complete witness has cleared `staged_visibility_active`, retained `min_write_ts_exclusive`, and every target voter observes the cleared descriptor; only then can target-local cleanup delete `!migstage|ready|` together with the per-job HLC floor and promotion proof (§6.4 step 5). A target voter that restarts or becomes leader before applying this guard must fail closed for the moving interval until the guard entry or the matching descriptor is loaded; it may not serve live-only by assuming the old local route snapshot is authoritative. - -1. **CUTOVER itself is one catalog write — no per-key work.** The CUTOVER FSM apply on the default group does **only**: - - (a) CAS-bump the catalog version, - - (b) remove the source's `WriteFenced(right)` route, - - (c) insert the target's `Active(right)` route with `raft_group_id = target`, - - (d) populate `cutover_version` on the SplitJob and stamp `staged_visibility_active = true`, `migration_job_id`, and `min_write_ts_exclusive=max(max_imported_ts, fence_ts)` on the new target route. - - No iteration over the migrated key range, no bulk rename, no per-key proposals. The Raft proposal carrying this apply is `O(1)` in the migrated data size — bounded by a handful of catalog descriptors and the SplitJob record. - -2. **Staged keyspace participates in MVCC merge — raw newest candidate wins (closes codex P1 line 408 and codex round-18 P1 line 705).** After CUTOVER the target FSM's read path treats the staged area as **a second source of raw MVCC versions for the same logical key**, not as an authoritative override and not as a reader-visible fallback. For a read for key `K` at `read_ts` in the moved range with `staged_visibility_active = true`: - - read the **raw newest live MVCC candidate of `K` with `commit_ts ≤ read_ts`**, including its `tombstone` and `expire_at` header, without applying the read-visible filter that hides deletes / expired TTL values; - - read the **raw newest staged MVCC candidate of `K` with `commit_ts ≤ read_ts`** under `!dist|migstage||` with the same raw header fields; - - choose the candidate with the greater `commit_ts` across both sources. If neither exists, return "not found". If the winning candidate has `tombstone=true`, return "not found". If the winning candidate has `expire_at != 0 && expire_at <= read_ts`, return "not found". Only otherwise return the winning candidate's value. - - The "raw candidate first, visibility filter after the cross-source max" ordering is required during partial promotion. A standard snapshot read would treat a live tombstone / expired version as absent, then incorrectly fall back to an older staged value. Example: staged has `K@10=value` and `K@20=tombstone`; the promoter copies/deletes the `K@20` row into live first and deletes that staged row, while staged `K@10` remains. At `read_ts >= 20`, the raw merge sees live `K@20` as the winner and returns not found. A reader-visible live lookup would see "absent" and resurrect staged `K@10` until the promoter reaches it. - - This is the standard MVCC visibility rule applied **after** combining two column-family-like sources, not a "stage-shadows-live" fallback. The earlier wording — "first look at staged, fall back to live when staged is absent" — was wrong precisely because §6.2.1's HLC floor guarantees a live write after CUTOVER has `commit_ts > max_imported_ts`, so for any `read_ts ≥ live_write.commit_ts` the live version is the most-recent committed value and a `staged-first-then-live-fallback` would return the older staged value while the staged entry still exists — exactly the visibility regression codex P1 line 408 flagged. The raw-merge form is correct because: - - For `read_ts < live_write.commit_ts`: live has no raw candidate ≤ read_ts (or only an older one), staged's imported `commit_ts ≤ max_imported_ts < live_write.commit_ts`, so the merge correctly surfaces whichever visible winning value is newest at `read_ts` (typically the staged one for `read_ts ∈ (max_imported_ts, live_write.commit_ts)`). - - For `read_ts ≥ live_write.commit_ts`: the live version's `commit_ts > max_imported_ts ≥ every staged version's commit_ts`, so the merge correctly returns the live value unless that live winner is itself a delete/expired value, in which case it correctly returns not found without falling back. - - For never-overwritten keys post-CUTOVER: only the staged raw candidate exists ≤ read_ts, so the merge returns it if visible and returns not found if the staged winner is a tombstone/expired TTL version. - - For a key with multiple staged versions (the §6.1.1 hot-key case): the staged raw-candidate iterator returns the newest staged version ≤ read_ts, the live raw-candidate iterator returns the newest live version ≤ read_ts, and the merge applies visibility only to the greater of the two. Correct in every interleaving, including partial promotion of hidden versions. - - Writes always land in the live keyspace — the CUTOVER bump already routed writers to the target group via the catalog, and §6.2.1's full-HLC floor guarantees their `commit_ts` is strictly greater than every imported `commit_ts`, so a live write shadowing a staged version is the *correct* MVCC visibility AND the merge rule above surfaces it correctly. This gives reads access to the imported data the instant CUTOVER lands, with **no promotion work blocking CUTOVER itself**, and with no staleness window where staged shadows a fresher live write. - - Per-family adapter helpers (Redis list head pointers, DynamoDB GSI shadow rows, etc.) follow the same merge rule on the corresponding internal-family keys — the merge operates on raw keys, so the per-family invariants the `key_family` dispatch (§6.3.1) already enforces still apply. - - `RawLatestCommitTS` / `ShardStore.LatestCommitTS` uses the same staged+live raw-candidate lookup for its point key while `staged_visibility_active=true`: return the maximum committed timestamp across live MVCC and the staged prefix for that logical key, including tombstone/TTL versions because the API asks for the latest commit timestamp, not read-visible value. After promotion clears staged visibility, the live MVCC result is sufficient, still guarded by `min_write_ts_exclusive` for future writes. A target that answers `LatestCommitTS` from live-only state before promotion would report missing/low timestamps for imported rows and break the read-fence atomicity guaranteed by §7.2. - - Write-side transaction probes also use the staged+live raw-candidate view while `staged_visibility_active=true`. New writes still create live MVCC versions/intents only, but any validation, deduplication, or status lookup that decides whether a write is idempotent or legal must search the same logical record across live and staged keyspaces before concluding "absent." This includes `txnCommitTS` commit-record checks, rollback markers, one-phase `PrevCommitTS` probes, primary/secondary lock status lookups, and adapter-specific internal keys that participate in OCC. Otherwise a retry after CUTOVER but before promotion reaches those internal keys could miss staged proof and create a contradictory live record. The proof result is type-specific: primary commit records, identity-bearing commit markers, or per-key transaction-success markers carrying `(primary_key,start_ts,commit_ts)` can drive idempotent COMMIT success; a plain committed MVCC version that lacks transaction identity is only a cross-check and must not be the sole success proof; rollback markers drive the idempotent abort/rollback result. The staged+live probe rule is cleared only after the default-group promotion-complete CAS clears `staged_visibility_active` and every target voter has ACKed the cleared descriptor (§6.4 step 5). - - **2a. Range-scan merge iterator (closes codex round-14 P1 line 562).** The point-read rule above generalises to range reads via a **raw-candidate merged iterator**, not by scanning live alone and not by merging only reader-visible rows. For `ScanAt(read_ts, scan_start, scan_end)` or `RawScanAt(...)` on a target route with `staged_visibility_active = true` and `scan` overlapping the route's `[routeStart, routeEnd)`, the target FSM constructs a merged iterator that: - - walks raw live MVCC candidates in `[scan_start, scan_end)` at `read_ts`, including tombstones and TTL headers, AND - - walks raw staged MVCC candidates under `!dist|migstage|||*` prefix in cursor-resumable chunks (the same `chunkBytes` / pacing knobs §6.1 and the migrator already use). Promotion progress is target-local Raft state under `!migstage|promote|`, never a cursor in the default-group SplitJob. For each staged batch the leader proposes one `PromoteStaged` Raft entry that: - - copies the staged key into its live position with the original imported `commit_ts` — **always**, regardless of whether a newer live version at a higher `commit_ts` exists. The MVCC store naturally holds both `K@commit_ts=3` (the staged version being promoted) and `K@commit_ts=10` (a later live write) at different commit timestamps; a snapshot read at any `read_ts ∈ [3, 10)` correctly sees the imported `K@3`, and a read at `read_ts ≥ 10` correctly sees `K@10`. The idempotency guard for this copy is "**no-op if a live version at the same `commit_ts` already exists**" — that means this exact staged version was already promoted in a prior batch before a leader flap re-tried it. It is **not** "no-op when a newer live version exists," which would silently drop the imported version from MVCC history and make snapshot reads at older `read_ts` return wrong results (closes claude round-3 P1, 5-round flag). History pruning belongs to `kv/compactor.go`, not the promoter, - - physically deletes the exact staged MVCC row it just copied, identified by `(job_id, raw_key, commit_ts)` under `!dist|migstage||*`. This is a storage-level remove of the staged version / control row, not a user-visible `DeleteAt` tombstone with a fresh timestamp. Leaving a tombstone in the staged prefix would keep the merge iterator and empty-prefix proof non-empty forever, and a normal MVCC tombstone could hide older staged rows incorrectly during partial promotion, - - advances the target-local `PromotionState.promote_cursor`. - Each batch is a small bounded proposal, exactly like BACKFILL — there is no single oversized apply. - -4. **Promotion is restart-safe.** The promoter is leader-only; a leader flap restarts from the target-local `PromotionState.promote_cursor`. Because the cursor update, live copy, and staged delete happen in the same target-group Raft apply, a crash cannot persist a cursor that skips rows whose promotion did not durably apply. Concurrent reads keep using the raw-candidate staged/live merge path (step 2) for whatever has not yet been promoted, so the user-visible state is correct throughout. - -5. **CLEANUP → DONE precondition and target-local cleanup.** The target group first completes promotion locally: a bounded target-group Raft apply verifies the staged prefix is empty and marks `!migstage|promote|.done = true` while retaining the import acks, HLC floor, readiness record, and promotion proof as recovery evidence. Only after the migrator observes that target-local `done=true` report does it issue a default-group promotion-complete CAS that clears `staged_visibility_active` / `migration_job_id` from the target route and records `target_promotion_done = true` plus `promotion_completed_ts`; the SplitJob remains in CLEANUP until the source and target cleanup ACKs below are durable. This ordered two-owner handoff is required: the default group must never clear the merge flag based on a cursor it owns, because a crash between target and default writes could otherwise skip unpromoted staged rows or hide staged rows before the target copy is durable. If the target completion apply lands but the default CAS does not, reads still pay the merge cost over an empty staged prefix until the migrator retries the CAS; correctness is preserved. - - After that default-group promotion-complete witness is durable, the migrator does **not** immediately delete target-local proofs. It first drives a target-group cleared-descriptor ACK barrier: for every current target voter it proposes/probes `AcknowledgeTargetClearedDescriptor(job_id, promotion_completed_ts)`, whose apply handler succeeds only after that voter's local serving route snapshot has applied the descriptor clearing `staged_visibility_active=false` and `migration_job_id=0` for the moved route while retaining `min_write_ts_exclusive`. Watcher heartbeats or "leader saw version X" watermarks do not count; the proof must be Raft-applied on each possible future target leader. Only after every target voter ACKs this `target_cleared_descriptor_ack_cursor` may the migrator run bounded target-local cleanup that deletes per-bracket import-dedup records under `!migstage|ack||*`, `!migstage|hlc_floor|`, `!migstage|ready|`, and `!migstage|promote|`. Deleting these before the default witness is forbidden: a restart between target completion and default CAS would otherwise lose the proof needed to decide whether the merge flag can be cleared safely. Deleting them before the target-voter ACK barrier is also forbidden: a stale target voter could later become leader with a route snapshot that still requires staged metadata or the per-job HLC proof, but the proof records it needs would already be gone. The durable descriptor floor is retained before the per-job HLC proof is deleted, so caller-supplied/replayed low timestamps remain rejected after cleanup. Only after this target-local cleanup ACK and §6.5's source disarm ACK are durable does the default group move the job to DONE history. - -6. **Edge cases.** - - **Hot key with many versions** (codex P2 on cursor granularity, §6.1.1): the promoter iterates the staged prefix in `(raw_key ASC, commit_ts DESC)` order using the same opaque-cursor codec; the raw-candidate staged/live merge path treats a key with multiple staged versions identically to the MVCC iterator. No version chain is materialised in one apply. - - **Concurrent writes during promotion**: a live write at `commit_ts > max_imported_ts` lands ahead of the staged version it shadows. The MVCC store holds **both** `K@commit_ts=staged_ts` (the promoted staged version) **and** `K@commit_ts=live_ts` (the live write) at different commit timestamps — there is no overwrite, only **version coexistence** (closes claude round-8 wording nit). `metaLastCommitTS` advances to track the Raft commit sequence on the live side but does **not** block `PromoteStaged`'s historical insert; phrasing this as "keeps the staged copy from overwriting" was misleading because it could be misread as a conditional gate that skips the staged copy entirely — the exact §6.4-step-3 bug the round-3 fix removed. The §6.4-step-2 merge read at any `read_ts ≥ live_write.commit_ts` returns the live value (newest wins); at any `read_ts ∈ (max_imported_ts, live_write.commit_ts)` returns the imported value (live has no version `≤ read_ts` at that point; staged has one). Both correct. - - **AbandonSplitJob after CUTOVER** is still rejected (§4); abandoning post-CUTOVER would leave half-promoted state visible. - -7. **Per-PR landing.** M2-PR6 ships the raw-candidate staged/live merge read path + `PromoteStaged` apply (mechanics); M2-PR7 ships the background promoter + CLEANUP precondition. PR4-PR5 land the catalog flag and the SplitJob fields needed by PR6/PR7 so the wire is in place before the runtime turns on. The incremental design is therefore a strict superset of the chunked BACKFILL/DELTA_COPY semantics — the same chunk sizing, the same cursor codec, the same leader-flap recovery — applied to the promotion step instead of one bulk apply at CUTOVER. - -The result: CUTOVER is a single bounded catalog write, never `O(migrated range)`, and the visibility fence is preserved by the per-route flag plus the raw-candidate staged/live merge. The migrator still gets resumable, bounded apply costs for the actual data move; the catalog switch itself is decoupled from the data volume. This closes codex P2 line 147 on PR #945 without weakening any consistency property already established by §6.2.1 (HLC floor) or §7.2 (read fence). - -### 6.5 Source cleanup uses the same disjoint bracket plan - -After the read-fence grace window (§7.2.4), after every current source voter has ACKed the active-read drain, and only after target promotion has made the moved range durable on the target, the source cleanup does **not** delete only the route's raw `[routeStart, routeEnd)` user-key interval. Internal families have the same raw-vs-routing split as export: DynamoDB metadata/items, Redis collection rows, list claims, SQS/S3 state, and txn records can all sort outside the logical route interval while still belonging to the moved route. Cleanup therefore reuses `PlanExportBrackets(routeStart, routeEnd)` with the same `RouteKeyFilter` and disjoint `familyUser` internal-prefix exclusion from §6.3.1. - -Each source cleanup batch is a source-group Raft proposal that scans one bracket with the same `maxScannedBytes` pacing and deletes only versions whose `commit_ts <= fence_ts` for rows accepted by the route-key filter. The cursor is bracket-scoped and persisted in the SplitJob, exactly like export progress. Unknown internal prefixes or reserved migration-control prefixes fail closed: they are either rejected by `SplitRange` before job creation (§3.2) or left untouched for a future adapter-specific bracket. A generic raw range delete is not an allowed implementation because it would miss internal-family state and could delete control records if a future prefix is added incorrectly. - -After the final source-data cleanup cursor reaches `done=true`, the source group still does **not** immediately disarm migration fences. The migrator first drives a source-group CUTOVER route-removal ACK barrier: for every current source voter it proposes/probes `AcknowledgeCutoverRouteRemoval(job_id, cutover_version)`, whose apply handler succeeds only after that voter's local route snapshot has applied at least `cutover_version` and no longer contains the moved `source.WriteFenced(right)` route. The barrier is tracked in `source_cutover_ack_cursor`; watcher heartbeats or `max_cutover_version_seen` watermarks alone do not count because they do not prove the FSM's serving snapshot has stopped accepting source-owned writes/reads for the moved range. The same ACK also proves the post-CUTOVER current-owner gates from §7.1 and §7.2.4 are active in that binary: a write footprint or read interval that resolves to a foreign route or to "no source-owned route" is rejected with `ErrRouteMoved` / `ErrRouteOwnershipUnknown` before MVCC, even when `ObservedRouteVersion == 0` and even after `!migfence` is cleared. Only after every source voter ACKs this proof does the source group run the separate Raft-applied disarm step for the job: delete `!migwrite||*`, clear `!migfence|` and any pending `ArmCutoverReadFence(job_id, expected_cutover_version)` entry, and release `source_retention_pin_ts`. The job must not move to DONE history until the route-removal ACK, this source disarm ACK, and the target-local cleanup ACK above are durable. This keeps the successful path symmetric with `AbandonSplitJob`: recovery never leaves a completed migration carrying write trackers, read fences, or retention pins that can reject unrelated future traffic, but it also never clears those guards while a source voter can still serve the pre-CUTOVER route snapshot or while clearing the explicit fence would leave no unconditional owner gate behind. - -## 7. Coordinator / FSM Integration - -### 7.1 FENCE write rejection - -Coordinator-side: - -- `kv/sharded_coordinator.go` consults the engine's per-route state. Routes in `WriteFenced` reject writes with a new sentinel `ErrRouteWriteFenced`, mapped to: - - gRPC: `codes.Aborted` + status detail `{kind:"route_write_fenced", route_id, retry_after_ms}`. - - Redis adapter: `MOVED`-style retry (existing path). - - DynamoDB adapter: throttled error so the SDK retries with backoff. - -FSM-side defense in depth — **must run on an unconditional apply path** (closes codex round-3 P1 on PR #945): - -- The naïve placement — extend `verifyOwnerFromSnapshot` so it also rejects routes in `WriteFenced` — does NOT work. `verifyComposed1` short-circuits when `ObservedRouteVersion == 0` (`kv/fsm.go:608-611`), and there are several **intentional zero-version write paths today**: read/write txns with `ReadKeys`, caller-supplied `StartTS`, and resolver-claimed keys all dispatch with `ObservedRouteVersion = 0` (`kv/sharded_coordinator.go:694-754`, M3 sibling §3.5 + auto-pin policy). A stale coordinator in any of those flows would forward a write to the source during FENCE and the FSM defense would never run. The fence must be enforced **before** any version-conditional early return. -- M2 therefore introduces a **separate, unconditional pre-gate** `verifyRouteNotFenced(mutations)` in `kv/fsm.go` that the apply path consults **before** `verifyComposed1`. It computes each mutation's write footprint against the **current** catalog snapshot and rejects with `ErrRouteWriteFenced` when **any** footprint intersects a route whose `RouteState == WriteFenced`. For ordinary `PUT` / `DEL` / prepare / one-phase commit mutations, the footprint is the single `routeKey(mut.Key)` point, matching `verifyOwnerFromSnapshot`. For raw `Op_DEL_PREFIX`, the footprint is a range write: an empty prefix means every non-transactional key (`kv/transcoder.go:10-12`, `kv/sharded_coordinator.go:631-635`, `kv/fsm.go:482-527`), and a non-empty prefix covers the half-open prefix interval `[prefix, prefixScanEnd(prefix))` with an empty end meaning `+∞`. The gate rejects if that interval, after the same route-key normalization used by scan ownership (§7.2.2b), intersects any WriteFenced route. It must not decide from `routeKey(prefix)` alone: a prefix can start outside the moving child while still matching keys inside it (closes codex round-21 P1). No dependency on `ObservedRouteVersion`, no early return on `0`; the gate runs on every applied write request unconditionally. The gate has the same `Phase_ABORT` bypass as the Composed-1 gate (for the same lock-release reason in §3.5 of the parent partial doc). `Phase_COMMIT` is **not** a blanket bypass: before the post-fence drain completes (`post_fence_drain_completed=false`), it is allowed only if every committed mutation resolves a matching pre-existing `!txn|lock|` row for the same `(primary_key, start_ts, commit_ts)` in the moving range. That narrow lane lets already-prepared txns resolve after the fence has stopped new prepares (closing codex round-18 P1's starvation issue) without letting a stale coordinator create new state. After the post-fence drain completes, **new/unprepared commits remain rejected**, but an idempotent COMMIT retry for a transaction that already committed must still be admitted when the FSM can prove the prior success from a durable record that encodes the exact transaction identity: the primary commit record, a commit marker, or a per-key transaction-success marker co-written with the version and carrying `(primary_key, start_ts, commit_ts)`. A plain committed MVCC version that stores only key/value/commit_ts is **not** sufficient proof by itself, because a caller-supplied timestamp collision cannot distinguish the transaction being retried from another writer; it may only be used as a consistency cross-check after an identity-bearing proof has been found. The same staged+live probe also checks rollback markers, but a rollback marker proves the transaction already aborted and must return the idempotent abort/rollback result (or reject the COMMIT as already rolled back), never COMMIT success. The idempotency proof uses the staged+live lookup rule from §6.4 while `staged_visibility_active` is true, so a response-lost COMMIT that succeeded before CUTOVER does not later surface as a fenced/moved failure merely because its lock row was deleted. -- Post-CUTOVER, `WriteFenced(right)` is removed from the source snapshot, so a fence-only gate is no longer enough. M2 therefore also introduces an unconditional **current write owner gate** `verifyCurrentWriteOwner(mutations)` on the same pre-version path. Point writes, one-phase writes, and prepares whose normalized key resolves to a foreign route return `ErrRouteMoved{new_route_version, new_group_id}`; if ownership cannot be normalized or loaded, they return `ErrRouteOwnershipUnknown` and do not apply. `Op_DEL_PREFIX` is split before this point: the coordinator resolves the full normalized prefix interval, rejects the whole user request if any intersecting route is `WriteFenced`, subtracts registered migration/control intervals, and sends each owner a clipped local delete plan. The FSM accepts only clipped subranges that resolve to the local Raft group; an unclipped prefix request that still includes foreign non-fenced routes and lacks the coordinator's split proof fails closed with `ErrRouteMoved`/`ErrRouteOwnershipUnknown`. This preserves broadcast deletes across many owners without letting a stale coordinator turn a moved-range delete into a source-side success/no-op. This gate runs even when there is no `WriteFenced` route and even when `ObservedRouteVersion == 0`, so clearing `!migfence` after §6.5 cannot reopen stale writes to the old source. The same narrow prepared-intent resolution lane described above applies before `post_fence_drain_completed`; after the drain, only idempotent COMMIT retries with a durable transaction-identity prior-success proof are allowed to return success, rollback proof returns the prior abort/rollback result, and every new/unprepared `Phase_COMMIT` for the moved route is rejected on source like any other write. -- Concretely, the apply path becomes: `verifyNoMigrationControlPrefix` → `verifyCurrentWriteOwner` → `verifyRouteNotFenced` → `verifyComposed1` → existing handlers. The owner/fence gates share a small `mutationFootprints(mut)` helper: point lookups call `OwnerOf(routeKey(mut.Key))`, while `Op_DEL_PREFIX` calls `IntersectingRoutes(normalizePrefixFootprint(mut.Key))` for global preflight and then carries a list of clipped local user-data subranges to `handleDelPrefix`. A raw user prefix normalizes to the raw prefix interval; known internal families (`!lst|meta|d|`, `!lst|claim|`, `!hs|`, `!st|`, `!zs|`, concrete stream prefixes `!stream|meta|` and `!stream|entry|`, DynamoDB/S3/SQS families) use the same decoders as the export bracket planner and read-side scan gate when the prefix contains enough bytes to recover logical owner bounds. A family-wide internal prefix that does not carry an owner segment, or any unrecognized internal prefix, fails closed with `ErrRouteOwnershipUnknown` for the owner gate and `ErrRouteWriteFenced` if the current snapshot contains any WriteFenced route, because the FSM cannot prove it is disjoint. `kv/sharded_coordinator.go` must preflight and split broadcast `DEL_PREFIX`; correctness remains source-FSM-local because every recipient re-runs the gates against its clipped plan before `handleDelPrefix`, so a stale coordinator cannot slip source tombstones in after FENCE or after CUTOVER. Cost is bounded by one route-interval lookup plus subrange clipping for prefix deletes and the existing per-mutation `OwnerOf` call for point writes. The gates stay independent so a future change to one cannot regress the others (same isolation rationale as `verifyReadOwner` next to `verifyOwnerFromSnapshot` in §7.2.5). -- Independently of `WriteFenced`, the same unconditional pre-gate sequence starts with `verifyNoMigrationControlPrefix(mutations)`. It rejects point keys and explicit user prefixes that target the registered migration/control prefixes from §3.2 (`!dist|`, `!dist|migstage|`, `!migstage|`, `!migwrite|`, `!migfence|`, etc.) before a key reaches `handleRawRequest`, `handleOnePhaseTxnRequest`, `handlePrepare`, or `handleDelPrefix`. For broadcast prefix deletes such as `Op_DEL_PREFIX ""`, intersection with control ranges is not by itself an error; those ranges are subtracted from the clipped user-data plan, and the FSM asserts that no control subrange reaches `handleDelPrefix`. A prefix that starts inside a control namespace, or an unrecognized internal prefix that cannot prove a user-data owner, is still rejected. The implementation may share the `mutationFootprints` interval helper with `verifyRouteNotFenced`, but it must not be conditional on there being an active WriteFenced route. Typed catalog/migration Raft proposals are the only writers to those prefixes. -- Forward-compatibility hook (M3): a follow-on change can flip the `ObservedRouteVersion == 0` short-circuit off entirely once every M2-shipped flow stamps a non-zero version, at which point `verifyRouteNotFenced` collapses into a `verifyComposed1` invariant. M2 ships the safe form (unconditional pre-gate) so the FENCE rejection cannot be silently bypassed before that cleanup lands. - -Reads during FENCE / DELTA_COPY are **not** rejected: `ShardStore.GetAt` continues to serve from source. Snapshot reads at any `commit_ts ≤ fence_ts` remain consistent because the source's MVCC history is unchanged through DELTA_COPY. The post-CUTOVER read-side fence is §7.2. - -### 7.2 Post-CUTOVER read fence (closes codex P1) - -Codex P1 on PR #945: today the read path resolves the group from the local engine and serves/proxies the read **without carrying an observed catalog version** (`kv/shard_store.go:38-53`, `kv/shard_store.go:1471-1477`), while `verifyComposed1`'s `verifyOwnerFromSnapshot` (`kv/fsm.go:753-773`) only inspects mutations. After CUTOVER, a coordinator or source-group leader that has not applied the new catalog yet would keep routing reads of the moved key to the source — returning the stale pre-cutover value while writes have already landed on the target. M2 must close this read path symmetrically to writes. - -#### 7.2.1 Read request carries an observed catalog version - -Add `read_route_version uint64` to the concrete read-path messages in §5.1.1: `proto/service.proto` `RawGetRequest` and `RawScanAtRequest`, plus the internal lease/proxy envelope used by `ShardStore.GetAt`, `.ScanAt`, and `.ReverseScanAt`. The coordinator stamps the engine's catalog version into every read at the moment it picks a group. This is the read-side equivalent of `ObservedRouteVersion` on writes. Range reads also carry the route-key-normalized interval when the caller has already decoded an internal-family raw scan; otherwise the source helper normalizes or fails closed before MVCC. - -#### 7.2.2 Source FSM rejects reads after CUTOVER — authoritative ownership check, not version comparison (closes codex P1 line 459) - -The original wording — "the FSM returns `ErrRouteMoved` when `read_route_version` is strictly less than the FSM's last-applied catalog version, and the key is no longer owned" — is **insufficient**. Codex P1 round-4 on PR #945: it misses the common post-CUTOVER case where the coordinator and the source FSM are **equally stale at the pre-CUTOVER version**. In that window: - -1. A *different* coordinator, refreshed against the new catalog, routes a write of key `K` to the target group. The write commits there with `commit_ts > max_imported_ts` (§6.2.1). -2. The original stale coordinator routes a read of `K` to the source FSM with `read_route_version == source_local_version` (both at the pre-CUTOVER version). -3. Under the strict-less-than rule, `read_route_version` equals `source_local_version`, the comparison passes, and the source serves the old value — even though the target has a fresher committed write. - -This is the same hazard the §7.2 read fence was added to prevent; the strict-less-than form simply doesn't catch the equal-stale case. The fix replaces version comparison with an **authoritative ownership check** scoped on either the local or the cutover-version snapshot, whichever exists: - -- If the source FSM **has applied** CUTOVER (`source_local_version ≥ cutover_version`): the source no longer owns the key under its current snapshot. **Unconditionally** return `ErrRouteMoved{new_route_version, new_group_id}` regardless of `read_route_version`. This is the simple case the old wording also handled. -- If the source FSM **has NOT applied** CUTOVER (`source_local_version < cutover_version`), but the **default group has** (the migrator already populated `cutover_version` on the SplitJob and that proposal has committed, observable via the watcher's view of the catalog version known to the source group leader): the source must consult an out-of-band authoritative signal because its local FSM snapshot still says it owns the key. M2 ships two layers: - 1. **Heartbeat-piggybacked watermark.** The default-group leader's existing distribution heartbeat to every node already carries `catalog_version` (§11.1 capability bit lives there). Extend it to also carry `max_cutover_version_seen` (monotone, never decreases). The source-group FSM stores this as a leader-local atomic and consults it on every read fence: if `max_cutover_version_seen > source_local_version`, the source treats itself as "potentially stale at the cutover boundary" and the read fence runs the next bullet. - 2. **Cutover-aware ownership query.** When the watermark indicates a possible stale read, the source-group leader makes a single deadline-bounded `GetRouteOwnership(routeKey(K), catalog_version=max_cutover_version_seen)` RPC to the default-group leader. The default-group leader answers from its current catalog snapshot. If the answer is "owned by another group" the source returns `ErrRouteMoved{new_route_version=max_cutover_version_seen, new_group_id}` to the coordinator; if owned by the source (no cutover landed on this key after all — the watermark covers an *unrelated* migration) the source returns the value normally. If the RPC times out, the default-group leader is unreachable, or the answer cannot be verified at `max_cutover_version_seen`, the source returns `ErrRouteOwnershipUnknown{route_version=max_cutover_version_seen, retry_after_ms}` and **must not read source MVCC**. Treating an unknown answer as "source-owned" would reopen the equal-stale stale-read window, so this path is fail-closed and retryable. -- Successful ownership answers are cached on the source-group leader per `(routeKey, max_cutover_version_seen)` for one second to keep the fence cheap; the cache invalidates the moment `max_cutover_version_seen` advances. Transport errors, timeouts, and `ErrRouteOwnershipUnknown` are not cached as source-owned answers. - -Crucially, this catches **the equal-stale window** the strict-less-than form missed: both coordinator and source are at the pre-CUTOVER version, but the heartbeat watermark on the source is already at the post-CUTOVER version (because the default-group leader propagated it as soon as the catalog write committed), so the source consults the default-group leader and gets the authoritative "no longer yours" verdict before serving a stale value. - -Two sub-cases of the legacy framing are preserved: - -- **Coordinator stale, source up-to-date**: source's `source_local_version ≥ cutover_version` and source no longer owns the key → unconditional `ErrRouteMoved`. Coordinator refreshes → re-routes to target → succeeds. -- **Source stale, coordinator further ahead (`read_route_version > source_local_version`)**: `source_local_version` is the distribution engine/catalog snapshot version that `CatalogWatcher.ApplySnapshot` installed for serving, not the source group's Raft `appliedIndex`. The read path therefore waits on a catalog-version waiter owned by the distribution engine (or polls `engine.CatalogVersion()` with a short bounded sleep) until `source_local_version >= read_route_version`, with a timeout of `min(remaining_lease_TTL, 200 ms)`. `CatalogWatcher.ApplySnapshot` / route-snapshot replacement must signal the waiter when the serving catalog version advances; a condition variable signalled only by the FSM apply goroutine is insufficient because the catalog watcher can catch up without a new source Raft apply. On wake/poll success the FSM re-evaluates ownership; if the source still owns the key under the new snapshot, the read proceeds. If the engine version is still behind on timeout, OR if the catch-up reveals the source no longer owns the key, the FSM returns `ErrRouteMoved` / `ErrRouteOwnershipUnknown` without touching MVCC. This avoids a wedge while keeping the read consistent (closes claude round-2 spec-mechanism flag, 4-round flag). - -#### 7.2.2a Equal-stale window — concrete walkthrough - -To make the §7.2.2 fix concrete: - -1. `t=0`: catalog at version `v`. Coordinators C1 and C2 both at `v`. Source FSM at `v`. CUTOVER not yet issued. -2. `t=1`: after §7.2.2e has already armed the source-side pending read fence for expected version `v+1`, the migrator issues CUTOVER on default group. Catalog committed at `v+1`. Default-group leader's outbound heartbeat starts carrying `max_cutover_version_seen = v+1` immediately. -3. `t=2`: source-group leader receives the heartbeat. Its local atomic `max_cutover_version_seen` advances to `v+1`. Its FSM-local `source_local_version` is **still `v`** because the watcher hasn't yet routed the catalog update through the source group's own Raft log. -4. `t=3`: C2 watches the catalog, sees `v+1`, refreshes its engine, routes its write of `K` to the target. Target commits the write with `commit_ts > max_imported_ts`. -5. `t=4`: C1 (still at `v`) routes a read of `K` to the source FSM with `read_route_version = v == source_local_version`. **Equal stale.** -6. `t=5`: source FSM sees `max_cutover_version_seen (v+1) > source_local_version (v)`. Runs `GetRouteOwnership(routeKey(K), v+1)` against the default-group leader. Answer: "owned by target group." Source returns `ErrRouteMoved{new_route_version=v+1, new_group_id=target}` to C1. -7. `t=6`: C1 refreshes, retries on target, sees C2's fresh write. - -Without the watermark + ownership-query path, step 6 would have returned the pre-CUTOVER value because the strict-less-than version check passed. - -#### 7.2.2b Scan-interval ownership gate — point-key check is insufficient for scans (closes codex P1 round-9 on PR #945) - -Codex round-9 P1: §7.2.2's ownership check resolves the scan's **start key** against `routeKey(scan_start)`, but a `ScanAt(start, end)` or `ReverseScanAt(start, end)` spans **a range**, and a stale-coordinator scan that straddles the post-CUTOVER split boundary can pass the point-key check while serving stale data for the moved sub-range. Concrete failure scenario (verified against `kv/shard_store.go:164-228`): - -1. After CUTOVER, the catalog is at version `v+1`: `source.Active(left: [A, M))`, `target.Active(right: [M, Z))`. -2. A coordinator stale at version `v` still sees `source.Active(full: [A, Z))` — one route. -3. Client calls `ScanAt([A, Z), ts)`. The coordinator's `routesForScan` calls `engine.GetIntersectingRoutes([A, Z))` against the stale engine → returns the single `source.Active(full)` route, and dispatches the entire scan to the source group. -4. Source FSM's §7.2.2 point-key check resolves `routeKey(A)` → still owned by source under its **current** catalog (left half is `source.Active([A, M))`). Check passes. -5. Source FSM executes the full `[A, Z)` scan over its MVCC store, returning **live data for `[A, M)` AND stale pre-CUTOVER data for `[M, Z)`** — the target has already received post-CUTOVER writes for that range, but the source has the historical versions for those keys still resident (CLEANUP grace window). Client sees a snapshot that mixes pre- and post-CUTOVER state across the boundary. This is the read-side analogue of the §3.2a write-loss window the source-side fence-apply barrier closes. - -This is not hypothetical — `ScanAt` / `ReverseScanAt` underlie Redis `KEYS` / `HSCAN` / `LRANGE`, DynamoDB `Scan` / GSI `Query`, S3 `ListObjects`, etc. Any such call straddling the split boundary in the equal-stale window would observe the regression. - -The fix is a **scan-interval ownership gate** on the source FSM, parallel to §7.2.2's point-key gate. The gate never proves ownership from raw MVCC scan bytes alone. Before either branch consults the catalog it first converts the request into one or more **route-key intervals**: - -- User-key scans use the normal `[start, end)` routing interval (`routeKey(raw) == raw`). -- Internal per-user scans (list meta/item/delta/claim, Redis hash/set/zset/stream, DynamoDB item/GSI, SQS/S3, etc.) decode their logical owner with the same `routeKey()` helpers as §6.3.1, then check that owner key or the corresponding logical interval. For example `!lst|meta|d|...`, `!lst|claim|...`, `!hs|...`, `!st|...`, and `!zs|...` are checked against the decoded Redis user key, not against their raw `!lst|...` / `!hs|...` prefixes. -- Family-wide internal scans (for example a compactor or migration bracket scanning an entire raw prefix) must carry explicit logical bounds `(route_start, route_end)` from the caller. If the FSM cannot derive or receive route-key intervals for a raw internal scan, it returns `ErrRouteOwnershipUnknown` and **must not scan source MVCC**. Treating raw `GetIntersectingRoutes(start, end)` as proof would misclassify internal rows whose raw prefix sorts outside their logical route. - -Like the point-key gate it has **two branches** keyed on the watermark, so it catches both the "coordinator-stale, source-up-to-date" and the "coordinator-and-source-equal-stale" cases (closes claude round-10 P1 on equal-stale scans, PR #945): - -- **Branch 1 — source has applied CUTOVER (`source_local_version >= cutover_version`).** The FSM calls `GetIntersectingRoutes(route_start, route_end)` for each normalized route-key interval against its **own** current catalog snapshot. If **every** intersecting route is owned by this group, the scan proceeds. If **any** intersecting route is owned by a different group, the FSM returns `ErrRouteMoved{new_route_version, new_group_id, sub_range_start, sub_range_end}` for the **first** foreign-owned sub-range encountered in route-key order. -- **Branch 2 — source has NOT applied CUTOVER but `max_cutover_version_seen > source_local_version` (equal-stale, watcher lag).** The local snapshot still says `source.Active(full: [A, Z))` — a single route, no foreign owner visible — so Branch 1 alone would silently pass a cross-boundary scan. The gate therefore additionally calls `GetIntersectingRoutes(route_start, route_end, catalog_version = max_cutover_version_seen)` for the normalized route-key intervals as a deadline-bounded RPC to the **default-group leader** (mirroring exactly §7.2.2's point-key extension — same watermark trigger, same RPC target, same 1 s cache TTL with cache key `(route_intervals, max_cutover_version_seen)`). The default-group leader answers from its current catalog snapshot. If any returned route is owned by another group, the source FSM returns `ErrRouteMoved{max_cutover_version_seen, foreign_group_id, sub_range_start, sub_range_end}` for the first foreign-owned sub-range. If every returned route is still source-owned (the elevated watermark covers an *unrelated* migration that did not touch this scan range), the scan proceeds normally. If interval normalization fails, the RPC times out, the default-group leader is unreachable, or the route set cannot be verified at `max_cutover_version_seen`, the FSM returns `ErrRouteOwnershipUnknown{route_version=max_cutover_version_seen, sub_range_start=route_start, sub_range_end=route_end, retry_after_ms}` and **must not scan source MVCC**. -- **Coordinator path identical in both branches.** The coordinator's existing `WatcherRefresh` + retry path (§7.2.3) re-issues the scan; on retry the fresh engine returns the correct two routes and `routesForScan` splits the scan into `[A, M)` to source and `[M, Z)` to target, both fenced correctly. -- **The §7.2.2 point-key gate still runs** for the scan's start key — the new check is **in addition to**, not in place of, so the watermark / equal-stale logic at the start key continues to apply. - -Implementation cost: route-interval normalization plus one additional `GetIntersectingRoutes` call per normalized interval on the FSM. Branch 1 (the common case once watcher catches up) is local-only. Branch 2 adds at most one cached successful RPC result from the default-group leader per `(route_intervals, max_cutover_version_seen)` per second — bounded by the same cache as §7.2.2's point-key extension. Failed / timed-out lookups and failed interval-normalization attempts fail closed and are not cached as proof of source ownership. Memory: bounded by the number of routes intersecting the normalized intervals (typically 1–2; the rare cross-multi-boundary scan is the same `O(routes_in_range)` the coordinator already pays). - -The §7.2.2a equal-stale walkthrough is extended in §7.2.2c (Branch 1 scenario) and §7.2.2d (Branch 2 / equal-stale scenario) with parallel scan-shape walkthroughs to demonstrate both branches catch their case. - -#### 7.2.2c Equal-stale scan walkthrough - -Continuing from §7.2.2a's setup, with the post-CUTOVER catalog v+1 = `source.Active(left: [A, M))`, `target.Active(right: [M, Z))`: - -1. `t=4`: stale coordinator C1 issues `ScanAt([A, Z), ts=now)` with `read_route_version = v`. -2. Coordinator's `routesForScan` against its v-catalog returns `[source.Active(full: [A, Z))]` → entire scan dispatched to source group. -3. `t=5`: source FSM applies the scan. §7.2.2 point-key gate resolves `routeKey(A)` against the current snapshot at `v+1` → owned by source (left half). Point-key check passes. -4. **§7.2.2b scan-interval gate fires**: source FSM calls `GetIntersectingRoutes([A, Z))` against its `v+1` catalog → returns `[source.Active(left: [A, M)), target.Active(right: [M, Z))]`. The second route is owned by target, not source. The FSM returns `ErrRouteMoved{v+1, target_group_id, M, Z}` for the moved sub-range. -5. `t=6`: C1 receives `ErrRouteMoved`, runs `WatcherRefresh`, advances its engine to `v+1`, and re-routes the scan as two sub-scans: `[A, M)` to source and `[M, Z)` to target. Both succeed; the merged result is the consistent post-CUTOVER snapshot. No stale data for `[M, Z)` was ever returned. - -The gate runs at apply time on the source FSM, not on the coordinator, so even a coordinator with a paused `WatcherRefresh` cannot serve mixed pre/post-CUTOVER data — the source FSM is the chokepoint that knows the authoritative current catalog. - -#### 7.2.2d Equal-stale cross-boundary scan walkthrough — Branch 2 - -Same `t=0..t=2` setup as §7.2.2a (the §7.2.2e pending source read fence was armed before CUTOVER, CUTOVER commits at `v+1`, default-group heartbeat carries `max_cutover_version_seen = v+1` immediately, source-group leader has received the heartbeat but its FSM watcher has **not yet** applied `v+1` — `source_local_version` is still `v`): - -1. `t=3`: source-group leader's local atomic `max_cutover_version_seen = v+1`, but FSM-local `source_local_version = v`. Source FSM snapshot still says `source.Active(full: [A, Z))`. -2. `t=4`: stale coordinator C1 issues `ScanAt([D, Z), ts=now)` where `D ∈ [A, M)` — scan starts in the non-moved left half and ends in the moved right half. `read_route_version = v`. -3. Coordinator's `routesForScan` against its v-catalog returns `[source.Active(full: [A, Z))]` → entire scan dispatched to source group. -4. Source FSM receives the scan. §7.2.2 **point-key gate** runs: `max_cutover_version_seen (v+1) > source_local_version (v)` → equal-stale watermark fires; FSM calls `GetRouteOwnership(routeKey(D), v+1)` to the default-group leader → answer: "owned by source (left half)". Point-key check passes (the scan's START key is still source-owned). -5. **§7.2.2b scan-interval gate Branch 2 fires** (because watermark is elevated): FSM calls `GetIntersectingRoutes([D, Z), catalog_version = v+1)` to the default-group leader → returns `[source.Active(left: [D, M)), target.Active(right: [M, Z))]`. The second route is owned by target. The FSM returns `ErrRouteMoved{v+1, target_group_id, M, Z}` for the moved sub-range. **The gap §7.2.2c's Branch 1 would have left open is now closed** — without Branch 2, step 5 would have run `GetIntersectingRoutes` against the FSM's own `v` snapshot, returned the single `source.Active(full)` route, and served the entire scan stale. -6. `t=5`: C1 receives `ErrRouteMoved`, runs `WatcherRefresh`, advances to `v+1`, and re-issues the scan as two sub-scans: `[D, M)` to source and `[M, Z)` to target. Both succeed; the merged result is consistent. No stale data for `[M, Z)` was ever returned. - -Successful Branch 2 answers from the default-group leader are cached for 1 second per `(route_intervals, max_cutover_version_seen)`, so a burst of cross-boundary scans during the watcher-lag window pays at most one successful RPC per second across all of them. Timeout / unreachable / unverifiable answers and interval-normalization failures return `ErrRouteOwnershipUnknown` and are not cached as source-owned. Once the watcher catches up (`source_local_version >= cutover_version`), Branch 1 takes over and the local snapshot is authoritative — the cached Branch 2 results are no longer consulted. - -#### 7.2.2e Source-side cutover read-fence arm — no admitted pre-watcher stale-read window - -Codex round-17 P1 on PR #945 flagged the remaining gap: if the default group publishes `target.Active` before the source group has either applied the CUTOVER catalog snapshot or received `max_cutover_version_seen`, a refreshed coordinator can already write/read on the target while a stale coordinator can still route a read or cross-boundary scan to the source. Branch 1 and Branch 2 above only fire after one of those source-side signals exists, so a design that merely accepts the watcher interval as "small" violates M2's atomic reader/writer cutover goal. - -M2 therefore makes target visibility depend on a synchronous source-side read barrier: - -1. After DELTA_COPY completes and after the target full-HLC floor proposal in §6.2.1, the migrator fixes `expected_cutover_version = current_catalog_version + 1`, persists `target_staged_readiness_state = ARMING`, and applies/probes §6.4 step 0's `ApplyTargetStagedReadiness` proposal on every current target voter. Only after `target_staged_readiness_ack_cursor` shows every current target voter has applied the guard does it record `target_staged_readiness_state = ARMED`. This closes the target-side watcher-lag gap before the source read barrier is armed; a follower that later becomes target leader already has the fail-closed readiness record or refuses to serve the moving interval until it catches up. -2. With target readiness armed, but **before** any source-group read-fence proposal, the migrator persists `cutover_read_fence_state = ARMING` on the default-group SplitJob. This default-group intent is durable proof that recovery must reconcile the source-side fence even if the leader dies during the next RPC. -3. With `ARMING` durable, the migrator proposes/probes `ArmCutoverReadFence(job_id, route_start, route_end, expected_cutover_version, target_group_id)` on every current **source group** voter. Each source FSM applies this as source-group Raft state: a pending-fence entry keyed by `(job_id, route_start, route_end, expected_cutover_version)`. The operation is idempotent for the same tuple and rejects a conflicting expected version for the same job. -4. After `source_cutover_read_fence_ack_cursor` shows every current source voter has applied the pending fence, the migrator CASes the default-group SplitJob from `ARMING` to `ARMED`. From the moment a source voter applies the fence until its watcher applies CUTOVER and normal §7.2.2 Branch 1 takes over, any point read or scan whose **route-key-normalized interval** intersects the moving range and whose `read_route_version < expected_cutover_version` **must not read source MVCC**. A source voter that restarts or wins leadership before applying the guard must fail closed for the moving interval until it has either applied the guard or loaded a catalog snapshot at/after `expected_cutover_version`; it may not serve from the stale pre-CUTOVER snapshot. If a raw internal scan cannot be normalized to route-key intervals, the source fails closed with `ErrRouteOwnershipUnknown` / retry rather than proving ownership from raw bytes. If the source has already learned that the default-group CAS committed (via watcher, heartbeat watermark, or the ownership RPC), it returns `ErrRouteMoved{expected_cutover_version, target_group_id}`. Otherwise it returns `ErrRouteCutoverPending{retry_after_ms}`; coordinators treat it like `ErrRouteMoved` with a short wait + `WatcherRefresh` retry. A transient retry is acceptable; serving stale source data is not. -5. Only after both `target_staged_readiness_state = ARMED` and `cutover_read_fence_state = ARMED` are durable does the migrator issue the default-group CUTOVER catalog CAS that removes `source.WriteFenced(right)` and inserts `target.Active(right)`. This ordering delays target visibility/writes until the target can fail closed for missing staged metadata, the source can fence stale reads, and the default group has recovery witnesses for both facts. -6. If the catalog CAS is **definitively not committed** (for example expected-version mismatch where a fresh default-group read proves `cutover_version` is still unset, the target route was not published, and the source route is still the pre-CUTOVER `WriteFenced` descriptor), the migrator may abort the attempt: CAS the SplitJob to `CLEARING`, then propose `ClearCutoverReadFence(job_id, expected_cutover_version)` to the source group and `ClearTargetStagedReadiness(job_id, expected_cutover_version)` to the target group, and only after both clear ACKs return set the two states to `NONE` before returning the job to DELTA_COPY/FAILED. A leadership loss, timeout, transport error, or ambiguous apply result is **not** proof that CUTOVER failed. Recovery from an ambiguous result must first reload the default-group catalog and SplitJob at a fresh read index. If `cutover_version` is populated, the target `Active(right)` descriptor exists, or the source `WriteFenced(right)` descriptor has been removed for the expected version, recovery treats CUTOVER as committed and resumes CUTOVER/CLEANUP with both target readiness and source read guards still armed; it must not enter `CLEARING`. While either fence is armed and CUTOVER is not proven absent, clients see bounded retry instead of stale reads or live-only target reads. - -Crash safety: because `ARMING` is persisted before each target/source proposal, a default-group leader loss after either side ACKs but before the corresponding `ARMED` write cannot strand an unowned guard. The new migrator first reconciles target readiness: at `target_staged_readiness_state = ARMING` it resumes `target_staged_readiness_ack_cursor`, probes/re-proposes `ApplyTargetStagedReadiness` idempotently for non-ACKed target voters, refreshes the target voter-set epoch, and records `ARMED` only after all current target voters ACK; at `ARMED` it reloads the default-group catalog/SplitJob and revalidates the voter-set epoch before deciding whether to retry CUTOVER or clear. It then reconciles the source fence the same way: at `cutover_read_fence_state = ARMING`, resumes `source_cutover_read_fence_ack_cursor`, probes/re-proposes `ArmCutoverReadFence` idempotently for non-ACKed source voters, refreshes the source voter-set epoch, and records `ARMED` only after all current source voters ACK; at `ARMED`, recovery either completes/resumes the same CUTOVER CAS or transitions through `CLEARING` only after the fresh catalog/job proof in step 6 shows CUTOVER did not land. If either voter set changes while a guard is `ARMED` but before CUTOVER or before proof cleanup, the state returns to the same re-ACK loop for the newly added/promoted voters before proceeding. At `CLEARING`, recovery keeps issuing the target/source clear proposals until both ACK under the current voter-set epoch, then records both states as `NONE`. Abandon/rollback follows the same clearing path for both `ARMING` and `ARMED`, but only for a pre-CUTOVER job whose absence of CUTOVER has been proven; it does not skip cleanup merely because an `ARMED` bit was never written. Source- or target-group leader changes are also safe because every current voter has applied the relevant guard before the target route is published; a new leader that has not loaded the guard must fail closed for the moving range until its Raft log / route snapshot catches up. - -This barrier is intentionally stronger than the heartbeat watermark. The watermark and authoritative ownership RPC remain as the steady-state read fence after CUTOVER, and they cover unrelated elevated-watermark cases, but they are no longer the only protection during the first watcher tick. - -#### 7.2.2f Read-serving chokepoint — every source read path must call the fence before touching MVCC - -The sections above say "source FSM" because the ownership decision is source-group-local, but the repo's read serving path is not a single Raft apply hook. M2 therefore places the read fence in a shared source-group read helper that runs before any local store lookup: - -- `kv.ShardStore.GetAt`, `ScanAt`, `ReverseScanAt`, and `LatestCommitTS` call the helper after the lease/read-index check selects the source group but **before** `g.Store.GetAt` / `ScanAt` / `LatestCommitTS` touches MVCC. -- `adapter/grpc.go` `RawGet` / `RawScanAt` / `RawLatestCommitTS` and any RawKV entrypoint that can read `r.store` directly must route through the same helper; direct calls into `store.MVCCStore` are allowed only for internal maintenance paths that do not serve client reads and do not overlap an Active/WriteFenced route. -- Local-leader reads and proxied reads both carry `read_route_version` and the normalized point key / scan intervals into the helper. A proxy target cannot downgrade or omit these fields. -- Point reads and `LatestCommitTS` run §7.2.2; scans run §7.2.2b. If normalization of an internal raw prefix is impossible, the helper returns `ErrRouteOwnershipUnknown` / retry and the caller must not fall back to direct source MVCC. - -The helper owns the pending `ArmCutoverReadFence` table, `max_cutover_version_seen`, the one-second successful ownership cache, and the fail-closed timeout behavior. This is a placement requirement, not a new semantic rule: the invariant is that every user-visible source read path has exactly one chance to prove route ownership before it can observe source MVCC. A red implementation that wires the check only into `kv/fsm.go` but leaves `ShardStore.GetAt` / `ScanAt` / `LatestCommitTS` or `RawGet` / `RawScanAt` / `RawLatestCommitTS` calling `Store` directly is incorrect and must fail the §10.1 read-fence tests. - -#### 7.2.3 Coordinator-side fallback - -If the engine's local catalog is behind the FSM's `new_route_version`, the coordinator does a single best-effort `WatcherRefresh` and retries once. `ErrRouteOwnershipUnknown` and `ErrRouteCutoverPending` use the same retry lane with the supplied `retry_after_ms`: wait briefly, run `WatcherRefresh`, and retry at the refreshed catalog version. Repeated `ErrRouteMoved` / `ErrRouteOwnershipUnknown` surfaces to the client only on persistent inconsistency or default-group unreachability (alert-worthy). The coordinator must not reinterpret either retryable error as permission to read from the source. - -#### 7.2.4 Grace window before CLEANUP - -CLEANUP deletes the moved user-data range from source's MVCC only after both of these are true: - -1. the configurable grace window has elapsed (`readFenceGrace`, default 30 s — chosen to comfortably exceed both lease TTL and watcher tick × 2), and -2. every current source voter has ACKed `AcknowledgeSourceReadDrain(job_id, fence_ts, cutover_version)`. - -The read-drain ACK is a source-group Raft-applied proof. The apply handler first proves the voter's local serving snapshot has installed the cutover read-fence epoch for `cutover_version`, so no new source read whose normalized point/range footprint intersects the moved route can be admitted. It then checks the voter's active read-pin table and succeeds only when **no active pin whose normalized point/range footprint intersects the moved route remains**, regardless of that pin's `read_ts`. A source read admitted during FENCE/DELTA_COPY can legitimately use a current/read timestamp greater than `fence_ts` while still streaming pre-CUTOVER source versions that cleanup would delete (`commit_ts <= fence_ts`), so `read_ts <= fence_ts` is only an audit field, not the drain predicate. A source read helper registers this interval pin before it touches MVCC and releases it only when the logical read is complete, including streaming or paginated reads that continue using the same `read_ts` across multiple storage calls. This explicitly covers long-lived paths such as S3 `getObject` / `streamObjectChunks`, Redis/DynamoDB paginated scans, and gRPC `RawScanAt` callers: a read admitted on the source before CUTOVER can finish against retained source history, but cleanup cannot delete the versions it may still need. - -The drain is not based on wall-clock age alone. If a client disconnects, the serving path releases the pin during context cancellation/close; if a node is wedged, the job remains in CLEANUP or FAILED with `retry_phase=CLEANUP` until the node recovers or is removed from the source Raft configuration. A stale leader cannot self-certify the drain: the ACK is persisted per voter in `source_read_drain_cursor`, and recovery resumes from the remaining voters. - -It never treats the default-group `!dist|` control namespace as migratable data; jobs that would intersect it are rejected before creation (§3.2). - -**No pre-watcher stale-read admission after CUTOVER (closes codex round-17 P1 on PR #945).** The `CatalogWatcher` may still take up to one polling interval (`defaultCatalogWatcherInterval = 100 ms`, `distribution/watcher.go`) to apply the CUTOVER catalog write locally on the source group, and the heartbeat-carried `max_cutover_version_seen` may arrive on a different schedule. M2 does **not** rely on either propagation path for the first instant after CUTOVER. The §7.2.2e source-side `ArmCutoverReadFence` barrier is already Raft-applied on the source before `target.Active` is published, so a stale coordinator that reaches the source during that watcher interval gets `ErrRouteMoved` / `ErrRouteCutoverPending` instead of source MVCC data. Fresh coordinators may route to the target only after the catalog CAS commits; by then stale-source reads are already fenced. The admitted stale-read window is therefore zero; the 30 s grace window is only about how long source retains physical data for retry/lag tolerance, not about allowing reads to observe it. - -**Fence disarm requires source voter CUTOVER proof.** The grace timer is also not a license to clear `!migfence` or the pending source read fence. Before §6.5's source disarm, every current source voter must ACK `AcknowledgeCutoverRouteRemoval(job_id, cutover_version)` after its FSM-local serving snapshot has applied the CUTOVER route-removal version. If a source voter's watcher/heartbeat path is stalled past `readFenceGrace`, the durable write fence and pending read fence remain armed and source cleanup stays in `CLEANUP` (or `FAILED` with `retry_phase=CLEANUP`) until the voter catches up or is removed from the source Raft configuration. This preserves fail-closed behavior: a lagging source voter may reject traffic longer than desired, but it cannot accept zero-version/stale writes or reads after target activation. - -During the grace window: - -- Source still has the data physically; the read fence above turns it into `ErrRouteMoved` rather than serving the value. -- After the window and read-drain ACKs, the data may be physically gone, but stale source reads still return retryable ownership errors before MVCC: once the source has applied `cutover_version`, `verifyReadOwner` resolves the moved interval to the target and returns `ErrRouteMoved{new_route_version, new_group_id}`; if ownership cannot be verified, it returns `ErrRouteOwnershipUnknown`. A missing source row is never surfaced as user-visible "not found" for a key/range now owned by the target. - -This protects the operational corner case where a coordinator's `WatcherRefresh` is paused (CPU saturation, GC pause) for tens of seconds. - -The post-CUTOVER read owner gate remains active after `!migfence` / pending read-fence records are disarmed by §6.5. Disarm removes per-job rejection metadata only after every source voter has applied the route-removal proof; it does not remove the unconditional "current owner or moved" check on source reads. Thus a stale coordinator that keeps sending reads to the old source after cleanup still gets `ErrRouteMoved` / `ErrRouteOwnershipUnknown` and refreshes, rather than seeing false-negative reads from deleted local MVCC rows. - -#### 7.2.5 Composed-1 alignment - -`verifyOwnerFromSnapshot` keeps its current write-only behaviour. The new read-fence lives on a parallel path (`verifyReadOwner` next to it in `kv/fsm.go`) so the two stay independent — a future change to read-fence policy doesn't risk regressing the write gate the Jepsen suite (PR #932) already trusts. - -### 7.3 CUTOVER coherence with Composed-1 - -The Composed-1 write gate compares observed catalog version against current. CUTOVER bumps the version by 1; any in-flight txn pinned at the pre-CUTOVER version that lands on the target sees its observed-version owner ≠ target → ErrComposed1Violation → coordinator retries at the new version → routes to the target's new active route. This is the same flow PR #927 exercised; M2 reuses it for writes, and §7.2 adds the parallel read mechanism. - -### 7.4 Engine route lookup invariant - -After CUTOVER, the engine's per-route ranges remain non-overlapping (per §3.2, the catalog only ever holds non-overlapping routes — the FENCE-time split out + the CUTOVER-time group rebind are each atomic and overlap-free). `Engine.GetRoute(routeKey(k))` therefore continues to return exactly one route, matching M1's invariant. No `OwnerOf` change is needed beyond what PR #932 already shipped. - -## 8. Migrator Runtime - -`distribution/migrator.go` (new): - -- Single goroutine per default-group leader. On leadership election the runtime starts it; on loss it cancels. -- Reads `!dist|job|*` on startup, picks up the oldest non-terminal job, drives it. After completion picks the next. -- Per-phase work uses the gRPC mesh (internal RPCs above) to the source/target group leaders. The migrator does **not** need direct Raft access. -- One job in flight at a time in M2 (simpler reasoning about FENCE / DELTA_COPY ordering). M3+ may relax this. - -Throttling: chunk-bytes default 1 MiB, inter-chunk pacing default 5 ms. Both configurable per-job via `StartSplitMigrationRequest.options` (string-map field; reserved for future without proto break). - -## 9. Idempotency and Resumability Matrix - -| Failure mode | Recovery action | Visible to client | -|---|---|---| -| Leader loss during BACKFILL | New leader resumes from persisted cursor; the source-group migration write tracker remains armed and the §6.1.0 step-2 running minimum (`snapshot_min_admitted_ts`) is re-read from tracker rows plus live locks, staying monotone-down across the flap. | None — job timeline lengthens. | -| Leader loss during FENCE | Catalog state is `WriteFenced` already, so coordinators still reject new writes/prepares while existing prepared txns may resolve through the narrow §7.1 lane. New migrator advances. The running minimum and `post_fence_drain_completed` are durable, so the new migrator re-reads the migration write tracker + route-faithful drain rather than re-deriving from a single stale scan. | Continued retryable rejection until DELTA_COPY → CUTOVER. | -| Leader loss between post-fence drain and `delta_floor` finalisation (§6.1.0 step 3) | `delta_floor` is the deterministic `min(snapshot_ts, snapshot_min_admitted_ts - 1)` of durable inputs, so the new migrator recomputes the identical value; no new raw/one-phase write or prepare can lower the running minimum because the fence already rejects them. | None. | -| Leader loss after target staged-readiness `ARMING` but before/while the target applies it | New migrator sees `target_staged_readiness_state = ARMING`, revalidates the expected catalog version, and idempotently probes/re-proposes `ApplyTargetStagedReadiness`. If the target already applied the record, the probe observes it; if not, the re-proposal creates it. The migrator records `ARMED` before arming the source read fence or attempting CUTOVER. | Target route is not published while readiness is missing; no live-only target reads. | -| Leader loss after target staged-readiness is `ARMED` but before default-group CUTOVER CAS | New migrator sees `target_staged_readiness_state = ARMED` and either completes the same CUTOVER CAS or transitions through `CLEARING` and clears `!migstage|ready|` on abort. If a fresh coordinator reaches the target during recovery after CUTOVER landed elsewhere, the target guard fails closed until the route descriptor with `staged_visibility_active` is local. | Brief retryable target reads/writes; no live-only target visibility. | -| Leader loss after default-group `ARMING` intent but before/while source-side cutover read fence is armed | New migrator sees `cutover_read_fence_state = ARMING`, revalidates the expected catalog version, and idempotently probes/re-proposes `ArmCutoverReadFence`. If the source had already applied the fence, the probe observes the matching pending entry; if not, the re-proposal creates it. The migrator then records `ARMED` before attempting CUTOVER. Target is not published while state is only `ARMING`. | Brief retryable reads only if the source fence already landed; no stale source value and no stranded fence. | -| Leader loss after source-side cutover read fence is `ARMED` but before default-group CUTOVER CAS | New migrator sees `cutover_read_fence_state = ARMED`, revalidates the expected catalog version, and either completes the same CUTOVER CAS or transitions through `CLEARING` and clears `ArmCutoverReadFence` on abort. Source returns `ErrRouteCutoverPending`/`ErrRouteMoved` rather than serving stale reads while the fence is armed. | Brief retryable reads; no stale source value. | -| Leader loss during CUTOVER | The CAS catalog write either landed (new version visible, job → CLEANUP) or not (job stays in DELTA_COPY, redo). Never partial. | None. | -| Leader loss during target promotion | The target leader resumes from target-local `!migstage|promote|.promote_cursor`; cursor advancement, live copy, and staged delete are one target-group Raft apply, so a persisted cursor cannot skip rows whose copy did not land. Default-group SplitJob state is only a witness and never drives the cursor. | None — reads keep using the raw staged/live merge until promotion finishes. | -| Target promotion finished but default-group promotion-complete CAS did not land | Target-local `PromotionState.done=true` and empty staged prefix are durable, and target-local ack / HLC-floor / readiness / promotion records are retained as recovery proof. The new migrator retries the default-group CAS that clears `staged_visibility_active` / `migration_job_id` and records `target_promotion_done=true`; only after the target observes the cleared descriptor does it delete those records. Until then, reads pay the merge cost over an empty staged prefix. | None. | -| Source cleanup reaches data-delete done before every source voter has applied CUTOVER | The source disarm step remains blocked on `source_cutover_ack_cursor`. The migrator keeps probing `AcknowledgeCutoverRouteRemoval(job_id, cutover_version)` for non-ACKed voters; if a voter is stalled or temporarily unreachable, the job stays in `CLEANUP` (or `FAILED` with `retry_phase=CLEANUP` if operator action is required) and keeps `!migfence`, the pending cutover read fence, and the retention pin armed. | Possible continued retryable rejects on stale source paths; no stale source writes/reads. | -| Target restart or snapshot restore after import / HLC-floor proposal | The restored target computes `fullFloor = max(store.LastCommitTS(), all !migstage|hlc_floor|*, all local RouteDescriptor.min_write_ts_exclusive values)`, then calls `SetPhysicalCeiling(hlcPhysicalMs(fullFloor))` and `Observe(fullFloor)` before accepting writes for routes with a non-zero migration high-water mark or durable descriptor floor. The descriptor term is load-bearing after promotion/cleanup deletes `!migstage|hlc_floor|`: an empty or sparse migrated range still retains `min_write_ts_exclusive >= fence_ts`, so a restart cannot issue or accept a caller-supplied timestamp below the source fence. | None. | -| Promotion copies a tombstone / expired version before older staged rows | The staged/live read path merges raw MVCC candidates before applying visibility. A hidden live winner suppresses the key and never falls back to an older staged candidate, so partial promotion cannot resurrect deleted or expired data. | None. | -| Target group temporarily unreachable | Migrator retries with backoff; phase doesn't advance. | None. | -| Source group temporarily unreachable | Same. Reads to source continue to fail with whatever transport error the adapter raises (independent of split). | Existing per-RPC retry. | -| Operator retry after guards are armed but CUTOVER CAS did not land | FAILED records `retry_phase=CUTOVER` and preserves `target_staged_readiness_state=ARMED` / `cutover_read_fence_state=ARMED`. `RetrySplitJob` returns to CUTOVER and retries the same catalog CAS or the explicit CLEARING path; it must not clear guards before retrying the CAS. | Bounded retries; no live-only target reads and no stale source reads. | -| Operator retry after post-CUTOVER cleanup failure | FAILED records `retry_phase=CLEANUP`. `RetrySplitJob` returns to CLEANUP and resumes target cleanup, source data cleanup, source CUTOVER ACK, and source disarm idempotently. `AbandonSplitJob` is rejected. | Migration remains active/guarded until cleanup completes. | -| Duplicate / out-of-order ImportRangeVersions RPC | Importer compares the request's `(bracket_id, batch_seq)` to the per-bracket contiguous high-water mark under `!migstage|ack||` (§6.1.1); drops any batch at or below it and returns the stored `acked_cursor`; rejects `batch_seq > stored+1` with `ErrImportBatchGap` before MVCC/HLC/cursor mutation. Empty progress chunks use the same accepted `stored+1` path, so a scan-budget-only chunk advances cursor / batch_seq without MVCC writes or HLC advancement. | None; migrator retries the gapped batch after the missing batch is acked. | -| Operator `AbandonSplitJob` during BACKFILL/FENCE/DELTA_COPY | Migrator first CASes the live job to `ABANDONING` with `abandon_from_phase`, then CASes the default-group catalog route back to `Active` if FENCE had landed and CUTOVER is proven absent, disarms the source write tracker / durable fence / retention pin, deletes target staged rows and target-local ack/HLC-floor/readiness/promotion records in bounded batches, and finally moves the `ABANDONING` job to `ABANDONED` history. | Brief retryable rejection clears; a crash after partial cleanup resumes abandon rather than retrying the old phase; no leaked per-job source/target state and no route left `WriteFenced` after abandon. | -| Crash between job state writes | Job state is **last-write-wins** on CAS; replaying a phase is safe by construction (idempotent imports, bounded ts windows). | None. | - -## 10. Test Strategy - -### 10.1 Unit - -- `distribution/migrator_test.go`: state machine — exhaustive phase transitions, abandon paths, CAS contention. Assert M2's live cap rejects a second non-terminal SplitJob (including `FAILED` and `ABANDONING`) with `ErrTooManyInFlightJobs`, and that the migrator never drives two jobs concurrently until an explicit M3 concurrency design lands. FAILED transitions must set durable `retry_phase`, `RetrySplitJob` must resume exactly that phase for `BACKFILL` / `FENCE` / `DELTA_COPY` / `CUTOVER` / `CLEANUP`, and a corrupt FAILED job with `retry_phase=NONE` must reject retry rather than deriving a phase from cursors or `last_error`. Abandon is accepted only before CUTOVER (or FAILED with a pre-CUTOVER retry phase), first persists `ABANDONING` + `abandon_from_phase` before source/target/catalog cleanup, rejects `RetrySplitJob` once `ABANDONING` is durable, resumes idempotent cleanup after a crash between cleanup bullets, and is rejected for `retry_phase=CUTOVER/CLEANUP`. -- `kv/migrator_snapshot_boundary_test.go`: §6.1.0 fence-before-boundary (codex round-15 P1). Assert (a) `ArmMigrationWriteTracker` and the conservative source retention pin are source-applied in the same Raft entry before `snapshot_ts` is chosen and BACKFILL opens; (b) `snapshot_min_admitted_ts` is a monotone-down running minimum across BACKFILL + FENCE-apply from tracker rows plus live pre-tracker locks — a raw point request using `r.Ts`, a raw `DEL_PREFIX` whose prefix interval intersects the moving range, a one-phase txn using `TxnMeta.CommitTS`, and a prepare that land **after** BACKFILL opens with low caller-supplied `commit_ts ≤ snapshot_ts` all lower it immediately via the tracker, even if the prepare lock resolves before the next migrator tick; (c) the active source retention pin remains at no-prune `min_ts=0` until every BACKFILL bracket is done, so compaction cannot prune older versions/tombstones still inside BACKFILL's `(0, snapshot_ts]` window; after `delta_floor` finalisation the pin relaxes to `delta_floor` and DELTA_COPY can still read `(delta_floor, fence_ts]`; (d) `delta_floor = min(snapshot_ts, snapshot_min_admitted_ts - 1)` is finalised **only** after `post_fence_drain_completed = true`, and once `verifyRouteNotFenced` rejects new writes/prepares the running minimum cannot move; (e) DELTA_COPY's window is `(delta_floor, fence_ts]`, so each post-scan low-ts committed version / range tombstone is `> delta_floor` and is exported; **(f) moved-secondary coverage (codex round-16 P1):** a multi-shard txn whose **primary key is outside** the moving range but whose **secondary key is inside** it, with `commit_ts ≤ snapshot_ts`, lowers `snapshot_min_admitted_ts` — proving the running minimum is keyed on `routeKey(lockedKey)` per key (via the write tracker and/or live secondary lock) and NOT only on the txn's primary. Red control 1: finalising `delta_floor = snapshot_ts` before the fence (the old single-pre-scan form) drops the late low-ts write — must fail. Red control 2: scoping the running minimum to moved *primaries* only (the pre-round-16 form) drops the moved-secondary's version — must fail. Red control 3: using a committed-MVCC-version scan over all `commit_ts ≤ snapshot_ts` to lower the floor makes an old retained version force near-full DELTA_COPY and must fail. Red control 4: tracking prepares only, without raw/one-phase apply tracker rows, drops a post-scan low-ts raw or one-phase write and must fail. Red control 5: treating `DEL_PREFIX` as `routeKey(prefix)` instead of a prefix range fails to lower the floor for a prefix that starts outside the moving route but tombstones keys inside it. Red control 6: relaxing the retention pin to `min(snapshot_ts, snapshot_min_admitted_ts - 1)` before BACKFILL completes lets compaction prune an old committed version/tombstone that a later BACKFILL bracket has not copied yet. Red control 7: arming the tracker before any source-local retention pin exists lets compaction prune a low-ts tracked version in the gap before `InstallMigrationRetentionPin`. Restart case: a flap between post-fence drain and finalisation recomputes the identical `delta_floor` from durable tracker/drain inputs (§9 row). -- `kv/migrator_lock_drain_test.go`: route-faithful txn-lock drain (§3.2a.0a). Table-driven over key families. **Must include a DynamoDB-item-lock case**: a lock stored at `!txn|lock|!ddb|item|
|` whose routing key is `!ddb|route|table|
` MUST be counted by a drain over the moving range `[!ddb|route|table|
, !ddb|route|table|
\xff)` — proving the drain scans the `!txn|lock|` family and filters by `routeKey(lockedKey)`, NOT by a raw `[txnLockKey(routeStart), txnLockKey(routeEnd))` bracket (which would sort `!txn|lock|!ddb|item|...` outside the bracket and falsely report the drain empty). Mirror cases for SQS (`!txn|lock|!sqs|...` → `!sqs|route|global`), Redis/list, and a plain user key (raw == routing, the trivial case). Negative cases: a same-table lock for a key whose routing falls outside `[routeStart, routeEnd)` is NOT counted; an unrelated-family lock is NOT counted. The same predicate backs §6.1.0's pre-BACKFILL live-lock scan and §3.2a step 0b's post-fence drain — assert both call sites share it. -- `kv/migrator_export_plan_test.go`: `PlanExportBrackets` includes every family whose `routeKey()` maps into the routing namespace. Required cases: DynamoDB table metadata (`!ddb|meta|table|`), DynamoDB generation (`!ddb|meta|gen|`), DynamoDB item/GSI, txn/list/Redis simple or legacy `!redis|...` rows, length-prefixed txn success markers (`!txn|ok||||...`) routed by decoded `locked_key`, including keys containing separator bytes, list base meta/item plus meta-delta/claim rows (`!lst|meta|`, `!lst|itm|`, `!lst|meta|d|`, `!lst|claim|`), Redis wide-column umbrella brackets for hash/set/zset (`!hs|`, `!st|`, `!zs|`) that cover meta, field/member/score, and delta rows, concrete stream brackets (`!stream|meta|`, `!stream|entry|`), concrete SQS/S3 storage prefixes, plus red controls where omitting the DynamoDB metadata brackets makes `loadTableSchemaAt` / `loadTableGenerationAt` fail after CUTOVER, omitting list delta/claim brackets loses list length/head deltas or POP claim protection after CUTOVER, omitting Redis wide-column brackets makes `HGETALL` / `SMEMBERS` / `ZRANGE` / `XRANGE` empty or partial after CUTOVER, omitting `!txn|ok|` makes a moved-secondary response-lost COMMIT retry lose local identity proof after CUTOVER, adding `!txn|lock|` to data export materializes an in-flight intent on the target and must fail, and adding a list/Redis wide-column bracket without its matching `routeKey()` decoder makes the `RouteKeyFilter` reject or mis-route every row in that bracket. Also assert `RouteKeyFilter(routeStart, nil)` and `RouteKeyFilter(routeStart, []byte{})` both treat the end as unbounded and accept keys in the rightmost route. Disjointness controls: a moving interval beginning at `!redis|` or another known internal prefix must not export that raw row through `familyUser`; it appears only in its explicit family bracket, `!txn|foo`, `!stream|foo`, `!ddb|foo`, `!sqs|foo`, and `!s3|foo` remain legal user keys exported by `familyUser` unless their adapters explicitly reserve them, and `routeKey()` / `sqsRouteKey` / `s3RouteKey` must keep those raw keys on their raw route rather than mapping them to global adapter routes. A red control that broadens SQS/S3 decoding to the whole `!sqs|` or `!s3|` umbrella must make `RouteKeyFilter` drop an otherwise legal raw key and fail. Concrete exported txn subprefixes (`!txn|int|`, `!txn|cmt|`, `!txn|rb|`, `!txn|ok|`, `!txn|meta|`), concrete DDB subprefixes (`!ddb|meta|table|`, `!ddb|meta|gen|`, `!ddb|item|`, `!ddb|gsi|`), concrete SQS/S3 storage subprefixes, and concrete `!stream|meta|` / `!stream|entry|` rows appear only in their brackets; `!txn|lock|` appears only in route-faithful drain scans, not export brackets; and reserved `!dist|` / `!migstage|` / `!migwrite|` / `!migfence|` intervals are rejected before planning. -- `distribution/catalog_test.go`: SplitJob codec round-trip + version-1 stability; RouteDescriptor codec matrix for v1 defaults, v2 non-zero `staged_visibility_active` / `migration_job_id` / `min_write_ts_exclusive`, finite and `End == nil` routes, zero-field descriptors still encoding as v1 during rollout, and truncated/extra v2 tails rejected; cross-group `StartSplitMigration` validation rejects any moving interval intersecting reserved migration/control namespaces (`!dist|`, `!dist|migstage|`, `!migstage|`, `!migwrite|`, `!migfence|`) before a SplitJob is created, including `!migstage|ready|` records. A mixed-cluster red test calls the old `SplitRange` RPC with an unknown `target_group_id` field encoded in the payload and asserts no SplitJob is created and no same-group split executes; cross-group work must enter only through `StartSplitMigration`. Same-group `SplitRange` rejects `ErrSplitJobOverlap` when its parent/child interval intersects any live SplitJob interval, including retryable `FAILED`, `ABANDONING`, and `CLEANUP` jobs; a disjoint same-group split remains allowed. -- `store/mvcc_store_export_test.go`: raw committed-version semantics — export walks MVCC versions in `(minCommitTS, maxCommitTS]` without the read-visible filter; includes delete tombstones and TTL versions even when `ExpireAt <= maxCommitTS`; preserves `expire_at`; excludes only intent locks; cursor monotonic; `maxScannedBytes` budget bounds a sparse-bracket call (returns a partial/empty chunk + advancing cursor at `done=false` instead of scanning the full prefix). Red control: a reader-visible export loses a DELTA_COPY delete tombstone after BACKFILL copied an older value. Retention-pin control: compact below the active migration pin before a bracket reaches an old tombstone/version and assert export still returns it; removing the pin must reproduce the miss. -- `store/mvcc_store_import_test.go`: idempotency under duplicate `(bracket_id, batch_seq)` (a replayed batch at or below the per-bracket high-water mark is a no-op that returns the stored `acked_cursor`; parallel brackets do not clobber each other's ack record) and gap rejection (`batch_seq > stored+1` returns `ErrImportBatchGap` without writing versions, clocks, or cursor, so a later batch cannot cause the missing earlier batch to be dropped); BACKFILL→DELTA_COPY phase transition clears each bracket cursor/done flag while preserving the monotone per-bracket batch sequence, so DELTA_COPY can rescan exhausted brackets without reusing `(job_id, bracket_id, batch_seq)`; zero-version progress chunk persists `(batch_seq, cursor)` only when it is the next contiguous batch and does so without MVCC writes, `metaLastCommitTS` / HLC advancement, or `max_imported_ts` changes; key_family dispatch; tombstone versions import via `DeleteAt`; non-tombstone TTL versions import via `PutAt(value, commit_ts, expire_at)` and reads after the original expiry hide the value. -- `kv/migrator_hlc_advance_test.go`: non-empty import advances `metaLastCommitTS`, target-local `!migstage|hlc_floor|`, physical ceiling, and in-memory `HLC.last`; the CUTOVER pre-write proposal floors the target at `max(max_imported_ts, fence_ts)` and writes the same value to `RouteDescriptor.min_write_ts_exclusive`, including empty/sparse moved ranges where `max_imported_ts < fence_ts`; restart/snapshot restore re-observes the full HLC floor (`ms<<16|logical`) before accepting writes; red control shows physical-ceiling-only restore can emit `ms<<16|0` below an imported version, and a supplied target write with `commit_ts <= fence_ts` is rejected both before and after promotion/cleanup deletes the per-job `!migstage|hlc_floor|` record. -- `kv/txn_codec_test.go`: M2-PR0 `txnLock` value round-trip with `commit_ts` plus backward-compat decode of an old lock value without the field (zero → migrator fallback path). -- `kv/txn_prepare_commit_ts_test.go`: M2-PR0 prepare plumbing. Assert the coordinator allocates/fences a non-zero `commit_ts` before PREPARE, the prepare request / txn metadata carry it to each participant, `handlePrepare` writes the same value into the txn lock, the migration write tracker records that value before the prepare is externally visible, COMMIT reuses it, and each participant commit writes the self-delimiting `!txn|ok|||||||` marker in the same apply as the committed version. Red controls: codec-only change leaves zero in `handlePrepare`; allocating a fresh commit timestamp during COMMIT disagrees with the tracker row; writing only the primary `!txn|cmt|` record leaves a moved secondary without local idempotency proof. -- `kv/fsm_route_fenced_test.go`: FENCE rejection at FSM (table-driven over key families, mirroring `fsm_composed1_test.go`). Include `Op_DEL_PREFIX` range footprints: coordinator global preflight rejects an empty/broadcast prefix whenever any intersecting route is WriteFenced; non-empty user prefixes reject on interval intersection; known internal prefixes normalize through the same route-key interval decoder as §7.2.2b; and a red control using only `routeKey(prefix)` allows a prefix outside the moving child to delete keys inside it. The source-group durable fence barrier is part of this test surface: after `ApplyMigrationWriteFence` commits, writes are rejected even if the catalog watcher snapshot is stale, and after snapshot restore / leadership acquisition the persisted `!migfence|` entry is loaded before serving writes. The migrator ACK test must advance `fence_ack_cursor` only when each voter has applied that durable source Raft entry; a watcher-only `catalog_version_applied >= fence_catalog_version` heartbeat must not count. Red controls that only wait for `distribution/watcher.go`'s in-memory `catalog_version_applied`, or that restore without loading `!migfence|`, can accept a write after restart and must fail. -- `kv/fsm_current_owner_gate_test.go`: post-CUTOVER owner gate for writes. After the source route is removed and `!migfence` is cleared, point writes, one-phase writes, prepares, and unclipped `DEL_PREFIX` ranges whose normalized footprint intersects the moved right child must return `ErrRouteMoved` / `ErrRouteOwnershipUnknown` on the old source even with `ObservedRouteVersion=0`. A broadcast `DEL_PREFIX` that the coordinator split under the current catalog applies only the local user-owned clipped subranges on each owner and no-ops owners with no local subrange; a red control that treats an unclipped foreign-only prefix as success/no-op on the old source must fail. Red controls: relying only on `verifyRouteNotFenced` accepts the write because no `WriteFenced` route remains; relying only on `verifyComposed1` accepts zero-version requests. -- `kv/fsm_migration_control_prefix_test.go`: user-plane mutation guard for reserved migration/control prefixes (§3.2 / §7.1). Assert raw `PUT`, `DEL`, one-phase txn, prepare, and explicit prefix deletes targeting `!dist|`, `!dist|migstage|`, `!migstage|`, `!migwrite|`, or `!migfence|` fail before `handleRawRequest` / `handlePrepare` creates MVCC versions or intents, even when no route is `WriteFenced`. Also assert broadcast `Op_DEL_PREFIX ""` subtracts those control intervals from the clipped user-data plan and never passes them to `handleDelPrefix`. Typed catalog/migration proposals remain allowed. Red controls: empty `DEL_PREFIX` deletes `!dist|job|*`, a user `PUT !migstage|hlc_floor|x` creates a fake floor, a broad prefix is rejected solely because it spans a control interval even though the control subrange would be clipped, and a prepare under `!migfence|` bypasses the gate. -- `kv/fsm_migration_write_tracker_test.go`: source-group `ArmMigrationWriteTracker` records raw point writes, `DEL_PREFIX` range tombstones, one-phase writes, and every prepare whose mutation/locked key routes into the moving range, including moved-secondary locks and same-tick land-and-resolve cases; `Phase_COMMIT` before `post_fence_drain_completed` is allowed only when the matching pre-existing intent lock is present. After the drain, new/unprepared commits are rejected, but an idempotent COMMIT retry for the same `(primary,start_ts,commit_ts)` is allowed only when an identity-bearing commit record / commit marker / per-key transaction-success marker proves the prior success, using staged+live probes while staged visibility is active. The moved-secondary case must succeed from `!txn|ok|` even when the primary `!txn|cmt|` routes outside the migrated interval. An ordinary committed MVCC version without `(primary,start_ts)` is not enough and is a red control. A rollback marker for the same tuple must prove the prior abort/rollback result and must not be treated as COMMIT success. New raw point writes, `DEL_PREFIX`, one-phase writes, and prepares after `WriteFenced` are rejected. Red controls: waiting for an empty drain while the route remains `Active` can starve under a steady prepare workload; closing the COMMIT lane solely because the drain completed makes a response-lost already-committed transaction return fenced/moved instead of idempotent success; treating rollback proof as success lets an aborted txn commit after the drain. -- `kv/fsm_cutover_read_fence_test.go`: §7.2.2e pending source read fence and §7.2.2 fail-closed ownership lookup. Assert point reads, `RawLatestCommitTS` / `ShardStore.LatestCommitTS`, and scans intersecting the moving range with `read_route_version < expected_cutover_version` never touch source MVCC; before the catalog CAS they return `ErrRouteCutoverPending`, after the CAS they return `ErrRouteMoved`; target-side `LatestCommitTS` reads the staged+live raw-candidate max while `staged_visibility_active=true`, so imported/staged versions are visible to timestamp reads before promotion. `cutover_read_fence_state` cannot become `ARMED` and CUTOVER cannot be issued until every current source voter has applied `ArmCutoverReadFence`; a source follower that becomes leader before applying the guard fails closed until the guard or a catalog snapshot at/after `expected_cutover_version` is loaded. When `read_route_version > source_local_version`, the wait is driven by `CatalogWatcher.ApplySnapshot` / engine catalog-version advancement, not Raft `appliedIndex`; a red control that only signals on FSM apply must time out despite the watcher advancing. When `max_cutover_version_seen > source_local_version` and the default-group ownership RPC times out/unreachable, both point reads and scan-interval checks return `ErrRouteOwnershipUnknown` and do not read source MVCC; non-intersecting left-half reads continue; `ClearCutoverReadFence` reopens reads only when a fresh default-group catalog/SplitJob read proves the CUTOVER CAS did not commit. External RawKV local-leader reads with zero `read_route_version` are first stamped server-side by the RawKV adapter / `ShardStore`; zero-version rejection applies only to forwarded/proxied internal requests that lost the stamp after `cap_migration_v2`. Recovery cases: crash after `ARMING` before all source voters ACK re-proposes/probes non-ACKed voters; crash after all source ACKs but before `ARMED` records `ARMED`; membership change after `ARMED` but before CUTOVER adds the new voter to the cursor and re-runs the guard before CUTOVER; leadership loss/timeout after issuing the CUTOVER CAS reloads catalog/job and, if `cutover_version` or `target.Active(right)` is visible, resumes CLEANUP with guards still armed rather than clearing; retry from FAILED with `retry_phase=CUTOVER` preserves ARMED target/source guards and retries/resumes the same CAS instead of clearing them; abandon/rollback from both states persists `CLEARING` and clears the source fence only after CUTOVER absence is proven. Internal scan cases must cover `!lst|meta|d|`, `!lst|claim|`, `!hs|`, `!st|`, and `!zs|` raw prefixes: the ownership gate normalizes to route-key intervals before `GetIntersectingRoutes`, and a red control that calls `GetIntersectingRoutes(raw_start, raw_end)` passes stale source ownership and must fail. Placement red controls must call the real read-serving entrypoints (`ShardStore.GetAt`, `ShardStore.ScanAt` / `ReverseScanAt`, `ShardStore.LatestCommitTS`, `adapter/grpc.go` RawGet, RawScanAt, and RawLatestCommitTS) with the source watcher paused; wiring the fence only into `kv/fsm.go` while those paths call `Store` directly must serve stale MVCC and fail. Red control: treating leadership loss as a failed CAS and clearing source/target guards after the CAS actually committed must expose stale source reads or live-only target reads. -- `kv/fsm_target_staged_readiness_test.go`: §6.4 step 0 target readiness. Assert `ApplyTargetStagedReadiness` is persisted before CUTOVER on every current target voter, survives restart/snapshot restore, and makes target reads/writes for the moving interval fail closed while the target route snapshot lacks matching `staged_visibility_active` / `migration_job_id`; `target_staged_readiness_state` cannot become `ARMED` and CUTOVER cannot be issued until every current target voter has ACKed the guard; a target follower that becomes leader before applying the guard fails closed until the guard or matching descriptor is loaded; a voter added/promoted after `ARMED` is added to the ACK cursor and must apply the guard before CUTOVER/cleanup proceeds. Once the staged descriptor arrives, reads use staged/live merge and writes enforce `min_write_ts_exclusive`; once the cleared descriptor arrives, the retained readiness record accepts it and serves ordinary MVCC with the durable floor rather than returning retry/unknown until cleanup. Recovery cases mirror the source read fence: crash at `ARMING`, crash after partial/all target ACKs before `ARMED`, rollback through `CLEARING`, and successful cleanup only after DONE. Red controls: fresh coordinator routes to target before target watcher catches up and target serves live-only empty data, accepts a low live write before loading the staged floor, or rejects a correctly cleared descriptor while `!migstage|ready|` is still retained for lagging voters. -- `kv/fsm_promote_staged_test.go`: target-local `PromotionState` (§6.4). Assert `PromoteStaged` copies a bounded staged batch, physically deletes the exact staged MVCC row (not a `DeleteAt` tombstone), and advances `promote_cursor` in one target-group apply; a replay is idempotent by `(raw_key, commit_ts)`; a leader flap resumes from the target-local cursor; the default-group promotion-complete CAS that clears `staged_visibility_active` is attempted only after target-local `done=true`; target-local import ACK / per-job HLC-floor / readiness / promotion records are deleted only after every current target voter ACKs `AcknowledgeTargetClearedDescriptor` for the cleared descriptor that still carries `min_write_ts_exclusive`; and the final DONE history move is attempted only after target/source cleanup ACKs. Include raw staged/live merge cases where staged has `K@10` plus `K@20` tombstone (and a TTL variant), the promoter copies/deletes only the hidden `K@20` row first, and point reads, `RawLatestCommitTS`, forward scans, `ReverseScanAt`, and `RawScanAt(reverse=true)` at `read_ts >= 20` return not found / omit the key or the correct latest timestamp rather than falling back to staged `K@10` / live-only. Write-side probe cases must cover a retry whose identity-bearing commit marker / `!txn|ok|` per-key transaction-success marker / one-phase `PrevCommitTS` proof exists only under `!dist|migstage||*` before promotion reaches that internal key; while `staged_visibility_active=true`, the write path must find that staged proof and return COMMIT success rather than creating a contradictory live record. An ordinary committed version without transaction identity must not be accepted as COMMIT success. A staged rollback marker for the tuple must return the idempotent abort/rollback result and must not be accepted as COMMIT success. Red controls: tombstoning the staged key instead of physically removing it leaves the staged prefix non-empty or hides older staged rows during partial promotion; deleting `!migstage|hlc_floor|` / readiness before a lagging target voter has observed the cleared descriptor lets that voter become leader and serve without the proof it still requires; clearing/deleting target proofs while the descriptor still has `staged_visibility_active=true` deadlocks cleanup or loses the staged metadata required by serving reads/writes; clearing `min_write_ts_exclusive` during promotion-complete lets a caller-supplied low timestamp hide behind migrated history. -- `kv/fsm_migration_timestamp_floor_test.go`: target-side post-CUTOVER timestamp floor (§6.2.1). With `staged_visibility_active=true`, `!migstage|hlc_floor|=T=max(max_imported_ts,fence_ts)`, and `RouteDescriptor.min_write_ts_exclusive=T`, coordinator-allocated writes after `Observe(T)` succeed with `commit_ts > T`, but raw / one-phase / prepare paths carrying caller-supplied `commit_ts <= T` are rejected before creating a live MVCC version or intent lock, including sparse/empty migrations where `max_imported_ts < fence_ts`. After promotion clears `staged_visibility_active` and target cleanup deletes `!migstage|hlc_floor|`, the descriptor floor still rejects the same low supplied timestamps. Red control: accepting the low supplied timestamp hides an acknowledged live write behind a higher staged version during the staged/live merge, violates the source fence boundary, or reopens the bug after cleanup. -- `kv/migrator_cleanup_test.go`: source cleanup walks the same disjoint bracket plan + `RouteKeyFilter` as export and deletes only versions `<= fence_ts`; red control using a raw `[routeStart, routeEnd)` delete leaves DynamoDB metadata, Redis collection rows, list claims, SQS/S3 rows, or txn records behind. Successful cleanup first waits for every current source voter to ACK both `AcknowledgeSourceReadDrain(job_id, fence_ts, cutover_version)` and `AcknowledgeCutoverRouteRemoval(job_id, cutover_version)` before disarming `!migwrite||*`, clearing `!migfence|` and the pending source cutover read fence, and releasing the source retention pin; a red control that deletes source versions after `readFenceGrace` while any pinned S3/getObject-style read whose footprint intersects the moved route is still streaming (even with `read_ts > fence_ts`) must start returning missing chunks, and a red control that clears fences without the route-removal/current-owner ACK must permit a watcher-stalled source voter to accept a stale/zero-version write. After cleanup, stale reads to the source must still return `ErrRouteMoved` / `ErrRouteOwnershipUnknown`, never user-visible not-found. After the default promotion-complete witness and the target cleared-descriptor ACK barrier, target-local ack / HLC-floor / readiness / promotion records are deleted in bounded batches. Abandon cleanup first persists `ABANDONING`, then does the same per-job source/target cleanup and CASes the default-group route descriptor back from `WriteFenced` to `Active` before moving the job to `ABANDONED` history; red controls: clearing side effects before `ABANDONING` lets recovery retry the old phase with missing guards, and clearing only source-local `!migfence` leaves the catalog route `WriteFenced` forever. Retryable `FAILED` jobs remain under `!dist|job|` and survive history GC until `RetrySplitJob` (or `AbandonSplitJob` only for pre-CUTOVER retry phases). -- `kv/sharded_coordinator_route_fenced_test.go`: coordinator-side rejection + retry-after surfacing. Include `DEL_PREFIX` broadcast preflight cases where one shard's current engine snapshot has a WriteFenced route intersecting the prefix, and assert the coordinator surfaces `ErrRouteWriteFenced` instead of broadcasting a tombstone request that only the source FSM later rejects. -- Property tests (`pgregory.net/rapid`): replay a sequence of (export-chunks, import-acks, leader-flap) and assert (a) all source keys land on target, (b) no duplicates committed at any commit_ts. - -### 10.2 Integration - -- `kv/integration_split_test.go`: same-group split (M1 regression) + cross-group split happy path. -- Migrator restart mid-phase: cursor recovery, no double-apply. -- Concurrent writes during BACKFILL (fence is not yet up; writes still serve from source, get exported with their commit_ts ≤ snapshot_ts excluded, delta-copied in (delta_floor, fence_ts]). -- **DELTA_COPY tombstone and TTL preservation (§6.1 / §6.2).** BACKFILL copies `K@10`, the source deletes `K@20 <= fence_ts`, and DELTA_COPY exports the delete tombstone so the target hides `K@10` after CUTOVER. A parallel TTL case writes `K@10` with `expire_at=30`, migrates before expiry, then asserts reads at `read_ts < 30` see the value and reads at `read_ts >= 30` do not. Controls: reader-visible export drops the tombstone; import without `expire_at` makes the TTL value non-expiring. -- **Target HLC floor survives restart/snapshot (§6.2.1).** Import a version whose `max_imported_ts = ms<<16|logical` with `logical > 0`, snapshot/restore or restart the target before the first post-CUTOVER write, then assert the restored target re-observes `max(max_imported_ts,fence_ts)` and its first `Next()` is greater than both witnesses. Include an empty/sparse moved range where `max_imported_ts < fence_ts` and a caller-supplied target write `<= fence_ts` is rejected. Controls: restoring only the physical ceiling emits `ms<<16|0` and fails; flooring only at `max_imported_ts` lets the sparse case write below the source fence. -- **Partial-promotion hidden-version merge (§6.4).** After CUTOVER, leave staged `K@10=value`, promote only a newer hidden version (`K@20` tombstone or TTL with `expire_at <= read_ts`) into live, and keep the older staged row unpromoted. Point reads, forward range scans, `ReverseScanAt`, and `RawScanAt(reverse=true)` at `read_ts >= 20` must return not found / omit the key. Control: standard snapshot reads over live plus staged fallback, or a live-only reverse iterator, resurrects `K@10`. -- **Redis collection migration (§6.3.1).** Build a list with uncompacted `!lst|meta|d|` rows and live `!lst|claim|` rows, a hash with multiple fields, a set with members, a zset with both member and score-index rows, and a stream with metadata plus entries. After cross-group CUTOVER, `LRANGE`/list length/POP behavior, `HGETALL`, `SMEMBERS`, `ZRANGE`, and `XRANGE` must match the source state. Controls: omitting the `!lst|meta|d|` / `!lst|claim|` / `!hs|` / `!st|` / `!zs|` / concrete `!stream|meta|` and `!stream|entry|` brackets, or keeping the bracket while omitting the matching `routeKey()` decoder, makes the collection read empty or partial or loses POP claim protection and must fail. Also create plain user keys `!stream|foo`, `!sqs|foo`, and `!s3|foo` and assert they migrate through `familyUser`, not stream/SQS/S3 internal brackets; a broad SQS/S3 umbrella decoder is a red control. -- **Post-scan low-ts writes (§6.1.0 fence-before-boundary, codex round-15 P1 + round-20 P1 + round-21 P1).** A raw point request, a raw `DEL_PREFIX` whose prefix starts outside the moving child but matches keys inside it, a one-phase txn, and a prepare each land after BACKFILL opens and before the fence with a caller-supplied / lagging-clock `commit_ts ≤ snapshot_ts`, then commit/apply after the BACKFILL iterator passed their keys. Assert: (a) the source-group migration write tracker records raw point / `DEL_PREFIX` / one-phase `commit_ts` at apply time and prepare `commit_ts` at admission time, even if the prepare lock resolves before the next migrator tick, (b) `delta_floor` is finalised below all four only after step 0b's post-fence drain returns empty, (c) DELTA_COPY's `(delta_floor, fence_ts]` window exports each committed version / tombstone, (d) they are present on the target after CUTOVER. A control case with the finalisation moved *before* the fence, with prepare-only tracking, or with `DEL_PREFIX` tested as a point `routeKey(prefix)` must drop at least one write or resurrect deleted data (red test guarding the fix's ordering and coverage). -- **Moved-secondary, outside-primary multi-shard prepare (§6.1.0 step 2, codex round-16 P1).** A 2PC txn whose primary key routes to a group **outside** the moving range and whose secondary key routes **into** the moving range, prepared with a low caller-supplied `commit_ts ≤ snapshot_ts`, then committed after BACKFILL passed the secondary key. Assert: (a) the running minimum is lowered by the secondary's `commit_ts` via the source-group migration write tracker even when the lock resolves before the next migrator tick, (b) DELTA_COPY's `(delta_floor, fence_ts]` window exports the committed secondary version, (c) it is present on the target after CUTOVER. Control: a build that scopes the running minimum to moved primaries (and relies on the primary-keyed `!txn|cmt|` record, which routes outside) must drop the secondary version — red test for this fix. -- **FENCE drain is non-starving (§3.2a / §7.1).** Run a steady workload that continuously prepares txns in the moving range. Assert `BACKFILL → FENCE` publishes `WriteFenced` before waiting for an empty drain, new prepares are rejected once every voter has applied the fence, existing prepares resolve through the narrow `Phase_COMMIT` / `Phase_ABORT` lane, the drain completes within one txn-TTL, and a response-lost COMMIT retry after the drain can return idempotent success only from durable transaction-identity proof; durable rollback proof returns abort/rollback, not success. In the same FENCE window, issue `DEL_PREFIX` with (a) empty prefix and (b) a prefix whose byte interval intersects only the moving right child after its start; both must return `ErrRouteWriteFenced` before `handleDelPrefix` writes tombstones. Controls: the old "wait empty while Active" design never reaches FENCE under the steady workload, a post-drain retry path that requires a live lock row makes an already-committed response-lost COMMIT return fenced/moved instead of success, a rollback marker accepted as COMMIT success resurrects an aborted txn, using an ordinary committed version without txn identity accepts a timestamp-collision retry, and a point-key-only fence lets the prefix delete land on source after `delta_floor` is fixed. -- **Reserved default catalog prefix is immovable and data-plane protected (§3.2).** When the source group is the default group, a cross-group split whose moving right child intersects `!dist|` fails with `ErrReservedRange` before creating a SplitJob. Separately, user-plane raw `PUT`, prepare, and explicit prefix deletes that target `!dist|`, `!dist|migstage|`, `!migstage|`, `!migwrite|`, or `!migfence|` fail even without an active migration, while `DEL_PREFIX ""` is split/clipped as a user-data flush and must leave those control prefixes untouched. Controls: allowing the split and then running CLEANUP would delete `!dist|meta|`, `!dist|route|`, or `!dist|job|` records; an unclipped empty prefix delete would wipe migration-control rows; rejecting the broad user-data flush solely because the raw interval spans control prefixes would break existing broadcast prefix-delete semantics. -- **CUTOVER target-readiness and source-read barrier (§6.4 step 0 / §7.2.2e, codex round-17 P1).** Pause one target voter before `ApplyTargetStagedReadiness` applies and assert `target_staged_readiness_state` does not become `ARMED` and CUTOVER is not issued; after every target voter ACKs, add/promote a new target voter and assert the ACK cursor reopens until that voter applies the same guard. Then pause the target catalog watcher and let another coordinator refresh against the default group after CUTOVER; target reads/writes for the moved interval must return retry/ownership-unknown until the target route descriptor with matching `staged_visibility_active` and retained `min_write_ts_exclusive` is local, never live-only empty data or a low live write; after promotion-complete, while `!migstage|ready|` is still retained for a lagging voter, a target that has the matching cleared descriptor serves ordinary MVCC instead of retrying. Then pause one source voter before `ArmCutoverReadFence` applies and assert `cutover_read_fence_state` does not become `ARMED` and CUTOVER is not issued; after every source voter ACKs, add/promote a new source voter and assert the ACK cursor reopens until that voter applies the read fence; then pause the source catalog watcher/heartbeat delivery before the source applies the CUTOVER snapshot. A stale coordinator's point read, `RawLatestCommitTS`, and cross-boundary scan routed to the source must return `ErrRouteMoved`/retry, never the pre-CUTOVER source value/timestamp; target `RawLatestCommitTS` must see staged rows before promotion. External RawKV `RawGet` / `RawScanAt` / `RawLatestCommitTS` calls that arrive without a version must be stamped server-side before this check; only internal forwarded/proxied requests that lose the stamp fail closed with `ErrRouteOwnershipUnknown`. When `read_route_version > source_local_version`, advance only the distribution engine catalog snapshot and assert the read fence wakes from the catalog-version waiter rather than Raft `appliedIndex`. Keep that source voter stalled beyond `readFenceGrace` and assert source cleanup does not delete source versions until `AcknowledgeSourceReadDrain` succeeds for all active pins intersecting the moved route, including a stream with `read_ts > fence_ts`, and does not clear `!migfence` / pending read fence / retention pin until `AcknowledgeCutoverRouteRemoval` succeeds on that voter; a stale zero-version write to the source must continue to fail closed. Keep one target voter stalled before the cleared descriptor and assert target-local per-job HLC/readiness/promotion proofs are not deleted until `AcknowledgeTargetClearedDescriptor` succeeds while the descriptor floor remains. Also inject default-group ownership RPC timeout/unreachable while `max_cutover_version_seen > source_local_version`; the source must return `ErrRouteOwnershipUnknown` for both point and scan requests without touching source MVCC, and the coordinator must retry through `WatcherRefresh`. Crash controls: kill the default leader after persisting target/source `ARMING` but before each proposal, and again after partial/all voter ACKs but before `ARMED`; recovery must either re-arm/probe non-ACKed voters and proceed to CUTOVER, or persist `CLEARING` and remove both guards only after a fresh default-group read proves CUTOVER did not land. Internal scan controls use raw prefixes such as `!lst|meta|d|`, `!lst|claim|`, `!hs|`, `!st|`, and `!zs|`; the passing implementation normalizes them to route-key intervals, while the red implementation that compares raw bounds against catalog routes serves stale collection rows. Control: disabling either barrier, accepting a single-leader ACK as `ARMED`, failing to re-arm after membership changes, treating timeout as source/target-owned, treating ambiguous CUTOVER as failed, skipping the `ARMING` witness, using raw scan bounds for internal ownership checks, or clearing source fences from a grace timer alone reproduces stale reads/writes or stranded pending fences. -- **Promotion ownership handoff (§6.4).** Kill the target leader after a `PromoteStaged` apply, and kill the default leader after target-local `PromotionState.done=true` but before the default-group CAS clears `staged_visibility_active`. Assert no staged rows are skipped, range reads keep using the staged/live merge until the CAS lands, and retrying the CAS reaches DONE without data changes. -- Cross-shard txn coordinated through `kv/transaction.go` straddling the split key — must serialize correctly across CUTOVER. - -### 10.3 Jepsen - -- New `jepsen/elastickv/split_workload.clj` (or extend the existing dynamodb workload): - - Linearizable register over a key that crosses the moving boundary. - - Nemeses: leader-kill on source group; partition source↔target mid-DELTA_COPY; abandon mid-FENCE. - - Pass criterion: `{:valid? true}` per existing Elle setup. -- M4 lands the broader hotspot + nemesis matrix; M2 only adds a deterministic "split happens at workload time T" scenario. - -### 10.4 Five-lens self-review (CLAUDE.md) - -| Lens | M2-specific risk | -|---|---| -| Data loss | CUTOVER atomicity; ImportRangeVersions idempotency; the BACKFILL/DELTA_COPY commit-ts windows have no gap; raw MVCC export preserves tombstones and TTL expiry headers. | -| Concurrency | Migrator goroutine vs leadership election; FENCE visibility timing on followers; target-local promotion cursor vs default-group visibility CAS; AbandonSplitJob race with phase advance. | -| Performance | Chunk size vs gRPC frame limit; pacing default vs prod throughput; cursor lookup cost per import. | -| Data consistency | Composed-1 alignment at CUTOVER; reader at `≤ fence_ts` on source vs reader at `≥ catalog_version+1` on target; target-local full-HLC floor at `max(max_imported_ts,fence_ts)` across restart/snapshot; raw staged/live merge suppresses hidden tombstone/TTL winners in forward and reverse scans; internal-key route mapping, including list delta/claim rows, Redis wide-column prefixes, concrete-only SQS/S3 decoding, matching `routeKey()` decoders, route-key-normalized scan ownership checks, server-stamped RawKV read versions, and `DEL_PREFIX` range-footprint fencing/tracking. | -| Test coverage | Property tests over chunk-then-restart sequences; FSM unit for FENCE; Redis collection migration controls; Jepsen split + nemesis. | - -## 11. Implementation Touchpoints - -Phased into reviewable PRs, each lands behind its own doc-or-test gate: - -| PR | Scope | Tests | -|---|---|---| -| M2-PR0 | Prepare-time `commit_ts` plumbing behind the PR7 cluster capability gate: decoder/helpers understand both legacy and new lock values; while the gate is closed, normal PREPARE/COMMIT remains byte-compatible with M1 and writes neither lock `commit_ts` fields nor per-key `!txn|ok|` markers; once the gate is open, the coordinator allocates/fences `commit_ts` before PREPARE, prepare request and txn metadata carry it to every participant, `handlePrepare` writes the same value into each `txnLock`, the migration write tracker records that value at prepare admission, and COMMIT reuses it while writing length-prefixed per-key `!txn|ok|` success markers for every committed participant. Keep decoder backward-compat for existing lock values without the field (zero → migrator fallback to direct commit-record lookup / pending drain). | `kv/txn_codec_test.go` round-trip + backward-compat case; `kv/txn_prepare_commit_ts_test.go` covers gate-closed mixed-cluster traffic emitting legacy locks/no `!txn|ok|`, then gate-open coordinator → prepare request/meta → `handlePrepare` → tracker path + per-key success marker; red controls for codec-only, commit-time allocation, primary-only commit proof, and writing PR0 side effects before the gate opens | -| M2-PR1 | SplitJob codec + reserved-key layout + catalog read/write helpers | catalog_test | -| M2-PR2 | `proto/distribution.proto` + `proto/internal.proto` regen (`MVCCVersion.expire_at`, bracket/batch identity, scan-budget fields); wire fields plumbed but unused | proto regen; build | -| M2-PR3 | `store/MVCCStore.ExportVersions` raw committed-version export + `ImportVersions` with idempotency, tombstone/`expire_at` preservation, target-local full-HLC floor persistence, durable RouteDescriptor v2 codec/dual decoder, and descriptor `min_write_ts_exclusive` plumbing, no migrator wiring | store unit incl. tombstone/TTL + HLC-floor restore cases + property; `distribution/catalog_test.go` v1/v2 route descriptor codec matrix | -| M2-PR4 | `distribution/migrator.go` state machine + export bracket planner, including list delta/claim, Redis wide-column brackets, concrete-only SQS/S3 brackets, txn success marker routing, drain-only txn locks, and matching `routeKey()` decoder coverage — same-group target (no-op data move) to lock the state shape | migrator unit + `migrator_export_plan_test` | -| M2-PR5 | Coordinator + FSM FENCE rejection (`ErrRouteWriteFenced`) with point and `DEL_PREFIX` range-footprint checks + route-faithful txn-lock drain (§3.2a.0a) + same-group `SplitRange` overlap rejection while a SplitJob is live | fsm + coordinator unit + `migrator_lock_drain_test` + `catalog_test` overlap red controls | -| M2-PR6 | Cross-group end-to-end: ExportRangeVersions / ImportVersions server-side handlers + migrator BACKFILL/DELTA_COPY + raw-candidate staged/live merge read path in both scan directions and `LatestCommitTS` + §7.2.2e source-side cutover read-fence arm with every-current-source-voter ACK, membership-epoch re-ACK, catalog-version waiter, route-key-normalized scan ownership checks, and server-stamped RawKV read versions | integration incl. delete/TTL DELTA_COPY, Redis list/hash/set/zset/stream migration, HLC restart/fence-floor cases + `kv/fsm_cutover_read_fence_test.go` | -| M2-PR7 | Target-local `PromotionState` + background promoter + ordered default-group promotion-complete CAS retaining `min_write_ts_exclusive` + source/target cleanup before DONE history move; readiness guard accepts matching cleared descriptors while retained; `AbandonSplitJob` with durable `ABANDONING` cleanup witness + CLEANUP GC + Jepsen split workload | `kv/fsm_promote_staged_test.go` raw hidden-version/LatestCommitTS merge + jepsen suite | -| M2-PR8 | Rename `*_proposed_*` → `*_partial_*` after PR1 ships; update parent partial doc M2 status; rename to `*_implemented_*` after PR7 | Completed after the runner, voter barriers, cleanup lifecycle, route publication, and Jepsen workload landed in PR #1096. | - -Each PR follows the five-lens self-review and is gated by its tests + `make lint`. - -### 11.1 Rolling-upgrade compatibility - -Closes gemini medium on PR #945: an M1 node that has not yet been upgraded does not understand `RouteStateWriteFenced` enforcement on writes, the read fence in §7.2, or the import RPCs. Starting a cross-group split while M1 nodes are still serving is a silent-write-loss hazard. - -Layered mitigations: - -1. **Capability advertisement.** Each node advertises a `node_capability_bitfield` in its periodic heartbeat to the default group (existing distribution heartbeat — extend by one field). PR0-PR6 binaries keep `cap_migration_v2` clear even though they can decode some new shapes; only PR7-complete binaries set the bit at boot, because only then can the full migration and its rollback/cleanup path complete. -2. **Cluster-readiness gate at the entry RPC.** `StartSplitMigrationRequest` is rejected at `adapter/distribution_server.go` with `ErrClusterNotReadyForMigration` unless **every active node** advertises `cap_migration_v2` and the default-group leader itself supports the new RPC. Same-group `SplitRange` (M1) keeps its semantics and wire shape, but it is not allowed to overlap a live cross-group SplitJob (§3.1.1): overlapping requests return `ErrSplitJobOverlap`, while disjoint same-group splits remain available. -3. **In-flight job invalidation on downgrade.** If the default-group leader observes any node losing `cap_migration_v2` mid-migration (rare — would only happen with a botched rollback), the job records `capability_regressed = true` / `ErrClusterCapabilityRegressed`, stops phase advancement, and makes every still-M2-capable coordinator fail closed for the moving route on the participating source/target groups until the regression is resolved. This is stronger than "pause the migrator": once the write tracker, source fence/read-fence, or target readiness guard may be armed, a downgraded voter that can become leader is unsafe because it may ignore `!migwrite`, `!migfence`, or `!migstage|ready`. Recovery must do one of three safe things before traffic resumes: (a) roll the node forward so it advertises `cap_migration_v2` again and then re-run the per-voter ACK barriers under the current voter-set epoch (§3.1); (b) remove/demote the regressed node from every default/source/target voter set that can serve the live SplitJob, then re-run the ACK barriers for the remaining voters; or (c) if and only if the job is still before any source/target guard or staged import became durable, abort through the normal pre-CUTOVER `AbandonSplitJob` cleanup. A hard rollback of a serving participant without roll-forward/removal is treated as cluster-unavailable for the moving range, not as a resumable paused job. -4. **M2-PR0..PR7 land in this order so the capability bit only opens when the full feature is present (closes claude round-11 P2 on PR #945).** PR0-PR3 are wire/codec/store additions with **no behaviour change for M1 while the gate is closed**: the existing `SplitRange` RPC remains same-group only; `StartSplitMigration` returns `UNIMPLEMENTED` or `ErrClusterNotReadyForMigration`; PR0's txn codec can read legacy/new shapes, but normal transaction apply still emits legacy lock values, does not write per-key `!txn|ok|` markers, and accepts legacy zero-`commit_ts` locks as ordinary traffic. PR4 introduces the state machine but no data move; PR5 lands FENCE rejection (`verifyRouteNotFenced` + `ErrRouteWriteFenced`) behind the same closed gate; PR6 lands `ExportRangeVersions` / `ImportRangeVersions` server-side handlers + §6.4 step 2 staged-merge read path + §7.2.2e source-side cutover read-fence arm; PR7 lands the background promoter + `AbandonSplitJob` + CLEANUP drain, starts advertising `cap_migration_v2`, and only then enables PR0's prepare-`commit_ts` persistence, migration write-tracker admission rows, and per-key transaction-success markers for normal traffic. **The `cap_migration_v2` bit goes live only in PR7 commit**, when the full end-to-end migration path can complete — not in PR5 (where FENCE is enforced but BACKFILL has no export handler yet, so a SplitJob would create successfully and then fail at BACKFILL, giving operators false assurance), and not in PR0 (where only transaction proof plumbing exists). The earlier-rounds intention was "the bit guards against rollout-skew silent execution," and the corrected intent is "the bit guards against operators or upgraded voters creating any migration-only side effects in a half-shipped cluster." If a finer-grained gate is desired later, OQ-18 records the option to split into `cap_txn_marker_v1` (PR0), `cap_fence_v2` (PR5), and `cap_data_move_v2` (PR6/PR7), with `StartSplitMigration` gated on the conjunction — out of scope for v1. -5. **Test plan**: `distribution/capability_test.go` exercises (a) gate accepts PR7-complete M2-only cluster, (b) gate rejects mixed cluster, (c) gate rejects on heartbeat staleness (no advertisement within 2× heartbeat interval treated as no capability), (d) mid-job capability regression after tracker/fence/readiness arming forces moving-route fail-closed behavior and blocks retry/resume until the regressed voter is rolled forward or removed, and (e) PR0-on/cluster-gate-closed transaction traffic produces legacy lock bytes and no `!txn|ok|` marker while the same test flips the all-voter gate open and observes non-zero prepare `commit_ts` plus per-key success markers; a pre-guard regression may abandon only through the normal cleanup proof. - -This is independent of the existing rolling-upgrade protocol for unrelated subsystems and adds zero overhead for clusters never using cross-group split. - -## 12. Open Questions - -1. **Job concurrency.** M2 caps at one in-flight migration. Confirm acceptable for first cut; M3 may want concurrent jobs for hotspot bursts. -2. **Pacing knobs.** Default 1 MiB chunk + 5 ms inter-chunk pacing — these are guesses; should we ship a metrics-emitting default and let operators tune via `StartSplitMigrationRequest.options`? -3. **Grace period (RESOLVED — `readFenceGrace = 30 s`).** §7.2.4 sets the default at 30 s with the rationale "comfortably exceeds both lease TTL and watcher tick × 2." All cross-references (§4 CLEANUP row) now match this value. Operators on extreme deployments (very long lease TTLs, or watcher polling intervals beyond the defaults) raise it via the existing `readFenceGrace` knob. -4. **`MVCCVersion.key_family` enum.** Add a strict enum in proto, or stay numeric and document constants in `kv/`? Strict enum costs proto churn for every new internal family; numeric + go-side constant has been the project pattern. -5. **`AbandonSplitJob` after CUTOVER.** Currently rejected. Worth offering a "reverse migration" RPC in M2, or defer to M3+? -6. **Backfill source visibility (RESOLVED — migration write tracker + fence-first drain; BACKFILL snapshot exclusion is intentional).** Two questions inside one OQ — resolved as follows. (a) The `Phase_COMMIT` stranding hazard (a prepared txn whose `Phase_COMMIT` arrives after FENCE rejection — flagged codex round-12 P1) is closed by §3.2a's fence-first drain: `BACKFILL → FENCE` publishes `WriteFenced` before waiting for an empty drain, so new prepares cannot starve the drain after every source voter applies the fence, while existing prepared txns can still resolve through §7.1's narrow `Phase_COMMIT` / `Phase_ABORT` lane until `post_fence_drain_completed=true`; after that point, only response-lost idempotent COMMIT retries with durable transaction-identity proof of the prior success may return success, while rollback proof returns the prior abort/rollback result. A committed MVCC version without `(primary,start_ts)` is not a success proof. (b) The BACKFILL-snapshot semantic question — *should the snapshot include intent locks or not?* — is closed in the negative: **excluding intent locks from BACKFILL is intentional and correct**. Any txn whose `commit_ts ≤ snapshot_ts` and whose prepare had resolved before the BACKFILL iterator reached its key is captured in BACKFILL as a fully-committed version; every other committed version for the moving range — anything with `commit_ts > snapshot_ts`, **and** any migration-window low-ts write with `commit_ts ≤ snapshot_ts` that committed/applied after BACKFILL passed its key, including raw point requests and `DEL_PREFIX` range tombstones at `r.Ts`, one-phase txns at `TxnMeta.CommitTS`, and prepared txns on the primary or on a secondary key that resolves into the moving range (§6.1.0 step 2's tracker-backed running minimum, keyed per moved key not only the primary and using range footprints for `DEL_PREFIX` — codex round-15 P1 + round-16 P1 + round-18 P1 + round-20 P1 + round-21 P1) — is captured by DELTA_COPY's `(delta_floor, fence_ts]` window, where `delta_floor ≤ snapshot_ts` is finalised only after the fence blocks new writes/prepares. Intent locks themselves are coordinator-managed state and don't need to migrate with the data — after CUTOVER the target's empty intent-lock space for the moved range is correct, because the FENCE drain guarantees no prepare remains unresolved across the boundary. -7. **Same-group observability.** Resolved for M2: same-group splits stay on the M1 `SplitRange` fast path and do not create `SplitJob` records, but they are rejected with `ErrSplitJobOverlap` when their route interval intersects a live cross-group SplitJob. If operators need same-group split audit records later, add a separate audit event rather than sending `target_group_id == source_group_id` through `StartSplitMigration`. - -## 13. Acceptance Criteria - -1. `StartSplitMigration` with `target_group_id != source_group_id` returns a `job_id` and the migration completes with `phase=DONE`. -1a. Same-group `SplitRange` requests whose parent/child interval overlaps any live SplitJob are rejected before changing the catalog; disjoint same-group splits still use the M1 fast path. -2. `ListRoutes` after a successful cross-group split shows two non-overlapping routes whose `raft_group_id` differs. -3. Writes to the moving range during FENCE / DELTA_COPY surface a retryable error to the client and are eventually accepted on the target after CUTOVER. -4. Killing the default-group leader at any point during a migration resumes the job to completion on the new leader without operator action. -5. Jepsen split workload (linearizable register straddling the boundary) passes under leader-kill and partition nemeses. -6. M1 same-group split path remains green (existing tests + manual via `elastickv-split` CLI). - -## 14. Lifecycle - -<<<<<<<< HEAD:docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md -This document is `*_implemented_*` because the M2 cross-group migration plane is complete as a central subsystem. The production runner now resumes durable jobs across leadership changes, drives every phase through `CLEANUP`, waits for current source and target voter proofs, publishes the target route, removes bounded migration state, and moves the job to `DONE` history. The deterministic Jepsen split workload exercises the cross-group operation alongside leader-kill and partition fault packages. - -The remaining work is intentionally outside M2's central scope: - -- M3 owns automatic hotspot detection, target selection, and split scheduling. -- M4 owns the broader route-shuffle nemesis matrix and production-scale fault campaigns. -- Reverse migration after CUTOVER and concurrent migration jobs remain future extensions. -======== -This document is `*_partial_*` because M2-PR1 and later component slices have landed, but the cross-group migration plane is not complete end to end. - -- Track per-PR landing under §11. -- Rename to `*_implemented_*` only after the acceptance criteria in §13 pass: cross-group `StartSplitMigration` reaches `phase=DONE`, `ListRoutes` shows the target-group child after cutover, leader-kill recovery is automatic, and the Jepsen split workload is green. ->>>>>>>> origin/design/hotspot-split-m2-promotion-complete:docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md - -`git mv` is used so history follows. diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 56130f1a3..6d3fa34ac 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -343,15 +343,9 @@ control-plane (`*_proposed_*` doc TBD).** - M1 standalone but doesn't enable cross-region writes — it just makes Raft survive cross-WAN partition. -<<<<<<< HEAD - M2's hotspot-split migration dependency is implemented in [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md), including the monotone-merge primitive. -======= -- M2 depends on the M2 hotspot-split migration contract - (`2026_06_11_partial_hotspot_split_milestone2_migration.md`) - being implemented so the monotone-merge primitive exists. ->>>>>>> origin/design/hotspot-split-m2-promotion-complete - M3 depends on M1's region-aware membership and M2's per-region ceiling. - M4 depends on M2 and M3. @@ -544,11 +538,7 @@ the ceiling shape: Composability invariant: **every monotone-merge happens via the same `SetPhysicalCeiling` + `Observe` primitive**. The M2 hotspot-split contract (§6.2.1 of -<<<<<<< HEAD [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md)) is the -======= -`2026_06_11_partial_hotspot_split_milestone2_migration.md`) is the ->>>>>>> origin/design/hotspot-split-m2-promotion-complete reference implementation; per-region and per-group merges reuse it. ### 7.2 Capability bits diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index fc16c992b..210d52c14 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -98,17 +98,10 @@ memory each group's private cache/memtable pins. - **Range split — distribute a range across groups.** Same-group split shipped in M1 (`distribution/`). Cross-group migration (the part that -<<<<<<< HEAD actually relocates data and reduces per-node volume) is implemented by the M2 stack recorded in [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md): a resumable `SplitJob` with -======= - actually relocates data and reduces per-node volume) is tracked by **PR #945** - and the active M2 stack - (`docs/design/2026_06_11_partial_hotspot_split_milestone2_migration.md`, - branch `docs/hotspot-split-m2-proposal`): a resumable `SplitJob` with ->>>>>>> origin/design/hotspot-split-m2-promotion-complete `PLANNED → BACKFILL → FENCE → DELTA_COPY → CUTOVER → CLEANUP → DONE` phases driven by a migrator on the default-group leader. M2 is the required ownership-migration mechanism, but it reduces per-node bytes only when the diff --git a/internal/raftengine/etcd/engine_applied_index_test.go b/internal/raftengine/etcd/engine_applied_index_test.go index b02e3bc43..5240ad148 100644 --- a/internal/raftengine/etcd/engine_applied_index_test.go +++ b/internal/raftengine/etcd/engine_applied_index_test.go @@ -390,12 +390,9 @@ func TestPersistReadyWithSnapshot_BumpsAppliedIndexAfterWALSnapshot(t *testing.T persist: persist, dataDir: t.TempDir(), fsmSnapDir: fsmSnapDir, -<<<<<<< HEAD -======= peers: map[uint64]Peer{ 1: {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, }, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } snap := appliedIndexTestSnapshot(index, encodeSnapshotToken(index, crc)) @@ -422,12 +419,9 @@ func TestPersistReadyWithSnapshot_BumpErrorAfterWALSnapshotSurfaces(t *testing.T persist: persist, dataDir: t.TempDir(), fsmSnapDir: fsmSnapDir, -<<<<<<< HEAD -======= peers: map[uint64]Peer{ 1: {NodeID: 1, ID: "n1", Address: "127.0.0.1:7001"}, }, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } snap := appliedIndexTestSnapshot(index, encodeSnapshotToken(index, crc)) diff --git a/internal/raftengine/etcd/grpc_transport.go b/internal/raftengine/etcd/grpc_transport.go index 1a1fdcd5c..fdf5cc0b3 100644 --- a/internal/raftengine/etcd/grpc_transport.go +++ b/internal/raftengine/etcd/grpc_transport.go @@ -148,15 +148,12 @@ type GRPCTransport struct { // dispatch workers run simultaneously. bridgeSem chan struct{} snapshotSendSem chan struct{} -<<<<<<< HEAD -======= sendStreamOpenCount atomic.Uint64 sendStreamReconnectCount atomic.Uint64 sendStreamMessageCount atomic.Uint64 snapshotStreamSendCount atomic.Uint64 snapshotPayloadByteCount atomic.Uint64 ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } func NewGRPCTransport(peers []Peer) *GRPCTransport { diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index 2188b2bee..a650285f2 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -108,11 +108,7 @@ (build-raft-service-map nodes grpc-port dynamo-port raft-groups)) (defn- start-node! -<<<<<<< HEAD - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine migration-enabled]}] -======= [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env migration-enabled]}] ->>>>>>> origin/design/hotspot-split-m2-promotion-complete (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -158,18 +154,7 @@ (c/on node (c/su (c/exec :mkdir :-p data-dir) -<<<<<<< HEAD - (apply cu/start-daemon! {:chdir bin-dir - :logfile log-file - :pidfile pid-file - :background? true - :env (when migration-enabled - {:ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE "true" - :ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE "true"})} - args))))) -======= (apply cu/start-daemon! daemon-opts server-bin args))))) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete (defn- stop-node! [node] diff --git a/jepsen/src/elastickv/split_workload.clj b/jepsen/src/elastickv/split_workload.clj index 8cc9d9526..b69fe89ed 100644 --- a/jepsen/src/elastickv/split_workload.clj +++ b/jepsen/src/elastickv/split_workload.clj @@ -23,11 +23,7 @@ (def default-nodes ["n1" "n2" "n3" "n4" "n5"]) (def ^:private split-key "m2-split") -<<<<<<< HEAD -(def ^:private register-keys ["m2-left" "m2-right"]) -======= (def ^:private register-keys ["m2-left" "m2-target"]) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete (defn- helper-path [name] (str (System/getProperty "user.dir") "/target/elastickv-jepsen/" name)) diff --git a/jepsen/test/elastickv/split_workload_test.clj b/jepsen/test/elastickv/split_workload_test.clj index 2e0e4a99b..de6cf89c2 100644 --- a/jepsen/test/elastickv/split_workload_test.clj +++ b/jepsen/test/elastickv/split_workload_test.clj @@ -17,15 +17,12 @@ (is (= #{:start-cross-group-split :verify-cross-group-split} (nemesis/fs (:nemesis package)))))) -<<<<<<< HEAD -======= (deftest split-register-keys-cover-both-routes (let [split-key (var-get (ns-resolve 'elastickv.split-workload 'split-key)) register-keys (var-get (ns-resolve 'elastickv.split-workload 'register-keys))] (is (some #(neg? (compare % split-key)) register-keys)) (is (some #(not (neg? (compare % split-key))) register-keys)))) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete (deftest split-nemesis-starts-cross-group-job (let [calls (atom [])] (with-redefs [shell/sh (fn [& args] diff --git a/kv/fsm.go b/kv/fsm.go index 123559906..d0808cf8c 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -760,12 +760,9 @@ func (f *kvFSM) verifyTargetReadinessForReadKeys(ctx context.Context, keys [][]b if isTxnInternalKey(key) { continue } -<<<<<<< HEAD -======= if err := f.verifySourceReadFenceForRange(ctx, key, nextScanCursor(key)); err != nil { return err } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if err := f.verifyTargetReadinessForRange(ctx, key, nextScanCursor(key)); err != nil { return err } @@ -773,8 +770,6 @@ func (f *kvFSM) verifyTargetReadinessForReadKeys(ctx context.Context, keys [][]b return nil } -<<<<<<< HEAD -======= func (f *kvFSM) verifySourceReadFenceForRange(ctx context.Context, start []byte, end []byte) error { reader, ok := f.store.(store.MigrationTargetReadinessReader) if !ok { @@ -791,7 +786,6 @@ func (f *kvFSM) verifySourceReadFenceForRange(ctx context.Context, start []byte, return nil } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func (f *kvFSM) verifyTargetReadinessForTxnFootprint(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, primaryKey []byte) error { if len(primaryKey) != 0 && !isTxnInternalKey(primaryKey) { if err := f.verifyTargetReadinessForRange(ctx, primaryKey, nextScanCursor(primaryKey)); err != nil { @@ -1013,20 +1007,12 @@ func rawPrefixMayContainRouteMappedKeys(prefix []byte) bool { return false } -<<<<<<< HEAD -var routeMappedRawPrefixes = [][]byte{ -======= var routeMappedRawPrefixes = append([][]byte{ ->>>>>>> origin/design/hotspot-split-m2-promotion-complete []byte(redisInternalRoutePrefix), []byte(DynamoTableMetaPrefix), []byte(DynamoTableGenerationPrefix), []byte(DynamoItemPrefix), []byte(DynamoGSIPrefix), -<<<<<<< HEAD - []byte(sqsInternalPrefix), -======= ->>>>>>> origin/design/hotspot-split-m2-promotion-complete []byte(store.ListMetaPrefix), []byte(store.ListItemPrefix), []byte(store.ListMetaDeltaPrefix), @@ -1051,11 +1037,7 @@ var routeMappedRawPrefixes = append([][]byte{ []byte(s3keys.BlobPrefix), []byte(s3keys.GCUploadPrefix), []byte(s3keys.RoutePrefix), -<<<<<<< HEAD -} -======= }, sqsConcreteInternalPrefixBytes...) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete var ErrNotImplemented = errors.New("not implemented") diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index a16d6d4c2..ea1431ef2 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -98,11 +98,7 @@ func applyTargetReadinessToFSM(t *testing.T, fsm *kvFSM, state store.TargetStage require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) } -<<<<<<< HEAD -func applySourceMigrationControlToFSM(t *testing.T, fsm *kvFSM, writeFence, readFence bool) { -======= func applySourceMigrationControlToFSM(t *testing.T, fsm *kvFSM, readFence bool) { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete t.Helper() applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ JobID: 10, @@ -111,11 +107,7 @@ func applySourceMigrationControlToFSM(t *testing.T, fsm *kvFSM, readFence bool) MigrationJobID: 10, MinWriteTSExclusive: 50, Armed: true, -<<<<<<< HEAD - SourceWriteFence: writeFence, -======= SourceWriteFence: true, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete SourceReadFence: readFence, RetentionPinTS: 40, }) @@ -314,11 +306,7 @@ func TestFSMDurableSourceFenceRejectsRawWriteDespiteCatalogBypass(t *testing.T) t.Parallel() fsm := newWriteFencedFSM(t) -<<<<<<< HEAD - applySourceMigrationControlToFSM(t, fsm, true, false) -======= applySourceMigrationControlToFSM(t, fsm, false) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete key := []byte("n") err := fsm.handleRawRequest(context.Background(), &pb.Request{ WriteFenceBypassKeys: [][]byte{key}, @@ -331,11 +319,7 @@ func TestFSMDurableSourceFenceRejectsPrefixWrite(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) -<<<<<<< HEAD - applySourceMigrationControlToFSM(t, fsm, true, false) -======= applySourceMigrationControlToFSM(t, fsm, false) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete err := fsm.handleRawRequest(context.Background(), &pb.Request{ Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, }, 60) @@ -348,33 +332,21 @@ func TestFSMWriteFenceBypassAllowsPinnedTxnOnNonOwningGroup(t *testing.T) { engine := distribution.NewEngine() applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, -<<<<<<< HEAD - {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, -======= {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced, MinWriteTSExclusive: 100}, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete }) fsm := newComposed1FSM(t, engine, 1) key := []byte("z") err := fsm.handleTxnRequest(context.Background(), &pb.Request{ IsTxn: true, Phase: pb.Phase_PREPARE, -<<<<<<< HEAD - Ts: 10, -======= Ts: 101, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete ObservedRouteVersion: 1, WriteFenceBypassKeys: [][]byte{key}, Mutations: []*pb.Mutation{ {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, LockTTLms: defaultTxnLockTTLms})}, {Op: pb.Op_DEL, Key: key}, }, -<<<<<<< HEAD - }, 10) -======= }, 101) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete require.NoError(t, err) } @@ -409,11 +381,7 @@ func TestFSMDurableSourceFenceRejectsPinnedOnePhaseTxn(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) -<<<<<<< HEAD - applySourceMigrationControlToFSM(t, fsm, true, false) -======= applySourceMigrationControlToFSM(t, fsm, false) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete key := []byte("n") err := fsm.handleTxnRequest(context.Background(), &pb.Request{ IsTxn: true, @@ -866,8 +834,6 @@ func TestFSMPrepareTxnChecksReadKeysForTargetReadiness(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } -<<<<<<< HEAD -======= func TestFSMOnePhaseTxnChecksSourceReadFenceForReadKeys(t *testing.T) { t.Parallel() @@ -887,7 +853,6 @@ func TestFSMOnePhaseTxnChecksSourceReadFenceForReadKeys(t *testing.T) { require.ErrorIs(t, getErr, store.ErrKeyNotFound) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestFSMPrepareTxnSkipsRemotePrimaryForTargetReadiness(t *testing.T) { t.Parallel() diff --git a/kv/shard_key.go b/kv/shard_key.go index e17c1dcf6..af3e14b61 100644 --- a/kv/shard_key.go +++ b/kv/shard_key.go @@ -77,10 +77,7 @@ var ( dynamoRouteKey, sqsRouteKey, s3keys.ExtractRouteKey, -<<<<<<< HEAD -======= fskeys.ExtractRouteKey, ->>>>>>> origin/design/hotspot-split-m2-promotion-complete listRouteKey, hashRouteKey, setRouteKey, @@ -115,11 +112,7 @@ func normalizeRouteKey(key []byte) []byte { return key } -<<<<<<< HEAD -func redisWideColumnScanRouteKey(key []byte) []byte { -======= func redisWideColumnScanRouteParts(key []byte) (prefix []byte, userKey []byte, userPrefix []byte, owned bool, parsed bool) { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete for _, prefix := range [][]byte{ []byte(store.HashMetaDeltaPrefix), []byte(store.HashMetaPrefix), @@ -174,13 +167,6 @@ func wideColumnScanUserKey(key []byte, prefix []byte) []byte { return nil } keyLen := binary.BigEndian.Uint32(rest[:wideColumnEncodedKeyLengthSize]) -<<<<<<< HEAD - if uint64(keyLen) > uint64(len(rest)-wideColumnEncodedKeyLengthSize) { //nolint:gosec // non-negative slice length fits uint64. - return nil - } - return rest[wideColumnEncodedKeyLengthSize : uint32(wideColumnEncodedKeyLengthSize)+keyLen] -} -======= rest = rest[wideColumnEncodedKeyLengthSize:] if uint64(keyLen) > uint64(len(rest)) { return nil @@ -188,7 +174,6 @@ func wideColumnScanUserKey(key []byte, prefix []byte) []byte { return rest[:keyLen] } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func redisRouteKey(key []byte) []byte { if !bytes.HasPrefix(key, redisInternalRoutePrefixBytes) { return nil diff --git a/kv/shard_key_test.go b/kv/shard_key_test.go index 829434046..d8107cc33 100644 --- a/kv/shard_key_test.go +++ b/kv/shard_key_test.go @@ -363,8 +363,6 @@ func TestRoutePrefixRangeTreatsBroadMappedPrefixesAsFullKeyspace(t *testing.T) { wantEnd: nil, }, { -<<<<<<< HEAD -======= name: "raw sqs-looking user prefix", prefix: []byte("!sqs|foo"), wantStart: []byte("!sqs|foo"), @@ -377,7 +375,6 @@ func TestRoutePrefixRangeTreatsBroadMappedPrefixesAsFullKeyspace(t *testing.T) { wantEnd: prefixScanEnd(sqsGlobalRouteKey), }, { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete name: "s3 bucket cleanup prefix", prefix: s3keys.ObjectManifestPrefixForBucket("bucket", 2), wantStart: s3keys.RoutePrefixForBucket("bucket", 2), diff --git a/kv/shard_store.go b/kv/shard_store.go index 16c46f941..fa852a2c7 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -38,10 +38,7 @@ var ( ErrRouteCutoverPending = errors.New("route cutover pending") ErrExplicitGroupStagedVisibilityUnresolved = errors.New("explicit group read cannot resolve staged visibility route") ErrReadRouteVersionUnavailable = errors.New("read route version is not locally available") -<<<<<<< HEAD -======= ErrFilesystemPlacementTargetNotFound = errors.New("filesystem placement target group has no routable home slot") ->>>>>>> origin/design/hotspot-split-m2-promotion-complete ErrRouteWriteBelowFloor = ErrRouteWriteTimestampTooLow ) @@ -189,12 +186,8 @@ func (s *ShardStore) GetAtWithReadFence(ctx context.Context, key []byte, ts uint if groupID != 0 { return s.getGroupAtWithReadFence(ctx, groupID, key, ts, readRouteVersion) } -<<<<<<< HEAD - route, g, ok := s.routeAndGroupForKey(key) -======= route, g, routeVersion, ok := s.routeAndGroupForKeyWithVersion(key) readRouteVersion = max(readRouteVersion, routeVersion) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if !ok || g.Store == nil { return nil, store.ErrKeyNotFound } @@ -768,11 +761,7 @@ func (s *ShardStore) ScanAtPhysicalLimit(ctx context.Context, start []byte, end if visibleLimit <= 0 || physicalLimit <= 0 { return []*store.KVPair{}, false, nil } -<<<<<<< HEAD - routes, clampToRoutes := s.routesForScan(start, end) -======= routes, clampToRoutes := s.routesForForwardScan(start, end) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if routesContainStagedVisibility(routes) { kvs, err := s.ScanAt(ctx, start, end, visibleLimit, ts) return kvs, false, err @@ -826,11 +815,7 @@ func (s *ShardStore) scanExplicitGroupRoutesAtWithReadFence(ctx context.Context, scanStart = clampScanStart(start, route.Start) scanEnd = clampScanEnd(end, route.End) } -<<<<<<< HEAD - kvs, err := s.scanRouteAtDirectionWithReadFence(ctx, route, scanStart, scanEnd, limit, ts, reverse, true, readRouteVersion, routeStart, routeEnd) -======= kvs, err := s.scanRouteAtDirectionWithS3StagedOwnerFilter(ctx, routes, route, scanStart, scanEnd, limit, ts, reverse, true, readRouteVersion, routeStart, routeEnd, dedupeByKey) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if err != nil { return nil, err } @@ -845,8 +830,6 @@ func (s *ShardStore) scanExplicitGroupRoutesAtWithReadFence(ctx context.Context, out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, reverse, dedupeByKey) } return out, nil -<<<<<<< HEAD -======= } // ReverseScanGroupAt reverse-scans a range on the explicitly selected Raft group. @@ -858,7 +841,6 @@ func (s *ShardStore) ReverseScanGroupAt(ctx context.Context, groupID uint64, sta ctx, distribution.Route{GroupID: groupID}, start, end, limit, ts, true, true, 0, nil, nil, ) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } // ScanGroupKeysAt scans keys on the explicitly selected Raft group without @@ -895,11 +877,7 @@ func (s *ShardStore) ReverseScanAtPhysicalLimit(ctx context.Context, start []byt if visibleLimit <= 0 || physicalLimit <= 0 { return []*store.KVPair{}, false, nil } -<<<<<<< HEAD - routes, clampToRoutes := s.routesForScan(start, end) -======= routes, clampToRoutes := s.routesForReverseScan(start, end) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if routesContainStagedVisibility(routes) { kvs, err := s.ReverseScanAt(ctx, start, end, visibleLimit, ts) return kvs, false, err @@ -911,71 +889,9 @@ func (s *ShardStore) ReverseScanAtPhysicalLimit(ctx context.Context, start []byt return s.scanRouteAtDirectionPhysicalLimit(ctx, routes[0], start, end, visibleLimit, physicalLimit, ts, true) } -<<<<<<< HEAD -func (s *ShardStore) AllowExactScanFallbackAfterPhysicalLimit(ctx context.Context, start []byte, end []byte, visibleLimit, physicalLimit int, _ uint64, _ bool) bool { - if visibleLimit <= 0 || physicalLimit <= 0 { - return false - } - g := s.exactFallbackPhysicalLimitGroup(start, end) - if g == nil { - return false - } - if _, ok := g.Store.(physicalLimitedStore); !ok { - return false - } - engine := engineForGroup(g) - return engine == nil || isLinearizableRaftLeader(ctx, engine) -} - -func (s *ShardStore) exactFallbackPhysicalLimitGroup(start []byte, end []byte) *ShardGroup { - if s == nil || s.engine == nil { - return nil - } - routes, clampToRoutes := s.routesForScan(start, end) - if len(routes) != 1 || clampToRoutes { - return nil - } - g, ok := s.groupForID(routes[0].GroupID) - if !ok || g == nil || g.Store == nil { - return nil - } - return g -} - -func (s *ShardStore) routesForScan(start []byte, end []byte) ([]distribution.Route, bool) { - if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { - return s.engine.GetIntersectingRoutes(routeStart, routeEnd), false - } - if routes, ok := s.routesForS3BucketAuxiliaryScan(start, end); ok { - return routes, false - } - if isBroadLegacyListDeltaScan(start) { - return s.engine.GetIntersectingRoutes(nil, nil), false - } - if routes, ok := s.routesForLegacyListDeltaScan(start, end); ok { - return routes, false - } - // For internal wide-column keys, shard routing is based on the logical - // user key rather than the raw key prefix. - if userKey := scanRouteUserKey(start); userKey != nil { - route, ok := s.engine.GetRoute(userKey) - if !ok { - return []distribution.Route{}, false - } - return []distribution.Route{route}, false - } - if userKey := redisWideColumnScanRouteKey(start); userKey != nil { - route, ok := s.engine.GetRoute(userKey) - if !ok { - return []distribution.Route{}, false - } - return []distribution.Route{route}, false - } -======= func (s *ShardStore) routesForForwardScan(start []byte, end []byte) ([]distribution.Route, bool) { return s.routesForScan(start, end) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func (s *ShardStore) routesForReverseScan(start []byte, end []byte) ([]distribution.Route, bool) { return s.routesForScan(start, end) @@ -1044,249 +960,6 @@ func (s *ShardStore) routesForScanWithVersion(start []byte, end []byte) ([]distr return routes, true, version } -<<<<<<< HEAD -func (s *ShardStore) routesForS3BucketAuxiliaryScan(start []byte, end []byte) ([]distribution.Route, bool) { - if s == nil || s.engine == nil || !s3BucketAuxiliaryScanBounds(start, end) { - return nil, false - } - routes := make([]distribution.Route, 0) - routes = append(routes, s.engine.GetIntersectingRoutes(start, end)...) - routeStart, routeEnd := s3BucketAuxiliaryScanRouteRange(start, end) - for _, route := range s.engine.GetIntersectingRoutes(routeStart, routeEnd) { - if routeHasStagedVisibility(route) { - routes = append(routes, route) - } - } - return routes, true -} - -func s3BucketAuxiliaryScanBounds(start []byte, end []byte) bool { - if !bytes.HasPrefix(start, []byte(s3keys.BucketMetaPrefix)) && - !bytes.HasPrefix(start, []byte(s3keys.BucketGenerationPrefix)) { - return false - } - if end == nil { - return true - } - return bytes.Compare(start, end) < 0 -} - -func s3BucketAuxiliaryScanRouteRange(start []byte, end []byte) ([]byte, []byte) { - if routeStart, routeEnd, ok := s3BucketAuxiliaryRouteRange(start); ok && end != nil && bytes.Compare(end, prefixScanEnd(start)) <= 0 { - return routeStart, routeEnd - } - routeStart := []byte(s3keys.RoutePrefix) - return routeStart, prefixScanEnd(routeStart) -} - -type repeatedRawScanRouteKey struct { - groupID uint64 - staged bool - migrationJobID uint64 - routeStart string - routeEnd string -} - -func dedupeRepeatedRawScanRoutes(routes []distribution.Route) []distribution.Route { - if len(routes) <= 1 { - return routes - } - out := make([]distribution.Route, 0, len(routes)) - seen := make(map[repeatedRawScanRouteKey]struct{}, len(routes)) - for _, route := range routes { - key := repeatedRawScanRouteDedupeKey(route) - if _, ok := seen[key]; ok { - continue - } - seen[key] = struct{}{} - out = append(out, route) - } - return out -} - -func repeatedRawScanRouteDedupeKey(route distribution.Route) repeatedRawScanRouteKey { - key := repeatedRawScanRouteKey{groupID: route.GroupID} - if routeHasStagedVisibility(route) { - key.staged = true - key.migrationJobID = route.MigrationJobID - key.routeStart = string(route.Start) - key.routeEnd = string(route.End) - } - return key -} - -func prepareUnclampedRawScanRoutes(routes []distribution.Route, dedupeByKey bool) ([]distribution.Route, bool) { - if routesContainStagedVisibility(routes) { - routes = dedupeRepeatedRawScanRoutes(routes) - return orderRawScanRoutesForStagedVisibility(routes), true - } - if !dedupeByKey { - routes = dedupeRepeatedRawScanRoutes(routes) - } - return routes, dedupeByKey -} - -func orderRawScanRoutesForStagedVisibility(routes []distribution.Route) []distribution.Route { - if !routesContainStagedVisibility(routes) { - return routes - } - out := make([]distribution.Route, 0, len(routes)) - staged := make([]distribution.Route, 0) - for _, route := range routes { - if routeHasStagedVisibility(route) { - staged = append(staged, route) - continue - } - out = append(out, route) - } - return append(out, staged...) -} - -func routesContainStagedVisibility(routes []distribution.Route) bool { - for _, route := range routes { - if routeHasStagedVisibility(route) { - return true - } - } - return false -} - -func (s *ShardStore) routesForExplicitGroupScanWithRouteBounds(groupID uint64, start []byte, end []byte, routeStart []byte, routeEnd []byte) ([]distribution.Route, bool, error) { - if routeScanBoundsPresent(routeStart, routeEnd) { - return s.routesForExplicitGroupRouteBounds(groupID, start, end, routeStart, routeEnd) - } - return s.routesForExplicitGroupScan(groupID, start, end) -} - -func (s *ShardStore) routesForExplicitGroupRouteBounds(groupID uint64, start []byte, end []byte, routeStart []byte, routeEnd []byte) ([]distribution.Route, bool, error) { - fallback := []distribution.Route{{GroupID: groupID}} - if s == nil || s.engine == nil { - return fallback, false, nil - } - routes := s.engine.GetIntersectingRoutes(routeStart, normalizedRouteScanEnd(routeEnd)) - matched := make([]distribution.Route, 0, len(routes)) - for _, route := range routes { - if route.GroupID == groupID { - matched = append(matched, route) - continue - } - if routeHasStagedVisibility(route) { - return nil, false, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d range=[%q,%q)", groupID, start, end) - } - } - if len(matched) == 0 { - return fallback, false, nil - } - return matched, false, nil -} - -func (s *ShardStore) routesForExplicitGroupScan(groupID uint64, start []byte, end []byte) ([]distribution.Route, bool, error) { - fallback := []distribution.Route{{GroupID: groupID}} - if s == nil || s.engine == nil { - return fallback, false, nil - } - routeStart, routeEnd, routeMapped := explicitGroupScanRouteBounds(start, end) - routes := s.engine.GetIntersectingRoutes(routeStart, routeEnd) - matched := make([]distribution.Route, 0, len(routes)) - for _, route := range routes { - if route.GroupID == groupID { - matched = append(matched, route) - continue - } - if routeHasStagedVisibility(route) { - return nil, false, errors.Wrapf(ErrExplicitGroupStagedVisibilityUnresolved, "group_id=%d range=[%q,%q)", groupID, start, end) - } - } - if len(matched) > 0 { - if routeMapped { - matched = dedupeRepeatedRawScanRoutes(matched) - } - return matched, !routeMapped, nil - } - return fallback, false, nil -} - -func explicitGroupScanRouteBounds(start []byte, end []byte) ([]byte, []byte, bool) { - routeStart := routeKey(start) - if len(start) == 0 { - routeStart = []byte("") - } - routeEnd := end - routeMapped := !bytes.Equal(routeStart, start) - if end != nil { - normalizedEnd := routeKey(end) - if !bytes.Equal(normalizedEnd, end) { - routeMapped = true - } - } - if routeMapped && len(routeStart) != 0 { - routeEnd = prefixScanEnd(routeStart) - } - return routeStart, routeEnd, routeMapped -} - -func (s *ShardStore) routesForLegacyListDeltaScan(start []byte, end []byte) ([]distribution.Route, bool) { - logicalUserKey := store.ExtractLegacyListUserKeyFromDeltaScanPrefix(start) - if logicalUserKey == nil { - return nil, false - } - routes := make([]distribution.Route, 0) - if route, ok := s.engine.GetRoute(logicalUserKey); ok { - routes = append(routes, route) - } - storedStart := store.ExtractListUserKey(start) - if storedStart != nil { - var storedEnd []byte - if len(end) > 0 { - storedEnd = store.ExtractListUserKey(end) - } - routes = append(routes, s.engine.GetIntersectingRoutes(storedStart, storedEnd)...) - } - return routes, true -} - -func isBroadLegacyListDeltaScan(start []byte) bool { - prefix := []byte(store.LegacyListMetaDeltaPrefix) - if !bytes.HasPrefix(start, prefix) { - return false - } - logicalUserKey := store.ExtractLegacyListUserKeyFromDeltaScanPrefix(start) - return logicalUserKey == nil || !bytes.Equal(start, store.LegacyListMetaDeltaScanPrefix(logicalUserKey)) -} - -func shouldMarkRouteGroupOnScan(start []byte, explicitGroup bool, routeStart []byte, routeEnd []byte) bool { - return !explicitGroup && !routeScanBoundsPresent(routeStart, routeEnd) && isBroadLegacyListDeltaScan(start) -} - -func scanRouteUserKey(start []byte) []byte { - for _, extract := range scanRouteUserKeyExtractors { - if userKey := extract(start); userKey != nil { - return userKey - } - } - return nil -} - -var scanRouteUserKeyExtractors = []func([]byte) []byte{ - store.ExtractListUserKeyFromDeltaScanPrefix, - store.ExtractListUserKey, - store.ExtractListUserKeyFromClaimScanPrefix, - store.ExtractHashUserKeyFromField, - store.ExtractHashUserKeyFromDeltaScanPrefix, - store.ExtractSetUserKeyFromMember, - store.ExtractSetUserKeyFromDeltaScanPrefix, - store.ExtractZSetUserKeyFromMember, - store.ExtractZSetUserKeyFromScore, - store.ExtractZSetUserKeyFromScoreScanPrefix, - store.ExtractZSetUserKeyFromDeltaScanPrefix, - store.ExtractStreamUserKeyFromMeta, - store.ExtractStreamUserKeyFromEntryScanPrefix, -} - -func (s *ShardStore) routesForFencedScan(start []byte, end []byte, routeStart []byte, routeEnd []byte) ([]distribution.Route, bool) { - if routeScanBoundsPresent(routeStart, routeEnd) { - return s.engine.GetIntersectingRoutes(routeStart, normalizedRouteScanEnd(routeEnd)), false -======= type internalScanRouteSelection struct { routes []distribution.Route version uint64 @@ -1296,7 +969,6 @@ func (s *ShardStore) routesForInternalScanWithVersion(start []byte, end []byte) if isBroadLegacyListDeltaScan(start) { routes, version := s.engine.GetIntersectingRoutesWithVersion(nil, nil) return internalScanRouteSelection{routes: routes, version: version}, true ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } if store.ExtractLegacyListUserKeyFromDeltaScanPrefix(start) != nil { catalogRoutes, version := s.engine.GetIntersectingRoutesWithVersion(nil, nil) @@ -1614,16 +1286,8 @@ func prepareScanRouteOwnerFilters(routes []distribution.Route, start []byte, end func (s *ShardStore) scanRoutesAtWithReadFence(ctx context.Context, routes []distribution.Route, start []byte, end []byte, limit int, ts uint64, clampToRoutes bool, readRouteVersion uint64, routeStart []byte, routeEnd []byte) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) -<<<<<<< HEAD - routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) - dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) - if !clampToRoutes && !routeFilterPresent { - routes, dedupeByKey = prepareUnclampedRawScanRoutes(routes, dedupeByKey) - } -======= plan := prepareScanRouteOwnerFilters(routes, start, end, clampToRoutes, routeStart, routeEnd) routes = plan.routes ->>>>>>> origin/design/hotspot-split-m2-promotion-complete for _, route := range routes { scanStart := start scanEnd := end @@ -1650,11 +1314,7 @@ func (s *ShardStore) scanRoutesAtWithReadFence(ctx context.Context, routes []dis } continue } -<<<<<<< HEAD - out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, false, dedupeByKey) -======= out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, false, plan.dedupeByKey) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } return out, nil } @@ -1966,16 +1626,8 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( routeEnd []byte, ) ([]*store.KVPair, error) { out := make([]*store.KVPair, 0) -<<<<<<< HEAD - routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) - dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) - if !clampToRoutes && !routeFilterPresent { - routes, dedupeByKey = prepareUnclampedRawScanRoutes(routes, dedupeByKey) - } -======= plan := prepareScanRouteOwnerFilters(routes, start, end, clampToRoutes, routeStart, routeEnd) routes = plan.routes ->>>>>>> origin/design/hotspot-split-m2-promotion-complete for i := 0; i < len(routes); i++ { route := routes[i] if clampToRoutes { @@ -1995,13 +1647,6 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( // shards), keys from different routes may interleave in descending order. // Fetch up to limit from every route and merge+sort descending so the // result honours the ReverseScanAt contract. -<<<<<<< HEAD - kvs, err := s.scanRouteAtDirectionWithReadFence(ctx, route, start, end, limit, ts, true, false, readRouteVersion, routeStart, routeEnd) - if err != nil { - return nil, err - } - out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, true, dedupeByKey) -======= kvs, err := s.scanRouteAtWithMigrationOwnerFilters( ctx, routes, route, start, end, limit, ts, true, true, readRouteVersion, routeStart, routeEnd, plan.filterUsageOwners, plan.dedupeByKey, @@ -2013,7 +1658,6 @@ func (s *ShardStore) reverseScanRoutesAtWithReadFence( kvs = markScanRouteGroup(kvs, route.GroupID, true) } out = mergeAndTrimScanResultsWithOptions(out, kvs, limit, true, plan.dedupeByKey) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } return out, nil } @@ -3163,19 +2807,6 @@ func (s *ShardStore) latestStagedVisibilityCandidates( ts uint64, ) (map[string]store.MVCCVersion, error) { keys := stagedVisibilityCandidateKeys(liveKVs, stagedKVs) -<<<<<<< HEAD - out := make(map[string]store.MVCCVersion, len(keys)) - for _, key := range keys { - live, liveOK, err := latestMVCCVersionAt(ctx, st, key, ts) - if err != nil { - return nil, err - } - stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, key) - staged, stagedOK, err := latestMVCCVersionAt(ctx, st, stagedKey, ts) - if err != nil { - return nil, err - } -======= liveVersions, err := latestCandidateVersionsAt(ctx, st, keys, ts) if err != nil { return nil, err @@ -3193,7 +2824,6 @@ func (s *ShardStore) latestStagedVisibilityCandidates( live, liveOK := liveVersions[string(key)] stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, key) staged, stagedOK := stagedVersions[string(stagedKey)] ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if stagedOK { staged.Key = bytes.Clone(key) } @@ -3204,8 +2834,6 @@ func (s *ShardStore) latestStagedVisibilityCandidates( return out, nil } -<<<<<<< HEAD -======= func latestCandidateVersionsAt(ctx context.Context, st store.MVCCStore, keys [][]byte, ts uint64) (map[string]store.MVCCVersion, error) { if len(keys) == 0 { return map[string]store.MVCCVersion{}, nil @@ -3257,7 +2885,6 @@ func latestCandidateVersionsAt(ctx context.Context, st store.MVCCStore, keys [][ return out, nil } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func stagedVisibilityCandidateKeys(liveKVs []*store.KVPair, stagedKVs []*store.KVPair) [][]byte { seen := make(map[string][]byte, len(liveKVs)+len(stagedKVs)) for _, kvp := range liveKVs { @@ -3294,19 +2921,11 @@ func isMigrationStagedDataKey(key []byte) bool { func stagedVisibilityScanBounds(jobID uint64, start []byte, end []byte) ([]byte, []byte) { prefix := distribution.MigrationStagedDataKeyPrefix(jobID) scanStart := prefix -<<<<<<< HEAD - if start != nil { - scanStart = distribution.MigrationStagedDataKey(jobID, start) - } - scanEnd := prefixScanEnd(prefix) - if end != nil { -======= if len(start) > 0 { scanStart = distribution.MigrationStagedDataKey(jobID, start) } scanEnd := prefixScanEnd(prefix) if len(end) > 0 { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete scanEnd = distribution.MigrationStagedDataKey(jobID, end) } return scanStart, scanEnd @@ -3368,8 +2987,6 @@ func (s *ShardStore) scanRouteAtLeaderRouteFilter( return nil, nil, err } resolved, err := s.resolveScanLocks(ctx, g, route, filteredKVs, lockKVs, ts) -<<<<<<< HEAD -======= if err == nil { sort.Slice(resolved, func(i, j int) bool { if reverse { @@ -3378,7 +2995,6 @@ func (s *ShardStore) scanRouteAtLeaderRouteFilter( return bytes.Compare(resolved[i].Key, resolved[j].Key) < 0 }) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete return resolved, kvs, err } @@ -3702,12 +3318,8 @@ func (s *ShardStore) LatestCommitTSWithReadFence(ctx context.Context, key []byte if err := s.awaitReadRouteVersion(ctx, readRouteVersion); err != nil { return 0, false, err } -<<<<<<< HEAD - route, g, ok := s.routeAndGroupForKey(key) -======= route, g, routeVersion, ok := s.routeAndGroupForKeyWithVersion(key) readRouteVersion = max(readRouteVersion, routeVersion) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if !ok || g.Store == nil { return 0, false, nil } @@ -4532,15 +4144,6 @@ func (s *ShardStore) readKeysWithStagedVisibilityAliases(group *ShardGroup, read if len(readKeys) == 0 { return readKeys } -<<<<<<< HEAD - for _, key := range readKeys { - readKeys = s.appendStagedVisibilityAlias(group, readKeys, key) - } - return readKeys -} - -func (s *ShardStore) readKeysWithStagedVisibilityMutationAliases(group *ShardGroup, readKeys [][]byte, mutations []*store.KVPairMutation) [][]byte { -======= out := readKeys copied := false for _, key := range readKeys { @@ -4560,32 +4163,10 @@ func (s *ShardStore) readKeysWithStagedVisibilityMutationAliases(group *ShardGro func (s *ShardStore) readKeysWithStagedVisibilityMutationAliases(group *ShardGroup, readKeys [][]byte, mutations []*store.KVPairMutation) [][]byte { out := readKeys copied := false ->>>>>>> origin/design/hotspot-split-m2-promotion-complete for _, mut := range mutations { if mut == nil { continue } -<<<<<<< HEAD - readKeys = s.appendStagedVisibilityAlias(group, readKeys, mut.Key) - } - return readKeys -} - -func (s *ShardStore) appendStagedVisibilityAlias(group *ShardGroup, readKeys [][]byte, key []byte) [][]byte { - if s == nil || s.engine == nil || group == nil { - return readKeys - } - alias, ok := s.stagedVisibilityReadKeyAlias(group, key) - if !ok { - return readKeys - } - out := append([][]byte(nil), readKeys...) - return append(out, alias) -} - -func (s *ShardStore) stagedVisibilityReadKeyAlias(group *ShardGroup, key []byte) ([]byte, bool) { - if len(key) == 0 { -======= alias, ok := s.stagedVisibilityReadKeyAlias(group, mut.Key) if !ok { continue @@ -4601,7 +4182,6 @@ func (s *ShardStore) stagedVisibilityReadKeyAlias(group *ShardGroup, key []byte) func (s *ShardStore) stagedVisibilityReadKeyAlias(group *ShardGroup, key []byte) ([]byte, bool) { if s == nil || s.engine == nil || group == nil || len(key) == 0 { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete return nil, false } if _, _, ok := distribution.MigrationStagedDataKeyParts(key); ok { @@ -4999,18 +4579,6 @@ func (s *ShardStore) verifyExplicitGroupRoutesForRange(ctx context.Context, grou } func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *ShardGroup, bool) { -<<<<<<< HEAD - if route, ok := s.stagedVisibilityRouteForS3BucketAuxiliaryKey(key); ok { - g, ok := s.groups[route.GroupID] - return route, g, ok - } - route, ok := s.engine.GetRoute(routeKey(key)) - if !ok { - return distribution.Route{}, nil, false - } - g, ok := s.groups[route.GroupID] - return route, g, ok -======= route, g, _, ok := s.routeAndGroupForKeyWithVersion(key) return route, g, ok } @@ -5042,7 +4610,6 @@ func (s *ShardStore) routeAndGroupForKeyWithVersion(key []byte) (distribution.Ro } g, ok := s.groups[route.GroupID] return route, g, version, ok ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } func (s *ShardStore) stagedVisibilityRouteForS3BucketAuxiliaryKey(key []byte) (distribution.Route, bool) { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 9ebed9e2d..1a51dd711 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -5,10 +5,7 @@ import ( "context" "errors" "fmt" -<<<<<<< HEAD -======= "sync" ->>>>>>> origin/design/hotspot-split-m2-promotion-complete "testing" "time" @@ -21,8 +18,6 @@ import ( "github.com/stretchr/testify/require" ) -<<<<<<< HEAD -======= type exportCountingStore struct { store.MVCCStore exportCalls int @@ -33,7 +28,6 @@ func (s *exportCountingStore) ExportVersions(ctx context.Context, opts store.Exp return s.MVCCStore.ExportVersions(ctx, opts) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func newStagedVisibilityShardStore(t *testing.T) (*ShardStore, *ShardGroup) { t.Helper() @@ -1160,8 +1154,6 @@ func TestShardStoreScanAndLatestCommitTS_MergeStagedVisibility(t *testing.T) { require.Equal(t, uint64(40), ts) } -<<<<<<< HEAD -======= func TestShardStoreStagedVisibilityScanUsesTwoRangeExports(t *testing.T) { t.Parallel() @@ -1203,7 +1195,6 @@ func TestShardStoreScanAt_FiltersStagedShadowRowsFromLiveCandidates(t *testing.T require.Equal(t, []*store.KVPair{{Key: rawKey, Value: []byte("staged")}}, kvs) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestShardStoreStagedVisibilityPreservesCompactionErrors(t *testing.T) { t.Parallel() @@ -1319,8 +1310,6 @@ func TestShardStoreScanAt_RoutesS3BucketAuxiliaryStagedVisibility(t *testing.T) } } -<<<<<<< HEAD -======= func TestShardStoreS3BucketAuxiliaryScanHonorsStagedTombstone(t *testing.T) { t.Parallel() @@ -1359,7 +1348,6 @@ func TestShardStoreS3BucketAuxiliaryScanHonorsStagedTombstone(t *testing.T) { require.Empty(t, kvs) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestShardStoreGetAt_ContinuesLatestVersionExportPages(t *testing.T) { t.Parallel() @@ -2688,8 +2676,6 @@ func TestShardStoreScanKeysRouteAtLeaderRefillsAfterTxnInternalKeys(t *testing.T require.NoError(t, g.Store.PutAt(ctx, []byte("a"), []byte("va"), 2, 0)) keys, err := st.scanKeysRouteAtLeader(ctx, g, distribution.Route{GroupID: 1}, []byte(""), nil, 1, ^uint64(0)) -<<<<<<< HEAD -======= require.NoError(t, err) require.Equal(t, [][]byte{[]byte("a")}, keys) } @@ -2707,7 +2693,6 @@ func TestShardStoreScanKeysRouteAtLeaderRefillsAfterStagedControlKeys(t *testing require.NoError(t, g.Store.PutAt(ctx, []byte("a"), []byte("visible"), 2, 0)) keys, err := st.scanKeysRouteAtLeader(ctx, g, distribution.Route{GroupID: 1}, []byte(""), nil, 1, ^uint64(0)) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete require.NoError(t, err) require.Equal(t, [][]byte{[]byte("a")}, keys) } diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 24a55b6b2..411d4c563 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1140,16 +1140,10 @@ func validateDelPrefixOnly(elems []*Elem[OP]) error { } // dispatchDelPrefixBroadcast validates and broadcasts DEL_PREFIX operations -<<<<<<< HEAD -// to every shard group. Each element becomes a separate pb.Request (the FSM's -// extractDelPrefix processes only the first DEL_PREFIX mutation per request). -// All requests are batched into a single Commit call per shard group. -======= // to every configured all-shard data group. Each element becomes a separate // pb.Request (the FSM's extractDelPrefix processes only the first DEL_PREFIX // mutation per request). All requests are batched into a single Commit call // per shard group. ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isTxn bool, elems []*Elem[OP], observedRouteVersion uint64) (*CoordinateResponse, error) { if isTxn { return nil, errors.Wrap(ErrInvalidRequest, "DEL_PREFIX not supported in transactions") @@ -1362,13 +1356,8 @@ func (c *ShardedCoordinator) rejectWriteTimestampFloorDelPrefix(prefix []byte, c return nil } -<<<<<<< HEAD -// broadcastToAllGroups sends the same set of requests to every shard group in -// parallel and returns the maximum commit index. -======= // broadcastToAllGroups sends the same set of requests to every configured // all-shard data group in parallel and returns the maximum commit index. ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func (c *ShardedCoordinator) broadcastToAllGroups(ctx context.Context, requests []*pb.Request) (*CoordinateResponse, error) { var ( maxIndex atomic.Uint64 @@ -1429,12 +1418,9 @@ func (c *ShardedCoordinator) dispatchTxn(ctx context.Context, startTS uint64, co if err != nil { return nil, err } -<<<<<<< HEAD -======= if err := ValidateElemCommitTSPatches(elems, commitTS); err != nil { return nil, err } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if err := c.rejectWriteTimestampFloorPointElems(elems, commitTS); err != nil { return nil, err } @@ -1447,12 +1433,9 @@ func (c *ShardedCoordinator) dispatchTxn(ctx context.Context, startTS uint64, co // preserving SSI. return c.dispatchSingleShardTxn(ctx, startTS, commitTS, prevCommitTS, primaryKey, gids[0], elems, readKeys, observedRouteVersion) } -<<<<<<< HEAD -======= if err := StampGroupedMutationCommitTS(grouped, commitTS); err != nil { return nil, err } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete return c.dispatchMultiShardTxn(ctx, startTS, commitTS, prevCommitTS, primaryKey, grouped, gids, readKeys, observedRouteVersion, bypassKeysByGroup) } @@ -1625,48 +1608,6 @@ func (c *ShardedCoordinator) readKeysWithStagedVisibilityMutationAliasesForGroup return out } -func (c *ShardedCoordinator) readKeysWithStagedVisibilityAliasesForGroup(gid uint64, readKeys [][]byte) [][]byte { - if len(readKeys) == 0 { - return readKeys - } - var out [][]byte - for _, key := range readKeys { - alias, ok := c.stagedVisibilityReadKeyAlias(gid, key) - if !ok { - continue - } - if out == nil { - out = append([][]byte(nil), readKeys...) - } - out = append(out, alias) - } - if out == nil { - return readKeys - } - return out -} - -func (c *ShardedCoordinator) readKeysWithStagedVisibilityMutationAliasesForGroup(gid uint64, readKeys [][]byte, elems []*Elem[OP]) [][]byte { - var out [][]byte - for _, elem := range elems { - if elem == nil { - continue - } - alias, ok := c.stagedVisibilityReadKeyAlias(gid, elem.Key) - if !ok { - continue - } - if out == nil { - out = append([][]byte(nil), readKeys...) - } - out = append(out, alias) - } - if out == nil { - return readKeys - } - return out -} - type preparedGroup struct { gid uint64 keys []*pb.Mutation @@ -2524,12 +2465,9 @@ func (c *ShardedCoordinator) verifyTargetReadinessForReadKeyOnShard(ctx context. return nil } routeStart, routeEnd := readinessRouteRange(key, nextScanCursor(key)) -<<<<<<< HEAD -======= if sourceReadFenceApplies(states, routeStart, routeEnd) { return errors.WithStack(ErrRouteCutoverPending) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete routes, catalogVersion, proof := c.currentShardRoutesForRouteRange(gid, routeStart, routeEnd) if targetReadinessStatesSatisfied(states, routes, routeStart, routeEnd, gid, catalogVersion, proof) { return nil @@ -2707,12 +2645,9 @@ func (c *ShardedCoordinator) txnLogs(ctx context.Context, reqs *OperationGroup[O if err != nil { return nil, err } -<<<<<<< HEAD -======= if err := StampGroupedMutationCommitTS(grouped, commitTS); err != nil { return nil, err } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete bypassKeysByGroup := c.writeFenceBypassKeysByGroup(reqs.Elems) return buildTxnLogs(reqs.StartTS, commitTS, grouped, gids, reqs.ObservedRouteVersion, bypassKeysByGroup) } diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index 56f1929c9..91bf9dd89 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -711,8 +711,6 @@ func TestShardedCoordinatorRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t } } -<<<<<<< HEAD -======= func TestShardedCoordinatorAllowsRawSQSLookingDelPrefixWhenUnrelatedRouteIsWriteFenced(t *testing.T) { t.Parallel() @@ -740,7 +738,6 @@ func TestShardedCoordinatorAllowsRawSQSLookingDelPrefixWhenUnrelatedRouteIsWrite require.NotEmpty(t, g2Txn.requests) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestShardedCoordinatorAllowsS3BucketDelPrefixWhenUnrelatedRouteIsWriteFenced(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 004342c27..445b2b6af 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -892,8 +892,6 @@ func TestValidateReadOnlyShards_FailsClosedOnTargetReadiness(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } -<<<<<<< HEAD -======= func TestValidateReadOnlyShards_FailsClosedOnSourceReadFence(t *testing.T) { t.Parallel() @@ -924,7 +922,6 @@ func TestValidateReadOnlyShards_FailsClosedOnSourceReadFence(t *testing.T) { require.ErrorIs(t, err, ErrRouteCutoverPending) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestValidateReadOnlyShards_ChecksStagedVisibilityLatestCommitTS(t *testing.T) { t.Parallel() engine := distribution.NewEngine() diff --git a/kv/transcoder.go b/kv/transcoder.go index ce3762ff8..0b92491aa 100644 --- a/kv/transcoder.go +++ b/kv/transcoder.go @@ -19,13 +19,10 @@ type Elem[T OP] struct { Op T Key []byte Value []byte -<<<<<<< HEAD -======= // CommitTSValueOffset, when non-zero, asks the coordinator or forwarded // leader to stamp the resolved transaction commit timestamp into Value at // this byte offset before committing the mutation. CommitTSValueOffset uint64 ->>>>>>> origin/design/hotspot-split-m2-promotion-complete // GroupID optionally pins this mutation to a shard group. Zero preserves // normal key-based routing. GroupID uint64 diff --git a/main.go b/main.go index 4a8bb6b96..010fa753b 100644 --- a/main.go +++ b/main.go @@ -56,14 +56,10 @@ const ( etcdMaxInflightMsg = 1024 defaultTSOBatchSize = 256 -<<<<<<< HEAD - splitMigrationCapabilityProbeTimeout = 2 * time.Second -======= defaultFilesystemRootMode = 0o755 defaultFilesystemPlacementScanInterval = 30 * time.Second defaultFilesystemLeaseReapInterval = 30 * time.Second splitMigrationCapabilityProbeTimeout = 2 * time.Second ->>>>>>> origin/design/hotspot-split-m2-promotion-complete ) func newRaftFactory(engineType raftEngineType, coldStartObs raftengine.ColdStartObserver) (raftengine.Factory, error) { @@ -537,8 +533,6 @@ func run() error { cleanup.Add(leadershipRefusalDeregister) eg, runCtx := errgroup.WithContext(ctx) startRaftEngineLifecycleWatchers(runCtx, eg, runtimes) -<<<<<<< HEAD -======= // setupDistributionCatalog + the Stage 7a process-start registration // gate are bundled so run() has a single startup-fault path: a // registry-read / behind-epoch failure fails the process @@ -597,7 +591,6 @@ func run() error { ) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) startFSMCompactorIfEnabled(runCtx, eg, runtimes, readTracker) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete // Stage 7c §3.1: build the encryption-aware // MembershipChangeInterceptor here where the concrete @@ -621,42 +614,17 @@ func run() error { slog.Default(), ) cleanup.Add(rotateOnStartupDeregister) - - serverInput := serversInput{ + if err := startServersAfterStartupRotation(waitRotateOnStartup, serversInput{ ctx: runCtx, eg: eg, cancel: cancel, lc: &lc, runtimes: runtimes, shardGroups: shardGroups, bootstrapServers: bootstrapCfg.adminSeed(cfg.defaultGroup), shardStore: shardStore, coordinate: coordinate, - readTracker: readTracker, + distServer: distServer, readTracker: readTracker, metricsRegistry: metricsRegistry, cfg: cfg, redisApplyObserver: redisApplyObserver, cleanup: &cleanup, encWiring: encWiring, keyvizSampler: sampler, encryptionConfChangeInterceptor: encryptionConfChangeInterceptor, - } - runtimeStartup, err := prepareDistributionRuntimeServer( - waitRotateOnStartup, serverInput, cfg.engine, runtimes, coordinate, readTracker) - if err != nil { - cancel() - return err - } - if err := startDistributionRuntimeAfterTransport(distributionRuntimeStartupInput{ - runCtx: runCtx, - eg: eg, - cancel: cancel, - runtimes: runtimes, - engine: cfg.engine, - coordinate: coordinate, - defaultGroup: shardGroups[cfg.defaultGroup], - encWiring: encWiring, - raftID: *raftId, - sidecarPath: *encryptionSidecarPath, - sampler: sampler, - clock: clock, - metricsRegistry: metricsRegistry, - readTracker: readTracker, - waitRotateOnStartup: waitRotateOnStartup, - prepared: runtimeStartup, }); err != nil { return err } @@ -1991,129 +1959,6 @@ type serversInput struct { encryptionConfChangeInterceptor internalraftadmin.MembershipChangeInterceptor } -<<<<<<< HEAD -type preparedRuntimeServer struct { - runner *runtimeServerRunner - connCache *kv.GRPCConnCache -} - -type preparedDistributionRuntime struct { - catalog *distribution.CatalogStore - serverInput serversInput - preparedServer *preparedRuntimeServer -} - -type distributionRuntimeStartupInput struct { - runCtx context.Context - eg *errgroup.Group - cancel context.CancelFunc - runtimes []*raftGroupRuntime - engine *distribution.Engine - coordinate *kv.ShardedCoordinator - defaultGroup *kv.ShardGroup - encWiring encryptionWriteWiring - raftID string - sidecarPath string - sampler *keyviz.MemSampler - clock *kv.HLC - metricsRegistry *monitoring.Registry - readTracker *kv.ActiveTimestampTracker - waitRotateOnStartup startupRotationWaiter - prepared *preparedDistributionRuntime -} - -func prepareDistributionRuntimeServer( - waitRotateOnStartup startupRotationWaiter, - in serversInput, - engine *distribution.Engine, - runtimes []*raftGroupRuntime, - coordinate *kv.ShardedCoordinator, - readTracker *kv.ActiveTimestampTracker, -) (*preparedDistributionRuntime, error) { - distCatalog, err := distributionCatalogStoreForEngine(runtimes, engine) - if err != nil { - return nil, err - } - splitPromotionConnCache := &kv.GRPCConnCache{} - in.eg.Go(func() error { - <-in.ctx.Done() - if err := splitPromotionConnCache.Close(); err != nil { - return errors.Wrap(err, "close split promotion gRPC connection cache") - } - return nil - }) - distOptions := []adapter.DistributionServerOption{ - adapter.WithDistributionCoordinator(coordinate), - adapter.WithDistributionActiveTimestampTracker(readTracker), - adapter.WithDistributionKnownRaftGroups(shardGroupIDs(in.shardGroups)...), - adapter.WithSplitMigrationCapabilityGate(newSplitMigrationCapabilityGate( - splitMigrationCapabilityPeerSourceForRuntimes(runtimes), - splitMigrationCapabilityProbeTimeout, - nil, - splitMigrationLocalReadinessGate, - )), - adapter.WithSplitPromotionClientFactory(splitPromotionClientFactory( - splitPromotionTargetLeaderResolver(in.shardGroups), - splitPromotionConnCache, - )), - adapter.WithSplitMigrationClientFactory(splitMigrationClientFactory( - splitMigrationGroupLeaderResolver(in.shardGroups), - splitPromotionConnCache, - )), - adapter.WithSplitMigrationVoterFactory(splitMigrationVoterFactory( - in.shardGroups, - splitPromotionConnCache, - )), - adapter.WithSplitJobRunnerReadinessGate(splitMigrationLocalReadinessGate), - adapter.WithSplitJobRunnerReady(), - } - in.distServer = adapter.NewDistributionServer(engine, distCatalog, distOptions...) - preparedServer, err := prepareRuntimeServerRunner(waitRotateOnStartup, in) - if err != nil { - return nil, err - } - return &preparedDistributionRuntime{ - catalog: distCatalog, - serverInput: in, - preparedServer: preparedServer, - }, nil -} - -func startDistributionRuntimeAfterTransport(in distributionRuntimeStartupInput) error { - prepared := in.prepared - if prepared == nil || prepared.preparedServer == nil || prepared.preparedServer.runner == nil { - return errors.New("distribution runtime server is not prepared") - } - if err := prepared.preparedServer.runner.startRaftTransport(); err != nil { - return err - } - if err := waitForRaftStartupAfterTransport(in.runCtx, in.runtimes); err != nil { - return prepared.preparedServer.runner.startupFailure(err) - } - // setupDistributionCatalog + the Stage 7a process-start registration - // gate are bundled so startup faults flow through one path. This now - // runs after raft transport is bound, but before public listeners serve. - if err := setupDistributionAndRegistration( - in.runCtx, in.eg, prepared.catalog, in.engine, - in.coordinate, in.defaultGroup, in.encWiring, in.raftID, in.sidecarPath); err != nil { - in.cancel() - return err - } - seedKeyVizRoutes(in.sampler, in.engine) - in.eg.Go(func() error { - return runDistributionCatalogWatcher(in.runCtx, prepared.catalog, in.engine) - }) - startKeyVizFlusher(in.runCtx, in.eg, in.sampler) - startKeyVizLeaderTermPublisher(in.runCtx, in.eg, in.sampler, in.runtimes) - startMemoryWatchdog(in.runCtx, in.eg, in.cancel) - startMonitoringCollectors(in.runCtx, in.metricsRegistry, in.runtimes, in.clock) - startFSMCompactorIfEnabled(in.runCtx, in.eg, in.runtimes, in.readTracker) - return startPreparedServerAfterStartupRotation( - in.waitRotateOnStartup, prepared.serverInput, prepared.preparedServer) -} - -======= ->>>>>>> origin/design/hotspot-split-m2-promotion-complete type splitPromotionLeaderResolver func(distribution.SplitJob) (string, error) type splitMigrationLeaderResolver func(uint64) (string, error) @@ -2208,11 +2053,7 @@ func splitMigrationVoterFactory(shardGroups map[uint64]*kv.ShardGroup, connCache } voters := make([]adapter.SplitMigrationVoter, 0, len(cfg.Servers)) for _, member := range cfg.Servers { -<<<<<<< HEAD - if member.Suffrage != "voter" { -======= if member.Suffrage != raftMemberSuffrageVoter { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete continue } voter, err := splitMigrationVoter(groupID, member, connCache) @@ -2247,12 +2088,9 @@ func splitMigrationLocalReadinessGate(context.Context) error { if !adapter.MigrationPromoteOpcodeEnabledFromEnv() { return errors.Errorf("%s is disabled", adapter.MigrationPromoteOpcodeEnv) } -<<<<<<< HEAD -======= if !adapter.MigrationCleanupOpcodeEnabledFromEnv() { return errors.Errorf("%s is disabled", adapter.MigrationCleanupOpcodeEnv) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete return nil } @@ -2265,18 +2103,14 @@ func startDistributionSplitJobRunner(ctx context.Context, eg *errgroup.Group, se }) } -<<<<<<< HEAD -func prepareRuntimeServerRunner(waitRotateOnStartup startupRotationWaiter, in serversInput) (*preparedRuntimeServer, error) { -======= // startServersAfterStartupRotation wires up the AdminServer, starts the // per-group Raft listeners needed for quorum traffic, waits for a fresh join // to catch up, prepares the public listeners, waits for any requested startup // rotation, then starts serving public traffic. func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, in serversInput) error { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler) if err != nil { - return nil, err + return err } // roleStore + connCache are gated on *adminEnabled. With admin // disabled, building either is wasted work AND a security @@ -2373,17 +2207,9 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, encryptionConfChangeInterceptor: in.encryptionConfChangeInterceptor, publicKVGate: publicKVGate, } - return &preparedRuntimeServer{runner: &runner, connCache: connCache}, nil -} - -func startPreparedServerAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, in serversInput, prepared *preparedRuntimeServer) error { - if prepared == nil || prepared.runner == nil { - return errors.New("runtime server runner is not prepared") + if err := runner.startRaftTransport(); err != nil { + return err } -<<<<<<< HEAD - runner := prepared.runner - if err := runner.preparePublicServices(); err != nil { -======= if err := preparePublicServicesAfterRaftJoinReady( in.ctx, in.runtimes, @@ -2391,7 +2217,6 @@ func startPreparedServerAfterStartupRotation(waitRotateOnStartup startupRotation strings.TrimSpace(*raftJoinMembers) != "", runner.preparePublicServices, ); err != nil { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete return runner.startupFailure(err) } // runner.startRaftTransport() has populated runner.dynamoServer for the @@ -2410,23 +2235,17 @@ func startPreparedServerAfterStartupRotation(waitRotateOnStartup startupRotation Nodes: parseCSV(*keyvizFanoutNodes), Timeout: *keyvizFanoutTimeout, } - adminHTTP, err := prepareAdminFromFlags(in.ctx, in.lc, in.runtimes, runner.dynamoServer, runner.s3Server, runner.sqsServer, runner.coordinate, prepared.connCache, in.keyvizSampler, fanoutCfg) + adminHTTP, err := prepareAdminFromFlags(in.ctx, in.lc, in.runtimes, runner.dynamoServer, runner.s3Server, runner.sqsServer, runner.coordinate, connCache, in.keyvizSampler, fanoutCfg) if err != nil { return runner.startupFailure(err) } runner.adminHTTP = adminHTTP - if runner.publicKVGate != nil { - runner.publicKVGate.blockMutator = waitRotateOnStartup.BlockMutators - } + publicKVGate.blockMutator = waitRotateOnStartup.BlockMutators if err := waitRotateOnStartup.Wait(in.ctx); err != nil { return runner.startupFailure(errors.Wrap(err, "encryption rotate-on-startup: wait before serving")) } startHLCLeaseRenewal(in.ctx, in.eg, in.coordinate) -<<<<<<< HEAD - runner.publicKVGate.markReady() -======= publicKVGate.markReady() ->>>>>>> origin/design/hotspot-split-m2-promotion-complete startDistributionSplitJobRunner(in.ctx, in.eg, in.distServer) if err := runner.startPublicServices(); err != nil { return err @@ -2511,23 +2330,6 @@ func configurationContainsMember(configuration raftengine.Configuration, localID return false } -func waitForRaftStartupAfterTransport(ctx context.Context, runtimes []*raftGroupRuntime) error { - for _, rt := range runtimes { - if rt == nil { - continue - } - engine := rt.snapshotEngine() - barrier, ok := engine.(raftengine.StartupBarrier) - if !ok { - continue - } - if err := barrier.WaitStarted(ctx); err != nil { - return errors.Wrapf(err, "wait for raft group %d startup", rt.spec.id) - } - } - return nil -} - func configureCoordinatorTSO(coordinate *kv.ShardedCoordinator) error { if !*tsoEnabled { return nil @@ -3020,7 +2822,6 @@ func startRaftServers( shardGroups map[uint64]*kv.ShardGroup, shardStore *kv.ShardStore, coordinate kv.Coordinator, - readGate func() bool, distServer *adapter.DistributionServer, routeEngine *distribution.Engine, relay *adapter.RedisPubSubRelay, @@ -3057,7 +2858,7 @@ func startRaftServers( } gs := grpc.NewServer(opts...) trx := kv.NewTransactionWithProposer(proposerForGroup(rt, shardGroups), kv.WithProposalObserver(observerForGroup(proposalObserverForGroup, rt.spec.id))) - grpcSvc := adapter.NewGRPCServer(shardStore, coordinate, adapter.WithGRPCReadGate(readGate)) + grpcSvc := adapter.NewGRPCServer(shardStore, coordinate) pb.RegisterRawKVServer(gs, grpcSvc) pb.RegisterTransactionalKVServer(gs, grpcSvc) pb.RegisterInternalServer(gs, adapter.NewInternalWithEngine( @@ -3335,7 +3136,8 @@ func distributionCatalogStoreForGroup(runtimes []*raftGroupRuntime, groupID uint return nil } -func distributionCatalogStoreForEngine( +func setupDistributionCatalog( + ctx context.Context, runtimes []*raftGroupRuntime, engine *distribution.Engine, ) (*distribution.CatalogStore, error) { @@ -3347,20 +3149,25 @@ func distributionCatalogStoreForEngine( if distCatalog == nil { return nil, errors.WithStack(errors.Newf("distribution catalog store is not available for group %d", catalogGroupID)) } - return distCatalog, nil -} - -func setupDistributionCatalog( - ctx context.Context, - runtimes []*raftGroupRuntime, - engine *distribution.Engine, -) (*distribution.CatalogStore, error) { - distCatalog, err := distributionCatalogStoreForEngine(runtimes, engine) - if err != nil { - return nil, err - } - if err := ensureDistributionCatalogSnapshot(ctx, distCatalog, engine); err != nil { - return nil, err + // EnsureCatalogSnapshot may Save through the direct (non-raft) write + // path. When the §7.1 storage envelope is active and this load's + // writer registration has not yet committed, that Save fails closed + // with store.ErrWriterNotRegistered (Stage 7a-2). retryUntilRegistered + // retries the bootstrap until the registration goroutine — armed + // before this call in setupDistributionAndRegistration — commits and + // the gate clears. The common cases (populated catalog → no-op Save, + // or pre-cutover → cleartext Save) never hit the gate and return on + // the first attempt. + // + // Idempotency requirement: the retry re-invokes EnsureCatalogSnapshot + // from scratch on each ErrWriterNotRegistered, so it MUST be + // re-entrant — on the populated-catalog path it is a version-unchanged + // no-op Save (no mutation, no nonce), so re-running it is safe. + if err := retryUntilRegistered(ctx, "distribution catalog bootstrap", func() error { + _, e := distribution.EnsureCatalogSnapshot(ctx, distCatalog, engine) + return errors.Wrap(e, "ensure catalog snapshot") + }); err != nil { + return nil, errors.Wrapf(err, "initialize distribution catalog") } return distCatalog, nil } @@ -3505,10 +3312,8 @@ func (r *runtimeServerRunner) startRaftTransport() error { return r.startupFailure(err) } adminGRPCOpts := r.adminGRPCOpts - var readGate func() bool if r.publicKVGate != nil { adminGRPCOpts.unary = append(adminGRPCOpts.unary, r.publicKVGate.unaryInterceptor) - readGate = r.publicKVGate.blocked } forwardDeps := adminForwardServerDeps{ tables: newDynamoTablesSource(r.dynamoServer), @@ -3527,7 +3332,6 @@ func (r *runtimeServerRunner) startRaftTransport() error { r.shardGroups, r.shardStore, r.coordinate, - readGate, r.distServer, r.routeEngine, r.pubsubRelay, diff --git a/main_catalog_test.go b/main_catalog_test.go index 1bda69aed..57af6a088 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -183,16 +183,10 @@ func TestSplitMigrationCapabilityGateFailsClosedWhenLocalReadinessFails(t *testi require.ErrorContains(t, err, "migration opcode disabled") } -<<<<<<< HEAD -func TestSplitMigrationLocalReadinessGateRequiresImportAndPromoteOpcodes(t *testing.T) { - t.Setenv(adapter.MigrationImportOpcodeEnv, "") - t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") -======= func TestSplitMigrationLocalReadinessGateRequiresMigrationOpcodes(t *testing.T) { t.Setenv(adapter.MigrationImportOpcodeEnv, "") t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") t.Setenv(adapter.MigrationCleanupOpcodeEnv, "1") ->>>>>>> origin/design/hotspot-split-m2-promotion-complete err := splitMigrationLocalReadinessGate(context.Background()) require.Error(t, err) require.ErrorContains(t, err, adapter.MigrationImportOpcodeEnv) @@ -204,15 +198,12 @@ func TestSplitMigrationLocalReadinessGateRequiresMigrationOpcodes(t *testing.T) require.ErrorContains(t, err, adapter.MigrationPromoteOpcodeEnv) t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") -<<<<<<< HEAD -======= t.Setenv(adapter.MigrationCleanupOpcodeEnv, "") err = splitMigrationLocalReadinessGate(context.Background()) require.Error(t, err) require.ErrorContains(t, err, adapter.MigrationCleanupOpcodeEnv) t.Setenv(adapter.MigrationCleanupOpcodeEnv, "1") ->>>>>>> origin/design/hotspot-split-m2-promotion-complete require.NoError(t, splitMigrationLocalReadinessGate(context.Background())) } diff --git a/monitoring/hotpath.go b/monitoring/hotpath.go index 33e12b44c..04e5a7776 100644 --- a/monitoring/hotpath.go +++ b/monitoring/hotpath.go @@ -104,8 +104,6 @@ func newHotPathMetrics(registerer prometheus.Registerer) *HotPathMetrics { prometheus.CounterOpts{ Name: "elastickv_raft_step_queue_full_total", Help: "Inbound raft messages that found the selected step queue full. Blocking replication messages wait for space; best-effort messages may still be rejected. Indicates the raft loop is starved.", -<<<<<<< HEAD -======= }, []string{"group"}, ), @@ -141,7 +139,6 @@ func newHotPathMetrics(registerer prometheus.Registerer) *HotPathMetrics { prometheus.CounterOpts{ Name: "elastickv_raft_snapshot_stream_payload_bytes_total", Help: "Payload bytes in outbound Raft snapshot streams acknowledged by peers.", ->>>>>>> origin/design/hotspot-split-m2-promotion-complete }, []string{"group"}, ), diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 19e5924a8..90c0943da 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -2034,12 +2034,8 @@ const file_internal_proto_rawDesc = "" + "\bMutation\x12\x13\n" + "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + -<<<<<<< HEAD - "\x05value\x18\x03 \x01(\fR\x05value\"\x81\x02\n" + -======= "\x05value\x18\x03 \x01(\fR\x05value\x123\n" + "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\x81\x02\n" + ->>>>>>> origin/design/hotspot-split-m2-promotion-complete "\aRequest\x12\x15\n" + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + diff --git a/store/lsm_migration.go b/store/lsm_migration.go index f9d0ed006..38334c931 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -219,12 +219,9 @@ func advancePebbleExportPastCurrentUserKey( } func pebbleExportCanStopAtEndKey(startKey, endKey, userKey []byte) bool { -<<<<<<< HEAD -======= // Pebble orders userKey||invertedCommitTS physically. The empty logical key // can therefore appear after non-empty keys; a leading range must keep // scanning or it can silently omit that key. ->>>>>>> origin/design/hotspot-split-m2-promotion-complete if len(startKey) == 0 { return false } diff --git a/store/migration_versions.go b/store/migration_versions.go index 9b7928677..4d95311a7 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -554,17 +554,10 @@ func appendMemoryExportVersion(opts ExportVersionsOptions, key []byte, version V if opts.AcceptKey != nil && !opts.AcceptKey(key) { return exportCursorTagScanned } -<<<<<<< HEAD - if opts.AcceptVersion != nil && !opts.AcceptVersion(key, version.Value) { - return exportCursorTagScanned - } - if opts.MaxCommitTSInclusive != 0 && version.TS > opts.MaxCommitTSInclusive { -======= if opts.MaxCommitTSInclusive != 0 && version.TS > opts.MaxCommitTSInclusive { return exportCursorTagScanned } if opts.AcceptVersion != nil && !opts.AcceptVersion(key, version.Value) { ->>>>>>> origin/design/hotspot-split-m2-promotion-complete return exportCursorTagScanned } result.Versions = append(result.Versions, MVCCVersion{ diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index e539a8af7..ff8a7a18a 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -125,8 +125,6 @@ func TestExportVersionsAcceptVersionFiltersByValue(t *testing.T) { }) } -<<<<<<< HEAD -======= func TestExportVersionsAppliesTimestampBoundBeforeAcceptVersion(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() @@ -150,7 +148,6 @@ func TestExportVersionsAppliesTimestampBoundBeforeAcceptVersion(t *testing.T) { }) } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete func TestExportVersionsCursorResumesWithinHotKey(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 61ace57b3..0d174e4df 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -899,15 +899,10 @@ func (s *mvccStore) writeSnapshotBodyLocked(f *os.File, version uint32) (uint32, if err := binary.Write(w, binary.LittleEndian, s.minRetainedTS); err != nil { return 0, errors.WithStack(err) } -<<<<<<< HEAD - if err := writeMVCCSnapshotReadinessStates(w, s.migrationReadinessCache); err != nil { - return 0, err -======= if version >= mvccSnapshotVersion { if err := writeMVCCSnapshotReadinessStates(w, s.migrationReadinessCache); err != nil { return 0, err } ->>>>>>> origin/design/hotspot-split-m2-promotion-complete } iter := s.tree.Iterator() for iter.Next() { diff --git a/store/mvcc_store_snapshot_test.go b/store/mvcc_store_snapshot_test.go index ad9195359..b23eec15a 100644 --- a/store/mvcc_store_snapshot_test.go +++ b/store/mvcc_store_snapshot_test.go @@ -72,10 +72,7 @@ func TestMVCCStore_SnapshotRestorePreservesTargetReadiness(t *testing.T) { require.NoError(t, err) defer snap.Close() raw := snapshotBytes(t, snap) -<<<<<<< HEAD -======= require.Equal(t, mvccSnapshotVersion, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) ->>>>>>> origin/design/hotspot-split-m2-promotion-complete dst := newTestMVCCStore(t) require.NoError(t, dst.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{