From 045905a6ea300b1bf3adc621b51c01d11bad71a0 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 21 Aug 2026 08:56:05 +0000 Subject: [PATCH 01/22] init Signed-off-by: wk989898 --- cmd/kafka-consumer/consumer.go | 2 + cmd/kafka-consumer/writer.go | 16 +- cmd/pulsar-consumer/consumer.go | 2 + cmd/pulsar-consumer/writer.go | 11 + cmd/pulsar-consumer/writer_test.go | 12 +- cmd/storage-consumer/consumer.go | 10 + cmd/util/event_group.go | 311 ++++++++++++++++++++++++++++- cmd/util/event_group_test.go | 123 +++++++----- 8 files changed, 420 insertions(+), 67 deletions(-) diff --git a/cmd/kafka-consumer/consumer.go b/cmd/kafka-consumer/consumer.go index 2bf26f1aee..c98125ddd0 100644 --- a/cmd/kafka-consumer/consumer.go +++ b/cmd/kafka-consumer/consumer.go @@ -170,6 +170,8 @@ func (c *consumer) readMessage(ctx context.Context) error { // Run the consumer, read data and write to the downstream target. func (c *consumer) Run(ctx context.Context) error { + defer c.writer.cleanupEventsGroups() + g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return c.writer.run(ctx) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index bbf8388405..1284f31927 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -147,6 +147,17 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } +func (w *writer) cleanupEventsGroups() { + for _, progress := range w.progresses { + for _, group := range progress.eventsGroup { + if err := group.Cleanup(); err != nil { + log.Warn("cleanup events group spill file failed", + zap.Int32("partition", progress.partition), zap.Error(err)) + } + } + } +} + func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { var ( done = make(chan struct{}, 1) @@ -628,8 +639,9 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti group = util.NewEventsGroup(progress.partition, tableID) progress.eventsGroup[tableID] = group } - message = w.messageWithPartitionCheck(message, progress.partition, offset) - group.AppendMessage(message) + group.AppendMessageWithPostRestore(message, func(message *common.DMLMessage) *common.DMLMessage { + return w.messageWithPartitionCheck(message, progress.partition, offset) + }) if commitTs < progress.watermark { log.Warn("DML event fallback row, since less than the partition watermark, append it and sort before flush", zap.Int64("tableID", tableID), zap.Int32("partition", group.Partition), diff --git a/cmd/pulsar-consumer/consumer.go b/cmd/pulsar-consumer/consumer.go index 8c75a8843b..c6b3d600ec 100644 --- a/cmd/pulsar-consumer/consumer.go +++ b/cmd/pulsar-consumer/consumer.go @@ -124,6 +124,8 @@ func (c *consumer) readMessage(ctx context.Context) error { // Run the consumer, read data and write to the downstream target. func (c *consumer) Run(ctx context.Context) error { + defer c.writer.cleanupEventsGroups() + g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return c.writer.run(ctx) diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index e0fb914e1e..e748c10a49 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -139,6 +139,17 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } +func (w *writer) cleanupEventsGroups() { + for _, progress := range w.progresses { + for _, group := range progress.eventsGroup { + if err := group.Cleanup(); err != nil { + log.Warn("cleanup events group spill file failed", + zap.Int32("partition", progress.partition), zap.Error(err)) + } + } + } +} + func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) error { var ( done = make(chan struct{}, 1) diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index 35c9c037e3..6775aecfba 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -381,20 +381,16 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTable(t *testing.T) { w.onDDL(ddl) require.True(t, w.partitionTableAccessor.IsPartitionTable("target", "dst")) - newDMLMessage := func(commitTs uint64) *codeccommon.DMLMessage { - return codeccommon.NewDMLMessage(1, "target", "dst", commitTs, common.RowTypeUpdate, nil) - } - progress := w.progresses[0] - w.appendMessage2Group(newDMLMessage(200), progress) - w.appendMessage2Group(newDMLMessage(100), progress) + w.appendMessage2Group(newDMLMessageForWriterTest(200), progress) + w.appendMessage2Group(newDMLMessageForWriterTest(100), progress) resolved := progress.eventsGroup[1].ResolveInto(150, nil) require.Len(t, resolved, 1) require.Equal(t, uint64(100), resolved[0].GetCommitTs()) } -func TestWriteMessageDefersDMLAssemblyUntilFlush(t *testing.T) { +func TestWriteMessageSpillsDMLImmediately(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) s := sinkmock.NewMockSink(ctrl) @@ -428,7 +424,7 @@ func TestWriteMessageDefersDMLAssemblyUntilFlush(t *testing.T) { require.Equal(t, 1, decoder.addKeyValueCount) require.Equal(t, 1, decoder.hasNextCount) require.Equal(t, 1, decoder.nextDMLMessageCount) - require.Zero(t, decoder.toDMLEventCount) + require.Equal(t, 1, decoder.toDMLEventCount) require.Len(t, progress.eventsGroup[1].ResolveInto(99, nil), 0) progress.watermark = 100 diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index d89de46cfe..9567ee57e0 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -449,6 +449,14 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { } } +func (c *consumer) cleanupEventsGroups() { + for _, group := range c.eventsGroup { + if err := group.Cleanup(); err != nil { + log.Warn("cleanup events group spill file failed", zap.Error(err)) + } + } +} + func (c *consumer) parseDMLIndexFile(ctx context.Context, path string, dmlkey cloudstorage.DMLPathKey) { if c.globalCheckpointTs > 0 && dmlkey.TableVersion > c.globalCheckpointTs { log.Debug("skip dml index file by checkpoint", @@ -767,6 +775,8 @@ func (c *consumer) handle(ctx context.Context) error { } func (c *consumer) run(ctx context.Context) error { + defer c.cleanupEventsGroups() + g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return c.sink.Run(ctx) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index d4d324282a..e310c429b9 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -14,21 +14,38 @@ package util import ( + "encoding/binary" + "encoding/json" "math" + "os" "sort" "github.com/pingcap/log" + commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/spill" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tidb/pkg/util/chunk" "go.uber.org/zap" ) +const eventsGroupSpillPattern = "ticdc-events-group-*.spill" + +type spilledMessage struct { + commitTs uint64 + handle spill.Handle + postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage +} + // EventsGroup stores change event messages. type EventsGroup struct { Partition int32 tableID int64 - messages []*codeccommon.DMLMessage + messages []spilledMessage + spillFile *spill.RecordFile outOfOrder bool HighWatermark uint64 } @@ -38,25 +55,69 @@ func NewEventsGroup(partition int32, tableID int64) *EventsGroup { return &EventsGroup{ Partition: partition, tableID: tableID, - messages: make([]*codeccommon.DMLMessage, 0, 1024), + messages: make([]spilledMessage, 0, 1024), } } -// AppendMessage appends a message to event groups. +// AppendMessage materializes a message and appends it to a local spill file. DMLMessage carries a +// decoder closure, so persisting its reconstructed event is necessary to release the decoder input +// retained by that closure. func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage) { + g.appendMessage(message, nil) +} + +// AppendMessageWithPostRestore appends a message and applies postRestore after it is read back from +// disk. It keeps consumer checks that intentionally run immediately before flushing out of the +// on-disk representation. +func (g *EventsGroup) AppendMessageWithPostRestore( + message *codeccommon.DMLMessage, + postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, +) { + g.appendMessage(message, postRestore) +} + +func (g *EventsGroup) appendMessage( + message *codeccommon.DMLMessage, + postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, +) { commitTs := message.GetCommitTs() - if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].GetCommitTs() { + if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].commitTs { g.outOfOrder = true } if commitTs > g.HighWatermark { g.HighWatermark = commitTs } - g.messages = append(g.messages, message) + + data, row, err := marshalDMLMessage(message) + if err != nil { + log.Panic("marshal DML message for spill failed", + zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + } + if g.spillFile == nil { + g.spillFile, err = spill.NewRecordFile(os.TempDir(), eventsGroupSpillPattern) + if err != nil { + log.Panic("create events group spill file failed", + zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + } + } + handle, err := g.spillFile.Append(data) + if err != nil { + log.Panic("write DML message to spill file failed", + zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + } + g.messages = append(g.messages, spilledMessage{ + commitTs: commitTs, + handle: handle, + postRestore: postRestore, + }) + // Codec decoders use this callback to release their pooled chunks. The event is durable in the + // spill file now, so the original in-memory event is no longer needed. + row.PostFlush() } // ResolveInto appends all messages with CommitTs <= resolve into dst in commit-ts order and removes -// them from the group. ResolveInto copies pointers into dst first, then clears the resolved messages -// so Go GC can reclaim them once downstream is done with them. +// them from the group. Resolved messages are restored from the spill file only when downstream needs +// them, keeping the buffered group out of heap memory. func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) []*codeccommon.DMLMessage { if len(g.messages) == 0 { return dst @@ -64,12 +125,12 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) if g.outOfOrder { sort.SliceStable(g.messages, func(i, j int) bool { - return g.messages[i].GetCommitTs() < g.messages[j].GetCommitTs() + return g.messages[i].commitTs < g.messages[j].commitTs }) } resolvedCount := sort.Search(len(g.messages), func(i int) bool { - return g.messages[i].GetCommitTs() > resolve + return g.messages[i].commitTs > resolve }) if g.outOfOrder { log.Warn("DML events are out of order before flush, sort them", @@ -83,13 +144,35 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) return dst } - dst = append(dst, g.messages[:resolvedCount]...) + for _, message := range g.messages[:resolvedCount] { + data, err := g.spillFile.Read(message.handle) + if err != nil { + log.Panic("read DML message from spill file failed", + zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + } + restored, err := unmarshalDMLMessage(data) + if err != nil { + log.Panic("unmarshal DML message from spill file failed", + zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + } + if message.postRestore != nil { + restored = message.postRestore(restored) + } + dst = append(dst, restored) + } remainingCount := len(g.messages) - resolvedCount copy(g.messages, g.messages[resolvedCount:]) clear(g.messages[remainingCount:]) g.messages = g.messages[:remainingCount] + if len(g.messages) == 0 { + if err := g.spillFile.Cleanup(); err != nil { + log.Panic("cleanup events group spill file failed", + zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + } + g.spillFile = nil + } if len(g.messages) != 0 { - firstCommitTs := g.messages[0].GetCommitTs() + firstCommitTs := g.messages[0].commitTs log.Debug("not all events resolved", zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Int("resolved", resolvedCount), zap.Int("remained", len(g.messages)), @@ -103,6 +186,212 @@ func (g *EventsGroup) GetAllMessages() []*codeccommon.DMLMessage { return g.ResolveInto(math.MaxUint64, nil) } +// Cleanup removes pending spill records when the consumer is stopping. +func (g *EventsGroup) Cleanup() error { + if g.spillFile == nil { + return nil + } + err := g.spillFile.Cleanup() + g.spillFile = nil + clear(g.messages) + g.messages = g.messages[:0] + return err +} + +func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commonEvent.DMLEvent, err error) { + if message == nil { + return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") + } + + row = message.ToDMLEvent() + if row == nil { + return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without event") + } + if row.Version == 0 { + row.Version = commonEvent.DMLEventVersion1 + } + eventData, err := row.Marshal() + if err != nil { + return nil, nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML event") + } + + var tableInfoData []byte + tableInfoStored := false + if row.TableInfo != nil { + tableInfoData, err = marshalDMLTableInfo(row.TableInfo) + if err != nil { + if row.Rows != nil && row.Rows.NumRows() > 0 { + return nil, nil, err + } + tableInfoData = nil + } else { + tableInfoStored = true + } + } + + var rowsData []byte + if row.Rows != nil && (row.Rows.NumRows() > 0 || tableInfoStored) { + rowsData, err = marshalDMLRows(row, tableInfoStored) + if err != nil { + if row.Rows.NumRows() > 0 { + return nil, nil, err + } + rowsData = nil + } + } + + checksumData, err := json.Marshal(row.Checksum) + if err != nil { + return nil, nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML checksums") + } + + data = make([]byte, 0, 7*8+len(eventData)+len(tableInfoData)+len(rowsData)+len(checksumData)) + data = appendUint64(data, uint64(len(eventData))) + data = append(data, eventData...) + data = appendUint64(data, uint64(len(tableInfoData))) + data = append(data, tableInfoData...) + data = appendUint64(data, uint64(len(rowsData))) + data = append(data, rowsData...) + data = appendUint64(data, uint64(len(checksumData))) + data = append(data, checksumData...) + if row.Rows != nil { + data = appendUint64(data, 1) + } else { + data = appendUint64(data, 0) + } + data = appendUint64(data, row.TableInfoVersion) + data = appendUint64(data, row.ReplicatingTs) + return data, row, nil +} + +func marshalDMLTableInfo(tableInfo *commonType.TableInfo) (data []byte, err error) { + defer func() { + if recover() != nil { + err = errors.ErrSpillFileOp.FastGenByArgs("marshal incomplete DML table info") + } + }() + + data, err = tableInfo.Marshal() + if err != nil { + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML table info") + } + return data, nil +} + +func marshalDMLRows(row *commonEvent.DMLEvent, tableInfoStored bool) (data []byte, err error) { + defer func() { + if recover() != nil { + err = errors.ErrSpillFileOp.FastGenByArgs("marshal DML rows with incomplete table info") + } + }() + + fieldTypes := []*types.FieldType(nil) + if tableInfoStored { + fieldTypes = row.TableInfo.GetFieldSlice() + } + return chunk.NewCodec(fieldTypes).Encode(row.Rows), nil +} + +func unmarshalDMLMessage(data []byte) (*codeccommon.DMLMessage, error) { + eventData, data, err := readSpilledField(data) + if err != nil { + return nil, err + } + tableInfoData, data, err := readSpilledField(data) + if err != nil { + return nil, err + } + rowsData, data, err := readSpilledField(data) + if err != nil { + return nil, err + } + checksumData, data, err := readSpilledField(data) + if err != nil { + return nil, err + } + rowsPresent, data, err := readSpilledUint64(data) + if err != nil { + return nil, err + } + if rowsPresent > 1 { + return nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill rows flag") + } + tableInfoVersion, data, err := readSpilledUint64(data) + if err != nil { + return nil, err + } + replicatingTs, data, err := readSpilledUint64(data) + if err != nil { + return nil, err + } + if len(data) != 0 { + return nil, errors.ErrSpillFileOp.FastGenByArgs("unexpected trailing DML spill data") + } + + row := &commonEvent.DMLEvent{} + if err := row.Unmarshal(eventData); err != nil { + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "unmarshal DML event") + } + if len(tableInfoData) != 0 { + tableInfo, err := commonType.UnmarshalJSONToTableInfo(tableInfoData) + if err != nil { + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "unmarshal DML table info") + } + row.TableInfo = tableInfo + } + if rowsPresent == 1 && len(rowsData) == 0 { + row.Rows = chunk.NewChunkWithCapacity(nil, 0) + } else if len(rowsData) != 0 { + fieldTypes := []*types.FieldType(nil) + if row.TableInfo != nil { + fieldTypes = row.TableInfo.GetFieldSlice() + } + row.Rows, _ = chunk.NewCodec(fieldTypes).Decode(rowsData) + } + row.TableInfoVersion = tableInfoVersion + row.ReplicatingTs = replicatingTs + if err := json.Unmarshal(checksumData, &row.Checksum); err != nil { + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "unmarshal DML checksums") + } + if len(row.RowTypes) == 0 { + return nil, errors.ErrSpillFileOp.FastGenByArgs("spilled DML event has no row type") + } + + var schema, table string + if row.TableInfo != nil { + schema = row.TableInfo.GetSchemaName() + table = row.TableInfo.GetTableName() + } + + return codeccommon.NewDMLMessage(row.PhysicalTableID, schema, table, row.CommitTs, row.RowTypes[0], func() *commonEvent.DMLEvent { + return row + }), nil +} + +func appendUint64(data []byte, value uint64) []byte { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], value) + return append(data, buf[:]...) +} + +func readSpilledField(data []byte) ([]byte, []byte, error) { + length, data, err := readSpilledUint64(data) + if err != nil { + return nil, nil, err + } + if length > uint64(len(data)) { + return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill field length") + } + return data[:length], data[length:], nil +} + +func readSpilledUint64(data []byte) (uint64, []byte, error) { + if len(data) < 8 { + return 0, nil, errors.ErrSpillFileOp.FastGenByArgs("truncated DML spill data") + } + return binary.BigEndian.Uint64(data[:8]), data[8:], nil +} + // AppendOrMergeDMLEvent appends a DML event, or merges it into the previous event // when both events belong to the same table group and have the same commit-ts. func AppendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index a756f297bd..4f37b8dbed 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -14,19 +14,25 @@ package util import ( + "os" "testing" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/integrity" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" ) func newTestDMLMessage(commitTs uint64) *codeccommon.DMLMessage { - return codeccommon.NewDMLMessage(1, "test", "t", commitTs, common.RowTypeInsert, nil) + return codeccommon.NewDMLMessageFromEvent(newTestDMLEvent(commitTs, common.RowTypeInsert)) } func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.DMLEvent { @@ -39,16 +45,16 @@ func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.D } } -func TestEventsGroupResolveIntoAppendsAndClearsResolvedMessages(t *testing.T) { +func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T) { // Scenario: A consumer resolves events by watermark/commit-ts and appends them into a downstream - // batch slice. We must clear resolved messages in the group's backing array to avoid retaining - // already-flushed events and causing unbounded memory growth. + // batch slice. Buffered messages are held only by a spill file, and the file must be cleaned once + // all of its records have been resolved. // // Steps: // 1. Append 3 events with increasing CommitTs. // 2. Call ResolveInto with resolve=2 and a nil dst. // 3. Verify (a) returned events are correct, (b) group keeps only the remaining event, - // (c) resolved messages in the original backing slice are cleared (nil'd). + // (c) the file survives the partial resolve. group := NewEventsGroup(0, 1) m1 := newTestDMLMessage(1) m2 := newTestDMLMessage(2) @@ -57,25 +63,23 @@ func TestEventsGroupResolveIntoAppendsAndClearsResolvedMessages(t *testing.T) { group.AppendMessage(m2) group.AppendMessage(m3) - // Keep a reference to the original slice header so we can validate that ResolveInto clears - // resolved messages in-place (this is what prevents GC retention of flushed events). - original := group.messages + spillPath := group.spillFile.Path() var dst []*codeccommon.DMLMessage dst = group.ResolveInto(2, dst) require.Len(t, dst, 2) - require.Same(t, m1, dst[0]) - require.Same(t, m2, dst[1]) + require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m2.GetCommitTs(), dst[1].GetCommitTs()) require.Len(t, group.messages, 1) - require.Same(t, m3, group.messages[0]) + require.Equal(t, m3.GetCommitTs(), group.messages[0].commitTs) + require.FileExists(t, spillPath) - // The unresolved event is compacted to the front, and the tail is cleared so the group - // doesn't keep flushed events alive via its backing array. - require.Same(t, m3, original[0]) - require.Nil(t, original[1]) - require.Nil(t, original[2]) + group.GetAllMessages() + require.Nil(t, group.spillFile) + _, err := os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) } func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { @@ -87,18 +91,13 @@ func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { group.AppendMessage(m1) group.AppendMessage(m2) - original := group.messages dst := make([]*codeccommon.DMLMessage, 0, 1) dst = group.ResolveInto(5, dst) require.Len(t, dst, 0) require.Len(t, group.messages, 2) - require.Same(t, m1, group.messages[0]) - require.Same(t, m2, group.messages[1]) - - // No prefix should be cleared because nothing was resolved. - require.Same(t, m1, original[0]) - require.Same(t, m2, original[1]) + require.Equal(t, m1.GetCommitTs(), group.messages[0].commitTs) + require.Equal(t, m2.GetCommitTs(), group.messages[1].commitTs) } func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { @@ -110,17 +109,18 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { group.AppendMessage(m1) group.AppendMessage(m2) - original := group.messages + spillPath := group.spillFile.Path() var dst []*codeccommon.DMLMessage dst = group.ResolveInto(100, dst) require.Len(t, dst, 2) - require.Same(t, m1, dst[0]) - require.Same(t, m2, dst[1]) + require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m2.GetCommitTs(), dst[1].GetCommitTs()) require.Len(t, group.messages, 0) - require.Nil(t, original[0]) - require.Nil(t, original[1]) + require.Nil(t, group.spillFile) + _, err := os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) } func TestEventsGroupResolveIntoSortsOutOfOrderResolvedMessages(t *testing.T) { @@ -132,19 +132,15 @@ func TestEventsGroupResolveIntoSortsOutOfOrderResolvedMessages(t *testing.T) { group.AppendMessage(m2) group.AppendMessage(m3) - original := group.messages var dst []*codeccommon.DMLMessage dst = group.ResolveInto(25, dst) require.Len(t, dst, 2) - require.Same(t, m2, dst[0]) - require.Same(t, m1, dst[1]) + require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m1.GetCommitTs(), dst[1].GetCommitTs()) require.Len(t, group.messages, 1) - require.Same(t, m3, group.messages[0]) - require.Same(t, m3, original[0]) - require.Nil(t, original[1]) - require.Nil(t, original[2]) + require.Equal(t, m3.GetCommitTs(), group.messages[0].commitTs) } func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { @@ -160,9 +156,9 @@ func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { dst = group.ResolveInto(20, dst) require.Len(t, dst, 3) - require.Same(t, m2, dst[0]) - require.Same(t, m1, dst[1]) - require.Same(t, m3, dst[2]) + require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m1.GetCommitTs(), dst[1].GetCommitTs()) + require.Equal(t, m3.GetCommitTs(), dst[2].GetCommitTs()) require.Empty(t, group.messages) } @@ -178,12 +174,49 @@ func TestEventsGroupGetAllMessagesSortsOutOfOrderMessages(t *testing.T) { messages := group.GetAllMessages() require.Len(t, messages, 3) - require.Same(t, m2, messages[0]) - require.Same(t, m1, messages[1]) - require.Same(t, m3, messages[2]) + require.Equal(t, m2.GetCommitTs(), messages[0].GetCommitTs()) + require.Equal(t, m1.GetCommitTs(), messages[1].GetCommitTs()) + require.Equal(t, m3.GetCommitTs(), messages[2].GetCommitTs()) require.Empty(t, group.messages) } +func TestEventsGroupRestoresSpilledEventRowsAndTableInfo(t *testing.T) { + tableInfo := common.WrapTableInfo("test", &model.TableInfo{ + ID: 1, + Name: ast.NewCIStr("t"), + Columns: []*model.ColumnInfo{ + { + ID: 1, + Name: ast.NewCIStr("id"), + FieldType: *types.NewFieldType(mysql.TypeLonglong), + }, + }, + }) + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 1) + rows.AppendInt64(0, 42) + event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) + event.Rows = rows + event.RowTypes = []common.RowType{common.RowTypeInsert} + event.Length = 1 + event.TableInfoVersion = 88 + event.ReplicatingTs = 99 + event.Checksum = []*integrity.Checksum{{Current: 1, Previous: 2, Corrupted: true, Version: 3}} + + group := NewEventsGroup(0, 1) + group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event)) + + messages := group.GetAllMessages() + require.Len(t, messages, 1) + restored := messages[0].ToDMLEvent() + require.Equal(t, uint64(100), restored.CommitTs) + require.Equal(t, uint64(88), restored.TableInfoVersion) + require.Equal(t, uint64(99), restored.ReplicatingTs) + require.Equal(t, event.Checksum, restored.Checksum) + require.Equal(t, "test", restored.TableInfo.GetSchemaName()) + require.Equal(t, "t", restored.TableInfo.GetTableName()) + require.Equal(t, int64(42), restored.Rows.GetRow(0).GetInt64(0)) +} + func BenchmarkEventsGroupResolveInto(b *testing.B) { const messageCount = 16 * 1024 @@ -215,16 +248,14 @@ func BenchmarkEventsGroupResolveInto(b *testing.B) { lastIndex := len(source) - 1 source[lastIndex-1], source[lastIndex] = source[lastIndex], source[lastIndex-1] } - group := NewEventsGroup(0, 1) - group.messages = make([]*codeccommon.DMLMessage, 0, messageCount) dst := make([]*codeccommon.DMLMessage, 0, messageCount) b.ReportAllocs() b.ResetTimer() for b.Loop() { - if len(group.messages) != messageCount { - group.messages = append(group.messages[:0], source...) - group.outOfOrder = benchmark.outOfOrder + group := NewEventsGroup(0, 1) + for _, message := range source { + group.AppendMessage(message) } dst = group.ResolveInto(benchmark.resolveTs, dst[:0]) } From 4295b183e946549c4b08f422da72e7099c49fdf1 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 21 Aug 2026 11:08:00 +0000 Subject: [PATCH 02/22] fix Signed-off-by: wk989898 --- cmd/kafka-consumer/consumer.go | 13 +++- cmd/kafka-consumer/writer.go | 63 +++++++++------ cmd/kafka-consumer/writer_test.go | 16 ++-- cmd/pulsar-consumer/consumer.go | 13 +++- cmd/pulsar-consumer/writer.go | 51 +++++++----- cmd/pulsar-consumer/writer_test.go | 28 +++++-- cmd/storage-consumer/consumer.go | 39 +++++++--- cmd/util/event_group.go | 120 ++++++++++++++++++----------- cmd/util/event_group_test.go | 44 ++++++++--- 9 files changed, 263 insertions(+), 124 deletions(-) diff --git a/cmd/kafka-consumer/consumer.go b/cmd/kafka-consumer/consumer.go index c98125ddd0..1c50034144 100644 --- a/cmd/kafka-consumer/consumer.go +++ b/cmd/kafka-consumer/consumer.go @@ -151,7 +151,10 @@ func (c *consumer) readMessage(ctx context.Context) error { log.Error("read message failed, just continue to retry", zap.Error(err)) continue } - needCommit := c.writer.WriteMessage(ctx, msg) + needCommit, err := c.writer.WriteMessage(ctx, msg) + if err != nil { + return err + } if !needCommit { continue } @@ -169,8 +172,12 @@ func (c *consumer) readMessage(ctx context.Context) error { } // Run the consumer, read data and write to the downstream target. -func (c *consumer) Run(ctx context.Context) error { - defer c.writer.cleanupEventsGroups() +func (c *consumer) Run(ctx context.Context) (err error) { + defer func() { + if cleanupErr := c.writer.cleanupEventsGroups(); err == nil && cleanupErr != nil { + err = cleanupErr + } + }() g, ctx := errgroup.WithContext(ctx) g.Go(func() error { diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index 1284f31927..6664ef5c7f 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -147,15 +147,20 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } -func (w *writer) cleanupEventsGroups() { +func (w *writer) cleanupEventsGroups() error { + var cleanupErr error for _, progress := range w.progresses { for _, group := range progress.eventsGroup { if err := group.Cleanup(); err != nil { log.Warn("cleanup events group spill file failed", zap.Int32("partition", progress.partition), zap.Error(err)) + if cleanupErr == nil { + cleanupErr = err + } } } } + return cleanupErr } func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { @@ -174,7 +179,10 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { if !ok { continue } - messages := g.ResolveInto(commitTs, nil) + messages, err := g.ResolveInto(commitTs, nil) + if err != nil { + return err + } events := make([]*event.DMLEvent, 0, len(messages)) for _, message := range messages { events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) @@ -286,7 +294,10 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { resolvedEvents := make([]*event.DMLEvent, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - messages := group.ResolveInto(watermark, nil) + messages, err := group.ResolveInto(watermark, nil) + if err != nil { + return err + } events := make([]*event.DMLEvent, 0, len(messages)) for _, message := range messages { events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) @@ -331,7 +342,7 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { // WriteMessage is to decode kafka message to event. // return true if the message is flushed to the downstream. // return error if flush messages failed. -func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool { +func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool, error) { var ( partition = message.TopicPartition.Partition offset = message.TopicPartition.Offset @@ -366,19 +377,21 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool log.Info("simple protocol cached event resolved, append to the group", zap.Int64("tableID", dmlMessage.TableID), zap.Uint64("commitTs", dmlMessage.GetCommitTs()), zap.Int32("partition", partition), zap.Any("offset", offset)) - w.appendMessage2Group(dmlMessage, progress, offset) + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + return false, err + } } } w.onDDL(ddl) // DDL is broadcast to all partitions, but only handle the DDL from partition-0. if partition != 0 { - return false + return false, nil } // the Query maybe empty if using simple protocol, it's comes from `bootstrap` event, no need to handle it. if ddl.Query == "" { - return false + return false, nil } w.appendDDL(ddl) log.Info("DDL event received", @@ -400,7 +413,9 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool break } - w.appendMessage2Group(dmlMessage, progress, offset) + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + return false, err + } counter++ for { _, hasNext = progress.decoder.HasNext() @@ -416,7 +431,9 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool log.Debug("DML message is nil, it's cached", zap.Int32("partition", partition), zap.Any("offset", offset)) break } - w.appendMessage2Group(dmlMessage, progress, offset) + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + return false, err + } counter++ } // If the message containing only one event exceeds the length limit, CDC will allow it and issue a warning. @@ -438,11 +455,11 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool if needFlush { return w.Write(ctx, messageType) } - return false + return false, nil } // Write will synchronously write data downstream -func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool { +func (w *writer) Write(ctx context.Context, messageType common.MessageType) (bool, error) { // DDL events can be received out of commit-ts order (e.g. due to protocol-level broadcasting and // buffering differences between DDL kinds). We must execute DDLs in commit-ts order; otherwise a // "future" DDL that is not yet eligible (commitTs > watermark) can block executing earlier DDLs @@ -490,8 +507,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool break } if err := w.flushDDLEvent(ctx, todoDDL); err != nil { - log.Panic("write DDL event failed", zap.Error(err), - zap.String("DDL", todoDDL.Query), zap.Uint64("commitTs", todoDDL.GetCommitTs())) + return false, err } } @@ -499,7 +515,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool // since watermark is broadcast to all partitions, so that each partition can flush events individually. err := w.flushDMLEventsByWatermark(ctx) if err != nil { - log.Panic("flush dml events by the watermark failed", zap.Error(err)) + return false, err } } @@ -509,9 +525,9 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool log.Info("some DDL events will be flushed in the future", zap.Uint64("watermark", watermark), zap.Int("length", len(w.ddlList))) - return false + return false, nil } - return true + return true, nil } func (w *writer) onDDL(ddl *event.DDLEvent) { @@ -611,7 +627,7 @@ func (w *writer) messageWithPartitionCheck(message *common.DMLMessage, partition }) } -func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress, offset kafka.Offset) { +func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress, offset kafka.Offset) error { // if the kafka cluster is normal, this should not hit. // else if the cluster is abnormal, the consumer may consume old message, then cause the watermark fallback. var ( @@ -631,7 +647,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } group := progress.eventsGroup[tableID] @@ -639,9 +655,11 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti group = util.NewEventsGroup(progress.partition, tableID) progress.eventsGroup[tableID] = group } - group.AppendMessageWithPostRestore(message, func(message *common.DMLMessage) *common.DMLMessage { + if err := group.AppendMessageWithPostRestore(message, func(message *common.DMLMessage) *common.DMLMessage { return w.messageWithPartitionCheck(message, progress.partition, offset) - }) + }); err != nil { + return err + } if commitTs < progress.watermark { log.Warn("DML event fallback row, since less than the partition watermark, append it and sort before flush", zap.Int64("tableID", tableID), zap.Int32("partition", group.Partition), @@ -651,7 +669,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } if commitTs >= group.HighWatermark { log.Debug("DML event append to the group", @@ -659,7 +677,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.Uint64("commitTs", commitTs), zap.Uint64("HighWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return + return nil } log.Warn("DML event commit ts fallback, append it and sort before flush", zap.Int32("partition", progress.partition), zap.Any("offset", offset), @@ -668,6 +686,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) + return nil } func openDB(ctx context.Context, dsn string) (*sql.DB, error) { diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index 5cb904abc4..b60075d1eb 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -299,7 +299,9 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { w.appendMessage2Group(newDMLMessageForWriterTest(20), p, kafka.Offset(3)) p.watermark = 20 - require.True(t, w.Write(ctx, codeccommon.MessageTypeResolved)) + needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) + require.NoError(t, err) + require.True(t, needCommit) require.Equal(t, []uint64{10, 20}, flushedCommitTs) } @@ -323,9 +325,10 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { maxMessageBytes: 1, } - needCommit := w.WriteMessage(ctx, &kafka.Message{ + needCommit, err := w.WriteMessage(ctx, &kafka.Message{ TopicPartition: kafka.TopicPartition{Partition: 0, Offset: kafka.Offset(10)}, }) + require.NoError(t, err) require.False(t, needCommit) require.Nil(t, progress.eventsGroup[1]) @@ -353,7 +356,8 @@ func TestAppendMessageKeepsFallbackDMLAboveGlobalWatermark(t *testing.T) { w.appendMessage2Group(newDMLMessageForWriterTest(10), progress, kafka.Offset(10)) require.NotNil(t, progress.eventsGroup[1]) - resolved := progress.eventsGroup[1].ResolveInto(20, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(20, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(10), resolved[0].GetCommitTs()) } @@ -404,7 +408,8 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTableForAvro(t *testing.T) { w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) - resolved := progress.eventsGroup[1].ResolveInto(150, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(100), resolved[0].GetCommitTs()) } @@ -452,7 +457,8 @@ func TestAppendRow2GroupKeepsDebeziumPartitionTableFallback(t *testing.T) { w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) - resolved := progress.eventsGroup[1].ResolveInto(150, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(100), resolved[0].GetCommitTs()) }) diff --git a/cmd/pulsar-consumer/consumer.go b/cmd/pulsar-consumer/consumer.go index c6b3d600ec..e9284cbb7f 100644 --- a/cmd/pulsar-consumer/consumer.go +++ b/cmd/pulsar-consumer/consumer.go @@ -110,7 +110,10 @@ func (c *consumer) readMessage(ctx context.Context) error { return errors.Trace(ctx.Err()) case consumerMsg := <-msgChan: log.Debug("Received message", zap.Stringer("msgId", consumerMsg.ID()), zap.ByteString("content", consumerMsg.Payload())) - needCommit := c.writer.WriteMessage(ctx, consumerMsg) + needCommit, writeErr := c.writer.WriteMessage(ctx, consumerMsg) + if writeErr != nil { + return writeErr + } if !needCommit { continue } @@ -123,8 +126,12 @@ func (c *consumer) readMessage(ctx context.Context) error { } // Run the consumer, read data and write to the downstream target. -func (c *consumer) Run(ctx context.Context) error { - defer c.writer.cleanupEventsGroups() +func (c *consumer) Run(ctx context.Context) (err error) { + defer func() { + if cleanupErr := c.writer.cleanupEventsGroups(); err == nil && cleanupErr != nil { + err = cleanupErr + } + }() g, ctx := errgroup.WithContext(ctx) g.Go(func() error { diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index e748c10a49..136d18719b 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -139,15 +139,20 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } -func (w *writer) cleanupEventsGroups() { +func (w *writer) cleanupEventsGroups() error { + var cleanupErr error for _, progress := range w.progresses { for _, group := range progress.eventsGroup { if err := group.Cleanup(); err != nil { log.Warn("cleanup events group spill file failed", zap.Int32("partition", progress.partition), zap.Error(err)) + if cleanupErr == nil { + cleanupErr = err + } } } } + return cleanupErr } func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) error { @@ -166,7 +171,10 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e if !ok { continue } - messages := g.ResolveInto(commitTs, nil) + messages, err := g.ResolveInto(commitTs, nil) + if err != nil { + return err + } events := make([]*commonEvent.DMLEvent, 0, len(messages)) for _, message := range messages { events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) @@ -278,7 +286,10 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { resolvedEvents := make([]*commonEvent.DMLEvent, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - messages := group.ResolveInto(watermark, nil) + messages, err := group.ResolveInto(watermark, nil) + if err != nil { + return err + } events := make([]*commonEvent.DMLEvent, 0, len(messages)) for _, message := range messages { events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) @@ -321,7 +332,7 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { // WriteMessage is to decode pulsar message to event. // return true if the message is flushed to the downstream. // return error if flush messages failed. -func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool { +func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) (bool, error) { progress := w.progresses[0] progress.decoder.AddKeyValue([]byte(message.Key()), message.Payload()) @@ -349,7 +360,7 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool // the Query maybe empty if using simple protocol, it's comes from `bootstrap` event, no need to handle it. if ddl.Query == "" { - return false + return false, nil } w.appendDDL(ddl) log.Info("DDL event received", @@ -362,18 +373,20 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool if dmlMessage == nil { log.Panic("DML message is nil, it's not expected") } - w.appendMessage2Group(dmlMessage, progress) + if err := w.appendMessage2Group(dmlMessage, progress); err != nil { + return false, err + } default: log.Panic("unknown message type", zap.Any("messageType", messageType)) } if needFlush { return w.Write(ctx, messageType) } - return false + return false, nil } // Write will synchronously write data downstream -func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool { +func (w *writer) Write(ctx context.Context, messageType common.MessageType) (bool, error) { // DDL events can be received out of commit-ts order (e.g. due to protocol-level broadcasting and // buffering differences between DDL kinds). We must execute DDLs in commit-ts order; otherwise a // "future" DDL that is not yet eligible (commitTs > watermark) can block executing earlier DDLs @@ -420,8 +433,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool break } if err := w.flushDDLEvent(ctx, todoDDL); err != nil { - log.Panic("write DDL event failed", zap.Error(err), - zap.String("DDL", todoDDL.Query), zap.Uint64("commitTs", todoDDL.GetCommitTs())) + return false, err } } @@ -429,7 +441,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool // since watermark is broadcast to all partitions, so that each partition can flush events individually. err := w.flushDMLEventsByWatermark(ctx) if err != nil { - log.Panic("flush dml events by the watermark failed", zap.Error(err)) + return false, err } } @@ -439,9 +451,9 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool log.Info("some DDL events will be flushed in the future", zap.Uint64("watermark", watermark), zap.Int("length", len(w.ddlList))) - return false + return false, nil } - return true + return true, nil } func (w *writer) onDDL(ddl *commonEvent.DDLEvent) { @@ -502,7 +514,7 @@ func (w *writer) addPartitionTable(schema, table string) { w.partitionTableAccessor.Add(schema, table) } -func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress) { +func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress) error { var ( tableID = message.TableID schema = message.Schema @@ -519,7 +531,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } group := progress.eventsGroup[tableID] @@ -527,7 +539,9 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti group = util.NewEventsGroup(progress.partition, tableID) progress.eventsGroup[tableID] = group } - group.AppendMessage(message) + if err := group.AppendMessage(message); err != nil { + return err + } if commitTs < progress.watermark { log.Warn("DML event fallback row, since less than the partition watermark, append it and sort before flush", zap.Int64("tableID", tableID), zap.Int32("partition", group.Partition), @@ -536,14 +550,14 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } if commitTs >= group.HighWatermark { log.Debug("DML event append to the group", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return + return nil } log.Warn("DML event commit ts fallback, append it and sort before flush", zap.Int32("partition", progress.partition), @@ -552,4 +566,5 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) + return nil } diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index 6775aecfba..6189b8511e 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -294,7 +294,9 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { w.appendMessage2Group(newDMLMessageForWriterTest(20), p) p.watermark = 20 - require.True(t, w.Write(ctx, codeccommon.MessageTypeResolved)) + needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) + require.NoError(t, err) + require.True(t, needCommit) require.Equal(t, []uint64{10, 20}, flushedCommitTs) } @@ -326,7 +328,8 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { protocol: config.ProtocolCanalJSON, } - needCommit := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + needCommit, err := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + require.NoError(t, err) require.False(t, needCommit) require.Nil(t, progress.eventsGroup[1]) @@ -349,7 +352,8 @@ func TestAppendMessageKeepsFallbackDMLAboveGlobalWatermark(t *testing.T) { w.appendMessage2Group(newDMLMessageForWriterTest(10), progress) require.NotNil(t, progress.eventsGroup[1]) - resolved := progress.eventsGroup[1].ResolveInto(20, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(20, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(10), resolved[0].GetCommitTs()) } @@ -385,7 +389,8 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTable(t *testing.T) { w.appendMessage2Group(newDMLMessageForWriterTest(200), progress) w.appendMessage2Group(newDMLMessageForWriterTest(100), progress) - resolved := progress.eventsGroup[1].ResolveInto(150, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(100), resolved[0].GetCommitTs()) } @@ -419,21 +424,28 @@ func TestWriteMessageSpillsDMLImmediately(t *testing.T) { protocol: config.ProtocolCanalJSON, } - needCommit := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + needCommit, err := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + require.NoError(t, err) require.False(t, needCommit) require.Equal(t, 1, decoder.addKeyValueCount) require.Equal(t, 1, decoder.hasNextCount) require.Equal(t, 1, decoder.nextDMLMessageCount) require.Equal(t, 1, decoder.toDMLEventCount) - require.Len(t, progress.eventsGroup[1].ResolveInto(99, nil), 0) + resolved, err := progress.eventsGroup[1].ResolveInto(99, nil) + require.NoError(t, err) + require.Len(t, resolved, 0) progress.watermark = 100 - require.True(t, w.Write(ctx, codeccommon.MessageTypeResolved)) + needCommit, err = w.Write(ctx, codeccommon.MessageTypeResolved) + require.NoError(t, err) + require.True(t, needCommit) require.Equal(t, 1, decoder.addKeyValueCount) require.Equal(t, 1, decoder.hasNextCount) require.Equal(t, 1, decoder.nextDMLMessageCount) require.Equal(t, 1, decoder.toDMLEventCount) - require.Empty(t, progress.eventsGroup[1].ResolveInto(100, nil)) + resolved, err = progress.eventsGroup[1].ResolveInto(100, nil) + require.NoError(t, err) + require.Empty(t, resolved) require.Equal(t, []byte(`{"fake":"row"}`), decoder.lastValue) } diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 9567ee57e0..71ed1a6d60 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -282,7 +282,7 @@ func (c *consumer) getNewFiles( return tableDMLMap, err } -func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAcrossNodes bool) { +func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAcrossNodes bool) error { var ( tableID = message.TableID schema = message.Schema @@ -295,20 +295,24 @@ func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAc c.eventsGroup[tableID] = group } if commitTs >= group.HighWatermark { - group.AppendMessage(message) + if err := group.AppendMessage(message); err != nil { + return err + } log.Debug("DML event append to the group", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return + return nil } if enableTableAcrossNodes { log.Warn("DML events fallback, but enableTableAcrossNodes is true, still append it", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - group.AppendMessage(message) - return + if err := group.AppendMessage(message); err != nil { + return err + } + return nil } log.Warn("dml event commit ts fallback, ignore", zap.Uint64("commitTs", commitTs), @@ -316,6 +320,7 @@ func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAc zap.String("schema", schema), zap.String("table", table), ) + return nil } // appendDMLEvents decodes RowChangedEvents from file content and append them to event group. @@ -372,7 +377,9 @@ func (c *consumer) appendDMLEvents( c.dmlCount.Add(1) message := decoder.NextDMLMessage() - c.appendMessage2Group(messageWithPhysicalTableID(message, tableID), fileIdx.EnableTableAcrossNodes) + if err := c.appendMessage2Group(messageWithPhysicalTableID(message, tableID), fileIdx.EnableTableAcrossNodes); err != nil { + return err + } filteredCnt++ } } @@ -398,7 +405,10 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { if group == nil { return nil } - messages := group.GetAllMessages() + messages, err := group.GetAllMessages() + if err != nil { + return err + } if len(messages) == 0 { return nil } @@ -449,12 +459,17 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { } } -func (c *consumer) cleanupEventsGroups() { +func (c *consumer) cleanupEventsGroups() error { + var cleanupErr error for _, group := range c.eventsGroup { if err := group.Cleanup(); err != nil { log.Warn("cleanup events group spill file failed", zap.Error(err)) + if cleanupErr == nil { + cleanupErr = err + } } } + return cleanupErr } func (c *consumer) parseDMLIndexFile(ctx context.Context, path string, dmlkey cloudstorage.DMLPathKey) { @@ -774,8 +789,12 @@ func (c *consumer) handle(ctx context.Context) error { } } -func (c *consumer) run(ctx context.Context) error { - defer c.cleanupEventsGroups() +func (c *consumer) run(ctx context.Context) (err error) { + defer func() { + if cleanupErr := c.cleanupEventsGroups(); err == nil && cleanupErr != nil { + err = cleanupErr + } + }() g, ctx := errgroup.WithContext(ctx) g.Go(func() error { diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index e310c429b9..9b8a13941e 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -62,8 +62,8 @@ func NewEventsGroup(partition int32, tableID int64) *EventsGroup { // AppendMessage materializes a message and appends it to a local spill file. DMLMessage carries a // decoder closure, so persisting its reconstructed event is necessary to release the decoder input // retained by that closure. -func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage) { - g.appendMessage(message, nil) +func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage) error { + return g.appendMessage(message, nil) } // AppendMessageWithPostRestore appends a message and applies postRestore after it is read back from @@ -72,38 +72,39 @@ func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage) { func (g *EventsGroup) AppendMessageWithPostRestore( message *codeccommon.DMLMessage, postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, -) { - g.appendMessage(message, postRestore) +) error { + return g.appendMessage(message, postRestore) } func (g *EventsGroup) appendMessage( message *codeccommon.DMLMessage, postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, -) { - commitTs := message.GetCommitTs() - if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].commitTs { - g.outOfOrder = true - } - if commitTs > g.HighWatermark { - g.HighWatermark = commitTs + +) error { + if message == nil { + return errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") } + commitTs := message.GetCommitTs() data, row, err := marshalDMLMessage(message) if err != nil { - log.Panic("marshal DML message for spill failed", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + return err } if g.spillFile == nil { g.spillFile, err = spill.NewRecordFile(os.TempDir(), eventsGroupSpillPattern) if err != nil { - log.Panic("create events group spill file failed", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + return err } } handle, err := g.spillFile.Append(data) if err != nil { - log.Panic("write DML message to spill file failed", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + return err + } + if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].commitTs { + g.outOfOrder = true + } + if commitTs > g.HighWatermark { + g.HighWatermark = commitTs } g.messages = append(g.messages, spilledMessage{ commitTs: commitTs, @@ -113,14 +114,15 @@ func (g *EventsGroup) appendMessage( // Codec decoders use this callback to release their pooled chunks. The event is durable in the // spill file now, so the original in-memory event is no longer needed. row.PostFlush() + return nil } // ResolveInto appends all messages with CommitTs <= resolve into dst in commit-ts order and removes // them from the group. Resolved messages are restored from the spill file only when downstream needs // them, keeping the buffered group out of heap memory. -func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) []*codeccommon.DMLMessage { +func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) ([]*codeccommon.DMLMessage, error) { if len(g.messages) == 0 { - return dst + return dst, nil } if g.outOfOrder { @@ -141,19 +143,20 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) g.outOfOrder = false } if resolvedCount == 0 { - return dst + return dst, nil + } + if g.spillFile == nil { + return dst, errors.ErrSpillFileOp.FastGenByArgs("events group spill file is missing") } for _, message := range g.messages[:resolvedCount] { data, err := g.spillFile.Read(message.handle) if err != nil { - log.Panic("read DML message from spill file failed", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + return dst, err } restored, err := unmarshalDMLMessage(data) if err != nil { - log.Panic("unmarshal DML message from spill file failed", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + return dst, err } if message.postRestore != nil { restored = message.postRestore(restored) @@ -166,8 +169,7 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) g.messages = g.messages[:remainingCount] if len(g.messages) == 0 { if err := g.spillFile.Cleanup(); err != nil { - log.Panic("cleanup events group spill file failed", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), zap.Error(err)) + return dst, err } g.spillFile = nil } @@ -178,11 +180,11 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) zap.Int("resolved", resolvedCount), zap.Int("remained", len(g.messages)), zap.Uint64("resolveTs", resolve), zap.Uint64("firstCommitTs", firstCommitTs)) } - return dst + return dst, nil } // GetAllMessages gets all messages. -func (g *EventsGroup) GetAllMessages() []*codeccommon.DMLMessage { +func (g *EventsGroup) GetAllMessages() ([]*codeccommon.DMLMessage, error) { return g.ResolveInto(math.MaxUint64, nil) } @@ -192,10 +194,13 @@ func (g *EventsGroup) Cleanup() error { return nil } err := g.spillFile.Cleanup() + if err != nil { + return err + } g.spillFile = nil clear(g.messages) g.messages = g.messages[:0] - return err + return nil } func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commonEvent.DMLEvent, err error) { @@ -223,6 +228,8 @@ func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commo if row.Rows != nil && row.Rows.NumRows() > 0 { return nil, nil, err } + log.Warn("spill DML event without table info", + zap.Int64("tableID", row.PhysicalTableID), zap.Uint64("commitTs", row.CommitTs), zap.Error(err)) tableInfoData = nil } else { tableInfoStored = true @@ -236,6 +243,8 @@ func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commo if row.Rows.NumRows() > 0 { return nil, nil, err } + log.Warn("spill DML event without row data", + zap.Int64("tableID", row.PhysicalTableID), zap.Uint64("commitTs", row.CommitTs), zap.Error(err)) rowsData = nil } } @@ -245,7 +254,7 @@ func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commo return nil, nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML checksums") } - data = make([]byte, 0, 7*8+len(eventData)+len(tableInfoData)+len(rowsData)+len(checksumData)) + data = make([]byte, 0, 10*8+len(eventData)+len(tableInfoData)+len(rowsData)+len(checksumData)+len(message.Schema)+len(message.Table)) data = appendUint64(data, uint64(len(eventData))) data = append(data, eventData...) data = appendUint64(data, uint64(len(tableInfoData))) @@ -254,6 +263,11 @@ func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commo data = append(data, rowsData...) data = appendUint64(data, uint64(len(checksumData))) data = append(data, checksumData...) + data = appendUint64(data, uint64(len(message.Schema))) + data = append(data, message.Schema...) + data = appendUint64(data, uint64(len(message.Table))) + data = append(data, message.Table...) + data = appendUint64(data, uint64(message.RowType)) if row.Rows != nil { data = appendUint64(data, 1) } else { @@ -309,6 +323,21 @@ func unmarshalDMLMessage(data []byte) (*codeccommon.DMLMessage, error) { if err != nil { return nil, err } + schemaData, data, err := readSpilledField(data) + if err != nil { + return nil, err + } + tableData, data, err := readSpilledField(data) + if err != nil { + return nil, err + } + rowType, data, err := readSpilledUint64(data) + if err != nil { + return nil, err + } + if rowType > uint64(^commonType.RowType(0)) { + return nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill row type") + } rowsPresent, data, err := readSpilledUint64(data) if err != nil { return nil, err @@ -346,26 +375,31 @@ func unmarshalDMLMessage(data []byte) (*codeccommon.DMLMessage, error) { if row.TableInfo != nil { fieldTypes = row.TableInfo.GetFieldSlice() } - row.Rows, _ = chunk.NewCodec(fieldTypes).Decode(rowsData) + rows, err := unmarshalDMLRows(rowsData, fieldTypes) + if err != nil { + return nil, err + } + row.Rows = rows } row.TableInfoVersion = tableInfoVersion row.ReplicatingTs = replicatingTs if err := json.Unmarshal(checksumData, &row.Checksum); err != nil { return nil, errors.WrapError(errors.ErrSpillFileOp, err, "unmarshal DML checksums") } - if len(row.RowTypes) == 0 { - return nil, errors.ErrSpillFileOp.FastGenByArgs("spilled DML event has no row type") - } - - var schema, table string - if row.TableInfo != nil { - schema = row.TableInfo.GetSchemaName() - table = row.TableInfo.GetTableName() - } + return codeccommon.NewDMLMessage(row.PhysicalTableID, string(schemaData), string(tableData), row.CommitTs, + commonType.RowType(rowType), func() *commonEvent.DMLEvent { + return row + }), nil +} - return codeccommon.NewDMLMessage(row.PhysicalTableID, schema, table, row.CommitTs, row.RowTypes[0], func() *commonEvent.DMLEvent { - return row - }), nil +func unmarshalDMLRows(data []byte, fieldTypes []*types.FieldType) (rows *chunk.Chunk, err error) { + defer func() { + if recover() != nil { + err = errors.ErrSpillFileOp.FastGenByArgs("decode DML spill rows") + } + }() + rows, _ = chunk.NewCodec(fieldTypes).Decode(data) + return rows, nil } func appendUint64(data []byte, value uint64) []byte { diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index 4f37b8dbed..c782b65773 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -66,7 +66,8 @@ func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T spillPath := group.spillFile.Path() var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(2, dst) + dst, err := group.ResolveInto(2, dst) + require.NoError(t, err) require.Len(t, dst, 2) require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) @@ -76,9 +77,10 @@ func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T require.Equal(t, m3.GetCommitTs(), group.messages[0].commitTs) require.FileExists(t, spillPath) - group.GetAllMessages() + _, err = group.GetAllMessages() + require.NoError(t, err) require.Nil(t, group.spillFile) - _, err := os.Stat(spillPath) + _, err = os.Stat(spillPath) require.True(t, os.IsNotExist(err)) } @@ -92,7 +94,8 @@ func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { group.AppendMessage(m2) dst := make([]*codeccommon.DMLMessage, 0, 1) - dst = group.ResolveInto(5, dst) + dst, err := group.ResolveInto(5, dst) + require.NoError(t, err) require.Len(t, dst, 0) require.Len(t, group.messages, 2) @@ -111,7 +114,8 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { spillPath := group.spillFile.Path() var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(100, dst) + dst, err := group.ResolveInto(100, dst) + require.NoError(t, err) require.Len(t, dst, 2) require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) @@ -119,7 +123,7 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { require.Len(t, group.messages, 0) require.Nil(t, group.spillFile) - _, err := os.Stat(spillPath) + _, err = os.Stat(spillPath) require.True(t, os.IsNotExist(err)) } @@ -133,7 +137,8 @@ func TestEventsGroupResolveIntoSortsOutOfOrderResolvedMessages(t *testing.T) { group.AppendMessage(m3) var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(25, dst) + dst, err := group.ResolveInto(25, dst) + require.NoError(t, err) require.Len(t, dst, 2) require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) @@ -153,7 +158,8 @@ func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { group.AppendMessage(m3) var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(20, dst) + dst, err := group.ResolveInto(20, dst) + require.NoError(t, err) require.Len(t, dst, 3) require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) @@ -171,7 +177,8 @@ func TestEventsGroupGetAllMessagesSortsOutOfOrderMessages(t *testing.T) { group.AppendMessage(m2) group.AppendMessage(m3) - messages := group.GetAllMessages() + messages, err := group.GetAllMessages() + require.NoError(t, err) require.Len(t, messages, 3) require.Equal(t, m2.GetCommitTs(), messages[0].GetCommitTs()) @@ -205,7 +212,8 @@ func TestEventsGroupRestoresSpilledEventRowsAndTableInfo(t *testing.T) { group := NewEventsGroup(0, 1) group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event)) - messages := group.GetAllMessages() + messages, err := group.GetAllMessages() + require.NoError(t, err) require.Len(t, messages, 1) restored := messages[0].ToDMLEvent() require.Equal(t, uint64(100), restored.CommitTs) @@ -252,12 +260,24 @@ func BenchmarkEventsGroupResolveInto(b *testing.B) { b.ReportAllocs() b.ResetTimer() + b.StopTimer() for b.Loop() { group := NewEventsGroup(0, 1) for _, message := range source { - group.AppendMessage(message) + if err := group.AppendMessage(message); err != nil { + b.Fatal(err) + } + } + b.StartTimer() + var err error + dst, err = group.ResolveInto(benchmark.resolveTs, dst[:0]) + b.StopTimer() + if err != nil { + b.Fatal(err) + } + if err := group.Cleanup(); err != nil { + b.Fatal(err) } - dst = group.ResolveInto(benchmark.resolveTs, dst[:0]) } }) } From 6f5720719fcc60228b26e266c105f0514bee224a Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 21 Aug 2026 11:10:03 +0000 Subject: [PATCH 03/22] update Signed-off-by: wk989898 --- cmd/util/event_group.go | 5 +---- cmd/util/event_group_test.go | 13 +++++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index 9b8a13941e..80e63c940f 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -86,7 +86,7 @@ func (g *EventsGroup) appendMessage( } commitTs := message.GetCommitTs() - data, row, err := marshalDMLMessage(message) + data, _, err := marshalDMLMessage(message) if err != nil { return err } @@ -111,9 +111,6 @@ func (g *EventsGroup) appendMessage( handle: handle, postRestore: postRestore, }) - // Codec decoders use this callback to release their pooled chunks. The event is durable in the - // spill file now, so the original in-memory event is no longer needed. - row.PostFlush() return nil } diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index c782b65773..fa6883bf0c 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -225,6 +225,19 @@ func TestEventsGroupRestoresSpilledEventRowsAndTableInfo(t *testing.T) { require.Equal(t, int64(42), restored.Rows.GetRow(0).GetInt64(0)) } +func TestEventsGroupSpillDoesNotSignalDownstreamCallbacks(t *testing.T) { + event := newTestDMLEvent(100, common.RowTypeInsert) + var enqueued, flushed int + event.AddPostEnqueueFunc(func() { enqueued++ }) + event.AddPostFlushFunc(func() { flushed++ }) + + group := NewEventsGroup(0, 1) + defer func() { require.NoError(t, group.Cleanup()) }() + require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) + require.Zero(t, enqueued) + require.Zero(t, flushed) +} + func BenchmarkEventsGroupResolveInto(b *testing.B) { const messageCount = 16 * 1024 From d84664e6c6f45c4c9d6f774fd204194b53d6ba8b Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 21 Aug 2026 11:13:33 +0000 Subject: [PATCH 04/22] fmt Signed-off-by: wk989898 --- cmd/util/event_group.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index 80e63c940f..f9349b2def 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -79,7 +79,6 @@ func (g *EventsGroup) AppendMessageWithPostRestore( func (g *EventsGroup) appendMessage( message *codeccommon.DMLMessage, postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, - ) error { if message == nil { return errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") From 74220635482c34837f6f076e5572ee5fe31cf0fd Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 24 Aug 2026 03:16:13 +0000 Subject: [PATCH 05/22] fix Signed-off-by: wk989898 --- cmd/util/event_group.go | 22 ++++++++++++-- cmd/util/event_group_test.go | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index f9349b2def..852319a1b3 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -211,7 +211,11 @@ func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commo if row.Version == 0 { row.Version = commonEvent.DMLEventVersion1 } - eventData, err := row.Marshal() + // Rows can be shared by several DML events. Persist only this event's rows + // below, so its offset must be reset in the serialized event as well. + event := *row + event.PreviousTotalOffset = 0 + eventData, err := event.Marshal() if err != nil { return nil, nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML event") } @@ -299,7 +303,21 @@ func marshalDMLRows(row *commonEvent.DMLEvent, tableInfoStored bool) (data []byt if tableInfoStored { fieldTypes = row.TableInfo.GetFieldSlice() } - return chunk.NewCodec(fieldTypes).Encode(row.Rows), nil + begin := row.PreviousTotalOffset + end := row.Rows.NumRows() + if len(row.RowTypes) != 0 { + end = begin + len(row.RowTypes) + } + if begin < 0 || end < begin || end > row.Rows.NumRows() { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event rows are outside the shared chunk") + } + if !tableInfoStored && begin != 0 { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event rows require table info") + } + + rows := chunk.NewChunkWithCapacity(fieldTypes, end-begin) + rows.Append(row.Rows, begin, end) + return chunk.NewCodec(fieldTypes).Encode(rows), nil } func unmarshalDMLMessage(data []byte) (*codeccommon.DMLMessage, error) { diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index fa6883bf0c..a446f308f0 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -225,6 +225,65 @@ func TestEventsGroupRestoresSpilledEventRowsAndTableInfo(t *testing.T) { require.Equal(t, int64(42), restored.Rows.GetRow(0).GetInt64(0)) } +func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { + tableInfo := common.WrapTableInfo("test", &model.TableInfo{ + ID: 1, + Name: ast.NewCIStr("t"), + Columns: []*model.ColumnInfo{ + {ID: 1, Name: ast.NewCIStr("id"), FieldType: *types.NewFieldType(mysql.TypeLonglong)}, + {ID: 2, Name: ast.NewCIStr("v1"), FieldType: *types.NewFieldType(mysql.TypeLong)}, + }, + }) + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 4) + for _, value := range []struct { + null bool + value int64 + }{ + {null: true}, {value: 1}, {null: true}, {value: 2}, + } { + rows.AppendInt64(0, 42) + if value.null { + rows.AppendNull(1) + } else { + rows.AppendInt64(1, value.value) + } + } + + group := NewEventsGroup(0, 1) + for _, offset := range []int{0, 2} { + event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) + event.Rows = rows + event.RowTypes = []common.RowType{common.RowTypeInsert, common.RowTypeInsert} + event.Length = 2 + event.PreviousTotalOffset = offset + require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) + } + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 2) + for _, message := range messages { + restored := message.ToDMLEvent() + require.Zero(t, restored.PreviousTotalOffset) + require.Len(t, restored.RowTypes, restored.Rows.NumRows()) + } + + second := messages[1].ToDMLEvent() + require.True(t, second.Rows.GetRow(0).IsNull(1)) + require.Equal(t, int64(2), second.Rows.GetRow(1).GetInt64(1)) + + var events []*commonEvent.DMLEvent + for _, message := range messages { + events = AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + } + require.Len(t, events, 1) + require.Len(t, events[0].RowTypes, events[0].Rows.NumRows()) + require.True(t, events[0].Rows.GetRow(0).IsNull(1)) + require.Equal(t, int64(1), events[0].Rows.GetRow(1).GetInt64(1)) + require.True(t, events[0].Rows.GetRow(2).IsNull(1)) + require.Equal(t, int64(2), events[0].Rows.GetRow(3).GetInt64(1)) +} + func TestEventsGroupSpillDoesNotSignalDownstreamCallbacks(t *testing.T) { event := newTestDMLEvent(100, common.RowTypeInsert) var enqueued, flushed int From 527b474298fb808378fd7e88e15a3f1e1d7c8196 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 24 Aug 2026 09:04:41 +0000 Subject: [PATCH 06/22] fix Signed-off-by: wk989898 --- cmd/kafka-consumer/writer.go | 15 +++++++++------ cmd/kafka-consumer/writer_test.go | 4 ++-- cmd/pulsar-consumer/writer.go | 15 +++++++++------ cmd/pulsar-consumer/writer_test.go | 4 ++-- cmd/storage-consumer/consumer.go | 4 +++- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index 6664ef5c7f..5712eac585 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -183,11 +183,14 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { if err != nil { return err } - events := make([]*event.DMLEvent, 0, len(messages)) + // A commit-ts is only a timestamp in a multi-source stream. It does + // not identify one transaction, so events from different upstreams can + // legitimately have the same value while carrying different schema + // snapshots. Keep restored events separate instead of merging their + // chunks by commit-ts. for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) } - resolvedEvents = append(resolvedEvents, events...) } } @@ -298,11 +301,11 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { if err != nil { return err } - events := make([]*event.DMLEvent, 0, len(messages)) + // See flushDDLEvent: events with an identical commit-ts can belong to + // different upstream transactions in a multi-source stream. for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) } - resolvedEvents = append(resolvedEvents, events...) } } total := len(resolvedEvents) diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index b60075d1eb..b7f0bb2ce7 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -276,7 +276,7 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { s.EXPECT().AddDMLEvent(gomock.Any()).Do(func(event *commonEvent.DMLEvent) { flushedCommitTs = append(flushedCommitTs, event.GetCommitTs()) event.PostFlush() - }).Times(2) + }).Times(3) replicaCfg := config.GetDefaultReplicaConfig() eventRouter, err := eventrouter.NewEventRouter(replicaCfg.Sink, "test-topic", false, false) @@ -302,7 +302,7 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) require.NoError(t, err) require.True(t, needCommit) - require.Equal(t, []uint64{10, 20}, flushedCommitTs) + require.Equal(t, []uint64{10, 20, 20}, flushedCommitTs) } func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index 136d18719b..aad1f06198 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -175,11 +175,14 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e if err != nil { return err } - events := make([]*commonEvent.DMLEvent, 0, len(messages)) + // A commit-ts is only a timestamp in a multi-source stream. It does + // not identify one transaction, so events from different upstreams can + // legitimately have the same value while carrying different schema + // snapshots. Keep restored events separate instead of merging their + // chunks by commit-ts. for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) } - resolvedEvents = append(resolvedEvents, events...) } } @@ -290,11 +293,11 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { if err != nil { return err } - events := make([]*commonEvent.DMLEvent, 0, len(messages)) + // See flushDDLEvent: events with an identical commit-ts can belong to + // different upstream transactions in a multi-source stream. for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) } - resolvedEvents = append(resolvedEvents, events...) } } total := len(resolvedEvents) diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index 6189b8511e..645f308176 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -276,7 +276,7 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { s.EXPECT().AddDMLEvent(gomock.Any()).Do(func(event *commonEvent.DMLEvent) { flushedCommitTs = append(flushedCommitTs, event.GetCommitTs()) event.PostFlush() - }).Times(2) + }).Times(3) p := &partitionProgress{ partition: 0, @@ -297,7 +297,7 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) require.NoError(t, err) require.True(t, needCommit) - require.Equal(t, []uint64{10, 20}, flushedCommitTs) + require.Equal(t, []uint64{10, 20, 20}, flushedCommitTs) } func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 71ed1a6d60..6d902407e0 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -412,9 +412,11 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { if len(messages) == 0 { return nil } + // A commit-ts is only a timestamp in a multi-source stream. Keep restored + // events separate instead of merging chunks from independent transactions. events := make([]*event.DMLEvent, 0, len(messages)) for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + events = append(events, message.ToDMLEvent()) } total := len(events) if total == 0 { From 8c2648d2bc269b8fbc6106f94320587dc0e0dcc2 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 24 Aug 2026 09:08:57 +0000 Subject: [PATCH 07/22] chore Signed-off-by: wk989898 --- cmd/util/event_group.go | 26 ---------------------- cmd/util/event_group_test.go | 42 ------------------------------------ 2 files changed, 68 deletions(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index 852319a1b3..8b3af9606d 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -439,29 +439,3 @@ func readSpilledUint64(data []byte) (uint64, []byte, error) { } return binary.BigEndian.Uint64(data[:8]), data[8:], nil } - -// AppendOrMergeDMLEvent appends a DML event, or merges it into the previous event -// when both events belong to the same table group and have the same commit-ts. -func AppendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { - var lastDMLEvent *commonEvent.DMLEvent - if len(events) > 0 { - lastDMLEvent = events[len(events)-1] - } - - if lastDMLEvent == nil || lastDMLEvent.GetCommitTs() < row.GetCommitTs() { - return append(events, row) - } - - if lastDMLEvent.GetCommitTs() == row.GetCommitTs() { - lastDMLEvent.Rows.Append(row.Rows, 0, row.Rows.NumRows()) - lastDMLEvent.RowTypes = append(lastDMLEvent.RowTypes, row.RowTypes...) - lastDMLEvent.Length += row.Length - lastDMLEvent.PostTxnFlushed = append(lastDMLEvent.PostTxnFlushed, row.PostTxnFlushed...) - return events - } - - log.Panic("append event with smaller commit ts", - zap.Int64("tableID", row.GetTableID()), - zap.Uint64("lastCommitTs", lastDMLEvent.GetCommitTs()), zap.Uint64("commitTs", row.GetCommitTs())) - return events -} diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index a446f308f0..352966344e 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -271,17 +271,6 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { second := messages[1].ToDMLEvent() require.True(t, second.Rows.GetRow(0).IsNull(1)) require.Equal(t, int64(2), second.Rows.GetRow(1).GetInt64(1)) - - var events []*commonEvent.DMLEvent - for _, message := range messages { - events = AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } - require.Len(t, events, 1) - require.Len(t, events[0].RowTypes, events[0].Rows.NumRows()) - require.True(t, events[0].Rows.GetRow(0).IsNull(1)) - require.Equal(t, int64(1), events[0].Rows.GetRow(1).GetInt64(1)) - require.True(t, events[0].Rows.GetRow(2).IsNull(1)) - require.Equal(t, int64(2), events[0].Rows.GetRow(3).GetInt64(1)) } func TestEventsGroupSpillDoesNotSignalDownstreamCallbacks(t *testing.T) { @@ -354,34 +343,3 @@ func BenchmarkEventsGroupResolveInto(b *testing.B) { }) } } - -func TestAppendOrMergeDMLEventMergesSameCommitTs(t *testing.T) { - var flushed []int - e1 := newTestDMLEvent(10, common.RowTypeInsert) - e1.AddPostFlushFunc(func() { flushed = append(flushed, 1) }) - e2 := newTestDMLEvent(10, common.RowTypeDelete) - e2.AddPostFlushFunc(func() { flushed = append(flushed, 2) }) - - events := AppendOrMergeDMLEvent(nil, e1) - events = AppendOrMergeDMLEvent(events, e2) - - require.Len(t, events, 1) - require.Same(t, e1, events[0]) - require.Equal(t, int32(2), events[0].Length) - require.Equal(t, []common.RowType{common.RowTypeInsert, common.RowTypeDelete}, events[0].RowTypes) - - events[0].PostFlush() - require.Equal(t, []int{1, 2}, flushed) -} - -func TestAppendOrMergeDMLEventAppendsDifferentCommitTs(t *testing.T) { - e1 := newTestDMLEvent(10, common.RowTypeInsert) - e2 := newTestDMLEvent(20, common.RowTypeDelete) - - events := AppendOrMergeDMLEvent(nil, e1) - events = AppendOrMergeDMLEvent(events, e2) - - require.Len(t, events, 2) - require.Same(t, e1, events[0]) - require.Same(t, e2, events[1]) -} From f4ede3de3f7442294f77c0a4ec8d54a8eb83287c Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 24 Aug 2026 10:33:22 +0000 Subject: [PATCH 08/22] fix Signed-off-by: wk989898 --- cmd/util/event_group.go | 12 +++++++++++- cmd/util/event_group_test.go | 29 +++++++++++------------------ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index 8b3af9606d..6516604d54 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -306,7 +306,17 @@ func marshalDMLRows(row *commonEvent.DMLEvent, tableInfoStored bool) (data []byt begin := row.PreviousTotalOffset end := row.Rows.NumRows() if len(row.RowTypes) != 0 { - end = begin + len(row.RowTypes) + end = begin + for _, rowType := range row.RowTypes { + switch rowType { + case commonType.RowTypeInsert, commonType.RowTypeDelete: + end++ + case commonType.RowTypeUpdate: + end += 2 + default: + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event has invalid row type") + } + } } if begin < 0 || end < begin || end > row.Rows.NumRows() { return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event rows are outside the shared chunk") diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index 352966344e..c70572540a 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -231,30 +231,19 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { Name: ast.NewCIStr("t"), Columns: []*model.ColumnInfo{ {ID: 1, Name: ast.NewCIStr("id"), FieldType: *types.NewFieldType(mysql.TypeLonglong)}, - {ID: 2, Name: ast.NewCIStr("v1"), FieldType: *types.NewFieldType(mysql.TypeLong)}, }, }) rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 4) - for _, value := range []struct { - null bool - value int64 - }{ - {null: true}, {value: 1}, {null: true}, {value: 2}, - } { - rows.AppendInt64(0, 42) - if value.null { - rows.AppendNull(1) - } else { - rows.AppendInt64(1, value.value) - } + for i := int64(0); i < 4; i++ { + rows.AppendInt64(0, i) } group := NewEventsGroup(0, 1) for _, offset := range []int{0, 2} { event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) event.Rows = rows - event.RowTypes = []common.RowType{common.RowTypeInsert, common.RowTypeInsert} - event.Length = 2 + event.RowTypes = []common.RowType{common.RowTypeUpdate} + event.Length = 1 event.PreviousTotalOffset = offset require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) } @@ -265,12 +254,16 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { for _, message := range messages { restored := message.ToDMLEvent() require.Zero(t, restored.PreviousTotalOffset) - require.Len(t, restored.RowTypes, restored.Rows.NumRows()) + require.Equal(t, 2, restored.Rows.NumRows()) } second := messages[1].ToDMLEvent() - require.True(t, second.Rows.GetRow(0).IsNull(1)) - require.Equal(t, int64(2), second.Rows.GetRow(1).GetInt64(1)) + row, ok := second.GetNextRow() + require.True(t, ok) + require.Equal(t, int64(2), row.PreRow.GetInt64(0)) + require.Equal(t, int64(3), row.Row.GetInt64(0)) + _, ok = second.GetNextRow() + require.False(t, ok) } func TestEventsGroupSpillDoesNotSignalDownstreamCallbacks(t *testing.T) { From 63557caaea4d4021bb13b4818d76a17bf5309900 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Mon, 24 Aug 2026 14:27:16 +0000 Subject: [PATCH 09/22] fix Signed-off-by: wk989898 --- cmd/util/event_group.go | 19 +++++++++++++++--- cmd/util/event_group_test.go | 39 +++++++++++++++++++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index 6516604d54..58a282c459 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -306,13 +306,26 @@ func marshalDMLRows(row *commonEvent.DMLEvent, tableInfoStored bool) (data []byt begin := row.PreviousTotalOffset end := row.Rows.NumRows() if len(row.RowTypes) != 0 { - end = begin + end = begin + len(row.RowTypes) + // Most decoders, including batched DML events, use one RowType entry per + // physical chunk row. An update consequently appears twice. The Avro + // decoder instead represents its single logical update with one entry, + // while retaining both rows in the chunk. Length distinguishes the two + // encodings: it is the number of logical row changes. + compactRowTypes := row.Length > 0 && len(row.RowTypes) == int(row.Length) + if compactRowTypes { + end = begin + } for _, rowType := range row.RowTypes { switch rowType { case commonType.RowTypeInsert, commonType.RowTypeDelete: - end++ + if compactRowTypes { + end++ + } case commonType.RowTypeUpdate: - end += 2 + if compactRowTypes { + end += 2 + } default: return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event has invalid row type") } diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index c70572540a..eb04d35e59 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -242,7 +242,9 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { for _, offset := range []int{0, 2} { event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) event.Rows = rows - event.RowTypes = []common.RowType{common.RowTypeUpdate} + // A decoded update occupies two RowTypes entries, matching its before + // and after rows in the shared chunk. + event.RowTypes = []common.RowType{common.RowTypeUpdate, common.RowTypeUpdate} event.Length = 1 event.PreviousTotalOffset = offset require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) @@ -266,6 +268,41 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { require.False(t, ok) } +func TestEventsGroupRestoresCompactUpdateRows(t *testing.T) { + tableInfo := common.WrapTableInfo("test", &model.TableInfo{ + ID: 1, + Name: ast.NewCIStr("t"), + Columns: []*model.ColumnInfo{ + {ID: 1, Name: ast.NewCIStr("id"), FieldType: *types.NewFieldType(mysql.TypeLonglong)}, + }, + }) + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 2) + rows.AppendInt64(0, 1) + rows.AppendInt64(0, 2) + + // The Avro decoder represents an update with one RowType even though the + // chunk still contains both before and after rows. + event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) + event.Rows = rows + event.RowTypes = []common.RowType{common.RowTypeUpdate} + event.Length = 1 + + group := NewEventsGroup(0, 1) + require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 1) + + restored := messages[0].ToDMLEvent() + require.Equal(t, 2, restored.Rows.NumRows()) + row, ok := restored.GetNextRow() + require.True(t, ok) + require.Equal(t, int64(1), row.PreRow.GetInt64(0)) + require.Equal(t, int64(2), row.Row.GetInt64(0)) + _, ok = restored.GetNextRow() + require.False(t, ok) +} + func TestEventsGroupSpillDoesNotSignalDownstreamCallbacks(t *testing.T) { event := newTestDMLEvent(100, common.RowTypeInsert) var enqueued, flushed int From 70880d05b2c8b71de462f5d5c63901120c7146a8 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 25 Aug 2026 08:14:14 +0000 Subject: [PATCH 10/22] fmt Signed-off-by: wk989898 --- cmd/storage-consumer/consumer.go | 5 +---- cmd/util/event_group_test.go | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 6d902407e0..41564babb2 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -309,10 +309,7 @@ func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAc zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - if err := group.AppendMessage(message); err != nil { - return err - } - return nil + return group.AppendMessage(message) } log.Warn("dml event commit ts fallback, ignore", zap.Uint64("commitTs", commitTs), diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index eb04d35e59..a162f3de91 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -234,7 +234,7 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { }, }) rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 4) - for i := int64(0); i < 4; i++ { + for i := range int64(4) { rows.AppendInt64(0, i) } From 50da712bdc621df8ccf5624717bc354abe03d6e2 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 26 Aug 2026 09:03:42 +0000 Subject: [PATCH 11/22] fix Signed-off-by: wk989898 --- cmd/kafka-consumer/writer.go | 15 ++-- cmd/kafka-consumer/writer_test.go | 7 +- cmd/pulsar-consumer/writer.go | 15 ++-- cmd/pulsar-consumer/writer_test.go | 7 +- cmd/storage-consumer/consumer.go | 4 +- cmd/util/event_group.go | 61 ++++++++++++++++ cmd/util/event_group_test.go | 108 +++++++++++++++++++++++++++++ 7 files changed, 192 insertions(+), 25 deletions(-) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index 5712eac585..6664ef5c7f 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -183,14 +183,11 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { if err != nil { return err } - // A commit-ts is only a timestamp in a multi-source stream. It does - // not identify one transaction, so events from different upstreams can - // legitimately have the same value while carrying different schema - // snapshots. Keep restored events separate instead of merging their - // chunks by commit-ts. + events := make([]*event.DMLEvent, 0, len(messages)) for _, message := range messages { - resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) } + resolvedEvents = append(resolvedEvents, events...) } } @@ -301,11 +298,11 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { if err != nil { return err } - // See flushDDLEvent: events with an identical commit-ts can belong to - // different upstream transactions in a multi-source stream. + events := make([]*event.DMLEvent, 0, len(messages)) for _, message := range messages { - resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) } + resolvedEvents = append(resolvedEvents, events...) } } total := len(resolvedEvents) diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index b7f0bb2ce7..c94fd3c4f2 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -273,10 +273,12 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { ctrl := gomock.NewController(t) s := sinkmock.NewMockSink(ctrl) flushedCommitTs := make([]uint64, 0) + flushedRowTypeCounts := make([]int, 0) s.EXPECT().AddDMLEvent(gomock.Any()).Do(func(event *commonEvent.DMLEvent) { flushedCommitTs = append(flushedCommitTs, event.GetCommitTs()) + flushedRowTypeCounts = append(flushedRowTypeCounts, len(event.RowTypes)) event.PostFlush() - }).Times(3) + }).Times(2) replicaCfg := config.GetDefaultReplicaConfig() eventRouter, err := eventrouter.NewEventRouter(replicaCfg.Sink, "test-topic", false, false) @@ -302,7 +304,8 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) require.NoError(t, err) require.True(t, needCommit) - require.Equal(t, []uint64{10, 20, 20}, flushedCommitTs) + require.Equal(t, []uint64{10, 20}, flushedCommitTs) + require.Equal(t, []int{1, 2}, flushedRowTypeCounts) } func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index aad1f06198..136d18719b 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -175,14 +175,11 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e if err != nil { return err } - // A commit-ts is only a timestamp in a multi-source stream. It does - // not identify one transaction, so events from different upstreams can - // legitimately have the same value while carrying different schema - // snapshots. Keep restored events separate instead of merging their - // chunks by commit-ts. + events := make([]*commonEvent.DMLEvent, 0, len(messages)) for _, message := range messages { - resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) } + resolvedEvents = append(resolvedEvents, events...) } } @@ -293,11 +290,11 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { if err != nil { return err } - // See flushDDLEvent: events with an identical commit-ts can belong to - // different upstream transactions in a multi-source stream. + events := make([]*commonEvent.DMLEvent, 0, len(messages)) for _, message := range messages { - resolvedEvents = append(resolvedEvents, message.ToDMLEvent()) + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) } + resolvedEvents = append(resolvedEvents, events...) } } total := len(resolvedEvents) diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index 645f308176..f44f5836b3 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -273,10 +273,12 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { ctrl := gomock.NewController(t) s := sinkmock.NewMockSink(ctrl) flushedCommitTs := make([]uint64, 0) + flushedRowTypeCounts := make([]int, 0) s.EXPECT().AddDMLEvent(gomock.Any()).Do(func(event *commonEvent.DMLEvent) { flushedCommitTs = append(flushedCommitTs, event.GetCommitTs()) + flushedRowTypeCounts = append(flushedRowTypeCounts, len(event.RowTypes)) event.PostFlush() - }).Times(3) + }).Times(2) p := &partitionProgress{ partition: 0, @@ -297,7 +299,8 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) require.NoError(t, err) require.True(t, needCommit) - require.Equal(t, []uint64{10, 20, 20}, flushedCommitTs) + require.Equal(t, []uint64{10, 20}, flushedCommitTs) + require.Equal(t, []int{1, 2}, flushedRowTypeCounts) } func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 41564babb2..61eed078aa 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -409,11 +409,9 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { if len(messages) == 0 { return nil } - // A commit-ts is only a timestamp in a multi-source stream. Keep restored - // events separate instead of merging chunks from independent transactions. events := make([]*event.DMLEvent, 0, len(messages)) for _, message := range messages { - events = append(events, message.ToDMLEvent()) + events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) } total := len(events) if total == 0 { diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index 58a282c459..e252488828 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -18,6 +18,7 @@ import ( "encoding/json" "math" "os" + "reflect" "sort" "github.com/pingcap/log" @@ -199,6 +200,66 @@ func (g *EventsGroup) Cleanup() error { return nil } +// AppendOrMergeDMLEvent appends row to events, or merges it into the preceding +// event when both are compatible parts of the same transaction. Events with the +// same commit-ts from different sources can use different table schemas, so a +// commit-ts alone is not enough to merge their chunks safely. +func AppendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { + if len(events) == 0 || !canMergeDMLEvents(events[len(events)-1], row) { + return append(events, row) + } + + last := events[len(events)-1] + lastRowTypeCount := len(last.RowTypes) + rowRowTypeCount := len(row.RowTypes) + last.Rows.Append(row.Rows, 0, row.Rows.NumRows()) + last.RowTypes = append(last.RowTypes, row.RowTypes...) + last.RowKeys = appendOptionalDMLValues(last.RowKeys, row.RowKeys, lastRowTypeCount, rowRowTypeCount) + last.Checksum = appendOptionalDMLValues(last.Checksum, row.Checksum, lastRowTypeCount, rowRowTypeCount) + last.Length += row.Length + last.ApproximateSize += row.ApproximateSize + last.PostTxnEnqueued = append(last.PostTxnEnqueued, row.PostTxnEnqueued...) + last.PostTxnFlushed = append(last.PostTxnFlushed, row.PostTxnFlushed...) + return events +} + +func canMergeDMLEvents(last, row *commonEvent.DMLEvent) bool { + if last == nil || row == nil || + last.CommitTs != row.CommitTs || + last.StartTs != row.StartTs || + last.DispatcherID != row.DispatcherID || + last.PhysicalTableID != row.PhysicalTableID || + last.TableInfoVersion != row.TableInfoVersion || + last.TableInfo == nil || row.TableInfo == nil || + last.TableInfo.GetSchemaName() != row.TableInfo.GetSchemaName() || + last.TableInfo.GetTableName() != row.TableInfo.GetTableName() || + last.TableInfo.GetUpdateTS() != row.TableInfo.GetUpdateTS() || + last.Rows == nil || row.Rows == nil || + last.Rows.NumCols() != row.Rows.NumCols() || + last.PreviousTotalOffset != 0 || row.PreviousTotalOffset != 0 || + !reflect.DeepEqual(last.TableInfo.GetFieldSlice(), row.TableInfo.GetFieldSlice()) { + return false + } + + return hasOptionalDMLValues(last.RowKeys, len(last.RowTypes)) && + hasOptionalDMLValues(row.RowKeys, len(row.RowTypes)) && + hasOptionalDMLValues(last.Checksum, len(last.RowTypes)) && + hasOptionalDMLValues(row.Checksum, len(row.RowTypes)) +} + +func hasOptionalDMLValues[T any](values []T, rowTypeCount int) bool { + return len(values) == 0 || len(values) == rowTypeCount +} + +func appendOptionalDMLValues[T any](last, row []T, lastRowTypeCount, rowRowTypeCount int) []T { + if len(last) == 0 && len(row) != 0 { + last = make([]T, lastRowTypeCount) + } else if len(last) != 0 && len(row) == 0 { + row = make([]T, rowRowTypeCount) + } + return append(last, row...) +} + func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commonEvent.DMLEvent, err error) { if message == nil { return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index a162f3de91..e116c86c32 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -14,6 +14,7 @@ package util import ( + "fmt" "os" "testing" @@ -45,6 +46,113 @@ func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.D } } +func newMergeTestTableInfo(tableID int64, updateTS uint64, columnCount int) *common.TableInfo { + columns := make([]*model.ColumnInfo, columnCount) + for i := range columns { + columns[i] = &model.ColumnInfo{ + ID: int64(i + 1), + Offset: i, + Name: ast.NewCIStr(fmt.Sprintf("c%d", i)), + FieldType: *types.NewFieldType(mysql.TypeLonglong), + } + } + return common.WrapTableInfo("test", &model.TableInfo{ + ID: tableID, + Name: ast.NewCIStr("t"), + UpdateTS: updateTS, + Columns: columns, + }) +} + +func newMergeTestDMLEvent( + commitTs uint64, tableInfo *common.TableInfo, value int64, +) *commonEvent.DMLEvent { + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 1) + for column := range tableInfo.GetFieldSlice() { + rows.AppendInt64(column, value) + } + return &commonEvent.DMLEvent{ + DispatcherID: common.DispatcherID{Low: 1}, + PhysicalTableID: tableInfo.TableName.TableID, + StartTs: commitTs - 1, + CommitTs: commitTs, + Length: 1, + RowTypes: []common.RowType{common.RowTypeInsert}, + Rows: rows, + TableInfo: tableInfo, + TableInfoVersion: tableInfo.GetUpdateTS(), + } +} + +func TestAppendOrMergeDMLEvent(t *testing.T) { + t.Run("merge compatible events", func(t *testing.T) { + tableInfo := newMergeTestTableInfo(1, 10, 1) + first := newMergeTestDMLEvent(100, tableInfo, 1) + second := newMergeTestDMLEvent(100, tableInfo, 2) + first.RowKeys = [][]byte{[]byte("first")} + second.RowKeys = [][]byte{[]byte("second")} + first.Checksum = []*integrity.Checksum{{Current: 1}} + second.Checksum = []*integrity.Checksum{{Current: 2}} + var flushed []int + first.AddPostFlushFunc(func() { flushed = append(flushed, 1) }) + second.AddPostFlushFunc(func() { flushed = append(flushed, 2) }) + + events := AppendOrMergeDMLEvent(nil, first) + events = AppendOrMergeDMLEvent(events, second) + + require.Len(t, events, 1) + require.Same(t, first, events[0]) + require.Equal(t, int32(2), first.Length) + require.Equal(t, 2, first.Rows.NumRows()) + require.Equal(t, []byte("second"), first.RowKeys[1]) + require.Equal(t, uint32(2), first.Checksum[1].Current) + first.PostFlush() + require.Equal(t, []int{1, 2}, flushed) + }) + + t.Run("keep different schema layouts separate", func(t *testing.T) { + first := newMergeTestDMLEvent(100, newMergeTestTableInfo(1, 10, 1), 1) + second := newMergeTestDMLEvent(100, newMergeTestTableInfo(1, 11, 2), 2) + + events := AppendOrMergeDMLEvent(nil, first) + events = AppendOrMergeDMLEvent(events, second) + + require.Len(t, events, 2) + require.Same(t, first, events[0]) + require.Same(t, second, events[1]) + }) + + t.Run("keep different dispatchers separate", func(t *testing.T) { + tableInfo := newMergeTestTableInfo(1, 10, 1) + first := newMergeTestDMLEvent(100, tableInfo, 1) + second := newMergeTestDMLEvent(100, tableInfo, 2) + second.DispatcherID = common.DispatcherID{Low: 2} + + events := AppendOrMergeDMLEvent(nil, first) + events = AppendOrMergeDMLEvent(events, second) + + require.Len(t, events, 2) + }) + + t.Run("merge compatible events restored from spill", func(t *testing.T) { + tableInfo := newMergeTestTableInfo(1, 10, 1) + group := NewEventsGroup(0, 1) + require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent( + newMergeTestDMLEvent(100, tableInfo, 1)))) + require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent( + newMergeTestDMLEvent(100, tableInfo, 2)))) + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 2) + events := AppendOrMergeDMLEvent(nil, messages[0].ToDMLEvent()) + events = AppendOrMergeDMLEvent(events, messages[1].ToDMLEvent()) + + require.Len(t, events, 1) + require.Equal(t, 2, events[0].Rows.NumRows()) + }) +} + func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T) { // Scenario: A consumer resolves events by watermark/commit-ts and appends them into a downstream // batch slice. Buffered messages are held only by a spill file, and the file must be cleaned once From deba1024f80f402025230923f94f14f719ec0098 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Thu, 27 Aug 2026 06:25:58 +0000 Subject: [PATCH 12/22] update Signed-off-by: wk989898 --- cmd/kafka-consumer/writer.go | 38 ++- cmd/kafka-consumer/writer_test.go | 1 + cmd/pulsar-consumer/writer.go | 27 +- cmd/pulsar-consumer/writer_test.go | 14 +- cmd/storage-consumer/consumer.go | 45 +++- cmd/util/event_group.go | 420 ++++++++++------------------- cmd/util/event_group_test.go | 44 +-- 7 files changed, 255 insertions(+), 334 deletions(-) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index 6664ef5c7f..09efed6df7 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -183,10 +183,7 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { if err != nil { return err } - events := make([]*event.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } + events := util.DMLMessagesToEvents(messages) resolvedEvents = append(resolvedEvents, events...) } } @@ -298,10 +295,7 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { if err != nil { return err } - events := make([]*event.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } + events := util.DMLMessagesToEvents(messages) resolvedEvents = append(resolvedEvents, events...) } } @@ -377,7 +371,9 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool log.Info("simple protocol cached event resolved, append to the group", zap.Int64("tableID", dmlMessage.TableID), zap.Uint64("commitTs", dmlMessage.GetCommitTs()), zap.Int32("partition", partition), zap.Any("offset", offset)) - if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + if err := w.appendMessage2Group(dmlMessage, progress, offset, util.DMLMessageSpillData{ + Restore: func([]byte) (*common.DMLMessage, error) { return dmlMessage, nil }, + }); err != nil { return false, err } } @@ -413,7 +409,8 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool break } - if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + if err := w.appendMessage2Group(dmlMessage, progress, offset, + util.NewDMLMessageSpillData(progress.decoder, message.Key, message.Value, uint64(counter))); err != nil { return false, err } counter++ @@ -431,7 +428,8 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool log.Debug("DML message is nil, it's cached", zap.Int32("partition", partition), zap.Any("offset", offset)) break } - if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + if err := w.appendMessage2Group(dmlMessage, progress, offset, + util.NewDMLMessageSpillData(progress.decoder, message.Key, message.Value, uint64(counter))); err != nil { return false, err } counter++ @@ -627,7 +625,12 @@ func (w *writer) messageWithPartitionCheck(message *common.DMLMessage, partition }) } -func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress, offset kafka.Offset) error { +func (w *writer) appendMessage2Group( + message *common.DMLMessage, + progress *partitionProgress, + offset kafka.Offset, + spillDataArgs ...util.DMLMessageSpillData, +) error { // if the kafka cluster is normal, this should not hit. // else if the cluster is abnormal, the consumer may consume old message, then cause the watermark fallback. var ( @@ -655,9 +658,16 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti group = util.NewEventsGroup(progress.partition, tableID) progress.eventsGroup[tableID] = group } - if err := group.AppendMessageWithPostRestore(message, func(message *common.DMLMessage) *common.DMLMessage { + spillData := util.DMLMessageSpillData{ + Restore: func([]byte) (*common.DMLMessage, error) { return message, nil }, + } + if len(spillDataArgs) > 0 { + spillData = spillDataArgs[0] + } + spillData.PostRestore = func(message *common.DMLMessage) *common.DMLMessage { return w.messageWithPartitionCheck(message, progress.partition, offset) - }); err != nil { + } + if err := group.AppendSpillMessage(message, spillData); err != nil { return err } if commitTs < progress.watermark { diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index c94fd3c4f2..e388522c01 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -472,6 +472,7 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { return codeccommon.NewDMLMessage(1, "test", "t", commitTs, common.RowTypeUpdate, func() *commonEvent.DMLEvent { return &commonEvent.DMLEvent{ PhysicalTableID: 1, + StartTs: commitTs - 1, CommitTs: commitTs, RowTypes: []common.RowType{common.RowTypeUpdate}, Rows: chunk.NewChunkWithCapacity(nil, 0), diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index 136d18719b..0d3272a274 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -175,10 +175,7 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e if err != nil { return err } - events := make([]*commonEvent.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } + events := util.DMLMessagesToEvents(messages) resolvedEvents = append(resolvedEvents, events...) } } @@ -290,10 +287,7 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { if err != nil { return err } - events := make([]*commonEvent.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } + events := util.DMLMessagesToEvents(messages) resolvedEvents = append(resolvedEvents, events...) } } @@ -373,7 +367,8 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) (bool if dmlMessage == nil { log.Panic("DML message is nil, it's not expected") } - if err := w.appendMessage2Group(dmlMessage, progress); err != nil { + if err := w.appendMessage2Group(dmlMessage, progress, + util.NewDMLMessageSpillData(progress.decoder, []byte(message.Key()), message.Payload(), 0)); err != nil { return false, err } default: @@ -514,7 +509,11 @@ func (w *writer) addPartitionTable(schema, table string) { w.partitionTableAccessor.Add(schema, table) } -func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress) error { +func (w *writer) appendMessage2Group( + message *common.DMLMessage, + progress *partitionProgress, + spillDataArgs ...util.DMLMessageSpillData, +) error { var ( tableID = message.TableID schema = message.Schema @@ -539,7 +538,13 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti group = util.NewEventsGroup(progress.partition, tableID) progress.eventsGroup[tableID] = group } - if err := group.AppendMessage(message); err != nil { + spillData := util.DMLMessageSpillData{ + Restore: func([]byte) (*common.DMLMessage, error) { return message, nil }, + } + if len(spillDataArgs) > 0 { + spillData = spillDataArgs[0] + } + if err := group.AppendSpillMessage(message, spillData); err != nil { return err } if commitTs < progress.watermark { diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index f44f5836b3..68a580f1c2 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -433,7 +433,7 @@ func TestWriteMessageSpillsDMLImmediately(t *testing.T) { require.Equal(t, 1, decoder.addKeyValueCount) require.Equal(t, 1, decoder.hasNextCount) require.Equal(t, 1, decoder.nextDMLMessageCount) - require.Equal(t, 1, decoder.toDMLEventCount) + require.Zero(t, decoder.toDMLEventCount) resolved, err := progress.eventsGroup[1].ResolveInto(99, nil) require.NoError(t, err) require.Len(t, resolved, 0) @@ -442,9 +442,9 @@ func TestWriteMessageSpillsDMLImmediately(t *testing.T) { needCommit, err = w.Write(ctx, codeccommon.MessageTypeResolved) require.NoError(t, err) require.True(t, needCommit) - require.Equal(t, 1, decoder.addKeyValueCount) - require.Equal(t, 1, decoder.hasNextCount) - require.Equal(t, 1, decoder.nextDMLMessageCount) + require.Equal(t, 2, decoder.addKeyValueCount) + require.Equal(t, 3, decoder.hasNextCount) + require.Equal(t, 2, decoder.nextDMLMessageCount) require.Equal(t, 1, decoder.toDMLEventCount) resolved, err = progress.eventsGroup[1].ResolveInto(100, nil) require.NoError(t, err) @@ -460,16 +460,18 @@ type deferredDMLDecoder struct { nextDMLMessageCount int toDMLEventCount int lastValue []byte + pending bool } func (d *deferredDMLDecoder) AddKeyValue(_, value []byte) { d.addKeyValueCount++ d.lastValue = append(d.lastValue[:0], value...) + d.pending = true } func (d *deferredDMLDecoder) HasNext() (codeccommon.MessageType, bool) { d.hasNextCount++ - return codeccommon.MessageTypeRow, true + return codeccommon.MessageTypeRow, d.pending } func (d *deferredDMLDecoder) NextResolvedEvent() uint64 { @@ -478,6 +480,7 @@ func (d *deferredDMLDecoder) NextResolvedEvent() uint64 { func (d *deferredDMLDecoder) NextDMLMessage() *codeccommon.DMLMessage { d.nextDMLMessageCount++ + d.pending = false return codeccommon.NewDMLMessage(1, "test", "t", d.row.CommitTs, common.RowTypeInsert, func() *commonEvent.DMLEvent { d.toDMLEventCount++ return d.row @@ -492,6 +495,7 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { return codeccommon.NewDMLMessage(1, "test", "t", commitTs, common.RowTypeUpdate, func() *commonEvent.DMLEvent { return &commonEvent.DMLEvent{ PhysicalTableID: 1, + StartTs: commitTs - 1, CommitTs: commitTs, RowTypes: []common.RowType{common.RowTypeUpdate}, Rows: chunk.NewChunkWithCapacity(nil, 0), diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 61eed078aa..defe151b11 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -282,7 +282,11 @@ func (c *consumer) getNewFiles( return tableDMLMap, err } -func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAcrossNodes bool) error { +func (c *consumer) appendMessage2Group( + message *common.DMLMessage, + enableTableAcrossNodes bool, + spillData util.DMLMessageSpillData, +) error { var ( tableID = message.TableID schema = message.Schema @@ -295,7 +299,7 @@ func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAc c.eventsGroup[tableID] = group } if commitTs >= group.HighWatermark { - if err := group.AppendMessage(message); err != nil { + if err := group.AppendSpillMessage(message, spillData); err != nil { return err } log.Debug("DML event append to the group", @@ -309,7 +313,7 @@ func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAc zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return group.AppendMessage(message) + return group.AppendSpillMessage(message, spillData) } log.Warn("dml event commit ts fallback, ignore", zap.Uint64("commitTs", commitTs), @@ -358,6 +362,7 @@ func (c *consumer) appendDMLEvents( } cnt := 0 + dmlIndex := uint64(0) filteredCnt := 0 for { tp, hasNext := decoder.HasNext() @@ -374,7 +379,12 @@ func (c *consumer) appendDMLEvents( c.dmlCount.Add(1) message := decoder.NextDMLMessage() - if err := c.appendMessage2Group(messageWithPhysicalTableID(message, tableID), fileIdx.EnableTableAcrossNodes); err != nil { + spillData := c.newDMLMessageSpillData(ctx, schemaFile, content, dmlIndex) + dmlIndex++ + spillData.PostRestore = func(message *common.DMLMessage) *common.DMLMessage { + return messageWithPhysicalTableID(message, tableID) + } + if err := c.appendMessage2Group(message, fileIdx.EnableTableAcrossNodes, spillData); err != nil { return err } filteredCnt++ @@ -389,6 +399,28 @@ func (c *consumer) appendDMLEvents( return err } +func (c *consumer) newDMLMessageSpillData( + ctx context.Context, + schemaFile cloudstorage.SchemaFile, + content []byte, + dmlIndex uint64, +) util.DMLMessageSpillData { + tableInfo := schemaFile.TableInfo() + selector := c.columnSelectors.GetForTableInfo(tableInfo) + return util.NewDMLMessageSpillDataWithDecoderFactory(nil, content, dmlIndex, + func(_ []byte, value []byte) (common.Decoder, error) { + switch c.codecCfg.Protocol { + case config.ProtocolCsv: + return csv.NewDecoderWithColumnSelector(ctx, c.codecCfg, tableInfo, value, selector) + case config.ProtocolCanalJSON: + decoder := canal.NewTxnDecoder(c.codecCfg) + return decoder, nil + default: + return nil, errors.ErrSpillFileOp.FastGenByArgs("unsupported storage DML spill protocol") + } + }) +} + func messageWithPhysicalTableID(message *common.DMLMessage, tableID int64) *common.DMLMessage { return common.NewDMLMessage(tableID, message.Schema, message.Table, message.GetCommitTs(), message.RowType, func() *event.DMLEvent { row := message.ToDMLEvent() @@ -409,10 +441,7 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { if len(messages) == 0 { return nil } - events := make([]*event.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } + events := util.DMLMessagesToEvents(messages) total := len(events) if total == 0 { return nil diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index e252488828..ad6f8590a5 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -15,20 +15,15 @@ package util import ( "encoding/binary" - "encoding/json" "math" "os" - "reflect" "sort" "github.com/pingcap/log" - commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/pingcap/ticdc/pkg/spill" - "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/chunk" "go.uber.org/zap" ) @@ -37,9 +32,18 @@ const eventsGroupSpillPattern = "ticdc-events-group-*.spill" type spilledMessage struct { commitTs uint64 handle spill.Handle + restore func([]byte) (*codeccommon.DMLMessage, error) postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage } +// DMLMessageSpillData is an opaque codec payload which can restore a lazy DML +// message after it is read from disk. Restore must not construct a DMLEvent. +type DMLMessageSpillData struct { + Data []byte + Restore func([]byte) (*codeccommon.DMLMessage, error) + PostRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage +} + // EventsGroup stores change event messages. type EventsGroup struct { Partition int32 @@ -60,42 +64,45 @@ func NewEventsGroup(partition int32, tableID int64) *EventsGroup { } } -// AppendMessage materializes a message and appends it to a local spill file. DMLMessage carries a -// decoder closure, so persisting its reconstructed event is necessary to release the decoder input -// retained by that closure. +// AppendMessage is kept for tests that already own a materialized event. Consumer +// code must use AppendSpillMessage so DMLEvent construction stays at flush time. func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage) error { - return g.appendMessage(message, nil) -} - -// AppendMessageWithPostRestore appends a message and applies postRestore after it is read back from -// disk. It keeps consumer checks that intentionally run immediately before flushing out of the -// on-disk representation. -func (g *EventsGroup) AppendMessageWithPostRestore( - message *codeccommon.DMLMessage, - postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, -) error { - return g.appendMessage(message, postRestore) + return g.AppendSpillMessage(message, DMLMessageSpillData{ + Restore: func([]byte) (*codeccommon.DMLMessage, error) { + return message, nil + }, + }) } -func (g *EventsGroup) appendMessage( +// AppendSpillMessage appends an opaque codec payload to the spill file. It does +// not call DMLMessage.ToDMLEvent; the restored message remains lazy until the +// consumer flushes it against a watermark or DDL barrier. +func (g *EventsGroup) AppendSpillMessage( message *codeccommon.DMLMessage, - postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, + spillData DMLMessageSpillData, ) error { if message == nil { return errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") } + if spillData.Restore == nil { + return errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without restore function") + } commitTs := message.GetCommitTs() - data, _, err := marshalDMLMessage(message) - if err != nil { - return err - } if g.spillFile == nil { + var err error g.spillFile, err = spill.NewRecordFile(os.TempDir(), eventsGroupSpillPattern) if err != nil { return err } } + data := spillData.Data + if len(data) == 0 { + // A lazy message may not need an input payload (for example, a Simple + // decoder message released from its table-info cache). RecordFile rejects + // empty records, so retain a marker while keeping the event lazy. + data = []byte{0} + } handle, err := g.spillFile.Append(data) if err != nil { return err @@ -109,7 +116,8 @@ func (g *EventsGroup) appendMessage( g.messages = append(g.messages, spilledMessage{ commitTs: commitTs, handle: handle, - postRestore: postRestore, + restore: spillData.Restore, + postRestore: spillData.PostRestore, }) return nil } @@ -119,6 +127,12 @@ func (g *EventsGroup) appendMessage( // them, keeping the buffered group out of heap memory. func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) ([]*codeccommon.DMLMessage, error) { if len(g.messages) == 0 { + if g.spillFile != nil { + if err := g.spillFile.Cleanup(); err != nil { + return dst, err + } + g.spillFile = nil + } return dst, nil } @@ -151,7 +165,7 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) if err != nil { return dst, err } - restored, err := unmarshalDMLMessage(data) + restored, err := message.restore(data) if err != nil { return dst, err } @@ -200,12 +214,18 @@ func (g *EventsGroup) Cleanup() error { return nil } -// AppendOrMergeDMLEvent appends row to events, or merges it into the preceding -// event when both are compatible parts of the same transaction. Events with the -// same commit-ts from different sources can use different table schemas, so a -// commit-ts alone is not enough to merge their chunks safely. -func AppendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { - if len(events) == 0 || !canMergeDMLEvents(events[len(events)-1], row) { +// DMLMessagesToEvents materializes messages and merges compatible adjacent +// messages before they are handed to the downstream sink. +func DMLMessagesToEvents(messages []*codeccommon.DMLMessage) []*commonEvent.DMLEvent { + events := make([]*commonEvent.DMLEvent, 0, len(messages)) + for _, message := range messages { + events = appendOrMergeDMLEvent(events, message.ToDMLEvent()) + } + return events +} + +func appendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { + if len(events) == 0 || !sameDMLTransaction(events[len(events)-1], row) { return append(events, row) } @@ -223,25 +243,21 @@ func AppendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLE return events } -func canMergeDMLEvents(last, row *commonEvent.DMLEvent) bool { - if last == nil || row == nil || - last.CommitTs != row.CommitTs || - last.StartTs != row.StartTs || - last.DispatcherID != row.DispatcherID || - last.PhysicalTableID != row.PhysicalTableID || - last.TableInfoVersion != row.TableInfoVersion || - last.TableInfo == nil || row.TableInfo == nil || - last.TableInfo.GetSchemaName() != row.TableInfo.GetSchemaName() || - last.TableInfo.GetTableName() != row.TableInfo.GetTableName() || - last.TableInfo.GetUpdateTS() != row.TableInfo.GetUpdateTS() || - last.Rows == nil || row.Rows == nil || - last.Rows.NumCols() != row.Rows.NumCols() || - last.PreviousTotalOffset != 0 || row.PreviousTotalOffset != 0 || - !reflect.DeepEqual(last.TableInfo.GetFieldSlice(), row.TableInfo.GetFieldSlice()) { - return false - } - - return hasOptionalDMLValues(last.RowKeys, len(last.RowTypes)) && +func sameDMLTransaction(last, row *commonEvent.DMLEvent) bool { + return last != nil && row != nil && + last.CommitTs == row.CommitTs && + last.StartTs == row.StartTs && + last.DispatcherID == row.DispatcherID && + last.PhysicalTableID == row.PhysicalTableID && + last.TableInfoVersion == row.TableInfoVersion && + last.TableInfo != nil && row.TableInfo != nil && + last.TableInfo.GetSchemaName() == row.TableInfo.GetSchemaName() && + last.TableInfo.GetTableName() == row.TableInfo.GetTableName() && + last.TableInfo.GetUpdateTS() == row.TableInfo.GetUpdateTS() && + last.Rows != nil && row.Rows != nil && + last.Rows.NumCols() == row.Rows.NumCols() && + last.PreviousTotalOffset == 0 && row.PreviousTotalOffset == 0 && + hasOptionalDMLValues(last.RowKeys, len(last.RowTypes)) && hasOptionalDMLValues(row.RowKeys, len(row.RowTypes)) && hasOptionalDMLValues(last.Checksum, len(last.RowTypes)) && hasOptionalDMLValues(row.Checksum, len(row.RowTypes)) @@ -260,264 +276,112 @@ func appendOptionalDMLValues[T any](last, row []T, lastRowTypeCount, rowRowTypeC return append(last, row...) } -func marshalDMLMessage(message *codeccommon.DMLMessage) (data []byte, row *commonEvent.DMLEvent, err error) { - if message == nil { - return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") - } - - row = message.ToDMLEvent() - if row == nil { - return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without event") - } - if row.Version == 0 { - row.Version = commonEvent.DMLEventVersion1 - } - // Rows can be shared by several DML events. Persist only this event's rows - // below, so its offset must be reset in the serialized event as well. - event := *row - event.PreviousTotalOffset = 0 - eventData, err := event.Marshal() - if err != nil { - return nil, nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML event") - } +// NewDMLMessageSpillData preserves the original codec input and the ordinal of +// its DML message. The decoder is intentionally used only during ResolveInto: +// this preserves the old behaviour where schema is selected at the DDL or +// watermark barrier, rather than while the message is appended. +func NewDMLMessageSpillData( + decoder codeccommon.Decoder, key, value []byte, dmlIndex uint64, +) DMLMessageSpillData { + return NewDMLMessageSpillDataWithDecoderFactory(key, value, dmlIndex, + func([]byte, []byte) (codeccommon.Decoder, error) { return decoder, nil }) +} - var tableInfoData []byte - tableInfoStored := false - if row.TableInfo != nil { - tableInfoData, err = marshalDMLTableInfo(row.TableInfo) - if err != nil { - if row.Rows != nil && row.Rows.NumRows() > 0 { - return nil, nil, err +// NewDMLMessageSpillDataWithDecoderFactory is for decoders such as CSV whose +// input is supplied during construction rather than through AddKeyValue. +func NewDMLMessageSpillDataWithDecoderFactory( + key, value []byte, + dmlIndex uint64, + decoderFactory func([]byte, []byte) (codeccommon.Decoder, error), +) DMLMessageSpillData { + data := make([]byte, 0, 3*8+len(key)+len(value)) + data = appendSpillUint64(data, dmlIndex) + data = appendSpillBytes(data, key) + data = appendSpillBytes(data, value) + return DMLMessageSpillData{ + Data: data, + Restore: func(data []byte) (*codeccommon.DMLMessage, error) { + dmlIndex, key, value, err := unmarshalDMLMessageSpillData(data) + if err != nil { + return nil, err } - log.Warn("spill DML event without table info", - zap.Int64("tableID", row.PhysicalTableID), zap.Uint64("commitTs", row.CommitTs), zap.Error(err)) - tableInfoData = nil - } else { - tableInfoStored = true - } - } - - var rowsData []byte - if row.Rows != nil && (row.Rows.NumRows() > 0 || tableInfoStored) { - rowsData, err = marshalDMLRows(row, tableInfoStored) - if err != nil { - if row.Rows.NumRows() > 0 { - return nil, nil, err + decoder, err := decoderFactory(key, value) + if err != nil { + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "create DML spill decoder") } - log.Warn("spill DML event without row data", - zap.Int64("tableID", row.PhysicalTableID), zap.Uint64("commitTs", row.CommitTs), zap.Error(err)) - rowsData = nil - } + return restoreDMLMessage(decoder, key, value, dmlIndex) + }, } - - checksumData, err := json.Marshal(row.Checksum) - if err != nil { - return nil, nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML checksums") - } - - data = make([]byte, 0, 10*8+len(eventData)+len(tableInfoData)+len(rowsData)+len(checksumData)+len(message.Schema)+len(message.Table)) - data = appendUint64(data, uint64(len(eventData))) - data = append(data, eventData...) - data = appendUint64(data, uint64(len(tableInfoData))) - data = append(data, tableInfoData...) - data = appendUint64(data, uint64(len(rowsData))) - data = append(data, rowsData...) - data = appendUint64(data, uint64(len(checksumData))) - data = append(data, checksumData...) - data = appendUint64(data, uint64(len(message.Schema))) - data = append(data, message.Schema...) - data = appendUint64(data, uint64(len(message.Table))) - data = append(data, message.Table...) - data = appendUint64(data, uint64(message.RowType)) - if row.Rows != nil { - data = appendUint64(data, 1) - } else { - data = appendUint64(data, 0) - } - data = appendUint64(data, row.TableInfoVersion) - data = appendUint64(data, row.ReplicatingTs) - return data, row, nil } -func marshalDMLTableInfo(tableInfo *commonType.TableInfo) (data []byte, err error) { - defer func() { - if recover() != nil { - err = errors.ErrSpillFileOp.FastGenByArgs("marshal incomplete DML table info") +func restoreDMLMessage( + decoder codeccommon.Decoder, key, value []byte, dmlIndex uint64, +) (*codeccommon.DMLMessage, error) { + decoder.AddKeyValue(key, value) + var restored *codeccommon.DMLMessage + for currentIndex := uint64(0); ; currentIndex++ { + messageType, hasNext := decoder.HasNext() + if !hasNext { + if restored == nil { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload has no message") + } + return restored, nil } - }() - - data, err = tableInfo.Marshal() - if err != nil { - return nil, errors.WrapError(errors.ErrSpillFileOp, err, "marshal DML table info") - } - return data, nil -} - -func marshalDMLRows(row *commonEvent.DMLEvent, tableInfoStored bool) (data []byte, err error) { - defer func() { - if recover() != nil { - err = errors.ErrSpillFileOp.FastGenByArgs("marshal DML rows with incomplete table info") + if messageType != codeccommon.MessageTypeRow { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload contains a non-DML message") } - }() - - fieldTypes := []*types.FieldType(nil) - if tableInfoStored { - fieldTypes = row.TableInfo.GetFieldSlice() - } - begin := row.PreviousTotalOffset - end := row.Rows.NumRows() - if len(row.RowTypes) != 0 { - end = begin + len(row.RowTypes) - // Most decoders, including batched DML events, use one RowType entry per - // physical chunk row. An update consequently appears twice. The Avro - // decoder instead represents its single logical update with one entry, - // while retaining both rows in the chunk. Length distinguishes the two - // encodings: it is the number of logical row changes. - compactRowTypes := row.Length > 0 && len(row.RowTypes) == int(row.Length) - if compactRowTypes { - end = begin + message := decoder.NextDMLMessage() + if message == nil { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload cannot be restored") } - for _, rowType := range row.RowTypes { - switch rowType { - case commonType.RowTypeInsert, commonType.RowTypeDelete: - if compactRowTypes { - end++ - } - case commonType.RowTypeUpdate: - if compactRowTypes { - end += 2 - } - default: - return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event has invalid row type") - } + if currentIndex == dmlIndex { + restored = message } } - if begin < 0 || end < begin || end > row.Rows.NumRows() { - return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event rows are outside the shared chunk") - } - if !tableInfoStored && begin != 0 { - return nil, errors.ErrSpillFileOp.FastGenByArgs("DML event rows require table info") - } +} - rows := chunk.NewChunkWithCapacity(fieldTypes, end-begin) - rows.Append(row.Rows, begin, end) - return chunk.NewCodec(fieldTypes).Encode(rows), nil +func appendSpillBytes(data, value []byte) []byte { + data = appendSpillUint64(data, uint64(len(value))) + return append(data, value...) } -func unmarshalDMLMessage(data []byte) (*codeccommon.DMLMessage, error) { - eventData, data, err := readSpilledField(data) - if err != nil { - return nil, err - } - tableInfoData, data, err := readSpilledField(data) - if err != nil { - return nil, err - } - rowsData, data, err := readSpilledField(data) - if err != nil { - return nil, err - } - checksumData, data, err := readSpilledField(data) - if err != nil { - return nil, err - } - schemaData, data, err := readSpilledField(data) - if err != nil { - return nil, err - } - tableData, data, err := readSpilledField(data) - if err != nil { - return nil, err - } - rowType, data, err := readSpilledUint64(data) - if err != nil { - return nil, err - } - if rowType > uint64(^commonType.RowType(0)) { - return nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill row type") - } - rowsPresent, data, err := readSpilledUint64(data) +func appendSpillUint64(data []byte, value uint64) []byte { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], value) + return append(data, buf[:]...) +} + +func unmarshalDMLMessageSpillData(data []byte) (uint64, []byte, []byte, error) { + dmlIndex, data, err := readSpillUint64(data) if err != nil { - return nil, err + return 0, nil, nil, err } - if rowsPresent > 1 { - return nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill rows flag") - } - tableInfoVersion, data, err := readSpilledUint64(data) + key, data, err := readSpillBytes(data) if err != nil { - return nil, err + return 0, nil, nil, err } - replicatingTs, data, err := readSpilledUint64(data) + value, data, err := readSpillBytes(data) if err != nil { - return nil, err + return 0, nil, nil, err } if len(data) != 0 { - return nil, errors.ErrSpillFileOp.FastGenByArgs("unexpected trailing DML spill data") - } - - row := &commonEvent.DMLEvent{} - if err := row.Unmarshal(eventData); err != nil { - return nil, errors.WrapError(errors.ErrSpillFileOp, err, "unmarshal DML event") - } - if len(tableInfoData) != 0 { - tableInfo, err := commonType.UnmarshalJSONToTableInfo(tableInfoData) - if err != nil { - return nil, errors.WrapError(errors.ErrSpillFileOp, err, "unmarshal DML table info") - } - row.TableInfo = tableInfo - } - if rowsPresent == 1 && len(rowsData) == 0 { - row.Rows = chunk.NewChunkWithCapacity(nil, 0) - } else if len(rowsData) != 0 { - fieldTypes := []*types.FieldType(nil) - if row.TableInfo != nil { - fieldTypes = row.TableInfo.GetFieldSlice() - } - rows, err := unmarshalDMLRows(rowsData, fieldTypes) - if err != nil { - return nil, err - } - row.Rows = rows - } - row.TableInfoVersion = tableInfoVersion - row.ReplicatingTs = replicatingTs - if err := json.Unmarshal(checksumData, &row.Checksum); err != nil { - return nil, errors.WrapError(errors.ErrSpillFileOp, err, "unmarshal DML checksums") + return 0, nil, nil, errors.ErrSpillFileOp.FastGenByArgs("unexpected trailing DML spill data") } - return codeccommon.NewDMLMessage(row.PhysicalTableID, string(schemaData), string(tableData), row.CommitTs, - commonType.RowType(rowType), func() *commonEvent.DMLEvent { - return row - }), nil -} - -func unmarshalDMLRows(data []byte, fieldTypes []*types.FieldType) (rows *chunk.Chunk, err error) { - defer func() { - if recover() != nil { - err = errors.ErrSpillFileOp.FastGenByArgs("decode DML spill rows") - } - }() - rows, _ = chunk.NewCodec(fieldTypes).Decode(data) - return rows, nil -} - -func appendUint64(data []byte, value uint64) []byte { - var buf [8]byte - binary.BigEndian.PutUint64(buf[:], value) - return append(data, buf[:]...) + return dmlIndex, key, value, nil } -func readSpilledField(data []byte) ([]byte, []byte, error) { - length, data, err := readSpilledUint64(data) +func readSpillBytes(data []byte) ([]byte, []byte, error) { + length, data, err := readSpillUint64(data) if err != nil { return nil, nil, err } if length > uint64(len(data)) { - return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill field length") + return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill payload length") } return data[:length], data[length:], nil } -func readSpilledUint64(data []byte) (uint64, []byte, error) { +func readSpillUint64(data []byte) (uint64, []byte, error) { if len(data) < 8 { return 0, nil, errors.ErrSpillFileOp.FastGenByArgs("truncated DML spill data") } diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index e116c86c32..159c977a2f 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -64,9 +64,7 @@ func newMergeTestTableInfo(tableID int64, updateTS uint64, columnCount int) *com }) } -func newMergeTestDMLEvent( - commitTs uint64, tableInfo *common.TableInfo, value int64, -) *commonEvent.DMLEvent { +func newMergeTestDMLEvent(commitTs uint64, tableInfo *common.TableInfo, value int64) *commonEvent.DMLEvent { rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 1) for column := range tableInfo.GetFieldSlice() { rows.AppendInt64(column, value) @@ -84,6 +82,11 @@ func newMergeTestDMLEvent( } } +func newMergeTestDMLMessage(event *commonEvent.DMLEvent) *codeccommon.DMLMessage { + return codeccommon.NewDMLMessage(event.GetTableID(), event.TableInfo.GetSchemaName(), event.TableInfo.GetTableName(), + event.GetCommitTs(), event.RowTypes[0], func() *commonEvent.DMLEvent { return event }) +} + func TestAppendOrMergeDMLEvent(t *testing.T) { t.Run("merge compatible events", func(t *testing.T) { tableInfo := newMergeTestTableInfo(1, 10, 1) @@ -97,8 +100,10 @@ func TestAppendOrMergeDMLEvent(t *testing.T) { first.AddPostFlushFunc(func() { flushed = append(flushed, 1) }) second.AddPostFlushFunc(func() { flushed = append(flushed, 2) }) - events := AppendOrMergeDMLEvent(nil, first) - events = AppendOrMergeDMLEvent(events, second) + events := DMLMessagesToEvents([]*codeccommon.DMLMessage{ + newMergeTestDMLMessage(first), + newMergeTestDMLMessage(second), + }) require.Len(t, events, 1) require.Same(t, first, events[0]) @@ -114,8 +119,10 @@ func TestAppendOrMergeDMLEvent(t *testing.T) { first := newMergeTestDMLEvent(100, newMergeTestTableInfo(1, 10, 1), 1) second := newMergeTestDMLEvent(100, newMergeTestTableInfo(1, 11, 2), 2) - events := AppendOrMergeDMLEvent(nil, first) - events = AppendOrMergeDMLEvent(events, second) + events := DMLMessagesToEvents([]*codeccommon.DMLMessage{ + newMergeTestDMLMessage(first), + newMergeTestDMLMessage(second), + }) require.Len(t, events, 2) require.Same(t, first, events[0]) @@ -128,8 +135,10 @@ func TestAppendOrMergeDMLEvent(t *testing.T) { second := newMergeTestDMLEvent(100, tableInfo, 2) second.DispatcherID = common.DispatcherID{Low: 2} - events := AppendOrMergeDMLEvent(nil, first) - events = AppendOrMergeDMLEvent(events, second) + events := DMLMessagesToEvents([]*codeccommon.DMLMessage{ + newMergeTestDMLMessage(first), + newMergeTestDMLMessage(second), + }) require.Len(t, events, 2) }) @@ -137,16 +146,15 @@ func TestAppendOrMergeDMLEvent(t *testing.T) { t.Run("merge compatible events restored from spill", func(t *testing.T) { tableInfo := newMergeTestTableInfo(1, 10, 1) group := NewEventsGroup(0, 1) - require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent( - newMergeTestDMLEvent(100, tableInfo, 1)))) - require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent( - newMergeTestDMLEvent(100, tableInfo, 2)))) + first := newMergeTestDMLEvent(100, tableInfo, 1) + second := newMergeTestDMLEvent(100, tableInfo, 2) + require.NoError(t, group.AppendMessage(newMergeTestDMLMessage(first))) + require.NoError(t, group.AppendMessage(newMergeTestDMLMessage(second))) messages, err := group.GetAllMessages() require.NoError(t, err) require.Len(t, messages, 2) - events := AppendOrMergeDMLEvent(nil, messages[0].ToDMLEvent()) - events = AppendOrMergeDMLEvent(events, messages[1].ToDMLEvent()) + events := DMLMessagesToEvents(messages) require.Len(t, events, 1) require.Equal(t, 2, events[0].Rows.NumRows()) @@ -361,10 +369,10 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { messages, err := group.GetAllMessages() require.NoError(t, err) require.Len(t, messages, 2) + require.Zero(t, messages[0].ToDMLEvent().PreviousTotalOffset) + require.Equal(t, 2, messages[1].ToDMLEvent().PreviousTotalOffset) for _, message := range messages { - restored := message.ToDMLEvent() - require.Zero(t, restored.PreviousTotalOffset) - require.Equal(t, 2, restored.Rows.NumRows()) + require.Equal(t, 4, message.ToDMLEvent().Rows.NumRows()) } second := messages[1].ToDMLEvent() From 88301df1eeaf0be6e8c16fc81ac076916daa42e8 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Thu, 27 Aug 2026 07:52:14 +0000 Subject: [PATCH 13/22] update Signed-off-by: wk989898 --- cmd/kafka-consumer/writer.go | 32 ++---- cmd/kafka-consumer/writer_test.go | 40 +++++-- cmd/pulsar-consumer/writer.go | 16 +-- cmd/pulsar-consumer/writer_test.go | 33 ++++-- cmd/storage-consumer/consumer.go | 39 ++++--- cmd/util/dml_message_decoder.go | 112 ++++++++++++++++++++ cmd/util/dml_message_decoder_test.go | 64 +++++++++++ cmd/util/event_group.go | 153 +++++++++++---------------- cmd/util/event_group_test.go | 106 ++++++++++++------- pkg/sink/codec/common/decoder.go | 46 ++++++++ 10 files changed, 441 insertions(+), 200 deletions(-) create mode 100644 cmd/util/dml_message_decoder.go create mode 100644 cmd/util/dml_message_decoder_test.go diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index 09efed6df7..f5d6603998 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -45,14 +45,14 @@ type partitionProgress struct { watermarkOffset kafka.Offset eventsGroup map[int64]*util.EventsGroup - decoder common.Decoder + decoder *util.DMLMessageDecoder } func newPartitionProgress(partition int32, decoder common.Decoder) *partitionProgress { return &partitionProgress{ partition: partition, eventsGroup: make(map[int64]*util.EventsGroup), - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } } @@ -343,6 +343,9 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool ) progress := w.progresses[partition] + progress.decoder.SetDMLMessageRestorer(func(message *common.DMLMessage) *common.DMLMessage { + return w.messageWithPartitionCheck(message, progress.partition, offset) + }) progress.decoder.AddKeyValue(message.Key, message.Value) messageType, hasNext := progress.decoder.HasNext() @@ -365,15 +368,14 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool // but all DDL event messages should be consumed. ddl := progress.decoder.NextDDLEvent() - if dec, ok := progress.decoder.(*simple.Decoder); ok { + if dec, ok := progress.decoder.Unwrap().(*simple.Decoder); ok { cachedMessages := dec.GetCachedMessages() for _, dmlMessage := range cachedMessages { log.Info("simple protocol cached event resolved, append to the group", zap.Int64("tableID", dmlMessage.TableID), zap.Uint64("commitTs", dmlMessage.GetCommitTs()), zap.Int32("partition", partition), zap.Any("offset", offset)) - if err := w.appendMessage2Group(dmlMessage, progress, offset, util.DMLMessageSpillData{ - Restore: func([]byte) (*common.DMLMessage, error) { return dmlMessage, nil }, - }); err != nil { + progress.decoder.AttachCachedDMLMessage(dmlMessage) + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { return false, err } } @@ -409,8 +411,7 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool break } - if err := w.appendMessage2Group(dmlMessage, progress, offset, - util.NewDMLMessageSpillData(progress.decoder, message.Key, message.Value, uint64(counter))); err != nil { + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { return false, err } counter++ @@ -428,8 +429,7 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool log.Debug("DML message is nil, it's cached", zap.Int32("partition", partition), zap.Any("offset", offset)) break } - if err := w.appendMessage2Group(dmlMessage, progress, offset, - util.NewDMLMessageSpillData(progress.decoder, message.Key, message.Value, uint64(counter))); err != nil { + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { return false, err } counter++ @@ -629,7 +629,6 @@ func (w *writer) appendMessage2Group( message *common.DMLMessage, progress *partitionProgress, offset kafka.Offset, - spillDataArgs ...util.DMLMessageSpillData, ) error { // if the kafka cluster is normal, this should not hit. // else if the cluster is abnormal, the consumer may consume old message, then cause the watermark fallback. @@ -658,16 +657,7 @@ func (w *writer) appendMessage2Group( group = util.NewEventsGroup(progress.partition, tableID) progress.eventsGroup[tableID] = group } - spillData := util.DMLMessageSpillData{ - Restore: func([]byte) (*common.DMLMessage, error) { return message, nil }, - } - if len(spillDataArgs) > 0 { - spillData = spillDataArgs[0] - } - spillData.PostRestore = func(message *common.DMLMessage) *common.DMLMessage { - return w.messageWithPartitionCheck(message, progress.partition, offset) - } - if err := group.AppendSpillMessage(message, spillData); err != nil { + if err := group.AppendMessage(message); err != nil { return err } if commitTs < progress.watermark { diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index e388522c01..b285230e4a 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -296,9 +296,16 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { protocol: config.ProtocolOpen, } - w.appendMessage2Group(newDMLMessageForWriterTest(20), p, kafka.Offset(1)) - w.appendMessage2Group(newDMLMessageForWriterTest(10), p, kafka.Offset(2)) - w.appendMessage2Group(newDMLMessageForWriterTest(20), p, kafka.Offset(3)) + for _, item := range []struct { + message *codeccommon.DMLMessage + offset kafka.Offset + }{ + {newDMLMessageForWriterTest(20), kafka.Offset(1)}, + {newDMLMessageForWriterTest(10), kafka.Offset(2)}, + {newDMLMessageForWriterTest(20), kafka.Offset(3)}, + } { + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(item.message), p, item.offset)) + } p.watermark = 20 needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) @@ -318,7 +325,7 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { partition: 0, eventsGroup: make(map[int64]*util.EventsGroup), watermark: 20, - decoder: &singleDMLDecoder{message: newDMLMessageForWriterTest(10)}, + decoder: util.NewDMLMessageDecoder(&singleDMLDecoder{message: newDMLMessageForWriterTest(10)}), } w := &writer{ progresses: []*partitionProgress{progress}, @@ -356,7 +363,8 @@ func TestAppendMessageKeepsFallbackDMLAboveGlobalWatermark(t *testing.T) { protocol: config.ProtocolOpen, } - w.appendMessage2Group(newDMLMessageForWriterTest(10), progress, kafka.Offset(10)) + message := newDMLMessageForWriterTest(10) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(message), progress, kafka.Offset(10))) require.NotNil(t, progress.eventsGroup[1]) resolved, err := progress.eventsGroup[1].ResolveInto(20, nil) @@ -408,8 +416,10 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTableForAvro(t *testing.T) { } progress := w.progresses[0] - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) + first := codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)) + second := codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(first), progress, kafka.Offset(10))) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(second), progress, kafka.Offset(11))) resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) require.NoError(t, err) @@ -457,8 +467,10 @@ func TestAppendRow2GroupKeepsDebeziumPartitionTableFallback(t *testing.T) { } progress := w.progresses[0] - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) + first := codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)) + second := codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(first), progress, kafka.Offset(10))) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(second), progress, kafka.Offset(11))) resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) require.NoError(t, err) @@ -483,6 +495,16 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { }) } +func attachDMLMessageDataForWriterTest(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte, uint64) (*codeccommon.DMLMessage, error) { + return message, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + type singleDMLDecoder struct { message *codeccommon.DMLMessage consumed bool diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index 0d3272a274..6e0a76f338 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -42,14 +42,14 @@ type partitionProgress struct { partition int32 watermark uint64 eventsGroup map[int64]*util.EventsGroup - decoder common.Decoder + decoder *util.DMLMessageDecoder } func newPartitionProgress(partition int32, decoder common.Decoder) *partitionProgress { return &partitionProgress{ partition: partition, eventsGroup: make(map[int64]*util.EventsGroup), - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } } @@ -367,8 +367,7 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) (bool if dmlMessage == nil { log.Panic("DML message is nil, it's not expected") } - if err := w.appendMessage2Group(dmlMessage, progress, - util.NewDMLMessageSpillData(progress.decoder, []byte(message.Key()), message.Payload(), 0)); err != nil { + if err := w.appendMessage2Group(dmlMessage, progress); err != nil { return false, err } default: @@ -512,7 +511,6 @@ func (w *writer) addPartitionTable(schema, table string) { func (w *writer) appendMessage2Group( message *common.DMLMessage, progress *partitionProgress, - spillDataArgs ...util.DMLMessageSpillData, ) error { var ( tableID = message.TableID @@ -538,13 +536,7 @@ func (w *writer) appendMessage2Group( group = util.NewEventsGroup(progress.partition, tableID) progress.eventsGroup[tableID] = group } - spillData := util.DMLMessageSpillData{ - Restore: func([]byte) (*common.DMLMessage, error) { return message, nil }, - } - if len(spillDataArgs) > 0 { - spillData = spillDataArgs[0] - } - if err := group.AppendSpillMessage(message, spillData); err != nil { + if err := group.AppendMessage(message); err != nil { return err } if commitTs < progress.watermark { diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index 68a580f1c2..ef40a1c404 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -291,9 +291,13 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { protocol: config.ProtocolCanalJSON, } - w.appendMessage2Group(newDMLMessageForWriterTest(20), p) - w.appendMessage2Group(newDMLMessageForWriterTest(10), p) - w.appendMessage2Group(newDMLMessageForWriterTest(20), p) + for _, message := range []*codeccommon.DMLMessage{ + newDMLMessageForWriterTest(20), + newDMLMessageForWriterTest(10), + newDMLMessageForWriterTest(20), + } { + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(message), p)) + } p.watermark = 20 needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) @@ -323,7 +327,7 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { partition: 0, eventsGroup: make(map[int64]*util.EventsGroup), watermark: 20, - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } w := &writer{ progresses: []*partitionProgress{progress}, @@ -352,7 +356,8 @@ func TestAppendMessageKeepsFallbackDMLAboveGlobalWatermark(t *testing.T) { protocol: config.ProtocolCanalJSON, } - w.appendMessage2Group(newDMLMessageForWriterTest(10), progress) + message := newDMLMessageForWriterTest(10) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(message), progress)) require.NotNil(t, progress.eventsGroup[1]) resolved, err := progress.eventsGroup[1].ResolveInto(20, nil) @@ -389,8 +394,10 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTable(t *testing.T) { require.True(t, w.partitionTableAccessor.IsPartitionTable("target", "dst")) progress := w.progresses[0] - w.appendMessage2Group(newDMLMessageForWriterTest(200), progress) - w.appendMessage2Group(newDMLMessageForWriterTest(100), progress) + first := newDMLMessageForWriterTest(200) + second := newDMLMessageForWriterTest(100) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(first), progress)) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(second), progress)) resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) require.NoError(t, err) @@ -419,7 +426,7 @@ func TestWriteMessageSpillsDMLImmediately(t *testing.T) { progress := &partitionProgress{ partition: 0, eventsGroup: make(map[int64]*util.EventsGroup), - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } w := &writer{ progresses: []*partitionProgress{progress}, @@ -506,6 +513,16 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { }) } +func attachDMLMessageDataForWriterTest(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte, uint64) (*codeccommon.DMLMessage, error) { + return message, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + type fakePulsarMessage struct { key string payload []byte diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index defe151b11..3b81602dbb 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -285,7 +285,6 @@ func (c *consumer) getNewFiles( func (c *consumer) appendMessage2Group( message *common.DMLMessage, enableTableAcrossNodes bool, - spillData util.DMLMessageSpillData, ) error { var ( tableID = message.TableID @@ -299,7 +298,7 @@ func (c *consumer) appendMessage2Group( c.eventsGroup[tableID] = group } if commitTs >= group.HighWatermark { - if err := group.AppendSpillMessage(message, spillData); err != nil { + if err := group.AppendMessage(message); err != nil { return err } log.Debug("DML event append to the group", @@ -313,7 +312,7 @@ func (c *consumer) appendMessage2Group( zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return group.AppendSpillMessage(message, spillData) + return group.AppendMessage(message) } log.Warn("dml event commit ts fallback, ignore", zap.Uint64("commitTs", commitTs), @@ -361,11 +360,15 @@ func (c *consumer) appendDMLEvents( decoder.AddKeyValue(nil, content) } + spillDecoder := util.NewDMLMessageDecoderWithDataFactory(decoder, + func(_ common.Decoder, key, value []byte) *common.DMLMessageData { + return c.newDMLMessageData(ctx, schemaFile, value, tableID) + }) + spillDecoder.SetRawMessage(nil, content) cnt := 0 - dmlIndex := uint64(0) filteredCnt := 0 for { - tp, hasNext := decoder.HasNext() + tp, hasNext := spillDecoder.HasNext() if err != nil { log.Error("failed to decode message", zap.Error(err)) return err @@ -378,13 +381,8 @@ func (c *consumer) appendDMLEvents( if tp == common.MessageTypeRow { c.dmlCount.Add(1) - message := decoder.NextDMLMessage() - spillData := c.newDMLMessageSpillData(ctx, schemaFile, content, dmlIndex) - dmlIndex++ - spillData.PostRestore = func(message *common.DMLMessage) *common.DMLMessage { - return messageWithPhysicalTableID(message, tableID) - } - if err := c.appendMessage2Group(message, fileIdx.EnableTableAcrossNodes, spillData); err != nil { + message := spillDecoder.NextDMLMessage() + if err := c.appendMessage2Group(message, fileIdx.EnableTableAcrossNodes); err != nil { return err } filteredCnt++ @@ -399,15 +397,15 @@ func (c *consumer) appendDMLEvents( return err } -func (c *consumer) newDMLMessageSpillData( +func (c *consumer) newDMLMessageData( ctx context.Context, schemaFile cloudstorage.SchemaFile, content []byte, - dmlIndex uint64, -) util.DMLMessageSpillData { + tableID int64, +) *common.DMLMessageData { tableInfo := schemaFile.TableInfo() selector := c.columnSelectors.GetForTableInfo(tableInfo) - return util.NewDMLMessageSpillDataWithDecoderFactory(nil, content, dmlIndex, + messageData := util.NewDMLMessageDataWithDecoderFactory(nil, content, func(_ []byte, value []byte) (common.Decoder, error) { switch c.codecCfg.Protocol { case config.ProtocolCsv: @@ -419,6 +417,15 @@ func (c *consumer) newDMLMessageSpillData( return nil, errors.ErrSpillFileOp.FastGenByArgs("unsupported storage DML spill protocol") } }) + restore := messageData.Restore + messageData.Restore = func(data []byte, dmlIndex uint64) (*common.DMLMessage, error) { + message, err := restore(data, dmlIndex) + if err != nil { + return nil, err + } + return messageWithPhysicalTableID(message, tableID), nil + } + return messageData } func messageWithPhysicalTableID(message *common.DMLMessage, tableID int64) *common.DMLMessage { diff --git a/cmd/util/dml_message_decoder.go b/cmd/util/dml_message_decoder.go new file mode 100644 index 0000000000..360f1ac947 --- /dev/null +++ b/cmd/util/dml_message_decoder.go @@ -0,0 +1,112 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + +// DMLMessageDataFactory creates data shared by DML messages decoded from one +// input. It is called lazily when the decoder first returns a DML message. +type DMLMessageDataFactory func(codeccommon.Decoder, []byte, []byte) *codeccommon.DMLMessageData + +// DMLMessageDecoder attaches spill data to DML messages as they are decoded. +// It keeps raw input only until EventsGroup has written it to the spill file. +type DMLMessageDecoder struct { + codeccommon.Decoder + + key, value []byte + data *codeccommon.DMLMessageData + factory DMLMessageDataFactory + restore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage +} + +// NewDMLMessageDecoder wraps a decoder with the standard raw-message restorer. +func NewDMLMessageDecoder(decoder codeccommon.Decoder) *DMLMessageDecoder { + return NewDMLMessageDecoderWithDataFactory(decoder, + func(decoder codeccommon.Decoder, key, value []byte) *codeccommon.DMLMessageData { + return NewDMLMessageData(decoder, key, value) + }) +} + +// NewDMLMessageDecoderWithDataFactory is for decoders such as CSV whose +// restore decoder must be constructed from the input value. +func NewDMLMessageDecoderWithDataFactory( + decoder codeccommon.Decoder, factory DMLMessageDataFactory, +) *DMLMessageDecoder { + return &DMLMessageDecoder{Decoder: decoder, factory: factory} +} + +// SetDMLMessageRestorer sets the per-input restore wrapper before AddKeyValue. +func (d *DMLMessageDecoder) SetDMLMessageRestorer( + restore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, +) { + d.restore = restore +} + +// AddKeyValue implements codeccommon.Decoder. +func (d *DMLMessageDecoder) AddKeyValue(key, value []byte) { + d.Decoder.AddKeyValue(key, value) + d.SetRawMessage(key, value) +} + +// SetRawMessage records input that was supplied while constructing a decoder, +// such as a CSV decoder. It does not pass the input to the wrapped decoder. +func (d *DMLMessageDecoder) SetRawMessage(key, value []byte) { + d.key = key + d.value = value + d.data = nil +} + +// NextDMLMessage implements codeccommon.Decoder. +func (d *DMLMessageDecoder) NextDMLMessage() *codeccommon.DMLMessage { + message := d.Decoder.NextDMLMessage() + if message != nil { + d.attachDMLMessage(message) + } + return message +} + +func (d *DMLMessageDecoder) attachDMLMessage(message *codeccommon.DMLMessage) { + if d.data == nil { + d.data = d.wrapRestore(d.factory(d.Decoder, d.key, d.value)) + } + d.data.AttachDMLMessage(message) +} + +// AttachCachedDMLMessage attaches data to a materialized DML message from +// Simple's DDL cache. It has no raw row payload to restore. +func (d *DMLMessageDecoder) AttachCachedDMLMessage(message *codeccommon.DMLMessage) { + data := codeccommon.NewDMLMessageData(nil, nil, + func([]byte, uint64) (*codeccommon.DMLMessage, error) { return message, nil }) + d.wrapRestore(data).AttachDMLMessage(message) +} + +func (d *DMLMessageDecoder) wrapRestore(data *codeccommon.DMLMessageData) *codeccommon.DMLMessageData { + if d.restore == nil { + return data + } + restore := data.Restore + data.Restore = func(data []byte, dmlIndex uint64) (*codeccommon.DMLMessage, error) { + message, err := restore(data, dmlIndex) + if err != nil { + return nil, err + } + return d.restore(message), nil + } + return data +} + +// Unwrap returns the decoder that produces protocol messages. +func (d *DMLMessageDecoder) Unwrap() codeccommon.Decoder { + return d.Decoder +} diff --git a/cmd/util/dml_message_decoder_test.go b/cmd/util/dml_message_decoder_test.go new file mode 100644 index 0000000000..7cf58d7c82 --- /dev/null +++ b/cmd/util/dml_message_decoder_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" +) + +func TestDMLMessageDecoderAttachesSharedData(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(11) + decoder := &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{first, second}} + wrapped := NewDMLMessageDecoder(decoder) + + wrapped.AddKeyValue([]byte("key"), []byte("value")) + decodedFirst := wrapped.NextDMLMessage() + firstData, firstIndex := decodedFirst.SpillData() + require.Equal(t, []byte("key"), firstData.Key) + require.Equal(t, []byte("value"), firstData.Value) + require.Zero(t, firstIndex) + + decodedSecond := wrapped.NextDMLMessage() + secondData, secondIndex := decodedSecond.SpillData() + require.Same(t, firstData, secondData) + require.Equal(t, uint64(1), secondIndex) +} + +type dmlMessageDecoderStub struct { + messages []*codeccommon.DMLMessage +} + +func (d *dmlMessageDecoderStub) AddKeyValue(_, _ []byte) {} + +func (d *dmlMessageDecoderStub) HasNext() (codeccommon.MessageType, bool) { + return codeccommon.MessageTypeRow, len(d.messages) > 0 +} + +func (d *dmlMessageDecoderStub) NextResolvedEvent() uint64 { return 0 } + +func (d *dmlMessageDecoderStub) NextDMLMessage() *codeccommon.DMLMessage { + if len(d.messages) == 0 { + return nil + } + message := d.messages[0] + d.messages = d.messages[1:] + return message +} + +func (d *dmlMessageDecoderStub) NextDDLEvent() *commonEvent.DDLEvent { return nil } diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index ad6f8590a5..f6246a63b9 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -30,18 +30,10 @@ import ( const eventsGroupSpillPattern = "ticdc-events-group-*.spill" type spilledMessage struct { - commitTs uint64 - handle spill.Handle - restore func([]byte) (*codeccommon.DMLMessage, error) - postRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage -} - -// DMLMessageSpillData is an opaque codec payload which can restore a lazy DML -// message after it is read from disk. Restore must not construct a DMLEvent. -type DMLMessageSpillData struct { - Data []byte - Restore func([]byte) (*codeccommon.DMLMessage, error) - PostRestore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage + commitTs uint64 + handle spill.Handle + dmlIndex uint64 + restore func([]byte, uint64) (*codeccommon.DMLMessage, error) } // EventsGroup stores change event messages. @@ -51,6 +43,7 @@ type EventsGroup struct { messages []spilledMessage spillFile *spill.RecordFile + spillHandles map[uint64]spill.Handle outOfOrder bool HighWatermark uint64 } @@ -64,27 +57,17 @@ func NewEventsGroup(partition int32, tableID int64) *EventsGroup { } } -// AppendMessage is kept for tests that already own a materialized event. Consumer -// code must use AppendSpillMessage so DMLEvent construction stays at flush time. -func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage) error { - return g.AppendSpillMessage(message, DMLMessageSpillData{ - Restore: func([]byte) (*codeccommon.DMLMessage, error) { - return message, nil - }, - }) -} - -// AppendSpillMessage appends an opaque codec payload to the spill file. It does +// AppendMessage appends an opaque codec payload to the spill file. It does // not call DMLMessage.ToDMLEvent; the restored message remains lazy until the // consumer flushes it against a watermark or DDL barrier. -func (g *EventsGroup) AppendSpillMessage( +func (g *EventsGroup) AppendMessage( message *codeccommon.DMLMessage, - spillData DMLMessageSpillData, ) error { if message == nil { return errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") } - if spillData.Restore == nil { + messageData, dmlIndex := message.SpillData() + if messageData == nil || messageData.Restore == nil { return errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without restore function") } commitTs := message.GetCommitTs() @@ -96,16 +79,24 @@ func (g *EventsGroup) AppendSpillMessage( return err } } - data := spillData.Data - if len(data) == 0 { - // A lazy message may not need an input payload (for example, a Simple - // decoder message released from its table-info cache). RecordFile rejects - // empty records, so retain a marker while keeping the event lazy. - data = []byte{0} + if g.spillHandles == nil { + g.spillHandles = make(map[uint64]spill.Handle) } - handle, err := g.spillFile.Append(data) - if err != nil { - return err + handle, ok := g.spillHandles[messageData.ID] + if !ok { + data := marshalDMLMessageData(messageData.Key, messageData.Value) + if len(data) == 0 { + // A lazy message may not need an input payload (for example, a Simple + // decoder message released from its table-info cache). RecordFile rejects + // empty records, so retain a marker while keeping the event lazy. + data = []byte{0} + } + var err error + handle, err = g.spillFile.Append(data) + if err != nil { + return err + } + g.spillHandles[messageData.ID] = handle } if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].commitTs { g.outOfOrder = true @@ -114,10 +105,10 @@ func (g *EventsGroup) AppendSpillMessage( g.HighWatermark = commitTs } g.messages = append(g.messages, spilledMessage{ - commitTs: commitTs, - handle: handle, - restore: spillData.Restore, - postRestore: spillData.PostRestore, + commitTs: commitTs, + handle: handle, + dmlIndex: dmlIndex, + restore: messageData.Restore, }) return nil } @@ -132,6 +123,7 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) return dst, err } g.spillFile = nil + clear(g.spillHandles) } return dst, nil } @@ -165,13 +157,10 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) if err != nil { return dst, err } - restored, err := message.restore(data) + restored, err := message.restore(data, message.dmlIndex) if err != nil { return dst, err } - if message.postRestore != nil { - restored = message.postRestore(restored) - } dst = append(dst, restored) } remainingCount := len(g.messages) - resolvedCount @@ -183,6 +172,7 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) return dst, err } g.spillFile = nil + clear(g.spillHandles) } if len(g.messages) != 0 { firstCommitTs := g.messages[0].commitTs @@ -209,6 +199,7 @@ func (g *EventsGroup) Cleanup() error { return err } g.spillFile = nil + clear(g.spillHandles) clear(g.messages) g.messages = g.messages[:0] return nil @@ -244,27 +235,7 @@ func appendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLE } func sameDMLTransaction(last, row *commonEvent.DMLEvent) bool { - return last != nil && row != nil && - last.CommitTs == row.CommitTs && - last.StartTs == row.StartTs && - last.DispatcherID == row.DispatcherID && - last.PhysicalTableID == row.PhysicalTableID && - last.TableInfoVersion == row.TableInfoVersion && - last.TableInfo != nil && row.TableInfo != nil && - last.TableInfo.GetSchemaName() == row.TableInfo.GetSchemaName() && - last.TableInfo.GetTableName() == row.TableInfo.GetTableName() && - last.TableInfo.GetUpdateTS() == row.TableInfo.GetUpdateTS() && - last.Rows != nil && row.Rows != nil && - last.Rows.NumCols() == row.Rows.NumCols() && - last.PreviousTotalOffset == 0 && row.PreviousTotalOffset == 0 && - hasOptionalDMLValues(last.RowKeys, len(last.RowTypes)) && - hasOptionalDMLValues(row.RowKeys, len(row.RowTypes)) && - hasOptionalDMLValues(last.Checksum, len(last.RowTypes)) && - hasOptionalDMLValues(row.Checksum, len(row.RowTypes)) -} - -func hasOptionalDMLValues[T any](values []T, rowTypeCount int) bool { - return len(values) == 0 || len(values) == rowTypeCount + return last != nil && row != nil && last.CommitTs == row.CommitTs } func appendOptionalDMLValues[T any](last, row []T, lastRowTypeCount, rowRowTypeCount int) []T { @@ -276,32 +247,22 @@ func appendOptionalDMLValues[T any](last, row []T, lastRowTypeCount, rowRowTypeC return append(last, row...) } -// NewDMLMessageSpillData preserves the original codec input and the ordinal of -// its DML message. The decoder is intentionally used only during ResolveInto: -// this preserves the old behaviour where schema is selected at the DDL or -// watermark barrier, rather than while the message is appended. -func NewDMLMessageSpillData( - decoder codeccommon.Decoder, key, value []byte, dmlIndex uint64, -) DMLMessageSpillData { - return NewDMLMessageSpillDataWithDecoderFactory(key, value, dmlIndex, +// NewDMLMessageData preserves one original codec input. The decoder is used +// only during ResolveInto, after the DDL or watermark barrier selects schema. +func NewDMLMessageData(decoder codeccommon.Decoder, key, value []byte) *codeccommon.DMLMessageData { + return NewDMLMessageDataWithDecoderFactory(key, value, func([]byte, []byte) (codeccommon.Decoder, error) { return decoder, nil }) } -// NewDMLMessageSpillDataWithDecoderFactory is for decoders such as CSV whose +// NewDMLMessageDataWithDecoderFactory is for decoders such as CSV whose // input is supplied during construction rather than through AddKeyValue. -func NewDMLMessageSpillDataWithDecoderFactory( +func NewDMLMessageDataWithDecoderFactory( key, value []byte, - dmlIndex uint64, decoderFactory func([]byte, []byte) (codeccommon.Decoder, error), -) DMLMessageSpillData { - data := make([]byte, 0, 3*8+len(key)+len(value)) - data = appendSpillUint64(data, dmlIndex) - data = appendSpillBytes(data, key) - data = appendSpillBytes(data, value) - return DMLMessageSpillData{ - Data: data, - Restore: func(data []byte) (*codeccommon.DMLMessage, error) { - dmlIndex, key, value, err := unmarshalDMLMessageSpillData(data) +) *codeccommon.DMLMessageData { + return codeccommon.NewDMLMessageData(key, value, + func(data []byte, dmlIndex uint64) (*codeccommon.DMLMessage, error) { + key, value, err := unmarshalDMLMessageData(data) if err != nil { return nil, err } @@ -310,8 +271,16 @@ func NewDMLMessageSpillDataWithDecoderFactory( return nil, errors.WrapError(errors.ErrSpillFileOp, err, "create DML spill decoder") } return restoreDMLMessage(decoder, key, value, dmlIndex) - }, + }) +} + +func marshalDMLMessageData(key, value []byte) []byte { + if len(key) == 0 && len(value) == 0 { + return nil } + data := make([]byte, 0, 2*8+len(key)+len(value)) + data = appendSpillBytes(data, key) + return appendSpillBytes(data, value) } func restoreDMLMessage( @@ -351,23 +320,19 @@ func appendSpillUint64(data []byte, value uint64) []byte { return append(data, buf[:]...) } -func unmarshalDMLMessageSpillData(data []byte) (uint64, []byte, []byte, error) { - dmlIndex, data, err := readSpillUint64(data) - if err != nil { - return 0, nil, nil, err - } +func unmarshalDMLMessageData(data []byte) ([]byte, []byte, error) { key, data, err := readSpillBytes(data) if err != nil { - return 0, nil, nil, err + return nil, nil, err } value, data, err := readSpillBytes(data) if err != nil { - return 0, nil, nil, err + return nil, nil, err } if len(data) != 0 { - return 0, nil, nil, errors.ErrSpillFileOp.FastGenByArgs("unexpected trailing DML spill data") + return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("unexpected trailing DML spill data") } - return dmlIndex, key, value, nil + return key, value, nil } func readSpillBytes(data []byte) ([]byte, []byte, error) { diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index 159c977a2f..77427c4d65 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -36,6 +36,16 @@ func newTestDMLMessage(commitTs uint64) *codeccommon.DMLMessage { return codeccommon.NewDMLMessageFromEvent(newTestDMLEvent(commitTs, common.RowTypeInsert)) } +func attachTestDMLMessageData(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte, uint64) (*codeccommon.DMLMessage, error) { + return message, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.DMLEvent { return &commonEvent.DMLEvent{ PhysicalTableID: 1, @@ -115,25 +125,10 @@ func TestAppendOrMergeDMLEvent(t *testing.T) { require.Equal(t, []int{1, 2}, flushed) }) - t.Run("keep different schema layouts separate", func(t *testing.T) { - first := newMergeTestDMLEvent(100, newMergeTestTableInfo(1, 10, 1), 1) - second := newMergeTestDMLEvent(100, newMergeTestTableInfo(1, 11, 2), 2) - - events := DMLMessagesToEvents([]*codeccommon.DMLMessage{ - newMergeTestDMLMessage(first), - newMergeTestDMLMessage(second), - }) - - require.Len(t, events, 2) - require.Same(t, first, events[0]) - require.Same(t, second, events[1]) - }) - - t.Run("keep different dispatchers separate", func(t *testing.T) { + t.Run("keep different commit timestamps separate", func(t *testing.T) { tableInfo := newMergeTestTableInfo(1, 10, 1) first := newMergeTestDMLEvent(100, tableInfo, 1) - second := newMergeTestDMLEvent(100, tableInfo, 2) - second.DispatcherID = common.DispatcherID{Low: 2} + second := newMergeTestDMLEvent(101, tableInfo, 2) events := DMLMessagesToEvents([]*codeccommon.DMLMessage{ newMergeTestDMLMessage(first), @@ -148,8 +143,10 @@ func TestAppendOrMergeDMLEvent(t *testing.T) { group := NewEventsGroup(0, 1) first := newMergeTestDMLEvent(100, tableInfo, 1) second := newMergeTestDMLEvent(100, tableInfo, 2) - require.NoError(t, group.AppendMessage(newMergeTestDMLMessage(first))) - require.NoError(t, group.AppendMessage(newMergeTestDMLMessage(second))) + firstMessage := newMergeTestDMLMessage(first) + secondMessage := newMergeTestDMLMessage(second) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(firstMessage))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(secondMessage))) messages, err := group.GetAllMessages() require.NoError(t, err) @@ -161,6 +158,31 @@ func TestAppendOrMergeDMLEvent(t *testing.T) { }) } +func TestEventsGroupSharesRawMessageData(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(10) + messageData := codeccommon.NewDMLMessageData(nil, []byte("raw message"), + func(_ []byte, dmlIndex uint64) (*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{first, second}[dmlIndex], nil + }, + ) + + group := NewEventsGroup(0, 1) + messageData.AttachDMLMessage(first) + require.NoError(t, group.AppendMessage(first)) + messageData.AttachDMLMessage(second) + require.NoError(t, group.AppendMessage(second)) + require.Len(t, group.messages, 2) + require.Equal(t, group.messages[0].handle, group.messages[1].handle) + require.Equal(t, uint64(0), group.messages[0].dmlIndex) + require.Equal(t, uint64(1), group.messages[1].dmlIndex) + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Same(t, first, messages[0]) + require.Same(t, second, messages[1]) +} + func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T) { // Scenario: A consumer resolves events by watermark/commit-ts and appends them into a downstream // batch slice. Buffered messages are held only by a spill file, and the file must be cleaned once @@ -175,9 +197,9 @@ func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T m1 := newTestDMLMessage(1) m2 := newTestDMLMessage(2) m3 := newTestDMLMessage(3) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) spillPath := group.spillFile.Path() @@ -206,8 +228,8 @@ func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { group := NewEventsGroup(0, 1) m1 := newTestDMLMessage(10) m2 := newTestDMLMessage(20) - group.AppendMessage(m1) - group.AppendMessage(m2) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) dst := make([]*codeccommon.DMLMessage, 0, 1) dst, err := group.ResolveInto(5, dst) @@ -225,8 +247,8 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { group := NewEventsGroup(0, 1) m1 := newTestDMLMessage(1) m2 := newTestDMLMessage(2) - group.AppendMessage(m1) - group.AppendMessage(m2) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) spillPath := group.spillFile.Path() var dst []*codeccommon.DMLMessage @@ -248,9 +270,9 @@ func TestEventsGroupResolveIntoSortsOutOfOrderResolvedMessages(t *testing.T) { m1 := newTestDMLMessage(20) m2 := newTestDMLMessage(10) m3 := newTestDMLMessage(30) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) var dst []*codeccommon.DMLMessage dst, err := group.ResolveInto(25, dst) @@ -269,9 +291,9 @@ func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { m1 := newTestDMLMessage(20) m2 := newTestDMLMessage(10) m3 := newTestDMLMessage(20) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) var dst []*codeccommon.DMLMessage dst, err := group.ResolveInto(20, dst) @@ -289,9 +311,9 @@ func TestEventsGroupGetAllMessagesSortsOutOfOrderMessages(t *testing.T) { m1 := newTestDMLMessage(20) m2 := newTestDMLMessage(10) m3 := newTestDMLMessage(30) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) messages, err := group.GetAllMessages() require.NoError(t, err) @@ -326,7 +348,8 @@ func TestEventsGroupRestoresSpilledEventRowsAndTableInfo(t *testing.T) { event.Checksum = []*integrity.Checksum{{Current: 1, Previous: 2, Corrupted: true, Version: 3}} group := NewEventsGroup(0, 1) - group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event)) + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) messages, err := group.GetAllMessages() require.NoError(t, err) @@ -363,7 +386,8 @@ func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { event.RowTypes = []common.RowType{common.RowTypeUpdate, common.RowTypeUpdate} event.Length = 1 event.PreviousTotalOffset = offset - require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) } messages, err := group.GetAllMessages() @@ -404,7 +428,8 @@ func TestEventsGroupRestoresCompactUpdateRows(t *testing.T) { event.Length = 1 group := NewEventsGroup(0, 1) - require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) messages, err := group.GetAllMessages() require.NoError(t, err) require.Len(t, messages, 1) @@ -427,7 +452,8 @@ func TestEventsGroupSpillDoesNotSignalDownstreamCallbacks(t *testing.T) { group := NewEventsGroup(0, 1) defer func() { require.NoError(t, group.Cleanup()) }() - require.NoError(t, group.AppendMessage(codeccommon.NewDMLMessageFromEvent(event))) + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) require.Zero(t, enqueued) require.Zero(t, flushed) } @@ -471,7 +497,7 @@ func BenchmarkEventsGroupResolveInto(b *testing.B) { for b.Loop() { group := NewEventsGroup(0, 1) for _, message := range source { - if err := group.AppendMessage(message); err != nil { + if err := group.AppendMessage(attachTestDMLMessageData(message)); err != nil { b.Fatal(err) } } diff --git a/pkg/sink/codec/common/decoder.go b/pkg/sink/codec/common/decoder.go index 86e5a8f5fa..c4803bea82 100644 --- a/pkg/sink/codec/common/decoder.go +++ b/pkg/sink/codec/common/decoder.go @@ -14,10 +14,39 @@ package common import ( + "sync/atomic" + commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" ) +// DMLMessageData keeps the original encoded input needed to restore a DML +// message after it has been spilled. One input can be attached to multiple +// DMLMessages; Attach assigns each message its ordinal in that input. +type DMLMessageData struct { + ID uint64 + Key []byte + Value []byte + Restore func([]byte, uint64) (*DMLMessage, error) + + nextDMLIndex uint64 +} + +var nextDMLMessageDataID atomic.Uint64 + +// NewDMLMessageData creates data shared by DMLMessages decoded from one input. +func NewDMLMessageData( + key, value []byte, + restore func([]byte, uint64) (*DMLMessage, error), +) *DMLMessageData { + return &DMLMessageData{ + ID: nextDMLMessageDataID.Add(1), + Key: key, + Value: value, + Restore: restore, + } +} + type DMLMessage struct { TableID int64 Schema string @@ -28,6 +57,8 @@ type DMLMessage struct { // toDMLEvent may be called after the decoder has consumed later messages. // It must only use data captured by this DMLMessage and must not depend on decoder cursor state. toDMLEvent func() *commonEvent.DMLEvent + spillData *DMLMessageData + dmlIndex uint64 } func NewDMLMessage( @@ -74,6 +105,21 @@ func (m *DMLMessage) ToDMLEvent() *commonEvent.DMLEvent { return m.toDMLEvent() } +// AttachDMLMessageData attaches the original input required to restore this +// message after spill. It must be called once for every decoded DML, including +// DMLs the consumer later discards. +func (d *DMLMessageData) AttachDMLMessage(message *DMLMessage) { + message.spillData = d + message.dmlIndex = d.nextDMLIndex + d.nextDMLIndex++ +} + +// SpillData returns the data and ordinal attached while this message was +// decoded. They are used by the consumer's in-memory events group only. +func (m *DMLMessage) SpillData() (*DMLMessageData, uint64) { + return m.spillData, m.dmlIndex +} + // Decoder is an abstraction for events decoder // this interface is only for testing now type Decoder interface { From 0e3ff677552d1820faf44e9c3a59d508b84b4658 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 28 Aug 2026 05:47:55 +0000 Subject: [PATCH 14/22] fmt Signed-off-by: wk989898 --- cmd/storage-consumer/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 3b81602dbb..79395d4e2d 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -361,7 +361,7 @@ func (c *consumer) appendDMLEvents( } spillDecoder := util.NewDMLMessageDecoderWithDataFactory(decoder, - func(_ common.Decoder, key, value []byte) *common.DMLMessageData { + func(_ common.Decoder, _, value []byte) *common.DMLMessageData { return c.newDMLMessageData(ctx, schemaFile, value, tableID) }) spillDecoder.SetRawMessage(nil, content) From fac1d9378fc643d40e6001a18055bf6b0dd5e6c5 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Fri, 28 Aug 2026 05:54:07 +0000 Subject: [PATCH 15/22] chore Signed-off-by: wk989898 --- tests/integration_tests/run_heavy_it_in_ci.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/integration_tests/run_heavy_it_in_ci.sh b/tests/integration_tests/run_heavy_it_in_ci.sh index f91efb5a57..4eea519108 100755 --- a/tests/integration_tests/run_heavy_it_in_ci.sh +++ b/tests/integration_tests/run_heavy_it_in_ci.sh @@ -194,10 +194,6 @@ echo "Group Number (parsed): ${group_num}" if [[ $group_num =~ ^[0-9]+$ ]] && [[ -n ${groups[10#${group_num}]} ]]; then # force use decimal index test_names="${groups[10#${group_num}]}" - if [[ "$sink_type" == "mysql" ]]; then - # Temporarily run the regression case in every MySQL shard. - test_names="ddl_for_split_tables_with_random_merge_and_split" - fi # Run test cases echo "Run cases: ${test_names}" export TICDC_NEWARCH=true From 2c4538d0f89d51b7aa973b17c25a817ba8b895d8 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 1 Sep 2026 07:26:03 +0000 Subject: [PATCH 16/22] fix Signed-off-by: wk989898 --- cmd/storage-consumer/consumer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 79395d4e2d..4febbe4e04 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -284,10 +284,10 @@ func (c *consumer) getNewFiles( func (c *consumer) appendMessage2Group( message *common.DMLMessage, + tableID int64, enableTableAcrossNodes bool, ) error { var ( - tableID = message.TableID schema = message.Schema table = message.Table commitTs = message.GetCommitTs() @@ -382,7 +382,7 @@ func (c *consumer) appendDMLEvents( c.dmlCount.Add(1) message := spillDecoder.NextDMLMessage() - if err := c.appendMessage2Group(message, fileIdx.EnableTableAcrossNodes); err != nil { + if err := c.appendMessage2Group(message, tableID, fileIdx.EnableTableAcrossNodes); err != nil { return err } filteredCnt++ From 9785419c2f25136c69ea687b430bae7f21feb258 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Tue, 1 Sep 2026 08:54:22 +0000 Subject: [PATCH 17/22] fix Signed-off-by: wk989898 --- cmd/util/event_group.go | 45 ++++++++++++++++++++++++------------ cmd/util/event_group_test.go | 33 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index f6246a63b9..bed6161379 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -18,6 +18,7 @@ import ( "math" "os" "sort" + "sync" "github.com/pingcap/log" commonEvent "github.com/pingcap/ticdc/pkg/common/event" @@ -260,17 +261,33 @@ func NewDMLMessageDataWithDecoderFactory( key, value []byte, decoderFactory func([]byte, []byte) (codeccommon.Decoder, error), ) *codeccommon.DMLMessageData { + var ( + once sync.Once + messages []*codeccommon.DMLMessage + err error + ) return codeccommon.NewDMLMessageData(key, value, func(data []byte, dmlIndex uint64) (*codeccommon.DMLMessage, error) { - key, value, err := unmarshalDMLMessageData(data) + once.Do(func() { + key, value, unmarshalErr := unmarshalDMLMessageData(data) + if unmarshalErr != nil { + err = unmarshalErr + return + } + decoder, decoderErr := decoderFactory(key, value) + if decoderErr != nil { + err = errors.WrapError(errors.ErrSpillFileOp, decoderErr, "create DML spill decoder") + return + } + messages, err = restoreDMLMessages(decoder, key, value) + }) if err != nil { return nil, err } - decoder, err := decoderFactory(key, value) - if err != nil { - return nil, errors.WrapError(errors.ErrSpillFileOp, err, "create DML spill decoder") + if dmlIndex >= uint64(len(messages)) { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill message index is out of range") } - return restoreDMLMessage(decoder, key, value, dmlIndex) + return messages[dmlIndex], nil }) } @@ -283,18 +300,18 @@ func marshalDMLMessageData(key, value []byte) []byte { return appendSpillBytes(data, value) } -func restoreDMLMessage( - decoder codeccommon.Decoder, key, value []byte, dmlIndex uint64, -) (*codeccommon.DMLMessage, error) { +func restoreDMLMessages( + decoder codeccommon.Decoder, key, value []byte, +) ([]*codeccommon.DMLMessage, error) { decoder.AddKeyValue(key, value) - var restored *codeccommon.DMLMessage - for currentIndex := uint64(0); ; currentIndex++ { + messages := make([]*codeccommon.DMLMessage, 0, 1) + for { messageType, hasNext := decoder.HasNext() if !hasNext { - if restored == nil { + if len(messages) == 0 { return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload has no message") } - return restored, nil + return messages, nil } if messageType != codeccommon.MessageTypeRow { return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload contains a non-DML message") @@ -303,9 +320,7 @@ func restoreDMLMessage( if message == nil { return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload cannot be restored") } - if currentIndex == dmlIndex { - restored = message - } + messages = append(messages, message) } } diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index 77427c4d65..ab7132551d 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -183,6 +183,39 @@ func TestEventsGroupSharesRawMessageData(t *testing.T) { require.Same(t, second, messages[1]) } +func TestEventsGroupRestoresSharedSpillInputOnce(t *testing.T) { + // A single canal-json input can contain thousands of DML messages. Restoring + // each ordinal must not re-decode the complete input. + inputMessages := []*codeccommon.DMLMessage{ + newTestDMLMessage(30), + newTestDMLMessage(10), + newTestDMLMessage(20), + } + var decoderCount int + messageData := NewDMLMessageDataWithDecoderFactory(nil, []byte("raw message"), + func(_, _ []byte) (codeccommon.Decoder, error) { + decoderCount++ + return &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{ + newTestDMLMessage(30), + newTestDMLMessage(10), + newTestDMLMessage(20), + }}, nil + }) + + group := NewEventsGroup(0, 1) + for _, message := range inputMessages { + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) + } + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Equal(t, 1, decoderCount) + require.Equal(t, []uint64{10, 20, 30}, []uint64{ + messages[0].GetCommitTs(), messages[1].GetCommitTs(), messages[2].GetCommitTs(), + }) +} + func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T) { // Scenario: A consumer resolves events by watermark/commit-ts and appends them into a downstream // batch slice. Buffered messages are held only by a spill file, and the file must be cleaned once From 90f2b06dc41f1fb1c7c2a92442ff71207e73dcc5 Mon Sep 17 00:00:00 2001 From: wk989898 Date: Wed, 2 Sep 2026 03:02:07 +0000 Subject: [PATCH 18/22] set memory limit Signed-off-by: wk989898 --- cmd/kafka-consumer/main.go | 2 +- cmd/pulsar-consumer/main.go | 2 +- cmd/storage-consumer/main.go | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/kafka-consumer/main.go b/cmd/kafka-consumer/main.go index d6d3e65058..efd9f71b2d 100644 --- a/cmd/kafka-consumer/main.go +++ b/cmd/kafka-consumer/main.go @@ -38,7 +38,7 @@ var ( ) func main() { - debug.SetMemoryLimit(14 * 1024 * 1024 * 1024) + debug.SetMemoryLimit(8 * 1024 * 1024 * 1024) var ( upstreamURIStr string configFile string diff --git a/cmd/pulsar-consumer/main.go b/cmd/pulsar-consumer/main.go index c1a7ed385d..25dff1ea32 100644 --- a/cmd/pulsar-consumer/main.go +++ b/cmd/pulsar-consumer/main.go @@ -39,7 +39,7 @@ var ( ) func main() { - debug.SetMemoryLimit(14 * 1024 * 1024 * 1024) + debug.SetMemoryLimit(8 * 1024 * 1024 * 1024) cmd := &cobra.Command{ Use: "pulsar consumer", Run: run, diff --git a/cmd/storage-consumer/main.go b/cmd/storage-consumer/main.go index 70256977f4..36a7c51ef0 100644 --- a/cmd/storage-consumer/main.go +++ b/cmd/storage-consumer/main.go @@ -20,6 +20,7 @@ import ( "net/url" "os" "os/signal" + "runtime/debug" "strings" "syscall" "time" @@ -82,6 +83,7 @@ func init() { } func main() { + debug.SetMemoryLimit(8 * 1024 * 1024 * 1024) var consumer *consumer var err error From 08a5713a08c34d7375d40b15613bb72ce04a87ae Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Wed, 2 Sep 2026 22:02:01 +0800 Subject: [PATCH 19/22] consumer: bound spill restore by encoded payload Share payload ownership across event groups so each encoded input is written, read, and decoded once. Restore bounded batches, release them only after downstream flush acknowledgement, and reclaim 128 MiB spill segments with 1 GiB/512 MiB drain watermarks. --- cmd/kafka-consumer/writer.go | 199 ++++++---- cmd/kafka-consumer/writer_test.go | 4 +- cmd/pulsar-consumer/writer.go | 199 ++++++---- cmd/pulsar-consumer/writer_test.go | 4 +- cmd/storage-consumer/consumer.go | 114 ++++-- cmd/util/dml_message_decoder.go | 21 +- cmd/util/event_group.go | 613 +++++++++++++++++++++++------ cmd/util/event_group_test.go | 346 +++++++++++++++- pkg/sink/codec/common/decoder.go | 20 +- 9 files changed, 1172 insertions(+), 348 deletions(-) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index f5d6603998..2c3e6dd1c7 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -87,6 +87,7 @@ type writer struct { maxBatchSize int mysqlSink sink.Sink enableTableAcrossNodes bool + spillStore *util.SpillStore } func newWriter(ctx context.Context, o *option) *writer { @@ -99,6 +100,7 @@ func newWriter(ctx context.Context, o *option) *writer { ddlList: make([]*event.DDLEvent, 0), ddlWithMaxCommitTs: make(map[int64]uint64), enableTableAcrossNodes: o.enableTableAcrossNodes, + spillStore: util.NewSpillStore(), } var ( db *sql.DB @@ -147,62 +149,129 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } +func (w *writer) getSpillStore() *util.SpillStore { + if w.spillStore == nil { + w.spillStore = util.NewSpillStore() + } + return w.spillStore +} + func (w *writer) cleanupEventsGroups() error { var cleanupErr error for _, progress := range w.progresses { for _, group := range progress.eventsGroup { - if err := group.Cleanup(); err != nil { - log.Warn("cleanup events group spill file failed", - zap.Int32("partition", progress.partition), zap.Error(err)) - if cleanupErr == nil { - cleanupErr = err - } - } + _ = group.Cleanup() } } + if err := w.getSpillStore().Cleanup(); err != nil { + cleanupErr = err + log.Warn("cleanup spill store failed", zap.Error(err)) + } return cleanupErr } func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { - var ( - done = make(chan struct{}, 1) - - flushed atomic.Int64 - ) - tableIDs := w.getBlockTableIDs(ddl) commitTs := ddl.GetCommitTs() - resolvedEvents := make([]*event.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for tableID := range tableIDs { for _, progress := range w.progresses { g, ok := progress.eventsGroup[tableID] if !ok { continue } - messages, err := g.ResolveInto(commitTs, nil) + groups = append(groups, g) + } + } + total, err := w.flushEventsFromGroups(ctx, groups, commitTs, + zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query)) + if err != nil { + return err + } + + if total != 0 { + log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Any("tables", tableIDs)) + } + return w.mysqlSink.WriteBlockEvent(ddl) +} + +func (w *writer) flushEventsFromGroups( + ctx context.Context, groups []*util.EventsGroup, resolveTs uint64, fields ...zap.Field, +) (int, error) { + limit := w.getSpillStore().ResolveLimit() + batchEvents := make([]*event.DMLEvent, 0, limit.MaxMessages) + batchMessages := 0 + var batchBytes int64 + total := 0 + prepared := make([]*util.ResolveBatch, 0, len(groups)) + flush := func() error { + if err := w.flushDMLBatch(ctx, batchEvents, fields...); err != nil { + return err + } + for _, batch := range prepared { + batch.Ack() + } + total += len(batchEvents) + batchEvents = nil + prepared = prepared[:0] + batchMessages = 0 + batchBytes = 0 + return nil + } + + for { + hasMoreGroups := false + preparedAny := false + for _, group := range groups { + if batchMessages >= limit.MaxMessages || batchBytes >= limit.MaxBytes { + if err := flush(); err != nil { + return 0, err + } + } + remaining := util.ResolveLimit{ + MaxBytes: limit.MaxBytes - batchBytes, + MaxMessages: limit.MaxMessages - batchMessages, + } + batch, hasMore, err := group.PrepareResolve(resolveTs, remaining) if err != nil { - return err + return 0, err } - events := util.DMLMessagesToEvents(messages) - resolvedEvents = append(resolvedEvents, events...) + hasMoreGroups = hasMoreGroups || hasMore + if batch != nil { + preparedAny = true + prepared = append(prepared, batch) + batchEvents = append(batchEvents, util.DMLMessagesToEvents(batch.Messages)...) + batchMessages += len(batch.Messages) + batchBytes += batch.ResolvedBytes + } + } + if err := flush(); err != nil { + return 0, err + } + if !hasMoreGroups || !preparedAny { + break } } + return total, nil +} - total := len(resolvedEvents) - if total == 0 { - return w.mysqlSink.WriteBlockEvent(ddl) +func (w *writer) flushDMLBatch(ctx context.Context, events []*event.DMLEvent, fields ...zap.Field) error { + if len(events) == 0 { + return nil } - for _, e := range resolvedEvents { + done := make(chan struct{}) + var flushed atomic.Int64 + for _, e := range events { e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { + if flushed.Inc() == int64(len(events)) { close(done) } }) w.mysqlSink.AddDMLEvent(e) } - - log.Info("flush DML events before DDL", zap.Uint64("DDLCommitTs", commitTs), zap.Int("total", total)) - start := time.Now() ticker := time.NewTicker(time.Minute) defer ticker.Stop() for { @@ -210,14 +279,10 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { case <-ctx.Done(): return context.Cause(ctx) case <-done: - log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), - zap.Int("total", total), zap.Duration("duration", time.Since(start)), - zap.Any("tables", tableIDs)) - return w.mysqlSink.WriteBlockEvent(ddl) + return nil case <-ticker.C: - log.Warn("DML events cannot be flushed in time", - zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + log.Warn("DML events cannot be flushed in time", append(fields, + zap.Int("total", len(events)), zap.Int64("flushed", flushed.Load()))...) } } } @@ -281,56 +346,30 @@ func (w *writer) globalWatermark() uint64 { } func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { - var ( - done = make(chan struct{}, 1) - - flushed atomic.Int64 - ) - watermark := w.globalWatermark() - resolvedEvents := make([]*event.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - messages, err := group.ResolveInto(watermark, nil) - if err != nil { - return err - } - events := util.DMLMessagesToEvents(messages) - resolvedEvents = append(resolvedEvents, events...) + groups = append(groups, group) } } - total := len(resolvedEvents) - if total == 0 { - return nil - } - for _, e := range resolvedEvents { - e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { - close(done) - } - }) - w.mysqlSink.AddDMLEvent(e) - log.Debug("flush DML event", zap.Int64("tableID", e.GetTableID()), - zap.Uint64("commitTs", e.GetCommitTs()), zap.Any("startTs", e.GetStartTs())) + total, err := w.flushEventsFromGroups(ctx, groups, watermark, zap.Uint64("watermark", watermark)) + if err != nil { + return err } - - log.Info("flush DML events by watermark", zap.Uint64("watermark", watermark), zap.Int("total", total)) - start := time.Now() - ticker := time.NewTicker(time.Minute) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return context.Cause(ctx) - case <-done: - log.Info("flush DML events done", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Duration("duration", time.Since(start))) - return nil - case <-ticker.C: - log.Warn("DML events cannot be flushed in time", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) - } + if total != 0 { + stats := w.getSpillStore().Stats() + log.Info("flush DML events done", zap.Uint64("watermark", watermark), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Int64("spillPayloadWriteBytes", stats.PayloadWriteBytes), + zap.Int64("spillPayloadReadBytes", stats.PayloadReadBytes), + zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), + zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), + zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), + zap.Int64("spillPendingBytes", stats.PendingBytes)) } + return nil } // WriteMessage is to decode kafka message to event. @@ -354,6 +393,7 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool } needFlush := false + wasDraining := w.getSpillStore().ShouldDrain() switch messageType { case common.MessageTypeResolved: newWatermark := progress.decoder.NextResolvedEvent() @@ -453,6 +493,11 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool if needFlush { return w.Write(ctx, messageType) } + if !wasDraining && w.getSpillStore().ShouldDrain() { + if err := w.flushDMLEventsByWatermark(ctx); err != nil { + return false, err + } + } return false, nil } @@ -654,7 +699,7 @@ func (w *writer) appendMessage2Group( group := progress.eventsGroup[tableID] if group == nil { - group = util.NewEventsGroup(progress.partition, tableID) + group = util.NewEventsGroup(progress.partition, tableID, w.getSpillStore()) progress.eventsGroup[tableID] = group } if err := group.AppendMessage(message); err != nil { diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index b285230e4a..f06fb120c6 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -497,8 +497,8 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { func attachDMLMessageDataForWriterTest(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { messageData := codeccommon.NewDMLMessageData(nil, nil, - func([]byte, uint64) (*codeccommon.DMLMessage, error) { - return message, nil + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil }, ) messageData.AttachDMLMessage(message) diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index 6e0a76f338..5aa2ed8d6b 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -77,6 +77,7 @@ type writer struct { protocol config.Protocol mysqlSink sink.Sink enableTableAcrossNodes bool + spillStore *util.SpillStore } func newWriter(ctx context.Context, o *option) *writer { @@ -87,6 +88,7 @@ func newWriter(ctx context.Context, o *option) *writer { ddlList: make([]*commonEvent.DDLEvent, 0), ddlWithMaxCommitTs: make(map[int64]uint64), enableTableAcrossNodes: putil.GetOrZero(o.replicaConfig.Scheduler.EnableTableAcrossNodes), + spillStore: util.NewSpillStore(), } var ( db *sql.DB @@ -139,62 +141,131 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } +func (w *writer) getSpillStore() *util.SpillStore { + if w.spillStore == nil { + w.spillStore = util.NewSpillStore() + } + return w.spillStore +} + func (w *writer) cleanupEventsGroups() error { var cleanupErr error for _, progress := range w.progresses { for _, group := range progress.eventsGroup { - if err := group.Cleanup(); err != nil { - log.Warn("cleanup events group spill file failed", - zap.Int32("partition", progress.partition), zap.Error(err)) - if cleanupErr == nil { - cleanupErr = err - } - } + _ = group.Cleanup() } } + if err := w.getSpillStore().Cleanup(); err != nil { + cleanupErr = err + log.Warn("cleanup spill store failed", zap.Error(err)) + } return cleanupErr } func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) error { - var ( - done = make(chan struct{}, 1) - - flushed atomic.Int64 - ) - tableIDs := w.getBlockTableIDs(ddl) commitTs := ddl.GetCommitTs() - resolvedEvents := make([]*commonEvent.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for tableID := range tableIDs { for _, progress := range w.progresses { g, ok := progress.eventsGroup[tableID] if !ok { continue } - messages, err := g.ResolveInto(commitTs, nil) + groups = append(groups, g) + } + } + total, err := w.flushEventsFromGroups(ctx, groups, commitTs, + zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query)) + if err != nil { + return err + } + + if total != 0 { + log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Any("tables", tableIDs)) + } + return w.mysqlSink.WriteBlockEvent(ddl) +} + +func (w *writer) flushEventsFromGroups( + ctx context.Context, groups []*util.EventsGroup, resolveTs uint64, fields ...zap.Field, +) (int, error) { + limit := w.getSpillStore().ResolveLimit() + batchEvents := make([]*commonEvent.DMLEvent, 0, limit.MaxMessages) + batchMessages := 0 + var batchBytes int64 + total := 0 + prepared := make([]*util.ResolveBatch, 0, len(groups)) + flush := func() error { + if err := w.flushDMLBatch(ctx, batchEvents, fields...); err != nil { + return err + } + for _, batch := range prepared { + batch.Ack() + } + total += len(batchEvents) + batchEvents = nil + prepared = prepared[:0] + batchMessages = 0 + batchBytes = 0 + return nil + } + + for { + hasMoreGroups := false + preparedAny := false + for _, group := range groups { + if batchMessages >= limit.MaxMessages || batchBytes >= limit.MaxBytes { + if err := flush(); err != nil { + return 0, err + } + } + remaining := util.ResolveLimit{ + MaxBytes: limit.MaxBytes - batchBytes, + MaxMessages: limit.MaxMessages - batchMessages, + } + batch, hasMore, err := group.PrepareResolve(resolveTs, remaining) if err != nil { - return err + return 0, err + } + hasMoreGroups = hasMoreGroups || hasMore + if batch != nil { + preparedAny = true + prepared = append(prepared, batch) + batchEvents = append(batchEvents, util.DMLMessagesToEvents(batch.Messages)...) + batchMessages += len(batch.Messages) + batchBytes += batch.ResolvedBytes } - events := util.DMLMessagesToEvents(messages) - resolvedEvents = append(resolvedEvents, events...) + } + if err := flush(); err != nil { + return 0, err + } + if !hasMoreGroups || !preparedAny { + break } } + return total, nil +} - total := len(resolvedEvents) - if total == 0 { - return w.mysqlSink.WriteBlockEvent(ddl) +func (w *writer) flushDMLBatch( + ctx context.Context, events []*commonEvent.DMLEvent, fields ...zap.Field, +) error { + if len(events) == 0 { + return nil } - for _, e := range resolvedEvents { + done := make(chan struct{}) + var flushed atomic.Int64 + for _, e := range events { e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { + if flushed.Inc() == int64(len(events)) { close(done) } }) w.mysqlSink.AddDMLEvent(e) } - - log.Info("flush DML events before DDL", zap.Uint64("DDLCommitTs", commitTs), zap.Int("total", total)) - start := time.Now() ticker := time.NewTicker(time.Minute) defer ticker.Stop() for { @@ -202,14 +273,10 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e case <-ctx.Done(): return context.Cause(ctx) case <-done: - log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), - zap.Int("total", total), zap.Duration("duration", time.Since(start)), - zap.Any("tables", tableIDs)) - return w.mysqlSink.WriteBlockEvent(ddl) + return nil case <-ticker.C: - log.Warn("DML events cannot be flushed in time", - zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + log.Warn("DML events cannot be flushed in time", append(fields, + zap.Int("total", len(events)), zap.Int64("flushed", flushed.Load()))...) } } } @@ -273,54 +340,30 @@ func (w *writer) globalWatermark() uint64 { } func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { - var ( - done = make(chan struct{}, 1) - - flushed atomic.Int64 - ) - watermark := w.globalWatermark() - resolvedEvents := make([]*commonEvent.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - messages, err := group.ResolveInto(watermark, nil) - if err != nil { - return err - } - events := util.DMLMessagesToEvents(messages) - resolvedEvents = append(resolvedEvents, events...) + groups = append(groups, group) } } - total := len(resolvedEvents) - if total == 0 { - return nil - } - for _, e := range resolvedEvents { - e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { - close(done) - } - }) - w.mysqlSink.AddDMLEvent(e) + total, err := w.flushEventsFromGroups(ctx, groups, watermark, zap.Uint64("watermark", watermark)) + if err != nil { + return err } - - log.Info("flush DML events by watermark", zap.Uint64("watermark", watermark), zap.Int("total", total)) - start := time.Now() - ticker := time.NewTicker(time.Minute) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return context.Cause(ctx) - case <-done: - log.Info("flush DML events done", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Duration("duration", time.Since(start))) - return nil - case <-ticker.C: - log.Warn("DML events cannot be flushed in time", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) - } + if total != 0 { + stats := w.getSpillStore().Stats() + log.Info("flush DML events done", zap.Uint64("watermark", watermark), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Int64("spillPayloadWriteBytes", stats.PayloadWriteBytes), + zap.Int64("spillPayloadReadBytes", stats.PayloadReadBytes), + zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), + zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), + zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), + zap.Int64("spillPendingBytes", stats.PendingBytes)) } + return nil } // WriteMessage is to decode pulsar message to event. @@ -336,6 +379,7 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) (bool } needFlush := false + wasDraining := w.getSpillStore().ShouldDrain() switch messageType { case common.MessageTypeResolved: newWatermark := progress.decoder.NextResolvedEvent() @@ -376,6 +420,11 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) (bool if needFlush { return w.Write(ctx, messageType) } + if !wasDraining && w.getSpillStore().ShouldDrain() { + if err := w.flushDMLEventsByWatermark(ctx); err != nil { + return false, err + } + } return false, nil } @@ -533,7 +582,7 @@ func (w *writer) appendMessage2Group( group := progress.eventsGroup[tableID] if group == nil { - group = util.NewEventsGroup(progress.partition, tableID) + group = util.NewEventsGroup(progress.partition, tableID, w.getSpillStore()) progress.eventsGroup[tableID] = group } if err := group.AppendMessage(message); err != nil { diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index ef40a1c404..0148efa4d8 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -515,8 +515,8 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { func attachDMLMessageDataForWriterTest(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { messageData := codeccommon.NewDMLMessageData(nil, nil, - func([]byte, uint64) (*codeccommon.DMLMessage, error) { - return message, nil + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil }, ) messageData.AttachDMLMessage(message) diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 4febbe4e04..41529fab55 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "math" "sort" "strconv" "strings" @@ -76,6 +77,7 @@ type consumer struct { // tableDMLIdxMap maintains a map of tableDMLIdxMap map[cloudstorage.DMLPathKey]fileIndexKeyMap eventsGroup map[int64]*util.EventsGroup + spillStore *util.SpillStore // tableDDLWatermark maintains a map of <`schema`.`table`, max executed DDL table version>. // DML files with smaller table versions are considered stale replays and should be ignored. tableDDLWatermark map[string]uint64 @@ -175,6 +177,7 @@ func newConsumer(ctx context.Context) (*consumer, error) { errCh: errCh, tableDMLIdxMap: make(map[cloudstorage.DMLPathKey]fileIndexKeyMap), eventsGroup: make(map[int64]*util.EventsGroup), + spillStore: util.NewSpillStore(), tableDDLWatermark: make(map[string]uint64), schemaFileMap: make(map[string]map[uint64]*cloudstorage.SchemaFile), tableIDGenerator: &fakeTableIDGenerator{ @@ -183,6 +186,13 @@ func newConsumer(ctx context.Context) (*consumer, error) { }, nil } +func (c *consumer) getSpillStore() *util.SpillStore { + if c.spillStore == nil { + c.spillStore = util.NewSpillStore() + } + return c.spillStore +} + // map1 - map2 func diffDMLMaps( map1, map2 map[cloudstorage.DMLPathKey]fileIndexKeyMap, @@ -294,7 +304,7 @@ func (c *consumer) appendMessage2Group( ) group := c.eventsGroup[tableID] if group == nil { - group = util.NewEventsGroup(0, tableID) + group = util.NewEventsGroup(0, tableID, c.getSpillStore()) c.eventsGroup[tableID] = group } if commitTs >= group.HighWatermark { @@ -417,13 +427,16 @@ func (c *consumer) newDMLMessageData( return nil, errors.ErrSpillFileOp.FastGenByArgs("unsupported storage DML spill protocol") } }) - restore := messageData.Restore - messageData.Restore = func(data []byte, dmlIndex uint64) (*common.DMLMessage, error) { - message, err := restore(data, dmlIndex) + decode := messageData.Decode + messageData.Decode = func(data []byte) ([]*common.DMLMessage, error) { + messages, err := decode(data) if err != nil { return nil, err } - return messageWithPhysicalTableID(message, tableID), nil + for i, message := range messages { + messages[i] = messageWithPhysicalTableID(message, tableID) + } + return messages, nil } return messageData } @@ -441,28 +454,62 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { if group == nil { return nil } - messages, err := group.GetAllMessages() - if err != nil { - return err + start := time.Now() + total := 0 + for { + batch, hasMore, err := group.PrepareResolve( + math.MaxUint64, c.getSpillStore().ResolveLimit()) + if err != nil { + return err + } + if batch == nil { + break + } + events := util.DMLMessagesToEvents(batch.Messages) + if len(events) != 0 { + fields := []zap.Field{zap.Int64("tableID", tableID)} + if events[0].TableInfo != nil { + fields = append(fields, + zap.String("schema", events[0].TableInfo.GetSchemaName()), + zap.String("table", events[0].TableInfo.GetTableName())) + } + if err := c.flushDMLBatch(ctx, events, fields...); err != nil { + return err + } + total += len(events) + } + batch.Ack() + if !hasMore { + break + } } - if len(messages) == 0 { - return nil + if total != 0 { + stats := c.getSpillStore().Stats() + log.Info("flush DML events done", zap.Int64("tableID", tableID), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Int64("spillPayloadWriteBytes", stats.PayloadWriteBytes), + zap.Int64("spillPayloadReadBytes", stats.PayloadReadBytes), + zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), + zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), + zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), + zap.Int64("spillPendingBytes", stats.PendingBytes), + zap.Int("spillLivePayloads", stats.LivePayloads), + zap.Int("spillLiveSegments", stats.LiveSegments)) } - events := util.DMLMessagesToEvents(messages) - total := len(events) - if total == 0 { + return nil +} + +func (c *consumer) flushDMLBatch( + ctx context.Context, events []*event.DMLEvent, fields ...zap.Field, +) error { + if len(events) == 0 { return nil } + total := len(events) var ( - schema string - table string + flushed atomic.Int64 + done = make(chan struct{}) ) - if events[0].TableInfo != nil { - schema = events[0].TableInfo.GetSchemaName() - table = events[0].TableInfo.GetTableName() - } - var flushed atomic.Int64 - done := make(chan struct{}) for _, e := range events { e.AddPostFlushFunc(func() { if flushed.Inc() == int64(total) { @@ -472,8 +519,6 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { c.sink.AddDMLEvent(e) } - // Make sure all events are flushed to downstream. - start := time.Now() ticker := time.NewTicker(defaultLogInterval) defer ticker.Stop() for { @@ -481,13 +526,10 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { case <-ctx.Done(): return context.Cause(ctx) case <-done: - log.Info("flush DML events done", - zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Int("total", total), zap.Duration("duration", time.Since(start))) return nil case <-ticker.C: - log.Warn("DML events cannot be flushed in time", - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + log.Warn("DML events cannot be flushed in time", append(fields, + zap.Int("total", total), zap.Int64("flushed", flushed.Load()))...) } } } @@ -495,12 +537,11 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { func (c *consumer) cleanupEventsGroups() error { var cleanupErr error for _, group := range c.eventsGroup { - if err := group.Cleanup(); err != nil { - log.Warn("cleanup events group spill file failed", zap.Error(err)) - if cleanupErr == nil { - cleanupErr = err - } - } + _ = group.Cleanup() + } + if err := c.getSpillStore().Cleanup(); err != nil { + cleanupErr = err + log.Warn("cleanup spill store failed", zap.Error(err)) } return cleanupErr } @@ -763,6 +804,11 @@ func (c *consumer) handleNewFiles( if err := c.appendDMLEvents(ctx, tableID, schemaFile, key, fileIndex); err != nil { return err } + if c.getSpillStore().ShouldDrain() { + if err := c.flushDMLEvents(ctx, tableID); err != nil { + return err + } + } } } if err := c.flushDMLEvents(ctx, tableID); err != nil { diff --git a/cmd/util/dml_message_decoder.go b/cmd/util/dml_message_decoder.go index 360f1ac947..dd1c2cba15 100644 --- a/cmd/util/dml_message_decoder.go +++ b/cmd/util/dml_message_decoder.go @@ -78,7 +78,7 @@ func (d *DMLMessageDecoder) NextDMLMessage() *codeccommon.DMLMessage { func (d *DMLMessageDecoder) attachDMLMessage(message *codeccommon.DMLMessage) { if d.data == nil { - d.data = d.wrapRestore(d.factory(d.Decoder, d.key, d.value)) + d.data = d.wrapDecode(d.factory(d.Decoder, d.key, d.value)) } d.data.AttachDMLMessage(message) } @@ -87,21 +87,26 @@ func (d *DMLMessageDecoder) attachDMLMessage(message *codeccommon.DMLMessage) { // Simple's DDL cache. It has no raw row payload to restore. func (d *DMLMessageDecoder) AttachCachedDMLMessage(message *codeccommon.DMLMessage) { data := codeccommon.NewDMLMessageData(nil, nil, - func([]byte, uint64) (*codeccommon.DMLMessage, error) { return message, nil }) - d.wrapRestore(data).AttachDMLMessage(message) + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }) + d.wrapDecode(data).AttachDMLMessage(message) } -func (d *DMLMessageDecoder) wrapRestore(data *codeccommon.DMLMessageData) *codeccommon.DMLMessageData { +func (d *DMLMessageDecoder) wrapDecode(data *codeccommon.DMLMessageData) *codeccommon.DMLMessageData { if d.restore == nil { return data } - restore := data.Restore - data.Restore = func(data []byte, dmlIndex uint64) (*codeccommon.DMLMessage, error) { - message, err := restore(data, dmlIndex) + decode := data.Decode + data.Decode = func(payload []byte) ([]*codeccommon.DMLMessage, error) { + messages, err := decode(payload) if err != nil { return nil, err } - return d.restore(message), nil + for i, message := range messages { + messages[i] = d.restore(message) + } + return messages, nil } return data } diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index bed6161379..b032179ea2 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -18,7 +18,6 @@ import ( "math" "os" "sort" - "sync" "github.com/pingcap/log" commonEvent "github.com/pingcap/ticdc/pkg/common/event" @@ -30,11 +29,168 @@ import ( const eventsGroupSpillPattern = "ticdc-events-group-*.spill" +const ( + defaultSpillSegmentBytes = 128 * 1024 * 1024 + defaultSpillSegmentMessages = 100000 + defaultPendingHighBytes = 1024 * 1024 * 1024 + defaultPendingLowBytes = 512 * 1024 * 1024 + defaultResolveBatchBytes = 64 * 1024 * 1024 + defaultResolveBatchMessages = 10000 + defaultMessageMetadataBytes = 128 + spillRecordLengthBytes = 8 +) + +type spillConfig struct { + segmentBytes int64 + segmentMessages int + pendingHighBytes int64 + pendingLowBytes int64 + resolveBatchBytes int64 + resolveBatchMessages int + messageMetadataBytes int64 +} + +func defaultSpillConfig() spillConfig { + return spillConfig{ + segmentBytes: defaultSpillSegmentBytes, + segmentMessages: defaultSpillSegmentMessages, + pendingHighBytes: defaultPendingHighBytes, + pendingLowBytes: defaultPendingLowBytes, + resolveBatchBytes: defaultResolveBatchBytes, + resolveBatchMessages: defaultResolveBatchMessages, + messageMetadataBytes: defaultMessageMetadataBytes, + } +} + +// ResolveLimit bounds one batch restored from spill. +type ResolveLimit struct { + MaxBytes int64 + MaxMessages int +} + +// SpillStats reports process-wide payload I/O and live spill state. +type SpillStats struct { + PendingBytes int64 + PayloadWriteBytes int64 + PayloadReadBytes int64 + PayloadWriteCount int64 + PayloadReadCount int64 + PayloadDecodeCount int64 + LivePayloads int + LiveSegments int +} + +// SpillStore owns spill payloads and segments across all event groups in one consumer. +// Its watermarks request draining; they are not a hard quota and never reject an append. +type SpillStore struct { + config spillConfig + segments map[*spillSegment]struct{} + activeSegment *spillSegment + payloads map[uint64]*spillPayload + pendingBytes int64 + draining bool + + readRecord func(*spill.RecordFile, spill.Handle) ([]byte, error) + stats SpillStats +} + +// NewSpillStore creates a process-wide store with the default spill limits. +func NewSpillStore() *SpillStore { + return newSpillStore(defaultSpillConfig()) +} + +func newSpillStore(config spillConfig) *SpillStore { + return &SpillStore{ + config: config, + segments: make(map[*spillSegment]struct{}), + payloads: make(map[uint64]*spillPayload), + readRecord: func(file *spill.RecordFile, handle spill.Handle) ([]byte, error) { + return file.Read(handle) + }, + } +} + +// ResolveLimit returns the configured per-batch restore limit. +func (s *SpillStore) ResolveLimit() ResolveLimit { + return ResolveLimit{ + MaxBytes: s.config.resolveBatchBytes, + MaxMessages: s.config.resolveBatchMessages, + } +} + +// PendingBytes returns conservatively accounted payload, decoded cache, and metadata bytes. +func (s *SpillStore) PendingBytes() int64 { + return s.pendingBytes +} + +// ShouldDrain reports whether pending spill data has crossed the high watermark and not yet fallen below the low watermark. +func (s *SpillStore) ShouldDrain() bool { + return s.draining +} + +// Stats returns a snapshot of process-wide spill activity. +func (s *SpillStore) Stats() SpillStats { + stats := s.stats + stats.PendingBytes = s.pendingBytes + stats.LivePayloads = len(s.payloads) + stats.LiveSegments = len(s.segments) + return stats +} + +func (s *SpillStore) addPending(bytes int64) { + if bytes <= 0 { + return + } + s.pendingBytes += bytes + if !s.draining && s.pendingBytes >= s.config.pendingHighBytes { + s.draining = true + log.Info("spill pending bytes reached high watermark", + zap.Int64("pendingBytes", s.pendingBytes), + zap.Int64("highWatermarkBytes", s.config.pendingHighBytes), + zap.Int64("lowWatermarkBytes", s.config.pendingLowBytes)) + } +} + +func (s *SpillStore) releasePending(bytes int64) { + if bytes <= 0 { + return + } + s.pendingBytes -= bytes + if s.pendingBytes < 0 { + log.Panic("spill pending bytes underflow", + zap.Int64("releasedBytes", bytes), zap.Int64("pendingBytes", s.pendingBytes)) + } + if s.draining && s.pendingBytes <= s.config.pendingLowBytes { + s.draining = false + log.Info("spill pending bytes fell below low watermark", + zap.Int64("pendingBytes", s.pendingBytes), + zap.Int64("highWatermarkBytes", s.config.pendingHighBytes), + zap.Int64("lowWatermarkBytes", s.config.pendingLowBytes)) + } +} + +type spillSegment struct { + file *spill.RecordFile + bytes int64 + payloadCount int + pendingPayloads int +} + +type spillPayload struct { + id uint64 + segment *spillSegment + handle spill.Handle + decode func([]byte) ([]*codeccommon.DMLMessage, error) + decoded []*codeccommon.DMLMessage + decodedBytes int64 + loaded bool + refs int +} + type spilledMessage struct { commitTs uint64 - handle spill.Handle + payload *spillPayload dmlIndex uint64 - restore func([]byte, uint64) (*codeccommon.DMLMessage, error) } // EventsGroup stores change event messages. @@ -43,19 +199,211 @@ type EventsGroup struct { tableID int64 messages []spilledMessage - spillFile *spill.RecordFile - spillHandles map[uint64]spill.Handle + store *SpillStore + batchPending bool outOfOrder bool HighWatermark uint64 } // NewEventsGroup will create new event group. -func NewEventsGroup(partition int32, tableID int64) *EventsGroup { +func NewEventsGroup(partition int32, tableID int64, stores ...*SpillStore) *EventsGroup { + store := NewSpillStore() + if len(stores) != 0 && stores[0] != nil { + store = stores[0] + } return &EventsGroup{ Partition: partition, tableID: tableID, messages: make([]spilledMessage, 0, 1024), + store: store, + } +} + +func (s *SpillStore) newSegment() error { + file, err := spill.NewRecordFile(os.TempDir(), eventsGroupSpillPattern) + if err != nil { + return err + } + segment := &spillSegment{file: file} + s.segments[segment] = struct{}{} + s.activeSegment = segment + return nil +} + +func (s *SpillStore) prepareSegment(recordBytes int64) error { + segment := s.activeSegment + if segment != nil && (segment.bytes+recordBytes > s.config.segmentBytes || + segment.payloadCount >= s.config.segmentMessages) { + s.activeSegment = nil + } + if s.activeSegment == nil { + return s.newSegment() + } + return nil +} + +func (s *SpillStore) sealFullSegment() { + segment := s.activeSegment + if segment == nil { + return + } + if segment.bytes >= s.config.segmentBytes || + segment.payloadCount >= s.config.segmentMessages { + s.activeSegment = nil + } +} + +func spillMessageDataSize(key, value []byte) int64 { + if len(key) == 0 && len(value) == 0 { + return spillRecordLengthBytes + 1 + } + return spillRecordLengthBytes + 2*8 + int64(len(key)) + int64(len(value)) +} + +func appendMessageData(file *spill.RecordFile, key, value []byte) (spill.Handle, error) { + if len(key) == 0 && len(value) == 0 { + return file.AppendChunks([]byte{0}) + } + var keyLen, valueLen [8]byte + binary.BigEndian.PutUint64(keyLen[:], uint64(len(key))) + binary.BigEndian.PutUint64(valueLen[:], uint64(len(value))) + return file.AppendChunks(keyLen[:], key, valueLen[:], value) +} + +func (s *SpillStore) acquirePayload(data *codeccommon.DMLMessageData) (*spillPayload, error) { + if payload, ok := s.payloads[data.ID]; ok { + return payload, nil + } + recordBytes := spillMessageDataSize(data.Key, data.Value) + if err := s.prepareSegment(recordBytes); err != nil { + return nil, err + } + segment := s.activeSegment + handle, err := appendMessageData(segment.file, data.Key, data.Value) + if err != nil { + return nil, err + } + payload := &spillPayload{ + id: data.ID, + segment: segment, + handle: handle, + decode: data.Decode, + } + s.payloads[data.ID] = payload + segment.bytes += recordBytes + segment.payloadCount++ + segment.pendingPayloads++ + s.addPending(recordBytes) + s.stats.PayloadWriteBytes += int64(handle.Length) + s.stats.PayloadWriteCount++ + s.sealFullSegment() + return payload, nil +} + +func (s *SpillStore) retainPayload(payload *spillPayload) { + payload.refs++ + s.addPending(s.config.messageMetadataBytes) +} + +func (s *SpillStore) loadPayload(payload *spillPayload) (int64, error) { + if payload.loaded { + return 0, nil + } + if payload.segment == nil || payload.segment.file == nil { + return 0, errors.ErrSpillFileOp.FastGenByArgs("spill payload segment is missing") + } + data, err := s.readRecord(payload.segment.file, payload.handle) + if err != nil { + return 0, err + } + messages, err := payload.decode(data) + if err != nil { + return 0, err } + payload.decoded = messages + payload.loaded = true + payload.decodedBytes = int64(payload.handle.Length) + s.addPending(payload.decodedBytes) + s.stats.PayloadReadBytes += int64(payload.handle.Length) + s.stats.PayloadReadCount++ + s.stats.PayloadDecodeCount++ + return int64(payload.handle.Length), nil +} + +func (s *SpillStore) messageAt(payload *spillPayload, index uint64) (*codeccommon.DMLMessage, int64, error) { + readBytes, err := s.loadPayload(payload) + if err != nil { + return nil, 0, err + } + if index >= uint64(len(payload.decoded)) { + return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("DML spill message index is out of range") + } + message := payload.decoded[index] + if message == nil { + return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("DML spill message was already released") + } + return message, readBytes, nil +} + +func (s *SpillStore) releasePayloadRef(payload *spillPayload, index uint64) { + if payload.loaded && index < uint64(len(payload.decoded)) { + payload.decoded[index] = nil + } + payload.refs-- + s.releasePending(s.config.messageMetadataBytes) + if payload.refs != 0 { + return + } + if payload.loaded { + s.releasePending(payload.decodedBytes) + } + payload.decoded = nil + payload.decode = nil + delete(s.payloads, payload.id) + + segment := payload.segment + segment.pendingPayloads-- + if segment.pendingPayloads == 0 { + s.cleanupSegment(segment) + } +} + +func (s *SpillStore) cleanupSegment(segment *spillSegment) { + if s.activeSegment == segment { + s.activeSegment = nil + } + if err := segment.file.Cleanup(); err != nil { + log.Warn("cleanup spill segment failed", zap.String("path", segment.file.Path()), zap.Error(err)) + return + } + s.releasePending(segment.bytes) + delete(s.segments, segment) +} + +// Cleanup removes all remaining payload and segment state when a consumer stops. +func (s *SpillStore) Cleanup() error { + for _, payload := range s.payloads { + s.releasePending(int64(payload.refs) * s.config.messageMetadataBytes) + if payload.loaded { + s.releasePending(payload.decodedBytes) + } + payload.decoded = nil + payload.decode = nil + } + clear(s.payloads) + s.activeSegment = nil + var cleanupErr error + for segment := range s.segments { + if err := segment.file.Cleanup(); err != nil { + if cleanupErr == nil { + cleanupErr = err + } + continue + } + s.releasePending(segment.bytes) + delete(s.segments, segment) + } + return cleanupErr } // AppendMessage appends an opaque codec payload to the spill file. It does @@ -68,36 +416,13 @@ func (g *EventsGroup) AppendMessage( return errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") } messageData, dmlIndex := message.SpillData() - if messageData == nil || messageData.Restore == nil { - return errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without restore function") + if messageData == nil || messageData.Decode == nil { + return errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without decode function") } commitTs := message.GetCommitTs() - - if g.spillFile == nil { - var err error - g.spillFile, err = spill.NewRecordFile(os.TempDir(), eventsGroupSpillPattern) - if err != nil { - return err - } - } - if g.spillHandles == nil { - g.spillHandles = make(map[uint64]spill.Handle) - } - handle, ok := g.spillHandles[messageData.ID] - if !ok { - data := marshalDMLMessageData(messageData.Key, messageData.Value) - if len(data) == 0 { - // A lazy message may not need an input payload (for example, a Simple - // decoder message released from its table-info cache). RecordFile rejects - // empty records, so retain a marker while keeping the event lazy. - data = []byte{0} - } - var err error - handle, err = g.spillFile.Append(data) - if err != nil { - return err - } - g.spillHandles[messageData.ID] = handle + payload, err := g.store.acquirePayload(messageData) + if err != nil { + return err } if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].commitTs { g.outOfOrder = true @@ -107,34 +432,46 @@ func (g *EventsGroup) AppendMessage( } g.messages = append(g.messages, spilledMessage{ commitTs: commitTs, - handle: handle, + payload: payload, dmlIndex: dmlIndex, - restore: messageData.Restore, }) + g.store.retainPayload(payload) return nil } -// ResolveInto appends all messages with CommitTs <= resolve into dst in commit-ts order and removes -// them from the group. Resolved messages are restored from the spill file only when downstream needs -// them, keeping the buffered group out of heap memory. -func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) ([]*codeccommon.DMLMessage, error) { - if len(g.messages) == 0 { - if g.spillFile != nil { - if err := g.spillFile.Cleanup(); err != nil { - return dst, err - } - g.spillFile = nil - clear(g.spillHandles) - } - return dst, nil +// ResolveBatch owns a prepared group prefix until the downstream confirms it. +type ResolveBatch struct { + Messages []*codeccommon.DMLMessage + ResolvedBytes int64 + group *EventsGroup + count int + acked bool +} + +// Ack releases a prepared group prefix after downstream flush callbacks complete. +func (b *ResolveBatch) Ack() { + if b == nil || b.acked { + return } + b.group.ack(b.count) + b.acked = true +} +// PrepareResolve restores one bounded group prefix without removing it. +func (g *EventsGroup) PrepareResolve( + resolve uint64, limit ResolveLimit, +) (*ResolveBatch, bool, error) { + if g.batchPending { + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("events group already has a pending resolve batch") + } + if len(g.messages) == 0 { + return nil, false, nil + } if g.outOfOrder { sort.SliceStable(g.messages, func(i, j int) bool { return g.messages[i].commitTs < g.messages[j].commitTs }) } - resolvedCount := sort.Search(len(g.messages), func(i int) bool { return g.messages[i].commitTs > resolve }) @@ -147,42 +484,100 @@ func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) g.outOfOrder = false } if resolvedCount == 0 { - return dst, nil - } - if g.spillFile == nil { - return dst, errors.ErrSpillFileOp.FastGenByArgs("events group spill file is missing") + return nil, false, nil } - for _, message := range g.messages[:resolvedCount] { - data, err := g.spillFile.Read(message.handle) - if err != nil { - return dst, err - } - restored, err := message.restore(data, message.dmlIndex) + batchCount := boundedResolvedBatch(g.messages[:resolvedCount], limit) + batch := &ResolveBatch{ + Messages: make([]*codeccommon.DMLMessage, 0, batchCount), + group: g, + count: batchCount, + } + for _, message := range g.messages[:batchCount] { + restored, readBytes, err := g.store.messageAt(message.payload, message.dmlIndex) if err != nil { - return dst, err + return nil, false, err } - dst = append(dst, restored) + batch.Messages = append(batch.Messages, restored) + batch.ResolvedBytes += readBytes } - remainingCount := len(g.messages) - resolvedCount - copy(g.messages, g.messages[resolvedCount:]) - clear(g.messages[remainingCount:]) - g.messages = g.messages[:remainingCount] - if len(g.messages) == 0 { - if err := g.spillFile.Cleanup(); err != nil { - return dst, err + g.batchPending = true + return batch, batchCount < resolvedCount, nil +} + +// ResolveInto appends all messages with CommitTs <= resolve into dst in commit-ts order. +func (g *EventsGroup) ResolveInto( + resolve uint64, dst []*codeccommon.DMLMessage, +) ([]*codeccommon.DMLMessage, error) { + dst, _, _, err := g.ResolveIntoBatch(resolve, dst, ResolveLimit{}) + return dst, err +} + +// ResolveIntoBatch appends one bounded batch of messages with CommitTs <= resolve into dst in commit-ts order. +// A single commit-ts group can exceed the limits so that one transaction is never split. +func (g *EventsGroup) ResolveIntoBatch( + resolve uint64, dst []*codeccommon.DMLMessage, limit ResolveLimit, +) ([]*codeccommon.DMLMessage, bool, int64, error) { + batch, hasMore, err := g.PrepareResolve(resolve, limit) + if err != nil || batch == nil { + return dst, hasMore, 0, err + } + dst = append(dst, batch.Messages...) + resolvedBytes := batch.ResolvedBytes + batch.Ack() + return dst, hasMore, resolvedBytes, nil +} + +func boundedResolvedBatch(messages []spilledMessage, limit ResolveLimit) int { + if len(messages) == 0 { + return 0 + } + maxBytes := limit.MaxBytes + if maxBytes <= 0 { + maxBytes = math.MaxInt64 + } + maxMessages := limit.MaxMessages + if maxMessages <= 0 { + maxMessages = int(^uint(0) >> 1) + } + + seenPayloads := make(map[*spillPayload]struct{}) + var bytes int64 + for i, message := range messages { + additionalBytes := int64(0) + if !message.payload.loaded { + if _, ok := seenPayloads[message.payload]; !ok { + additionalBytes = int64(message.payload.handle.Length) + } } - g.spillFile = nil - clear(g.spillHandles) + exceedsLimit := i > 0 && (i >= maxMessages || bytes+additionalBytes > maxBytes) + if exceedsLimit && message.commitTs != messages[i-1].commitTs { + return i + } + bytes += additionalBytes + seenPayloads[message.payload] = struct{}{} + } + return len(messages) +} + +func (g *EventsGroup) ack(count int) { + for i := range count { + message := &g.messages[i] + g.store.releasePayloadRef(message.payload, message.dmlIndex) } - if len(g.messages) != 0 { - firstCommitTs := g.messages[0].commitTs - log.Debug("not all events resolved", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), - zap.Int("resolved", resolvedCount), zap.Int("remained", len(g.messages)), - zap.Uint64("resolveTs", resolve), zap.Uint64("firstCommitTs", firstCommitTs)) + remainingCount := len(g.messages) - count + copy(g.messages, g.messages[count:]) + clear(g.messages[remainingCount:]) + if remainingCount == 0 { + g.messages = nil + } else if cap(g.messages) > 2*remainingCount && cap(g.messages) > 1024 { + remaining := make([]spilledMessage, remainingCount) + copy(remaining, g.messages[:remainingCount]) + g.messages = remaining + } else { + g.messages = g.messages[:remainingCount] } - return dst, nil + g.batchPending = false } // GetAllMessages gets all messages. @@ -192,17 +587,13 @@ func (g *EventsGroup) GetAllMessages() ([]*codeccommon.DMLMessage, error) { // Cleanup removes pending spill records when the consumer is stopping. func (g *EventsGroup) Cleanup() error { - if g.spillFile == nil { - return nil - } - err := g.spillFile.Cleanup() - if err != nil { - return err + for i := range g.messages { + message := &g.messages[i] + g.store.releasePayloadRef(message.payload, message.dmlIndex) } - g.spillFile = nil - clear(g.spillHandles) clear(g.messages) - g.messages = g.messages[:0] + g.messages = nil + g.batchPending = false return nil } @@ -261,45 +652,20 @@ func NewDMLMessageDataWithDecoderFactory( key, value []byte, decoderFactory func([]byte, []byte) (codeccommon.Decoder, error), ) *codeccommon.DMLMessageData { - var ( - once sync.Once - messages []*codeccommon.DMLMessage - err error - ) return codeccommon.NewDMLMessageData(key, value, - func(data []byte, dmlIndex uint64) (*codeccommon.DMLMessage, error) { - once.Do(func() { - key, value, unmarshalErr := unmarshalDMLMessageData(data) - if unmarshalErr != nil { - err = unmarshalErr - return - } - decoder, decoderErr := decoderFactory(key, value) - if decoderErr != nil { - err = errors.WrapError(errors.ErrSpillFileOp, decoderErr, "create DML spill decoder") - return - } - messages, err = restoreDMLMessages(decoder, key, value) - }) + func(data []byte) ([]*codeccommon.DMLMessage, error) { + key, value, err := unmarshalDMLMessageData(data) if err != nil { return nil, err } - if dmlIndex >= uint64(len(messages)) { - return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill message index is out of range") + decoder, err := decoderFactory(key, value) + if err != nil { + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "create DML spill decoder") } - return messages[dmlIndex], nil + return restoreDMLMessages(decoder, key, value) }) } -func marshalDMLMessageData(key, value []byte) []byte { - if len(key) == 0 && len(value) == 0 { - return nil - } - data := make([]byte, 0, 2*8+len(key)+len(value)) - data = appendSpillBytes(data, key) - return appendSpillBytes(data, value) -} - func restoreDMLMessages( decoder codeccommon.Decoder, key, value []byte, ) ([]*codeccommon.DMLMessage, error) { @@ -324,17 +690,6 @@ func restoreDMLMessages( } } -func appendSpillBytes(data, value []byte) []byte { - data = appendSpillUint64(data, uint64(len(value))) - return append(data, value...) -} - -func appendSpillUint64(data []byte, value uint64) []byte { - var buf [8]byte - binary.BigEndian.PutUint64(buf[:], value) - return append(data, buf[:]...) -} - func unmarshalDMLMessageData(data []byte) ([]byte, []byte, error) { key, data, err := readSpillBytes(data) if err != nil { diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index ab7132551d..22deeb5606 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -14,7 +14,9 @@ package util import ( + "errors" "fmt" + "math" "os" "testing" @@ -23,6 +25,7 @@ import ( commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/integrity" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/spill" "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/ast" "github.com/pingcap/tidb/pkg/parser/mysql" @@ -38,8 +41,20 @@ func newTestDMLMessage(commitTs uint64) *codeccommon.DMLMessage { func attachTestDMLMessageData(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { messageData := codeccommon.NewDMLMessageData(nil, nil, - func([]byte, uint64) (*codeccommon.DMLMessage, error) { - return message, nil + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + +func attachTestDMLMessageDataWithPayload( + message *codeccommon.DMLMessage, key, value []byte, +) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(key, value, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil }, ) messageData.AttachDMLMessage(message) @@ -162,8 +177,8 @@ func TestEventsGroupSharesRawMessageData(t *testing.T) { first := newTestDMLMessage(10) second := newTestDMLMessage(10) messageData := codeccommon.NewDMLMessageData(nil, []byte("raw message"), - func(_ []byte, dmlIndex uint64) (*codeccommon.DMLMessage, error) { - return []*codeccommon.DMLMessage{first, second}[dmlIndex], nil + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{first, second}, nil }, ) @@ -173,7 +188,8 @@ func TestEventsGroupSharesRawMessageData(t *testing.T) { messageData.AttachDMLMessage(second) require.NoError(t, group.AppendMessage(second)) require.Len(t, group.messages, 2) - require.Equal(t, group.messages[0].handle, group.messages[1].handle) + require.Same(t, group.messages[0].payload, group.messages[1].payload) + require.Equal(t, group.messages[0].payload.handle, group.messages[1].payload.handle) require.Equal(t, uint64(0), group.messages[0].dmlIndex) require.Equal(t, uint64(1), group.messages[1].dmlIndex) @@ -192,9 +208,13 @@ func TestEventsGroupRestoresSharedSpillInputOnce(t *testing.T) { newTestDMLMessage(20), } var decoderCount int - messageData := NewDMLMessageDataWithDecoderFactory(nil, []byte("raw message"), - func(_, _ []byte) (codeccommon.Decoder, error) { + key := []byte("raw key") + value := []byte("raw message") + messageData := NewDMLMessageDataWithDecoderFactory(key, value, + func(restoredKey, restoredValue []byte) (codeccommon.Decoder, error) { decoderCount++ + require.Equal(t, key, restoredKey) + require.Equal(t, value, restoredValue) return &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{ newTestDMLMessage(30), newTestDMLMessage(10), @@ -216,6 +236,306 @@ func TestEventsGroupRestoresSharedSpillInputOnce(t *testing.T) { }) } +func TestEventsGroupReadsLargeSharedPayloadOnceAcrossBatches(t *testing.T) { + const messageCount = 27020 + + config := defaultSpillConfig() + config.resolveBatchMessages = 10000 + store := newSpillStore(config) + group := NewEventsGroup(0, 1, store) + originalMessages := make([]*codeccommon.DMLMessage, messageCount) + restoredMessages := make([]*codeccommon.DMLMessage, messageCount) + for i := range messageCount { + commitTs := uint64(i + 1) + originalMessages[i] = newTestDMLMessage(commitTs) + restoredMessages[i] = newTestDMLMessage(commitTs) + } + decodeCount := 0 + messageData := codeccommon.NewDMLMessageData(nil, []byte("one large object payload"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + decodeCount++ + return restoredMessages, nil + }) + for _, message := range originalMessages { + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) + } + + readRecord := store.readRecord + readCount := 0 + store.readRecord = func(file *spill.RecordFile, handle spill.Handle) ([]byte, error) { + readCount++ + return readRecord(file, handle) + } + + resolved := 0 + batchCount := 0 + for { + batch, hasMore, err := group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + require.NotNil(t, batch) + batchCount++ + resolved += len(batch.Messages) + if batchCount == 1 { + require.Equal(t, int64(group.messages[0].payload.handle.Length), batch.ResolvedBytes) + } else { + require.Zero(t, batch.ResolvedBytes) + } + batch.Ack() + if !hasMore { + break + } + } + + require.Equal(t, messageCount, resolved) + require.Equal(t, 3, batchCount) + require.Equal(t, 1, readCount) + require.Equal(t, 1, decodeCount) + require.Equal(t, int64(1), store.Stats().PayloadWriteCount) + require.Equal(t, int64(1), store.Stats().PayloadReadCount) + require.Equal(t, int64(1), store.Stats().PayloadDecodeCount) + require.Equal(t, store.Stats().PayloadWriteBytes, store.Stats().PayloadReadBytes) + require.Zero(t, store.PendingBytes()) + require.Empty(t, store.payloads) + require.Empty(t, store.segments) +} + +func TestEventsGroupsSharePayloadUntilEveryGroupAcks(t *testing.T) { + store := NewSpillStore() + firstGroup := NewEventsGroup(0, 1, store) + secondGroup := NewEventsGroup(0, 2, store) + first := newTestDMLMessage(1) + second := newTestDMLMessage(2) + decodeCount := 0 + messageData := codeccommon.NewDMLMessageData(nil, []byte("shared across groups"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + decodeCount++ + return []*codeccommon.DMLMessage{first, second}, nil + }) + messageData.AttachDMLMessage(first) + require.NoError(t, firstGroup.AppendMessage(first)) + messageData.AttachDMLMessage(second) + require.NoError(t, secondGroup.AppendMessage(second)) + + readRecord := store.readRecord + readCount := 0 + store.readRecord = func(file *spill.RecordFile, handle spill.Handle) ([]byte, error) { + readCount++ + return readRecord(file, handle) + } + + firstBatch, _, err := firstGroup.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + secondBatch, _, err := secondGroup.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + require.Equal(t, 1, readCount) + require.Equal(t, 1, decodeCount) + require.Len(t, store.payloads, 1) + require.Len(t, store.segments, 1) + spillPath := firstGroup.messages[0].payload.segment.file.Path() + + firstBatch.Ack() + require.Len(t, store.payloads, 1) + require.FileExists(t, spillPath) + secondBatch.Ack() + require.Empty(t, store.payloads) + require.Empty(t, store.segments) + _, err = os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) +} + +func TestEventsGroupPrepareDoesNotReleaseBeforeAck(t *testing.T) { + store := NewSpillStore() + group := NewEventsGroup(0, 1, store) + message := attachTestDMLMessageDataWithPayload(newTestDMLMessage(1), nil, []byte("payload")) + require.NoError(t, group.AppendMessage(message)) + spillPath := group.messages[0].payload.segment.file.Path() + + batch, _, err := group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + require.Len(t, batch.Messages, 1) + require.Len(t, group.messages, 1) + require.FileExists(t, spillPath) + _, _, err = group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.Error(t, err) + + batch.Ack() + require.Empty(t, group.messages) + _, err = os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) +} + +func TestEventsGroupSegmentsAndPendingWatermarks(t *testing.T) { + config := defaultSpillConfig() + config.segmentBytes = 64 + config.pendingHighBytes = 82 + config.pendingLowBytes = 41 + config.messageMetadataBytes = 1 + store := newSpillStore(config) + group := NewEventsGroup(0, 1, store) + + first := attachTestDMLMessageDataWithPayload(newTestDMLMessage(1), nil, make([]byte, 16)) + second := attachTestDMLMessageDataWithPayload(newTestDMLMessage(2), nil, make([]byte, 16)) + require.NoError(t, group.AppendMessage(first)) + firstSegment := group.messages[0].payload.segment + firstPath := firstSegment.file.Path() + require.NoError(t, group.AppendMessage(second)) + secondPath := group.messages[1].payload.segment.file.Path() + + require.Len(t, store.segments, 2) + require.Equal(t, int64(82), store.PendingBytes()) + require.True(t, store.ShouldDrain()) + + messages, hasMore, _, err := group.ResolveIntoBatch(1, nil, ResolveLimit{MaxMessages: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, messages, 1) + require.Equal(t, int64(41), store.PendingBytes()) + require.False(t, store.ShouldDrain()) + require.Len(t, store.segments, 1) + _, err = os.Stat(firstPath) + require.True(t, os.IsNotExist(err)) + require.FileExists(t, secondPath) + + require.NoError(t, group.Cleanup()) + require.Zero(t, store.PendingBytes()) + _, err = os.Stat(secondPath) + require.True(t, os.IsNotExist(err)) +} + +func TestEventsGroupKeepsSharedPayloadInOneSegment(t *testing.T) { + config := defaultSpillConfig() + config.segmentMessages = 1 + store := newSpillStore(config) + group := NewEventsGroup(0, 1, store) + first := newTestDMLMessage(1) + second := newTestDMLMessage(2) + messageData := codeccommon.NewDMLMessageData(nil, []byte("shared payload"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{first, second}, nil + }) + + messageData.AttachDMLMessage(first) + require.NoError(t, group.AppendMessage(first)) + messageData.AttachDMLMessage(second) + require.NoError(t, group.AppendMessage(second)) + + require.Len(t, store.segments, 1) + require.Same(t, group.messages[0].payload, group.messages[1].payload) + require.Same(t, group.messages[0].payload.segment, group.messages[1].payload.segment) + require.Equal(t, group.messages[0].payload.handle, group.messages[1].payload.handle) + require.NoError(t, group.Cleanup()) +} + +func TestEventsGroupAllowsOversizeSegment(t *testing.T) { + config := defaultSpillConfig() + config.segmentBytes = 16 + store := newSpillStore(config) + group := NewEventsGroup(0, 1, store) + + message := attachTestDMLMessageDataWithPayload( + newTestDMLMessage(1), []byte("oversize-key"), []byte("oversize-value")) + require.NoError(t, group.AppendMessage(message)) + require.Len(t, store.segments, 1) + require.Nil(t, store.activeSegment) + require.Greater(t, store.PendingBytes(), config.segmentBytes) + + messages, hasMore, _, err := group.ResolveIntoBatch( + math.MaxUint64, nil, ResolveLimit{MaxBytes: 1, MaxMessages: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, messages, 1) + require.Zero(t, store.PendingBytes()) +} + +func TestEventsGroupRestoreErrorDoesNotReleasePendingData(t *testing.T) { + config := defaultSpillConfig() + store := newSpillStore(config) + group := NewEventsGroup(0, 1, store) + wantErr := errors.New("restore failed") + messageData := codeccommon.NewDMLMessageData([]byte("key"), []byte("value"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + return nil, wantErr + }) + message := newTestDMLMessage(1) + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) + pendingBytes := store.PendingBytes() + + _, _, _, err := group.ResolveIntoBatch(math.MaxUint64, nil, store.ResolveLimit()) + require.ErrorIs(t, err, wantErr) + require.Len(t, group.messages, 1) + require.Equal(t, pendingBytes, store.PendingBytes()) + require.NoError(t, group.Cleanup()) + require.Zero(t, store.PendingBytes()) +} + +func TestEventsGroupResolveIntoBatchBounds(t *testing.T) { + group := NewEventsGroup(0, 1) + for _, commitTs := range []uint64{1, 2, 2, 3} { + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + + messages, hasMore, _, err := group.ResolveIntoBatch( + math.MaxUint64, nil, ResolveLimit{MaxMessages: 2}) + require.NoError(t, err) + require.True(t, hasMore) + require.Len(t, messages, 3) + require.Equal(t, []uint64{1, 2, 2}, []uint64{ + messages[0].GetCommitTs(), messages[1].GetCommitTs(), messages[2].GetCommitTs(), + }) + + messages, hasMore, _, err = group.ResolveIntoBatch( + math.MaxUint64, nil, ResolveLimit{MaxBytes: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, messages, 1) + require.Equal(t, uint64(3), messages[0].GetCommitTs()) +} + +func TestSpillStoreAllowsPendingAboveHighWatermark(t *testing.T) { + config := defaultSpillConfig() + config.pendingHighBytes = 10 + config.pendingLowBytes = 5 + store := newSpillStore(config) + + store.addPending(11) + require.True(t, store.ShouldDrain()) + store.addPending(100) + require.Equal(t, int64(111), store.PendingBytes()) + require.True(t, store.ShouldDrain()) + + store.releasePending(106) + require.Equal(t, int64(5), store.PendingBytes()) + require.False(t, store.ShouldDrain()) +} + +func TestSpillStoreDefaults(t *testing.T) { + store := NewSpillStore() + require.Equal(t, int64(128*1024*1024), store.config.segmentBytes) + require.Equal(t, int64(1024*1024*1024), store.config.pendingHighBytes) + require.Equal(t, int64(512*1024*1024), store.config.pendingLowBytes) + require.Equal(t, ResolveLimit{MaxBytes: 64 * 1024 * 1024, MaxMessages: 10000}, store.ResolveLimit()) +} + +func TestEventsGroupShrinksResolvedMetadata(t *testing.T) { + group := NewEventsGroup(0, 1) + const messageCount = 2048 + for i := 1; i <= messageCount; i++ { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(uint64(i))))) + } + require.Greater(t, cap(group.messages), 1024) + + messages, hasMore, _, err := group.ResolveIntoBatch(1536, nil, ResolveLimit{}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, messages, 1536) + require.Len(t, group.messages, 512) + require.Equal(t, len(group.messages), cap(group.messages)) + require.NoError(t, group.Cleanup()) +} + func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T) { // Scenario: A consumer resolves events by watermark/commit-ts and appends them into a downstream // batch slice. Buffered messages are held only by a spill file, and the file must be cleaned once @@ -234,7 +554,7 @@ func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) - spillPath := group.spillFile.Path() + spillPath := group.store.activeSegment.file.Path() var dst []*codeccommon.DMLMessage dst, err := group.ResolveInto(2, dst) @@ -250,7 +570,8 @@ func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T _, err = group.GetAllMessages() require.NoError(t, err) - require.Nil(t, group.spillFile) + require.Nil(t, group.store.activeSegment) + require.Empty(t, group.store.segments) _, err = os.Stat(spillPath) require.True(t, os.IsNotExist(err)) } @@ -283,7 +604,7 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) - spillPath := group.spillFile.Path() + spillPath := group.store.activeSegment.file.Path() var dst []*codeccommon.DMLMessage dst, err := group.ResolveInto(100, dst) require.NoError(t, err) @@ -293,7 +614,8 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { require.Equal(t, m2.GetCommitTs(), dst[1].GetCommitTs()) require.Len(t, group.messages, 0) - require.Nil(t, group.spillFile) + require.Nil(t, group.store.activeSegment) + require.Empty(t, group.store.segments) _, err = os.Stat(spillPath) require.True(t, os.IsNotExist(err)) } @@ -527,7 +849,7 @@ func BenchmarkEventsGroupResolveInto(b *testing.B) { b.ReportAllocs() b.ResetTimer() b.StopTimer() - for b.Loop() { + for range b.N { group := NewEventsGroup(0, 1) for _, message := range source { if err := group.AppendMessage(attachTestDMLMessageData(message)); err != nil { diff --git a/pkg/sink/codec/common/decoder.go b/pkg/sink/codec/common/decoder.go index c4803bea82..d32acec8c1 100644 --- a/pkg/sink/codec/common/decoder.go +++ b/pkg/sink/codec/common/decoder.go @@ -24,10 +24,12 @@ import ( // message after it has been spilled. One input can be attached to multiple // DMLMessages; Attach assigns each message its ordinal in that input. type DMLMessageData struct { - ID uint64 - Key []byte - Value []byte - Restore func([]byte, uint64) (*DMLMessage, error) + ID uint64 + Key []byte + Value []byte + // Decode restores every DML message from one encoded input. The spill + // store owns the decoded result and maps messages by their attached ordinal. + Decode func([]byte) ([]*DMLMessage, error) nextDMLIndex uint64 } @@ -37,13 +39,13 @@ var nextDMLMessageDataID atomic.Uint64 // NewDMLMessageData creates data shared by DMLMessages decoded from one input. func NewDMLMessageData( key, value []byte, - restore func([]byte, uint64) (*DMLMessage, error), + decode func([]byte) ([]*DMLMessage, error), ) *DMLMessageData { return &DMLMessageData{ - ID: nextDMLMessageDataID.Add(1), - Key: key, - Value: value, - Restore: restore, + ID: nextDMLMessageDataID.Add(1), + Key: key, + Value: value, + Decode: decode, } } From f364db2ddbafa4c2a530dee21d36a0d0747fb684 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Thu, 3 Sep 2026 12:24:33 +0800 Subject: [PATCH 20/22] consumer: bound spill memory with disk-backed event index --- cmd/kafka-consumer/writer.go | 21 +- cmd/pulsar-consumer/writer.go | 11 +- cmd/storage-consumer/consumer.go | 13 +- cmd/util/dml_message_decoder.go | 49 +- cmd/util/dml_message_decoder_test.go | 83 +++ cmd/util/event_group.go | 943 ++++++++++++++++++++------- cmd/util/event_group_test.go | 220 +++++-- pkg/sink/codec/common/decoder.go | 74 ++- 8 files changed, 1099 insertions(+), 315 deletions(-) diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index 2c3e6dd1c7..057fc2c1d1 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -212,7 +212,9 @@ func (w *writer) flushEventsFromGroups( return err } for _, batch := range prepared { - batch.Ack() + if err := batch.Ack(); err != nil { + return err + } } total += len(batchEvents) batchEvents = nil @@ -367,7 +369,12 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), - zap.Int64("spillPendingBytes", stats.PendingBytes)) + zap.Int64("spillIndexWriteCount", stats.IndexWriteCount), + zap.Int64("spillIndexReadCount", stats.IndexReadCount), + zap.Int64("spillAppliedEventCount", stats.AppliedEventCount), + zap.Int64("spillPendingBytes", stats.PendingBytes), + zap.Int("spillLivePayloads", stats.LivePayloads), + zap.Int("spillLiveSegments", stats.LiveSegments)) } return nil } @@ -382,9 +389,7 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool ) progress := w.progresses[partition] - progress.decoder.SetDMLMessageRestorer(func(message *common.DMLMessage) *common.DMLMessage { - return w.messageWithPartitionCheck(message, progress.partition, offset) - }) + progress.decoder.SetSourcePosition(int64(offset)) progress.decoder.AddKeyValue(message.Key, message.Value) messageType, hasNext := progress.decoder.HasNext() @@ -700,8 +705,14 @@ func (w *writer) appendMessage2Group( group := progress.eventsGroup[tableID] if group == nil { group = util.NewEventsGroup(progress.partition, tableID, w.getSpillStore()) + group.SetPostRestore(func(message *common.DMLMessage, sourcePosition int64) *common.DMLMessage { + return w.messageWithPartitionCheck(message, progress.partition, kafka.Offset(sourcePosition)) + }) progress.eventsGroup[tableID] = group } + if messageData, _ := message.SpillData(); messageData != nil { + messageData.SourcePosition = int64(offset) + } if err := group.AppendMessage(message); err != nil { return err } diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index 5aa2ed8d6b..a520feefa2 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -204,7 +204,9 @@ func (w *writer) flushEventsFromGroups( return err } for _, batch := range prepared { - batch.Ack() + if err := batch.Ack(); err != nil { + return err + } } total += len(batchEvents) batchEvents = nil @@ -361,7 +363,12 @@ func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), - zap.Int64("spillPendingBytes", stats.PendingBytes)) + zap.Int64("spillIndexWriteCount", stats.IndexWriteCount), + zap.Int64("spillIndexReadCount", stats.IndexReadCount), + zap.Int64("spillAppliedEventCount", stats.AppliedEventCount), + zap.Int64("spillPendingBytes", stats.PendingBytes), + zap.Int("spillLivePayloads", stats.LivePayloads), + zap.Int("spillLiveSegments", stats.LiveSegments)) } return nil } diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index 41529fab55..b57c08c043 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -427,8 +427,8 @@ func (c *consumer) newDMLMessageData( return nil, errors.ErrSpillFileOp.FastGenByArgs("unsupported storage DML spill protocol") } }) - decode := messageData.Decode - messageData.Decode = func(data []byte) ([]*common.DMLMessage, error) { + decode := messageData.Restorer.Decode + messageData.Restorer = common.NewDMLMessageRestorer(func(data []byte) ([]*common.DMLMessage, error) { messages, err := decode(data) if err != nil { return nil, err @@ -437,7 +437,7 @@ func (c *consumer) newDMLMessageData( messages[i] = messageWithPhysicalTableID(message, tableID) } return messages, nil - } + }) return messageData } @@ -478,7 +478,9 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { } total += len(events) } - batch.Ack() + if err := batch.Ack(); err != nil { + return err + } if !hasMore { break } @@ -492,6 +494,9 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), + zap.Int64("spillIndexWriteCount", stats.IndexWriteCount), + zap.Int64("spillIndexReadCount", stats.IndexReadCount), + zap.Int64("spillAppliedEventCount", stats.AppliedEventCount), zap.Int64("spillPendingBytes", stats.PendingBytes), zap.Int("spillLivePayloads", stats.LivePayloads), zap.Int("spillLiveSegments", stats.LiveSegments)) diff --git a/cmd/util/dml_message_decoder.go b/cmd/util/dml_message_decoder.go index dd1c2cba15..5b8c4ea76d 100644 --- a/cmd/util/dml_message_decoder.go +++ b/cmd/util/dml_message_decoder.go @@ -27,15 +27,19 @@ type DMLMessageDecoder struct { key, value []byte data *codeccommon.DMLMessageData factory DMLMessageDataFactory - restore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage + restorer *codeccommon.DMLMessageRestorer + share bool + position int64 } // NewDMLMessageDecoder wraps a decoder with the standard raw-message restorer. func NewDMLMessageDecoder(decoder codeccommon.Decoder) *DMLMessageDecoder { - return NewDMLMessageDecoderWithDataFactory(decoder, + d := NewDMLMessageDecoderWithDataFactory(decoder, func(decoder codeccommon.Decoder, key, value []byte) *codeccommon.DMLMessageData { return NewDMLMessageData(decoder, key, value) }) + d.share = true + return d } // NewDMLMessageDecoderWithDataFactory is for decoders such as CSV whose @@ -46,11 +50,11 @@ func NewDMLMessageDecoderWithDataFactory( return &DMLMessageDecoder{Decoder: decoder, factory: factory} } -// SetDMLMessageRestorer sets the per-input restore wrapper before AddKeyValue. -func (d *DMLMessageDecoder) SetDMLMessageRestorer( - restore func(*codeccommon.DMLMessage) *codeccommon.DMLMessage, -) { - d.restore = restore +// SetSourcePosition records broker-specific source metadata, such as a Kafka +// offset. The position is persisted in every event descriptor and can be used +// by an EventsGroup post-restore hook without retaining a closure per input. +func (d *DMLMessageDecoder) SetSourcePosition(position int64) { + d.position = position } // AddKeyValue implements codeccommon.Decoder. @@ -78,7 +82,15 @@ func (d *DMLMessageDecoder) NextDMLMessage() *codeccommon.DMLMessage { func (d *DMLMessageDecoder) attachDMLMessage(message *codeccommon.DMLMessage) { if d.data == nil { - d.data = d.wrapDecode(d.factory(d.Decoder, d.key, d.value)) + d.data = d.factory(d.Decoder, d.key, d.value) + if d.share { + if d.restorer == nil { + d.restorer = d.data.Restorer + } else { + d.data.Restorer = d.restorer + } + } + d.data.SourcePosition = d.position } d.data.AttachDMLMessage(message) } @@ -90,25 +102,8 @@ func (d *DMLMessageDecoder) AttachCachedDMLMessage(message *codeccommon.DMLMessa func([]byte) ([]*codeccommon.DMLMessage, error) { return []*codeccommon.DMLMessage{message}, nil }) - d.wrapDecode(data).AttachDMLMessage(message) -} - -func (d *DMLMessageDecoder) wrapDecode(data *codeccommon.DMLMessageData) *codeccommon.DMLMessageData { - if d.restore == nil { - return data - } - decode := data.Decode - data.Decode = func(payload []byte) ([]*codeccommon.DMLMessage, error) { - messages, err := decode(payload) - if err != nil { - return nil, err - } - for i, message := range messages { - messages[i] = d.restore(message) - } - return messages, nil - } - return data + data.SourcePosition = d.position + data.AttachDMLMessage(message) } // Unwrap returns the decoder that produces protocol messages. diff --git a/cmd/util/dml_message_decoder_test.go b/cmd/util/dml_message_decoder_test.go index 7cf58d7c82..fe756fafe9 100644 --- a/cmd/util/dml_message_decoder_test.go +++ b/cmd/util/dml_message_decoder_test.go @@ -40,6 +40,63 @@ func TestDMLMessageDecoderAttachesSharedData(t *testing.T) { require.Equal(t, uint64(1), secondIndex) } +func TestDMLMessageDecoderSharesRestorerAcrossInputs(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(11) + decoder := &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{first, second}} + wrapped := NewDMLMessageDecoder(decoder) + + wrapped.SetSourcePosition(100) + wrapped.AddKeyValue([]byte("first-key"), []byte("first-value")) + firstData, _ := wrapped.NextDMLMessage().SpillData() + + wrapped.SetSourcePosition(101) + wrapped.AddKeyValue([]byte("second-key"), []byte("second-value")) + secondData, _ := wrapped.NextDMLMessage().SpillData() + + require.NotSame(t, firstData, secondData) + require.Same(t, firstData.Restorer, secondData.Restorer) + require.Equal(t, int64(100), firstData.SourcePosition) + require.Equal(t, int64(101), secondData.SourcePosition) +} + +func TestDMLMessageDecoderKeepsCustomRestorersPerInput(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(11) + decoder := &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{first, second}} + wrapped := NewDMLMessageDecoderWithDataFactory(decoder, + func(_ codeccommon.Decoder, key, value []byte) *codeccommon.DMLMessageData { + return codeccommon.NewDMLMessageData(key, value, + func([]byte) ([]*codeccommon.DMLMessage, error) { return nil, nil }) + }) + + wrapped.AddKeyValue([]byte("first-key"), []byte("first-value")) + firstData, _ := wrapped.NextDMLMessage().SpillData() + wrapped.AddKeyValue([]byte("second-key"), []byte("second-value")) + secondData, _ := wrapped.NextDMLMessage().SpillData() + + require.NotSame(t, firstData.Restorer, secondData.Restorer) +} + +func TestSharedRestorerDecodesMultipleInputs(t *testing.T) { + decoder := &resettableDMLDecoder{} + wrapped := NewDMLMessageDecoder(decoder) + group := NewEventsGroup(0, 1) + + for _, commitTs := range []byte{20, 10} { + wrapped.AddKeyValue(nil, []byte{commitTs}) + message := wrapped.NextDMLMessage() + require.NotNil(t, message) + require.NoError(t, group.AppendMessage(message)) + } + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Equal(t, []uint64{10, 20}, []uint64{ + messages[0].GetCommitTs(), messages[1].GetCommitTs(), + }) +} + type dmlMessageDecoderStub struct { messages []*codeccommon.DMLMessage } @@ -62,3 +119,29 @@ func (d *dmlMessageDecoderStub) NextDMLMessage() *codeccommon.DMLMessage { } func (d *dmlMessageDecoderStub) NextDDLEvent() *commonEvent.DDLEvent { return nil } + +type resettableDMLDecoder struct { + message *codeccommon.DMLMessage +} + +func (d *resettableDMLDecoder) AddKeyValue(_, value []byte) { + if len(value) == 0 { + d.message = nil + return + } + d.message = newTestDMLMessage(uint64(value[0])) +} + +func (d *resettableDMLDecoder) HasNext() (codeccommon.MessageType, bool) { + return codeccommon.MessageTypeRow, d.message != nil +} + +func (d *resettableDMLDecoder) NextResolvedEvent() uint64 { return 0 } + +func (d *resettableDMLDecoder) NextDMLMessage() *codeccommon.DMLMessage { + message := d.message + d.message = nil + return message +} + +func (d *resettableDMLDecoder) NextDDLEvent() *commonEvent.DDLEvent { return nil } diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index b032179ea2..6639dd418b 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -14,20 +14,29 @@ package util import ( + "container/list" "encoding/binary" + "fmt" "math" "os" - "sort" + "path/filepath" + "sync/atomic" + "github.com/cockroachdb/pebble" "github.com/pingcap/log" commonEvent "github.com/pingcap/ticdc/pkg/common/event" "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" "github.com/pingcap/ticdc/pkg/spill" "go.uber.org/zap" + "golang.org/x/sys/unix" ) -const eventsGroupSpillPattern = "ticdc-events-group-*.spill" +const ( + eventsGroupSpillDirPattern = "ticdc-events-group-*" + payloadSpillPattern = "payload-*.spill" + spillIndexDir = "index" +) const ( defaultSpillSegmentBytes = 128 * 1024 * 1024 @@ -37,7 +46,14 @@ const ( defaultResolveBatchBytes = 64 * 1024 * 1024 defaultResolveBatchMessages = 10000 defaultMessageMetadataBytes = 128 + defaultIndexBatchMessages = 10000 + defaultDiskCheckBytes = 16 * 1024 * 1024 + defaultDiskUsageLimit = 0.90 + defaultIndexCacheBytes = 32 * 1024 * 1024 + defaultIndexMemTableBytes = 16 * 1024 * 1024 spillRecordLengthBytes = 8 + eventIndexKeyBytes = 3 * 8 + eventIndexValueBytes = 6 * 8 ) type spillConfig struct { @@ -48,6 +64,9 @@ type spillConfig struct { resolveBatchBytes int64 resolveBatchMessages int messageMetadataBytes int64 + indexBatchMessages int + diskCheckBytes int64 + diskUsageLimit float64 } func defaultSpillConfig() spillConfig { @@ -59,6 +78,9 @@ func defaultSpillConfig() spillConfig { resolveBatchBytes: defaultResolveBatchBytes, resolveBatchMessages: defaultResolveBatchMessages, messageMetadataBytes: defaultMessageMetadataBytes, + indexBatchMessages: defaultIndexBatchMessages, + diskCheckBytes: defaultDiskCheckBytes, + diskUsageLimit: defaultDiskUsageLimit, } } @@ -78,22 +100,92 @@ type SpillStats struct { PayloadDecodeCount int64 LivePayloads int LiveSegments int + IndexWriteCount int64 + IndexReadCount int64 + AppliedEventCount int64 +} + +type filesystemUsage struct { + usedBytes uint64 + totalBytes uint64 +} + +type payloadLocation struct { + segmentID uint64 + handle spill.Handle +} + +type spillSegment struct { + id uint64 + file *spill.RecordFile + bytes int64 + payloadCount int + pendingEvents int64 +} + +type registeredRestorer struct { + decode func([]byte) ([]*codeccommon.DMLMessage, error) + refs int64 +} + +type payloadCacheKey struct { + segmentID uint64 + offset int64 +} + +type payloadCacheEntry struct { + key payloadCacheKey + messages []*codeccommon.DMLMessage + bytes int64 + pins int + element *list.Element } -// SpillStore owns spill payloads and segments across all event groups in one consumer. -// Its watermarks request draining; they are not a hard quota and never reject an append. +type spilledMessage struct { + key []byte + commitTs uint64 + location payloadLocation + dmlIndex uint64 + restorerID uint64 + sourcePosition int64 +} + +// SpillStore owns a disk-backed ordered event index and append-only payload +// segments shared by all event groups in one consumer. Pending DML descriptors +// live in Pebble rather than one Go object per row. Only a bounded restore cache +// and segment-level reference counts remain in memory. Filesystem usage is +// checked periodically while appending and before every applied-range delete; +// crossing the hard limit latches an error so the consumer terminates. type SpillStore struct { - config spillConfig - segments map[*spillSegment]struct{} - activeSegment *spillSegment - payloads map[uint64]*spillPayload - pendingBytes int64 - draining bool + id uint64 + config spillConfig + rootDir string + index *pebble.DB + indexCache *pebble.Cache + indexBatch *pebble.Batch + indexBatchCount int + segments map[uint64]*spillSegment + activeSegment *spillSegment + restorers map[uint64]*registeredRestorer + cache map[payloadCacheKey]*payloadCacheEntry + cacheLRU list.List + cacheBytes int64 + pendingBytes int64 + livePayloads int + draining bool + nextSegmentID uint64 + nextGroupID uint64 + nextSequence uint64 + bytesSinceDiskCheck int64 + terminalErr error readRecord func(*spill.RecordFile, spill.Handle) ([]byte, error) + diskUsage func(string) (filesystemUsage, error) stats SpillStats } +var nextSpillStoreID atomic.Uint64 + // NewSpillStore creates a process-wide store with the default spill limits. func NewSpillStore() *SpillStore { return newSpillStore(defaultSpillConfig()) @@ -101,12 +193,15 @@ func NewSpillStore() *SpillStore { func newSpillStore(config spillConfig) *SpillStore { return &SpillStore{ - config: config, - segments: make(map[*spillSegment]struct{}), - payloads: make(map[uint64]*spillPayload), + id: nextSpillStoreID.Add(1), + config: config, + segments: make(map[uint64]*spillSegment), + restorers: make(map[uint64]*registeredRestorer), + cache: make(map[payloadCacheKey]*payloadCacheEntry), readRecord: func(file *spill.RecordFile, handle spill.Handle) ([]byte, error) { return file.Read(handle) }, + diskUsage: getFilesystemUsage, } } @@ -118,7 +213,8 @@ func (s *SpillStore) ResolveLimit() ResolveLimit { } } -// PendingBytes returns conservatively accounted payload, decoded cache, and metadata bytes. +// PendingBytes returns conservatively accounted payload, decoded cache, and +// logical index bytes. Physical index usage is covered by the filesystem guard. func (s *SpillStore) PendingBytes() int64 { return s.pendingBytes } @@ -132,7 +228,7 @@ func (s *SpillStore) ShouldDrain() bool { func (s *SpillStore) Stats() SpillStats { stats := s.stats stats.PendingBytes = s.pendingBytes - stats.LivePayloads = len(s.payloads) + stats.LivePayloads = s.livePayloads stats.LiveSegments = len(s.segments) return stats } @@ -169,68 +265,114 @@ func (s *SpillStore) releasePending(bytes int64) { } } -type spillSegment struct { - file *spill.RecordFile - bytes int64 - payloadCount int - pendingPayloads int -} - -type spillPayload struct { - id uint64 - segment *spillSegment - handle spill.Handle - decode func([]byte) ([]*codeccommon.DMLMessage, error) - decoded []*codeccommon.DMLMessage - decodedBytes int64 - loaded bool - refs int -} - -type spilledMessage struct { - commitTs uint64 - payload *spillPayload - dmlIndex uint64 -} - // EventsGroup stores change event messages. type EventsGroup struct { Partition int32 tableID int64 - messages []spilledMessage - store *SpillStore - batchPending bool - outOfOrder bool - HighWatermark uint64 + id uint64 + store *SpillStore + ownsStore bool + pendingCount int64 + segmentRefs map[uint64]int64 + restorerRefs map[uint64]int64 + batchPending bool + outOfOrder bool + lastAppendedTs uint64 + resolvedTs uint64 + appliedTs uint64 + postRestore func(*codeccommon.DMLMessage, int64) *codeccommon.DMLMessage + HighWatermark uint64 } // NewEventsGroup will create new event group. func NewEventsGroup(partition int32, tableID int64, stores ...*SpillStore) *EventsGroup { store := NewSpillStore() + ownsStore := true if len(stores) != 0 && stores[0] != nil { store = stores[0] + ownsStore = false } + store.nextGroupID++ return &EventsGroup{ - Partition: partition, - tableID: tableID, - messages: make([]spilledMessage, 0, 1024), - store: store, + Partition: partition, + tableID: tableID, + id: store.nextGroupID, + store: store, + ownsStore: ownsStore, + segmentRefs: make(map[uint64]int64), + restorerRefs: make(map[uint64]int64), + } +} + +// SetPostRestore installs a bounded group-level hook for source metadata that +// must be applied after lazy decoding. Unlike the old per-input closure, this +// hook does not grow with the backlog. +func (g *EventsGroup) SetPostRestore( + restore func(*codeccommon.DMLMessage, int64) *codeccommon.DMLMessage, +) { + g.postRestore = restore +} + +func (s *SpillStore) ensureOpen() error { + if s.terminalErr != nil { + return s.terminalErr + } + if s.index != nil { + return nil } + + rootDir, err := os.MkdirTemp(os.TempDir(), eventsGroupSpillDirPattern) + if err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "create spill store directory") + } + s.rootDir = rootDir + if err := s.checkDiskUsage(0, true); err != nil { + return err + } + + cache := pebble.NewCache(defaultIndexCacheBytes) + options := &pebble.Options{ + Cache: cache, + DisableWAL: true, + MaxOpenFiles: 128, + MemTableSize: defaultIndexMemTableBytes, + MemTableStopWritesThreshold: 2, + MaxConcurrentCompactions: func() int { return 1 }, + } + index, err := pebble.Open(filepath.Join(rootDir, spillIndexDir), options) + if err != nil { + cache.Unref() + return errors.WrapError(errors.ErrSpillFileOp, err, "open spill event index") + } + s.indexCache = cache + s.index = index + s.indexBatch = index.NewBatch() + return nil } func (s *SpillStore) newSegment() error { - file, err := spill.NewRecordFile(os.TempDir(), eventsGroupSpillPattern) + if err := s.ensureOpen(); err != nil { + return err + } + if err := s.checkDiskUsage(0, true); err != nil { + return err + } + file, err := spill.NewRecordFile(s.rootDir, payloadSpillPattern) if err != nil { return err } - segment := &spillSegment{file: file} - s.segments[segment] = struct{}{} + s.nextSegmentID++ + segment := &spillSegment{id: s.nextSegmentID, file: file} + s.segments[segment.id] = segment s.activeSegment = segment return nil } func (s *SpillStore) prepareSegment(recordBytes int64) error { + if err := s.ensureOpen(); err != nil { + return err + } segment := s.activeSegment if segment != nil && (segment.bytes+recordBytes > s.config.segmentBytes || segment.payloadCount >= s.config.segmentMessages) { @@ -270,139 +412,335 @@ func appendMessageData(file *spill.RecordFile, key, value []byte) (spill.Handle, return file.AppendChunks(keyLen[:], key, valueLen[:], value) } -func (s *SpillStore) acquirePayload(data *codeccommon.DMLMessageData) (*spillPayload, error) { - if payload, ok := s.payloads[data.ID]; ok { - return payload, nil +func (s *SpillStore) acquirePayload(data *codeccommon.DMLMessageData) (payloadLocation, error) { + if segmentID, offset, length, ok := data.SpillLocation(s.id); ok { + if _, exists := s.segments[segmentID]; exists { + return payloadLocation{ + segmentID: segmentID, + handle: spill.Handle{Offset: offset, Length: length}, + }, nil + } } recordBytes := spillMessageDataSize(data.Key, data.Value) + if err := s.checkDiskUsage(recordBytes, false); err != nil { + return payloadLocation{}, err + } if err := s.prepareSegment(recordBytes); err != nil { - return nil, err + return payloadLocation{}, err } segment := s.activeSegment handle, err := appendMessageData(segment.file, data.Key, data.Value) if err != nil { - return nil, err - } - payload := &spillPayload{ - id: data.ID, - segment: segment, - handle: handle, - decode: data.Decode, + return payloadLocation{}, err } - s.payloads[data.ID] = payload + data.SetSpillLocation(s.id, segment.id, handle.Offset, handle.Length) segment.bytes += recordBytes segment.payloadCount++ - segment.pendingPayloads++ + s.livePayloads++ s.addPending(recordBytes) s.stats.PayloadWriteBytes += int64(handle.Length) s.stats.PayloadWriteCount++ s.sealFullSegment() - return payload, nil + return payloadLocation{segmentID: segment.id, handle: handle}, nil } -func (s *SpillStore) retainPayload(payload *spillPayload) { - payload.refs++ - s.addPending(s.config.messageMetadataBytes) +func (s *SpillStore) cleanupSegment(segment *spillSegment) { + if s.activeSegment == segment { + s.activeSegment = nil + } + s.evictSegmentCache(segment.id) + if err := segment.file.Cleanup(); err != nil { + log.Warn("cleanup spill segment failed", zap.String("path", segment.file.Path()), zap.Error(err)) + return + } + s.releasePending(segment.bytes) + s.livePayloads -= segment.payloadCount + delete(s.segments, segment.id) } -func (s *SpillStore) loadPayload(payload *spillPayload) (int64, error) { - if payload.loaded { - return 0, nil +func (s *SpillStore) appendEventIndex(key, value []byte) error { + if err := s.ensureOpen(); err != nil { + return err } - if payload.segment == nil || payload.segment.file == nil { - return 0, errors.ErrSpillFileOp.FastGenByArgs("spill payload segment is missing") + if err := s.checkDiskUsage(int64(len(key)+len(value)), false); err != nil { + return err } - data, err := s.readRecord(payload.segment.file, payload.handle) - if err != nil { - return 0, err + if err := s.indexBatch.Set(key, value, nil); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "append spill event index") } - messages, err := payload.decode(data) - if err != nil { - return 0, err + s.indexBatchCount++ + if s.indexBatchCount >= s.config.indexBatchMessages { + return s.flushEventIndex() } - payload.decoded = messages - payload.loaded = true - payload.decodedBytes = int64(payload.handle.Length) - s.addPending(payload.decodedBytes) - s.stats.PayloadReadBytes += int64(payload.handle.Length) - s.stats.PayloadReadCount++ - s.stats.PayloadDecodeCount++ - return int64(payload.handle.Length), nil + return nil } -func (s *SpillStore) messageAt(payload *spillPayload, index uint64) (*codeccommon.DMLMessage, int64, error) { - readBytes, err := s.loadPayload(payload) - if err != nil { - return nil, 0, err +func (s *SpillStore) flushEventIndex() error { + if s.indexBatchCount == 0 { + return nil } - if index >= uint64(len(payload.decoded)) { - return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("DML spill message index is out of range") + if err := s.indexBatch.Commit(pebble.NoSync); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "commit spill event index") } - message := payload.decoded[index] - if message == nil { - return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("DML spill message was already released") + if err := s.indexBatch.Close(); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "close committed spill event index batch") } - return message, readBytes, nil + s.indexBatch = s.index.NewBatch() + s.indexBatchCount = 0 + return nil +} + +func encodeEventIndexKey(groupID, commitTs, sequence uint64) []byte { + key := make([]byte, eventIndexKeyBytes) + binary.BigEndian.PutUint64(key[0:8], groupID) + binary.BigEndian.PutUint64(key[8:16], commitTs) + binary.BigEndian.PutUint64(key[16:24], sequence) + return key } -func (s *SpillStore) releasePayloadRef(payload *spillPayload, index uint64) { - if payload.loaded && index < uint64(len(payload.decoded)) { - payload.decoded[index] = nil +func eventIndexBounds(groupID uint64) ([]byte, []byte) { + lower := make([]byte, 8) + upper := make([]byte, 8) + binary.BigEndian.PutUint64(lower, groupID) + binary.BigEndian.PutUint64(upper, groupID+1) + return lower, upper +} + +func encodeEventIndexValue( + location payloadLocation, dmlIndex, restorerID uint64, sourcePosition int64, +) []byte { + value := make([]byte, eventIndexValueBytes) + binary.BigEndian.PutUint64(value[0:8], location.segmentID) + binary.BigEndian.PutUint64(value[8:16], uint64(location.handle.Offset)) + binary.BigEndian.PutUint64(value[16:24], location.handle.Length) + binary.BigEndian.PutUint64(value[24:32], dmlIndex) + binary.BigEndian.PutUint64(value[32:40], restorerID) + binary.BigEndian.PutUint64(value[40:48], uint64(sourcePosition)) + return value +} + +func decodeEventIndexCommitTs(key []byte) (uint64, error) { + if len(key) != eventIndexKeyBytes { + return 0, errors.ErrSpillFileOp.FastGenByArgs("invalid spill event index key") + } + return binary.BigEndian.Uint64(key[8:16]), nil +} + +func decodeSpilledMessage(key, value []byte) (spilledMessage, error) { + commitTs, err := decodeEventIndexCommitTs(key) + if err != nil { + return spilledMessage{}, err + } + if len(value) != eventIndexValueBytes { + return spilledMessage{}, errors.ErrSpillFileOp.FastGenByArgs("invalid spill event index value") } - payload.refs-- - s.releasePending(s.config.messageMetadataBytes) - if payload.refs != 0 { + return spilledMessage{ + key: append([]byte(nil), key...), + commitTs: commitTs, + location: payloadLocation{ + segmentID: binary.BigEndian.Uint64(value[0:8]), + handle: spill.Handle{ + Offset: int64(binary.BigEndian.Uint64(value[8:16])), + Length: binary.BigEndian.Uint64(value[16:24]), + }, + }, + dmlIndex: binary.BigEndian.Uint64(value[24:32]), + restorerID: binary.BigEndian.Uint64(value[32:40]), + sourcePosition: int64(binary.BigEndian.Uint64(value[40:48])), + }, nil +} + +func (s *SpillStore) trimPayloadCache() { + byteLimit := s.config.resolveBatchBytes + if byteLimit <= 0 { + byteLimit = defaultResolveBatchBytes + } + entryLimit := s.config.resolveBatchMessages + if entryLimit <= 0 { + entryLimit = defaultResolveBatchMessages + } + for s.cacheBytes > byteLimit || len(s.cache) > entryLimit { + var victim *payloadCacheEntry + for element := s.cacheLRU.Back(); element != nil; element = element.Prev() { + entry := element.Value.(*payloadCacheEntry) + if entry.pins == 0 { + victim = entry + break + } + } + if victim == nil { + return + } + s.removePayloadCacheEntry(victim) + } +} + +func (s *SpillStore) removePayloadCacheEntry(entry *payloadCacheEntry) { + if entry == nil || entry.element == nil { return } - if payload.loaded { - s.releasePending(payload.decodedBytes) + delete(s.cache, entry.key) + s.cacheLRU.Remove(entry.element) + entry.element = nil + entry.messages = nil + s.cacheBytes -= entry.bytes + s.releasePending(entry.bytes) +} + +func (s *SpillStore) unpinPayloads(payloads []*payloadCacheEntry) { + for _, payload := range payloads { + payload.pins-- + if payload.pins < 0 { + log.Panic("spill payload cache pin underflow", + zap.Uint64("segmentID", payload.key.segmentID), zap.Int64("offset", payload.key.offset)) + } } - payload.decoded = nil - payload.decode = nil - delete(s.payloads, payload.id) + s.trimPayloadCache() +} - segment := payload.segment - segment.pendingPayloads-- - if segment.pendingPayloads == 0 { - s.cleanupSegment(segment) +func (s *SpillStore) evictSegmentCache(segmentID uint64) { + for _, entry := range s.cache { + if entry.key.segmentID == segmentID { + s.removePayloadCacheEntry(entry) + } } } -func (s *SpillStore) cleanupSegment(segment *spillSegment) { - if s.activeSegment == segment { - s.activeSegment = nil +func (g *EventsGroup) releaseEvent(segmentID, restorerID uint64, count int64) { + g.releaseSegmentRefs(segmentID, count) + g.releaseRestorerRefs(restorerID, count) +} + +func (g *EventsGroup) releaseSegmentRefs(segmentID uint64, count int64) { + segment := g.store.segments[segmentID] + if segment == nil || segment.pendingEvents < count || g.segmentRefs[segmentID] < count { + log.Panic("spill segment reference underflow", + zap.Uint64("segmentID", segmentID), zap.Int64("released", count)) } - if err := segment.file.Cleanup(); err != nil { - log.Warn("cleanup spill segment failed", zap.String("path", segment.file.Path()), zap.Error(err)) - return + segment.pendingEvents -= count + g.segmentRefs[segmentID] -= count + if g.segmentRefs[segmentID] == 0 { + delete(g.segmentRefs, segmentID) + } + if segment.pendingEvents == 0 { + g.store.cleanupSegment(segment) + } +} + +func (g *EventsGroup) releaseRestorerRefs(restorerID uint64, count int64) { + restorer := g.store.restorers[restorerID] + if restorer == nil || restorer.refs < count || g.restorerRefs[restorerID] < count { + log.Panic("spill restorer reference underflow", + zap.Uint64("restorerID", restorerID), zap.Int64("released", count)) + } + restorer.refs -= count + g.restorerRefs[restorerID] -= count + if g.restorerRefs[restorerID] == 0 { + delete(g.restorerRefs, restorerID) + } + if restorer.refs == 0 { + delete(g.store.restorers, restorerID) } - s.releasePending(segment.bytes) - delete(s.segments, segment) } -// Cleanup removes all remaining payload and segment state when a consumer stops. +func getFilesystemUsage(path string) (filesystemUsage, error) { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return filesystemUsage{}, errors.WrapError(errors.ErrSpillFileOp, err, "read spill filesystem usage") + } + if stat.Bsize <= 0 { + return filesystemUsage{}, errors.ErrSpillFileOp.FastGenByArgs("spill filesystem has invalid block size") + } + blockSize := uint64(stat.Bsize) + totalBytes := uint64(stat.Blocks) * blockSize + availableBytes := uint64(stat.Bavail) * blockSize + if availableBytes > totalBytes { + return filesystemUsage{}, errors.ErrSpillFileOp.FastGenByArgs("spill filesystem has invalid available blocks") + } + return filesystemUsage{usedBytes: totalBytes - availableBytes, totalBytes: totalBytes}, nil +} + +func (s *SpillStore) checkDiskUsage(nextWriteBytes int64, force bool) error { + if s.terminalErr != nil { + return s.terminalErr + } + s.bytesSinceDiskCheck += nextWriteBytes + if !force && s.config.diskCheckBytes > 0 && s.bytesSinceDiskCheck < s.config.diskCheckBytes { + return nil + } + path := s.rootDir + if path == "" { + path = os.TempDir() + } + usage, err := s.diskUsage(path) + if err != nil { + return err + } + s.bytesSinceDiskCheck = 0 + if usage.totalBytes == 0 { + return errors.ErrSpillFileOp.FastGenByArgs("spill filesystem has zero capacity") + } + projectedUsed := usage.usedBytes + if nextWriteBytes > 0 && uint64(nextWriteBytes) <= ^uint64(0)-projectedUsed { + projectedUsed += uint64(nextWriteBytes) + } + ratio := float64(projectedUsed) / float64(usage.totalBytes) + if ratio <= s.config.diskUsageLimit { + return nil + } + message := fmt.Sprintf( + "spill filesystem usage %.2f%% exceeds %.2f%% limit: path=%s usedBytes=%d totalBytes=%d", + ratio*100, s.config.diskUsageLimit*100, path, projectedUsed, usage.totalBytes) + s.terminalErr = errors.ErrSpillFileOp.FastGenByArgs(message) + log.Error("spill filesystem usage limit exceeded; stop consumer", + zap.String("path", path), + zap.Uint64("usedBytes", projectedUsed), + zap.Uint64("totalBytes", usage.totalBytes), + zap.Float64("usagePercent", ratio*100), + zap.Float64("limitPercent", s.config.diskUsageLimit*100)) + return s.terminalErr +} + +// Cleanup removes all temporary index and payload state when a consumer stops. func (s *SpillStore) Cleanup() error { - for _, payload := range s.payloads { - s.releasePending(int64(payload.refs) * s.config.messageMetadataBytes) - if payload.loaded { - s.releasePending(payload.decodedBytes) + var cleanupErr error + if s.indexBatch != nil { + if err := s.indexBatch.Close(); err != nil && cleanupErr == nil { + cleanupErr = errors.WrapError(errors.ErrSpillFileOp, err, "close spill event index batch") } - payload.decoded = nil - payload.decode = nil + s.indexBatch = nil } - clear(s.payloads) - s.activeSegment = nil - var cleanupErr error - for segment := range s.segments { - if err := segment.file.Cleanup(); err != nil { - if cleanupErr == nil { - cleanupErr = err - } - continue + if s.index != nil { + if err := s.index.Close(); err != nil && cleanupErr == nil { + cleanupErr = errors.WrapError(errors.ErrSpillFileOp, err, "close spill event index") } - s.releasePending(segment.bytes) - delete(s.segments, segment) + s.index = nil + } + if s.indexCache != nil { + s.indexCache.Unref() + s.indexCache = nil } + for _, segment := range s.segments { + if err := segment.file.Cleanup(); err != nil && cleanupErr == nil { + cleanupErr = err + } + } + if s.rootDir != "" { + if err := os.RemoveAll(s.rootDir); err != nil && cleanupErr == nil { + cleanupErr = errors.WrapError(errors.ErrSpillFileOp, err, "remove spill store directory") + } + } + s.rootDir = "" + s.activeSegment = nil + clear(s.segments) + clear(s.restorers) + clear(s.cache) + s.cacheLRU.Init() + s.cacheBytes = 0 + s.pendingBytes = 0 + s.livePayloads = 0 + s.draining = false + s.indexBatchCount = 0 return cleanupErr } @@ -416,26 +754,41 @@ func (g *EventsGroup) AppendMessage( return errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") } messageData, dmlIndex := message.SpillData() - if messageData == nil || messageData.Decode == nil { + if messageData == nil || messageData.Restorer == nil || messageData.Restorer.Decode == nil { return errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without decode function") } commitTs := message.GetCommitTs() - payload, err := g.store.acquirePayload(messageData) + location, err := g.store.acquirePayload(messageData) if err != nil { return err } - if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].commitTs { + if g.pendingCount > 0 && commitTs < g.lastAppendedTs { g.outOfOrder = true } if commitTs > g.HighWatermark { g.HighWatermark = commitTs } - g.messages = append(g.messages, spilledMessage{ - commitTs: commitTs, - payload: payload, - dmlIndex: dmlIndex, - }) - g.store.retainPayload(payload) + g.lastAppendedTs = commitTs + g.store.nextSequence++ + key := encodeEventIndexKey(g.id, commitTs, g.store.nextSequence) + value := encodeEventIndexValue(location, dmlIndex, messageData.Restorer.ID, messageData.SourcePosition) + if err := g.store.appendEventIndex(key, value); err != nil { + return err + } + + segment := g.store.segments[location.segmentID] + segment.pendingEvents++ + restorer := g.store.restorers[messageData.Restorer.ID] + if restorer == nil { + restorer = ®isteredRestorer{decode: messageData.Restorer.Decode} + g.store.restorers[messageData.Restorer.ID] = restorer + } + restorer.refs++ + g.pendingCount++ + g.segmentRefs[location.segmentID]++ + g.restorerRefs[messageData.Restorer.ID]++ + g.store.addPending(g.store.config.messageMetadataBytes) + g.store.stats.IndexWriteCount++ return nil } @@ -444,17 +797,21 @@ type ResolveBatch struct { Messages []*codeccommon.DMLMessage ResolvedBytes int64 group *EventsGroup - count int + entries []spilledMessage + payloads []*payloadCacheEntry acked bool } // Ack releases a prepared group prefix after downstream flush callbacks complete. -func (b *ResolveBatch) Ack() { +func (b *ResolveBatch) Ack() error { if b == nil || b.acked { - return + return nil + } + if err := b.group.ack(b); err != nil { + return err } - b.group.ack(b.count) b.acked = true + return nil } // PrepareResolve restores one bounded group prefix without removing it. @@ -464,45 +821,180 @@ func (g *EventsGroup) PrepareResolve( if g.batchPending { return nil, false, errors.ErrSpillFileOp.FastGenByArgs("events group already has a pending resolve batch") } - if len(g.messages) == 0 { + if resolve > g.resolvedTs { + g.resolvedTs = resolve + } + if g.pendingCount == 0 { return nil, false, nil } - if g.outOfOrder { - sort.SliceStable(g.messages, func(i, j int) bool { - return g.messages[i].commitTs < g.messages[j].commitTs - }) + if err := g.store.flushEventIndex(); err != nil { + return nil, false, err + } + + lower, upper := eventIndexBounds(g.id) + iterator, err := g.store.index.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) + if err != nil { + return nil, false, errors.WrapError(errors.ErrSpillFileOp, err, "create spill event iterator") + } + defer iterator.Close() + + entries := make([]spilledMessage, 0, boundedMessageCapacity(limit)) + seenPayloads := make(map[payloadCacheKey]struct{}) + var plannedBytes int64 + var lastCommitTs uint64 + valid := iterator.First() + for valid { + commitTs, err := decodeEventIndexCommitTs(iterator.Key()) + if err != nil { + return nil, false, err + } + if commitTs > g.resolvedTs { + break + } + entry, err := decodeSpilledMessage(iterator.Key(), iterator.Value()) + if err != nil { + return nil, false, err + } + payloadKey := payloadCacheKey{segmentID: entry.location.segmentID, offset: entry.location.handle.Offset} + additionalBytes := int64(0) + if _, ok := seenPayloads[payloadKey]; !ok { + if _, cached := g.store.cache[payloadKey]; !cached { + additionalBytes = int64(entry.location.handle.Length) + } + } + if len(entries) > 0 && exceedsResolveLimit(len(entries), plannedBytes, additionalBytes, limit) && + commitTs != lastCommitTs { + break + } + entries = append(entries, entry) + plannedBytes += additionalBytes + seenPayloads[payloadKey] = struct{}{} + lastCommitTs = commitTs + valid = iterator.Next() + } + if err := iterator.Error(); err != nil { + return nil, false, errors.WrapError(errors.ErrSpillFileOp, err, "iterate spill event index") + } + if len(entries) == 0 { + return nil, false, nil + } + hasMore := false + if valid { + nextCommitTs, err := decodeEventIndexCommitTs(iterator.Key()) + if err != nil { + return nil, false, err + } + hasMore = nextCommitTs <= g.resolvedTs } - resolvedCount := sort.Search(len(g.messages), func(i int) bool { - return g.messages[i].commitTs > resolve - }) if g.outOfOrder { - log.Warn("DML events are out of order before flush, sort them", + log.Warn("DML events were appended out of order; restore from ordered spill index", zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), - zap.Uint64("resolveTs", resolve), - zap.Int("resolved", resolvedCount)) + zap.Uint64("resolveTs", g.resolvedTs), + zap.Int("resolved", len(entries))) g.outOfOrder = false } - if resolvedCount == 0 { - return nil, false, nil - } - - batchCount := boundedResolvedBatch(g.messages[:resolvedCount], limit) batch := &ResolveBatch{ - Messages: make([]*codeccommon.DMLMessage, 0, batchCount), + Messages: make([]*codeccommon.DMLMessage, 0, len(entries)), group: g, - count: batchCount, - } - for _, message := range g.messages[:batchCount] { - restored, readBytes, err := g.store.messageAt(message.payload, message.dmlIndex) - if err != nil { - return nil, false, err + entries: entries, + } + loaded := make(map[payloadCacheKey]*payloadCacheEntry) + for _, message := range entries { + payloadKey := payloadCacheKey{segmentID: message.location.segmentID, offset: message.location.handle.Offset} + payload := loaded[payloadKey] + readBytes := int64(0) + if payload == nil { + payload, readBytes, err = g.store.loadAndPinPayload(message) + if err != nil { + g.store.unpinPayloads(batch.payloads) + return nil, false, err + } + loaded[payloadKey] = payload + batch.payloads = append(batch.payloads, payload) + } + if message.dmlIndex >= uint64(len(payload.messages)) { + g.store.unpinPayloads(batch.payloads) + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("DML spill message index is out of range") + } + restored := payload.messages[message.dmlIndex] + if restored == nil { + g.store.unpinPayloads(batch.payloads) + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("DML spill message is nil") + } + if g.postRestore != nil { + restored = g.postRestore(restored, message.sourcePosition) + } + if restored == nil { + g.store.unpinPayloads(batch.payloads) + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("post-restore returned nil DML message") } batch.Messages = append(batch.Messages, restored) batch.ResolvedBytes += readBytes } + g.store.stats.IndexReadCount += int64(len(entries)) g.batchPending = true - return batch, batchCount < resolvedCount, nil + return batch, hasMore, nil +} + +func boundedMessageCapacity(limit ResolveLimit) int { + if limit.MaxMessages <= 0 || limit.MaxMessages > defaultResolveBatchMessages { + return defaultResolveBatchMessages + } + return limit.MaxMessages +} + +func exceedsResolveLimit(count int, bytes, additionalBytes int64, limit ResolveLimit) bool { + maxBytes := limit.MaxBytes + if maxBytes <= 0 { + maxBytes = math.MaxInt64 + } + maxMessages := limit.MaxMessages + if maxMessages <= 0 { + maxMessages = int(^uint(0) >> 1) + } + return count >= maxMessages || bytes+additionalBytes > maxBytes +} + +func (s *SpillStore) loadAndPinPayload(message spilledMessage) (*payloadCacheEntry, int64, error) { + key := payloadCacheKey{segmentID: message.location.segmentID, offset: message.location.handle.Offset} + if cached := s.cache[key]; cached != nil { + cached.pins++ + s.cacheLRU.MoveToFront(cached.element) + return cached, 0, nil + } + segment := s.segments[message.location.segmentID] + if segment == nil || segment.file == nil { + return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("spill payload segment is missing") + } + restorer := s.restorers[message.restorerID] + if restorer == nil || restorer.decode == nil { + return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("DML spill restorer is missing") + } + data, err := s.readRecord(segment.file, message.location.handle) + if err != nil { + return nil, 0, err + } + messages, err := restorer.decode(data) + if err != nil { + return nil, 0, err + } + entry := &payloadCacheEntry{ + key: key, + messages: messages, + bytes: int64(message.location.handle.Length) + + int64(len(messages))*s.config.messageMetadataBytes, + pins: 1, + } + entry.element = s.cacheLRU.PushFront(entry) + s.cache[key] = entry + s.cacheBytes += entry.bytes + s.addPending(entry.bytes) + s.stats.PayloadReadBytes += int64(message.location.handle.Length) + s.stats.PayloadReadCount++ + s.stats.PayloadDecodeCount++ + s.trimPayloadCache() + return entry, int64(message.location.handle.Length), nil } // ResolveInto appends all messages with CommitTs <= resolve into dst in commit-ts order. @@ -524,60 +1016,42 @@ func (g *EventsGroup) ResolveIntoBatch( } dst = append(dst, batch.Messages...) resolvedBytes := batch.ResolvedBytes - batch.Ack() + if err := batch.Ack(); err != nil { + return dst, hasMore, resolvedBytes, err + } return dst, hasMore, resolvedBytes, nil } -func boundedResolvedBatch(messages []spilledMessage, limit ResolveLimit) int { - if len(messages) == 0 { - return 0 - } - maxBytes := limit.MaxBytes - if maxBytes <= 0 { - maxBytes = math.MaxInt64 +func (g *EventsGroup) ack(batch *ResolveBatch) error { + if len(batch.entries) == 0 { + g.batchPending = false + g.store.unpinPayloads(batch.payloads) + return nil } - maxMessages := limit.MaxMessages - if maxMessages <= 0 { - maxMessages = int(^uint(0) >> 1) + if err := g.store.checkDiskUsage(eventIndexKeyBytes+eventIndexValueBytes, true); err != nil { + return err } - - seenPayloads := make(map[*spillPayload]struct{}) - var bytes int64 - for i, message := range messages { - additionalBytes := int64(0) - if !message.payload.loaded { - if _, ok := seenPayloads[message.payload]; !ok { - additionalBytes = int64(message.payload.handle.Length) - } - } - exceedsLimit := i > 0 && (i >= maxMessages || bytes+additionalBytes > maxBytes) - if exceedsLimit && message.commitTs != messages[i-1].commitTs { - return i - } - bytes += additionalBytes - seenPayloads[message.payload] = struct{}{} + start := batch.entries[0].key + end := append(append([]byte(nil), batch.entries[len(batch.entries)-1].key...), 0) + if err := g.store.index.DeleteRange(start, end, pebble.NoSync); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "delete applied spill event range") } - return len(messages) -} - -func (g *EventsGroup) ack(count int) { - for i := range count { - message := &g.messages[i] - g.store.releasePayloadRef(message.payload, message.dmlIndex) + for _, message := range batch.entries { + g.releaseEvent(message.location.segmentID, message.restorerID, 1) } - remainingCount := len(g.messages) - count - copy(g.messages, g.messages[count:]) - clear(g.messages[remainingCount:]) - if remainingCount == 0 { - g.messages = nil - } else if cap(g.messages) > 2*remainingCount && cap(g.messages) > 1024 { - remaining := make([]spilledMessage, remainingCount) - copy(remaining, g.messages[:remainingCount]) - g.messages = remaining - } else { - g.messages = g.messages[:remainingCount] + g.pendingCount -= int64(len(batch.entries)) + g.store.releasePending(int64(len(batch.entries)) * g.store.config.messageMetadataBytes) + g.store.stats.AppliedEventCount += int64(len(batch.entries)) + lastCommitTs := batch.entries[len(batch.entries)-1].commitTs + if lastCommitTs > g.appliedTs { + g.appliedTs = lastCommitTs } g.batchPending = false + g.store.unpinPayloads(batch.payloads) + if g.pendingCount == 0 && g.ownsStore { + return g.store.Cleanup() + } + return nil } // GetAllMessages gets all messages. @@ -587,13 +1061,32 @@ func (g *EventsGroup) GetAllMessages() ([]*codeccommon.DMLMessage, error) { // Cleanup removes pending spill records when the consumer is stopping. func (g *EventsGroup) Cleanup() error { - for i := range g.messages { - message := &g.messages[i] - g.store.releasePayloadRef(message.payload, message.dmlIndex) + if g.batchPending { + return errors.ErrSpillFileOp.FastGenByArgs("cannot clean events group with pending resolve batch") } - clear(g.messages) - g.messages = nil + if g.pendingCount != 0 && g.store.index != nil { + if err := g.store.flushEventIndex(); err != nil { + return err + } + lower, upper := eventIndexBounds(g.id) + if err := g.store.index.DeleteRange(lower, upper, pebble.NoSync); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "delete spill event group") + } + for segmentID, count := range g.segmentRefs { + g.releaseSegmentRefs(segmentID, count) + } + for restorerID, count := range g.restorerRefs { + g.releaseRestorerRefs(restorerID, count) + } + g.store.releasePending(g.pendingCount * g.store.config.messageMetadataBytes) + } + g.pendingCount = 0 + clear(g.segmentRefs) + clear(g.restorerRefs) g.batchPending = false + if g.ownsStore { + return g.store.Cleanup() + } return nil } diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index 22deeb5606..38212b4a4c 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -20,6 +20,7 @@ import ( "os" "testing" + "github.com/cockroachdb/pebble" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" @@ -61,6 +62,24 @@ func attachTestDMLMessageDataWithPayload( return message } +func readGroupIndex(t *testing.T, group *EventsGroup) []spilledMessage { + t.Helper() + require.NoError(t, group.store.flushEventIndex()) + lower, upper := eventIndexBounds(group.id) + iterator, err := group.store.index.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) + require.NoError(t, err) + defer func() { require.NoError(t, iterator.Close()) }() + + entries := make([]spilledMessage, 0) + for valid := iterator.First(); valid; valid = iterator.Next() { + entry, err := decodeSpilledMessage(iterator.Key(), iterator.Value()) + require.NoError(t, err) + entries = append(entries, entry) + } + require.NoError(t, iterator.Error()) + return entries +} + func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.DMLEvent { return &commonEvent.DMLEvent{ PhysicalTableID: 1, @@ -187,11 +206,11 @@ func TestEventsGroupSharesRawMessageData(t *testing.T) { require.NoError(t, group.AppendMessage(first)) messageData.AttachDMLMessage(second) require.NoError(t, group.AppendMessage(second)) - require.Len(t, group.messages, 2) - require.Same(t, group.messages[0].payload, group.messages[1].payload) - require.Equal(t, group.messages[0].payload.handle, group.messages[1].payload.handle) - require.Equal(t, uint64(0), group.messages[0].dmlIndex) - require.Equal(t, uint64(1), group.messages[1].dmlIndex) + entries := readGroupIndex(t, group) + require.Len(t, entries, 2) + require.Equal(t, entries[0].location, entries[1].location) + require.Equal(t, uint64(0), entries[0].dmlIndex) + require.Equal(t, uint64(1), entries[1].dmlIndex) messages, err := group.GetAllMessages() require.NoError(t, err) @@ -242,6 +261,7 @@ func TestEventsGroupReadsLargeSharedPayloadOnceAcrossBatches(t *testing.T) { config := defaultSpillConfig() config.resolveBatchMessages = 10000 store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) group := NewEventsGroup(0, 1, store) originalMessages := make([]*codeccommon.DMLMessage, messageCount) restoredMessages := make([]*codeccommon.DMLMessage, messageCount) @@ -277,11 +297,11 @@ func TestEventsGroupReadsLargeSharedPayloadOnceAcrossBatches(t *testing.T) { batchCount++ resolved += len(batch.Messages) if batchCount == 1 { - require.Equal(t, int64(group.messages[0].payload.handle.Length), batch.ResolvedBytes) + require.Positive(t, batch.ResolvedBytes) } else { require.Zero(t, batch.ResolvedBytes) } - batch.Ack() + require.NoError(t, batch.Ack()) if !hasMore { break } @@ -296,12 +316,13 @@ func TestEventsGroupReadsLargeSharedPayloadOnceAcrossBatches(t *testing.T) { require.Equal(t, int64(1), store.Stats().PayloadDecodeCount) require.Equal(t, store.Stats().PayloadWriteBytes, store.Stats().PayloadReadBytes) require.Zero(t, store.PendingBytes()) - require.Empty(t, store.payloads) + require.Empty(t, store.restorers) require.Empty(t, store.segments) } func TestEventsGroupsSharePayloadUntilEveryGroupAcks(t *testing.T) { store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) firstGroup := NewEventsGroup(0, 1, store) secondGroup := NewEventsGroup(0, 2, store) first := newTestDMLMessage(1) @@ -330,15 +351,16 @@ func TestEventsGroupsSharePayloadUntilEveryGroupAcks(t *testing.T) { require.NoError(t, err) require.Equal(t, 1, readCount) require.Equal(t, 1, decodeCount) - require.Len(t, store.payloads, 1) + require.Equal(t, 1, store.Stats().LivePayloads) require.Len(t, store.segments, 1) - spillPath := firstGroup.messages[0].payload.segment.file.Path() + entry := readGroupIndex(t, firstGroup)[0] + spillPath := store.segments[entry.location.segmentID].file.Path() - firstBatch.Ack() - require.Len(t, store.payloads, 1) + require.NoError(t, firstBatch.Ack()) + require.Equal(t, 1, store.Stats().LivePayloads) require.FileExists(t, spillPath) - secondBatch.Ack() - require.Empty(t, store.payloads) + require.NoError(t, secondBatch.Ack()) + require.Zero(t, store.Stats().LivePayloads) require.Empty(t, store.segments) _, err = os.Stat(spillPath) require.True(t, os.IsNotExist(err)) @@ -346,21 +368,23 @@ func TestEventsGroupsSharePayloadUntilEveryGroupAcks(t *testing.T) { func TestEventsGroupPrepareDoesNotReleaseBeforeAck(t *testing.T) { store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) group := NewEventsGroup(0, 1, store) message := attachTestDMLMessageDataWithPayload(newTestDMLMessage(1), nil, []byte("payload")) require.NoError(t, group.AppendMessage(message)) - spillPath := group.messages[0].payload.segment.file.Path() + entry := readGroupIndex(t, group)[0] + spillPath := store.segments[entry.location.segmentID].file.Path() batch, _, err := group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) require.NoError(t, err) require.Len(t, batch.Messages, 1) - require.Len(t, group.messages, 1) + require.Equal(t, int64(1), group.pendingCount) require.FileExists(t, spillPath) _, _, err = group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) require.Error(t, err) - batch.Ack() - require.Empty(t, group.messages) + require.NoError(t, batch.Ack()) + require.Zero(t, group.pendingCount) _, err = os.Stat(spillPath) require.True(t, os.IsNotExist(err)) } @@ -372,15 +396,18 @@ func TestEventsGroupSegmentsAndPendingWatermarks(t *testing.T) { config.pendingLowBytes = 41 config.messageMetadataBytes = 1 store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) group := NewEventsGroup(0, 1, store) first := attachTestDMLMessageDataWithPayload(newTestDMLMessage(1), nil, make([]byte, 16)) second := attachTestDMLMessageDataWithPayload(newTestDMLMessage(2), nil, make([]byte, 16)) require.NoError(t, group.AppendMessage(first)) - firstSegment := group.messages[0].payload.segment + firstEntry := readGroupIndex(t, group)[0] + firstSegment := store.segments[firstEntry.location.segmentID] firstPath := firstSegment.file.Path() require.NoError(t, group.AppendMessage(second)) - secondPath := group.messages[1].payload.segment.file.Path() + entries := readGroupIndex(t, group) + secondPath := store.segments[entries[1].location.segmentID].file.Path() require.Len(t, store.segments, 2) require.Equal(t, int64(82), store.PendingBytes()) @@ -407,6 +434,7 @@ func TestEventsGroupKeepsSharedPayloadInOneSegment(t *testing.T) { config := defaultSpillConfig() config.segmentMessages = 1 store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) group := NewEventsGroup(0, 1, store) first := newTestDMLMessage(1) second := newTestDMLMessage(2) @@ -420,10 +448,9 @@ func TestEventsGroupKeepsSharedPayloadInOneSegment(t *testing.T) { messageData.AttachDMLMessage(second) require.NoError(t, group.AppendMessage(second)) + entries := readGroupIndex(t, group) require.Len(t, store.segments, 1) - require.Same(t, group.messages[0].payload, group.messages[1].payload) - require.Same(t, group.messages[0].payload.segment, group.messages[1].payload.segment) - require.Equal(t, group.messages[0].payload.handle, group.messages[1].payload.handle) + require.Equal(t, entries[0].location, entries[1].location) require.NoError(t, group.Cleanup()) } @@ -431,6 +458,7 @@ func TestEventsGroupAllowsOversizeSegment(t *testing.T) { config := defaultSpillConfig() config.segmentBytes = 16 store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) group := NewEventsGroup(0, 1, store) message := attachTestDMLMessageDataWithPayload( @@ -451,6 +479,7 @@ func TestEventsGroupAllowsOversizeSegment(t *testing.T) { func TestEventsGroupRestoreErrorDoesNotReleasePendingData(t *testing.T) { config := defaultSpillConfig() store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) group := NewEventsGroup(0, 1, store) wantErr := errors.New("restore failed") messageData := codeccommon.NewDMLMessageData([]byte("key"), []byte("value"), @@ -464,7 +493,7 @@ func TestEventsGroupRestoreErrorDoesNotReleasePendingData(t *testing.T) { _, _, _, err := group.ResolveIntoBatch(math.MaxUint64, nil, store.ResolveLimit()) require.ErrorIs(t, err, wantErr) - require.Len(t, group.messages, 1) + require.Equal(t, int64(1), group.pendingCount) require.Equal(t, pendingBytes, store.PendingBytes()) require.NoError(t, group.Cleanup()) require.Zero(t, store.PendingBytes()) @@ -512,27 +541,130 @@ func TestSpillStoreAllowsPendingAboveHighWatermark(t *testing.T) { func TestSpillStoreDefaults(t *testing.T) { store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) require.Equal(t, int64(128*1024*1024), store.config.segmentBytes) require.Equal(t, int64(1024*1024*1024), store.config.pendingHighBytes) require.Equal(t, int64(512*1024*1024), store.config.pendingLowBytes) require.Equal(t, ResolveLimit{MaxBytes: 64 * 1024 * 1024, MaxMessages: 10000}, store.ResolveLimit()) + require.Equal(t, 0.90, store.config.diskUsageLimit) +} + +func TestSpillStoreRejectsWriteAboveDiskUsageLimit(t *testing.T) { + config := defaultSpillConfig() + config.diskCheckBytes = 1 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + store.diskUsage = func(string) (filesystemUsage, error) { + return filesystemUsage{usedBytes: 91, totalBytes: 100}, nil + } + group := NewEventsGroup(0, 1, store) + + err := group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(1))) + require.ErrorContains(t, err, "spill filesystem usage") + require.ErrorContains(t, err, "90.00% limit") + require.Same(t, store.terminalErr, err) + require.Zero(t, group.pendingCount) + require.Empty(t, store.segments) + require.NoError(t, store.Cleanup()) +} + +func TestEventsGroupTracksResolvedAndAppliedFrontiers(t *testing.T) { + store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(1)))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(2)))) + + batch, hasMore, err := group.PrepareResolve(1, store.ResolveLimit()) + require.NoError(t, err) + require.False(t, hasMore) + require.Equal(t, uint64(2), group.HighWatermark) + require.Equal(t, uint64(1), group.resolvedTs) + require.Zero(t, group.appliedTs) + require.Equal(t, int64(2), group.pendingCount) + + require.NoError(t, batch.Ack()) + require.Equal(t, uint64(1), group.appliedTs) + require.Equal(t, int64(1), group.pendingCount) + require.Equal(t, int64(1), store.Stats().AppliedEventCount) + require.NoError(t, group.Cleanup()) + require.NoError(t, store.Cleanup()) +} + +func TestEventsGroupRestoresPersistedSourcePosition(t *testing.T) { + group := NewEventsGroup(3, 1) + message := newTestDMLMessage(10) + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }) + messageData.SourcePosition = 42 + messageData.AttachDMLMessage(message) + var restoredPosition int64 + group.SetPostRestore(func(message *codeccommon.DMLMessage, position int64) *codeccommon.DMLMessage { + restoredPosition = position + return message + }) + + require.NoError(t, group.AppendMessage(message)) + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 1) + require.Equal(t, int64(42), restoredPosition) } -func TestEventsGroupShrinksResolvedMetadata(t *testing.T) { +func TestSpillStoreRestoreCacheHasByteAndEntryBounds(t *testing.T) { + config := defaultSpillConfig() + config.resolveBatchBytes = 1 << 20 + config.resolveBatchMessages = 2 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + for commitTs := uint64(1); commitTs <= 4; commitTs++ { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageDataWithPayload(newTestDMLMessage(commitTs), nil, []byte{byte(commitTs)}))) + } + + for commitTs := uint64(1); commitTs <= 3; commitTs++ { + batch, hasMore, err := group.PrepareResolve(4, ResolveLimit{MaxMessages: 1}) + require.NoError(t, err) + require.True(t, hasMore) + require.Equal(t, commitTs, batch.Messages[0].GetCommitTs()) + require.NoError(t, batch.Ack()) + require.LessOrEqual(t, len(store.cache), 2) + require.LessOrEqual(t, store.cacheBytes, config.resolveBatchBytes) + } + + batch, hasMore, err := group.PrepareResolve(4, ResolveLimit{MaxMessages: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.NoError(t, batch.Ack()) + require.Empty(t, store.cache) +} + +func TestEventsGroupKeepsPerEventMetadataOnDisk(t *testing.T) { group := NewEventsGroup(0, 1) const messageCount = 2048 + messages := make([]*codeccommon.DMLMessage, 0, messageCount) for i := 1; i <= messageCount; i++ { - require.NoError(t, group.AppendMessage( - attachTestDMLMessageData(newTestDMLMessage(uint64(i))))) + messages = append(messages, newTestDMLMessage(uint64(i))) + } + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { return messages, nil }) + for _, message := range messages { + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) } - require.Greater(t, cap(group.messages), 1024) + require.Equal(t, int64(messageCount), group.pendingCount) + require.Len(t, group.segmentRefs, 1) + require.Len(t, group.restorerRefs, 1) - messages, hasMore, _, err := group.ResolveIntoBatch(1536, nil, ResolveLimit{}) + resolved, hasMore, _, err := group.ResolveIntoBatch(1536, nil, ResolveLimit{}) require.NoError(t, err) require.False(t, hasMore) - require.Len(t, messages, 1536) - require.Len(t, group.messages, 512) - require.Equal(t, len(group.messages), cap(group.messages)) + require.Len(t, resolved, 1536) + require.Equal(t, int64(512), group.pendingCount) + require.Len(t, readGroupIndex(t, group), 512) require.NoError(t, group.Cleanup()) } @@ -564,8 +696,10 @@ func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) require.Equal(t, m2.GetCommitTs(), dst[1].GetCommitTs()) - require.Len(t, group.messages, 1) - require.Equal(t, m3.GetCommitTs(), group.messages[0].commitTs) + require.Equal(t, int64(1), group.pendingCount) + entries := readGroupIndex(t, group) + require.Len(t, entries, 1) + require.Equal(t, m3.GetCommitTs(), entries[0].commitTs) require.FileExists(t, spillPath) _, err = group.GetAllMessages() @@ -590,9 +724,10 @@ func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { require.NoError(t, err) require.Len(t, dst, 0) - require.Len(t, group.messages, 2) - require.Equal(t, m1.GetCommitTs(), group.messages[0].commitTs) - require.Equal(t, m2.GetCommitTs(), group.messages[1].commitTs) + require.Equal(t, int64(2), group.pendingCount) + entries := readGroupIndex(t, group) + require.Equal(t, m1.GetCommitTs(), entries[0].commitTs) + require.Equal(t, m2.GetCommitTs(), entries[1].commitTs) } func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { @@ -613,7 +748,7 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) require.Equal(t, m2.GetCommitTs(), dst[1].GetCommitTs()) - require.Len(t, group.messages, 0) + require.Zero(t, group.pendingCount) require.Nil(t, group.store.activeSegment) require.Empty(t, group.store.segments) _, err = os.Stat(spillPath) @@ -637,8 +772,9 @@ func TestEventsGroupResolveIntoSortsOutOfOrderResolvedMessages(t *testing.T) { require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) require.Equal(t, m1.GetCommitTs(), dst[1].GetCommitTs()) - require.Len(t, group.messages, 1) - require.Equal(t, m3.GetCommitTs(), group.messages[0].commitTs) + require.Equal(t, int64(1), group.pendingCount) + entries := readGroupIndex(t, group) + require.Equal(t, m3.GetCommitTs(), entries[0].commitTs) } func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { @@ -658,7 +794,7 @@ func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) require.Equal(t, m1.GetCommitTs(), dst[1].GetCommitTs()) require.Equal(t, m3.GetCommitTs(), dst[2].GetCommitTs()) - require.Empty(t, group.messages) + require.Zero(t, group.pendingCount) } func TestEventsGroupGetAllMessagesSortsOutOfOrderMessages(t *testing.T) { @@ -677,7 +813,7 @@ func TestEventsGroupGetAllMessagesSortsOutOfOrderMessages(t *testing.T) { require.Equal(t, m2.GetCommitTs(), messages[0].GetCommitTs()) require.Equal(t, m1.GetCommitTs(), messages[1].GetCommitTs()) require.Equal(t, m3.GetCommitTs(), messages[2].GetCommitTs()) - require.Empty(t, group.messages) + require.Zero(t, group.pendingCount) } func TestEventsGroupRestoresSpilledEventRowsAndTableInfo(t *testing.T) { diff --git a/pkg/sink/codec/common/decoder.go b/pkg/sink/codec/common/decoder.go index d32acec8c1..9553daea9f 100644 --- a/pkg/sink/codec/common/decoder.go +++ b/pkg/sink/codec/common/decoder.go @@ -24,31 +24,85 @@ import ( // message after it has been spilled. One input can be attached to multiple // DMLMessages; Attach assigns each message its ordinal in that input. type DMLMessageData struct { - ID uint64 - Key []byte - Value []byte - // Decode restores every DML message from one encoded input. The spill - // store owns the decoded result and maps messages by their attached ordinal. - Decode func([]byte) ([]*DMLMessage, error) + ID uint64 + Key []byte + Value []byte + Restorer *DMLMessageRestorer + SourcePosition int64 nextDMLIndex uint64 + + spillStoreID uint64 + spillSegmentID uint64 + spillOffset int64 + spillLength uint64 } var nextDMLMessageDataID atomic.Uint64 +// DMLMessageRestorer restores every DML message from one encoded input. A +// restorer can be shared by all inputs decoded by the same decoder context, so +// a spill queue does not need to retain one closure per input message. +type DMLMessageRestorer struct { + ID uint64 + Decode func([]byte) ([]*DMLMessage, error) +} + +var nextDMLMessageRestorerID atomic.Uint64 + +// NewDMLMessageRestorer creates a decoder context that can be shared by many +// DMLMessageData values. +func NewDMLMessageRestorer(decode func([]byte) ([]*DMLMessage, error)) *DMLMessageRestorer { + return &DMLMessageRestorer{ + ID: nextDMLMessageRestorerID.Add(1), + Decode: decode, + } +} + // NewDMLMessageData creates data shared by DMLMessages decoded from one input. func NewDMLMessageData( key, value []byte, decode func([]byte) ([]*DMLMessage, error), +) *DMLMessageData { + return NewDMLMessageDataWithRestorer(key, value, NewDMLMessageRestorer(decode)) +} + +// NewDMLMessageDataWithRestorer creates input data using a reusable decoder +// context. +func NewDMLMessageDataWithRestorer( + key, value []byte, + restorer *DMLMessageRestorer, ) *DMLMessageData { return &DMLMessageData{ - ID: nextDMLMessageDataID.Add(1), - Key: key, - Value: value, - Decode: decode, + ID: nextDMLMessageDataID.Add(1), + Key: key, + Value: value, + Restorer: restorer, } } +// SpillLocation returns the payload record already written by storeID. +func (d *DMLMessageData) SpillLocation(storeID uint64) ( + segmentID uint64, offset int64, length uint64, ok bool, +) { + if d == nil || d.spillStoreID != storeID || d.spillSegmentID == 0 { + return 0, 0, 0, false + } + return d.spillSegmentID, d.spillOffset, d.spillLength, true +} + +// SetSpillLocation records the payload location while this input is being +// appended. The input object is short-lived; the location is copied into each +// disk-backed event descriptor. +func (d *DMLMessageData) SetSpillLocation( + storeID, segmentID uint64, offset int64, length uint64, +) { + d.spillStoreID = storeID + d.spillSegmentID = segmentID + d.spillOffset = offset + d.spillLength = length +} + type DMLMessage struct { TableID int64 Schema string From 1d2edb3208d1e7561f6189b2f3cfaf80689f1d31 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 4 Sep 2026 01:50:34 +0800 Subject: [PATCH 21/22] consumer: avoid repeated scans over applied spill index --- cmd/util/event_group.go | 124 +++++++++++++++++++++++++------ cmd/util/event_group_test.go | 137 ++++++++++++++++++++++++++++++++++- 2 files changed, 236 insertions(+), 25 deletions(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index 6639dd418b..acd432f5cf 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -14,6 +14,7 @@ package util import ( + "bytes" "container/list" "encoding/binary" "fmt" @@ -47,6 +48,7 @@ const ( defaultResolveBatchMessages = 10000 defaultMessageMetadataBytes = 128 defaultIndexBatchMessages = 10000 + defaultIndexCleanupMessages = 100000 defaultDiskCheckBytes = 16 * 1024 * 1024 defaultDiskUsageLimit = 0.90 defaultIndexCacheBytes = 32 * 1024 * 1024 @@ -65,6 +67,7 @@ type spillConfig struct { resolveBatchMessages int messageMetadataBytes int64 indexBatchMessages int + indexCleanupMessages int64 diskCheckBytes int64 diskUsageLimit float64 } @@ -79,6 +82,7 @@ func defaultSpillConfig() spillConfig { resolveBatchMessages: defaultResolveBatchMessages, messageMetadataBytes: defaultMessageMetadataBytes, indexBatchMessages: defaultIndexBatchMessages, + indexCleanupMessages: defaultIndexCleanupMessages, diskCheckBytes: defaultDiskCheckBytes, diskUsageLimit: defaultDiskUsageLimit, } @@ -182,6 +186,8 @@ type SpillStore struct { readRecord func(*spill.RecordFile, spill.Handle) ([]byte, error) diskUsage func(string) (filesystemUsage, error) stats SpillStats + + indexDeleteRangeCount int64 } var nextSpillStoreID atomic.Uint64 @@ -283,6 +289,12 @@ type EventsGroup struct { appliedTs uint64 postRestore func(*codeccommon.DMLMessage, int64) *codeccommon.DMLMessage HighWatermark uint64 + + // indexCursor is the first logical key not acknowledged by the downstream. + indexCursor []byte + // indexCleanupStart begins the acknowledged prefix not yet deleted from Pebble. + indexCleanupStart []byte + indexCleanupCount int64 } // NewEventsGroup will create new event group. @@ -294,14 +306,17 @@ func NewEventsGroup(partition int32, tableID int64, stores ...*SpillStore) *Even ownsStore = false } store.nextGroupID++ + lower, _ := eventIndexBounds(store.nextGroupID) return &EventsGroup{ - Partition: partition, - tableID: tableID, - id: store.nextGroupID, - store: store, - ownsStore: ownsStore, - segmentRefs: make(map[uint64]int64), - restorerRefs: make(map[uint64]int64), + Partition: partition, + tableID: tableID, + id: store.nextGroupID, + store: store, + ownsStore: ownsStore, + segmentRefs: make(map[uint64]int64), + restorerRefs: make(map[uint64]int64), + indexCursor: lower, + indexCleanupStart: append([]byte(nil), lower...), } } @@ -506,6 +521,10 @@ func eventIndexBounds(groupID uint64) ([]byte, []byte) { return lower, upper } +func nextEventIndexKey(key []byte) []byte { + return append(append([]byte(nil), key...), 0) +} + func encodeEventIndexValue( location payloadLocation, dmlIndex, restorerID uint64, sourcePosition int64, ) []byte { @@ -701,6 +720,43 @@ func (s *SpillStore) checkDiskUsage(nextWriteBytes int64, force bool) error { return s.terminalErr } +func (g *EventsGroup) deleteAppliedIndex(end []byte) error { + if bytes.Compare(g.indexCleanupStart, end) >= 0 { + return nil + } + if err := g.store.checkDiskUsage(eventIndexKeyBytes+eventIndexValueBytes, true); err != nil { + return err + } + if err := g.store.index.DeleteRange(g.indexCleanupStart, end, pebble.NoSync); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "delete applied spill event range") + } + g.store.indexDeleteRangeCount++ + return nil +} + +// prepareIndexAppend preserves ordering if a caller appends an event older +// than the already applied cursor. Consumers normally reject such events by +// their global watermark. Keeping the fallback here prevents the generic +// EventsGroup from skipping a late event or deleting it with a future +// coalesced range tombstone. +func (g *EventsGroup) prepareIndexAppend(key []byte) error { + if bytes.Compare(key, g.indexCursor) >= 0 { + return nil + } + if g.indexCleanupCount != 0 { + if err := g.store.flushEventIndex(); err != nil { + return err + } + if err := g.deleteAppliedIndex(g.indexCursor); err != nil { + return err + } + } + g.indexCursor = append(g.indexCursor[:0], key...) + g.indexCleanupStart = append(g.indexCleanupStart[:0], key...) + g.indexCleanupCount = 0 + return nil +} + // Cleanup removes all temporary index and payload state when a consumer stops. func (s *SpillStore) Cleanup() error { var cleanupErr error @@ -758,6 +814,11 @@ func (g *EventsGroup) AppendMessage( return errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without decode function") } commitTs := message.GetCommitTs() + g.store.nextSequence++ + key := encodeEventIndexKey(g.id, commitTs, g.store.nextSequence) + if err := g.prepareIndexAppend(key); err != nil { + return err + } location, err := g.store.acquirePayload(messageData) if err != nil { return err @@ -769,8 +830,6 @@ func (g *EventsGroup) AppendMessage( g.HighWatermark = commitTs } g.lastAppendedTs = commitTs - g.store.nextSequence++ - key := encodeEventIndexKey(g.id, commitTs, g.store.nextSequence) value := encodeEventIndexValue(location, dmlIndex, messageData.Restorer.ID, messageData.SourcePosition) if err := g.store.appendEventIndex(key, value); err != nil { return err @@ -831,7 +890,8 @@ func (g *EventsGroup) PrepareResolve( return nil, false, err } - lower, upper := eventIndexBounds(g.id) + _, upper := eventIndexBounds(g.id) + lower := g.indexCursor iterator, err := g.store.index.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) if err != nil { return nil, false, errors.WrapError(errors.ErrSpillFileOp, err, "create spill event iterator") @@ -1028,13 +1088,17 @@ func (g *EventsGroup) ack(batch *ResolveBatch) error { g.store.unpinPayloads(batch.payloads) return nil } - if err := g.store.checkDiskUsage(eventIndexKeyBytes+eventIndexValueBytes, true); err != nil { - return err + end := nextEventIndexKey(batch.entries[len(batch.entries)-1].key) + indexCleanupCount := g.indexCleanupCount + int64(len(batch.entries)) + cleanupMessages := g.store.config.indexCleanupMessages + if cleanupMessages <= 0 { + cleanupMessages = defaultIndexCleanupMessages } - start := batch.entries[0].key - end := append(append([]byte(nil), batch.entries[len(batch.entries)-1].key...), 0) - if err := g.store.index.DeleteRange(start, end, pebble.NoSync); err != nil { - return errors.WrapError(errors.ErrSpillFileOp, err, "delete applied spill event range") + indexDeleted := indexCleanupCount >= cleanupMessages + if indexDeleted { + if err := g.deleteAppliedIndex(end); err != nil { + return err + } } for _, message := range batch.entries { g.releaseEvent(message.location.segmentID, message.restorerID, 1) @@ -1046,6 +1110,13 @@ func (g *EventsGroup) ack(batch *ResolveBatch) error { if lastCommitTs > g.appliedTs { g.appliedTs = lastCommitTs } + g.indexCursor = end + if indexDeleted { + g.indexCleanupStart = append(g.indexCleanupStart[:0], end...) + g.indexCleanupCount = 0 + } else { + g.indexCleanupCount = indexCleanupCount + } g.batchPending = false g.store.unpinPayloads(batch.payloads) if g.pendingCount == 0 && g.ownsStore { @@ -1064,7 +1135,7 @@ func (g *EventsGroup) Cleanup() error { if g.batchPending { return errors.ErrSpillFileOp.FastGenByArgs("cannot clean events group with pending resolve batch") } - if g.pendingCount != 0 && g.store.index != nil { + if g.store.index != nil { if err := g.store.flushEventIndex(); err != nil { return err } @@ -1072,17 +1143,22 @@ func (g *EventsGroup) Cleanup() error { if err := g.store.index.DeleteRange(lower, upper, pebble.NoSync); err != nil { return errors.WrapError(errors.ErrSpillFileOp, err, "delete spill event group") } - for segmentID, count := range g.segmentRefs { - g.releaseSegmentRefs(segmentID, count) - } - for restorerID, count := range g.restorerRefs { - g.releaseRestorerRefs(restorerID, count) - } - g.store.releasePending(g.pendingCount * g.store.config.messageMetadataBytes) + g.store.indexDeleteRangeCount++ + } + for segmentID, count := range g.segmentRefs { + g.releaseSegmentRefs(segmentID, count) + } + for restorerID, count := range g.restorerRefs { + g.releaseRestorerRefs(restorerID, count) } + g.store.releasePending(g.pendingCount * g.store.config.messageMetadataBytes) g.pendingCount = 0 clear(g.segmentRefs) clear(g.restorerRefs) + lower, _ := eventIndexBounds(g.id) + g.indexCursor = lower + g.indexCleanupStart = append(g.indexCleanupStart[:0], lower...) + g.indexCleanupCount = 0 g.batchPending = false if g.ownsStore { return g.store.Cleanup() diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index 38212b4a4c..1b94d7ca2c 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -63,9 +63,13 @@ func attachTestDMLMessageDataWithPayload( } func readGroupIndex(t *testing.T, group *EventsGroup) []spilledMessage { + return readGroupIndexFrom(t, group, group.indexCursor) +} + +func readGroupIndexFrom(t *testing.T, group *EventsGroup, lower []byte) []spilledMessage { t.Helper() require.NoError(t, group.store.flushEventIndex()) - lower, upper := eventIndexBounds(group.id) + _, upper := eventIndexBounds(group.id) iterator, err := group.store.index.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) require.NoError(t, err) defer func() { require.NoError(t, iterator.Close()) }() @@ -80,6 +84,92 @@ func readGroupIndex(t *testing.T, group *EventsGroup) []spilledMessage { return entries } +func TestEventsGroupResolvesFromAppliedIndexCursor(t *testing.T) { + config := defaultSpillConfig() + config.resolveBatchMessages = 1 + config.indexCleanupMessages = 1000 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + const messageCount = 129 + for commitTs := uint64(1); commitTs <= messageCount; commitTs++ { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + for resolveTs := uint64(1); resolveTs < messageCount; resolveTs++ { + batch, _, err := group.PrepareResolve(resolveTs, store.ResolveLimit()) + require.NoError(t, err) + require.NoError(t, batch.Ack()) + } + + require.Zero(t, store.indexDeleteRangeCount) + lower, _ := eventIndexBounds(group.id) + require.Len(t, readGroupIndexFrom(t, group, lower), messageCount) + entries := readGroupIndex(t, group) + require.Len(t, entries, 1) + require.Equal(t, uint64(messageCount), entries[0].commitTs) +} + +func TestEventsGroupCoalescesAppliedIndexCleanup(t *testing.T) { + config := defaultSpillConfig() + config.resolveBatchMessages = 1 + config.indexCleanupMessages = 64 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + const messageCount = 257 + for commitTs := uint64(1); commitTs <= messageCount; commitTs++ { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + + for resolveTs := uint64(1); resolveTs < messageCount; resolveTs++ { + batch, _, err := group.PrepareResolve(resolveTs, store.ResolveLimit()) + require.NoError(t, err) + require.NotNil(t, batch) + require.Len(t, batch.Messages, 1) + require.Equal(t, resolveTs, batch.Messages[0].GetCommitTs()) + require.NoError(t, batch.Ack()) + } + + require.Equal(t, int64(4), store.indexDeleteRangeCount) + require.Zero(t, group.indexCleanupCount) + require.Equal(t, int64(1), group.pendingCount) + entries := readGroupIndex(t, group) + require.Len(t, entries, 1) + require.Equal(t, uint64(messageCount), entries[0].commitTs) +} + +func TestEventsGroupRestoresLateEventBehindAppliedCursor(t *testing.T) { + config := defaultSpillConfig() + config.indexCleanupMessages = 100 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + for _, commitTs := range []uint64{10, 20, 30} { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + resolved, err := group.ResolveInto(20, nil) + require.NoError(t, err) + require.Equal(t, []uint64{10, 20}, []uint64{ + resolved[0].GetCommitTs(), resolved[1].GetCommitTs(), + }) + require.Zero(t, store.indexDeleteRangeCount) + + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(15)))) + require.Equal(t, int64(1), store.indexDeleteRangeCount) + resolved, err = group.GetAllMessages() + require.NoError(t, err) + require.Equal(t, []uint64{15, 30}, []uint64{ + resolved[0].GetCommitTs(), resolved[1].GetCommitTs(), + }) +} + func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.DMLEvent { return &commonEvent.DMLEvent{ PhysicalTableID: 1, @@ -546,6 +636,7 @@ func TestSpillStoreDefaults(t *testing.T) { require.Equal(t, int64(1024*1024*1024), store.config.pendingHighBytes) require.Equal(t, int64(512*1024*1024), store.config.pendingLowBytes) require.Equal(t, ResolveLimit{MaxBytes: 64 * 1024 * 1024, MaxMessages: 10000}, store.ResolveLimit()) + require.Equal(t, int64(100000), store.config.indexCleanupMessages) require.Equal(t, 0.90, store.config.diskUsageLimit) } @@ -1006,3 +1097,47 @@ func BenchmarkEventsGroupResolveInto(b *testing.B) { }) } } + +func BenchmarkEventsGroupResolveIncrementally(b *testing.B) { + const ( + messageCount = 32 * 1024 + batchMessages = 16 + ) + + oldLogLevel := log.GetLevel() + log.SetLevel(zapcore.FatalLevel) + b.Cleanup(func() { log.SetLevel(oldLogLevel) }) + + b.ReportAllocs() + for range b.N { + b.StopTimer() + config := defaultSpillConfig() + config.resolveBatchMessages = batchMessages + store := newSpillStore(config) + group := NewEventsGroup(0, 1, store) + for i := 1; i <= messageCount; i++ { + if err := group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(uint64(i)))); err != nil { + b.Fatal(err) + } + } + + b.StartTimer() + for { + batch, hasMore, err := group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + if err != nil { + b.Fatal(err) + } + if err := batch.Ack(); err != nil { + b.Fatal(err) + } + if !hasMore { + break + } + } + b.StopTimer() + if err := store.Cleanup(); err != nil { + b.Fatal(err) + } + } +} From e1cba2d67520a776c15da60669f54abea1018a66 Mon Sep 17 00:00:00 2001 From: dongmen <414110582@qq.com> Date: Fri, 4 Sep 2026 02:37:42 +0800 Subject: [PATCH 22/22] consumer: allocate spill resolve batches lazily --- cmd/util/event_group.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index acd432f5cf..50ac0d2191 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -898,7 +898,9 @@ func (g *EventsGroup) PrepareResolve( } defer iterator.Close() - entries := make([]spilledMessage, 0, boundedMessageCapacity(limit)) + // Most groups have no event below a given global watermark. Allocate the + // batch lazily so polling those groups does not create a full-sized slice. + var entries []spilledMessage seenPayloads := make(map[payloadCacheKey]struct{}) var plannedBytes int64 var lastCommitTs uint64 @@ -926,6 +928,13 @@ func (g *EventsGroup) PrepareResolve( commitTs != lastCommitTs { break } + if entries == nil { + capacity := boundedMessageCapacity(limit) + if int64(capacity) > g.pendingCount { + capacity = int(g.pendingCount) + } + entries = make([]spilledMessage, 0, capacity) + } entries = append(entries, entry) plannedBytes += additionalBytes seenPayloads[payloadKey] = struct{}{}