diff --git a/cmd/kafka-consumer/consumer.go b/cmd/kafka-consumer/consumer.go index 2bf26f1aee..1c50034144 100644 --- a/cmd/kafka-consumer/consumer.go +++ b/cmd/kafka-consumer/consumer.go @@ -151,7 +151,10 @@ func (c *consumer) readMessage(ctx context.Context) error { log.Error("read message failed, just continue to retry", zap.Error(err)) continue } - needCommit := c.writer.WriteMessage(ctx, msg) + needCommit, err := c.writer.WriteMessage(ctx, msg) + if err != nil { + return err + } if !needCommit { continue } @@ -169,7 +172,13 @@ func (c *consumer) readMessage(ctx context.Context) error { } // Run the consumer, read data and write to the downstream target. -func (c *consumer) Run(ctx context.Context) error { +func (c *consumer) Run(ctx context.Context) (err error) { + defer func() { + if cleanupErr := c.writer.cleanupEventsGroups(); err == nil && cleanupErr != nil { + err = cleanupErr + } + }() + g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return c.writer.run(ctx) diff --git a/cmd/kafka-consumer/main.go b/cmd/kafka-consumer/main.go index d6d3e65058..efd9f71b2d 100644 --- a/cmd/kafka-consumer/main.go +++ b/cmd/kafka-consumer/main.go @@ -38,7 +38,7 @@ var ( ) func main() { - debug.SetMemoryLimit(14 * 1024 * 1024 * 1024) + debug.SetMemoryLimit(8 * 1024 * 1024 * 1024) var ( upstreamURIStr string configFile string diff --git a/cmd/kafka-consumer/writer.go b/cmd/kafka-consumer/writer.go index bbf8388405..057fc2c1d1 100644 --- a/cmd/kafka-consumer/writer.go +++ b/cmd/kafka-consumer/writer.go @@ -45,14 +45,14 @@ type partitionProgress struct { watermarkOffset kafka.Offset eventsGroup map[int64]*util.EventsGroup - decoder common.Decoder + decoder *util.DMLMessageDecoder } func newPartitionProgress(partition int32, decoder common.Decoder) *partitionProgress { return &partitionProgress{ partition: partition, eventsGroup: make(map[int64]*util.EventsGroup), - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } } @@ -87,6 +87,7 @@ type writer struct { maxBatchSize int mysqlSink sink.Sink enableTableAcrossNodes bool + spillStore *util.SpillStore } func newWriter(ctx context.Context, o *option) *writer { @@ -99,6 +100,7 @@ func newWriter(ctx context.Context, o *option) *writer { ddlList: make([]*event.DDLEvent, 0), ddlWithMaxCommitTs: make(map[int64]uint64), enableTableAcrossNodes: o.enableTableAcrossNodes, + spillStore: util.NewSpillStore(), } var ( db *sql.DB @@ -147,46 +149,131 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } -func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { - var ( - done = make(chan struct{}, 1) +func (w *writer) getSpillStore() *util.SpillStore { + if w.spillStore == nil { + w.spillStore = util.NewSpillStore() + } + return w.spillStore +} - flushed atomic.Int64 - ) +func (w *writer) cleanupEventsGroups() error { + var cleanupErr error + for _, progress := range w.progresses { + for _, group := range progress.eventsGroup { + _ = group.Cleanup() + } + } + if err := w.getSpillStore().Cleanup(); err != nil { + cleanupErr = err + log.Warn("cleanup spill store failed", zap.Error(err)) + } + return cleanupErr +} +func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { tableIDs := w.getBlockTableIDs(ddl) commitTs := ddl.GetCommitTs() - resolvedEvents := make([]*event.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for tableID := range tableIDs { for _, progress := range w.progresses { g, ok := progress.eventsGroup[tableID] if !ok { continue } - messages := g.ResolveInto(commitTs, nil) - events := make([]*event.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + groups = append(groups, g) + } + } + total, err := w.flushEventsFromGroups(ctx, groups, commitTs, + zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query)) + if err != nil { + return err + } + + if total != 0 { + log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Any("tables", tableIDs)) + } + return w.mysqlSink.WriteBlockEvent(ddl) +} + +func (w *writer) flushEventsFromGroups( + ctx context.Context, groups []*util.EventsGroup, resolveTs uint64, fields ...zap.Field, +) (int, error) { + limit := w.getSpillStore().ResolveLimit() + batchEvents := make([]*event.DMLEvent, 0, limit.MaxMessages) + batchMessages := 0 + var batchBytes int64 + total := 0 + prepared := make([]*util.ResolveBatch, 0, len(groups)) + flush := func() error { + if err := w.flushDMLBatch(ctx, batchEvents, fields...); err != nil { + return err + } + for _, batch := range prepared { + if err := batch.Ack(); err != nil { + return err + } + } + total += len(batchEvents) + batchEvents = nil + prepared = prepared[:0] + batchMessages = 0 + batchBytes = 0 + return nil + } + + for { + hasMoreGroups := false + preparedAny := false + for _, group := range groups { + if batchMessages >= limit.MaxMessages || batchBytes >= limit.MaxBytes { + if err := flush(); err != nil { + return 0, err + } + } + remaining := util.ResolveLimit{ + MaxBytes: limit.MaxBytes - batchBytes, + MaxMessages: limit.MaxMessages - batchMessages, + } + batch, hasMore, err := group.PrepareResolve(resolveTs, remaining) + if err != nil { + return 0, err } - resolvedEvents = append(resolvedEvents, events...) + hasMoreGroups = hasMoreGroups || hasMore + if batch != nil { + preparedAny = true + prepared = append(prepared, batch) + batchEvents = append(batchEvents, util.DMLMessagesToEvents(batch.Messages)...) + batchMessages += len(batch.Messages) + batchBytes += batch.ResolvedBytes + } + } + if err := flush(); err != nil { + return 0, err + } + if !hasMoreGroups || !preparedAny { + break } } + return total, nil +} - total := len(resolvedEvents) - if total == 0 { - return w.mysqlSink.WriteBlockEvent(ddl) +func (w *writer) flushDMLBatch(ctx context.Context, events []*event.DMLEvent, fields ...zap.Field) error { + if len(events) == 0 { + return nil } - for _, e := range resolvedEvents { + done := make(chan struct{}) + var flushed atomic.Int64 + for _, e := range events { e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { + if flushed.Inc() == int64(len(events)) { close(done) } }) w.mysqlSink.AddDMLEvent(e) } - - log.Info("flush DML events before DDL", zap.Uint64("DDLCommitTs", commitTs), zap.Int("total", total)) - start := time.Now() ticker := time.NewTicker(time.Minute) defer ticker.Stop() for { @@ -194,14 +281,10 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *event.DDLEvent) error { case <-ctx.Done(): return context.Cause(ctx) case <-done: - log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), - zap.Int("total", total), zap.Duration("duration", time.Since(start)), - zap.Any("tables", tableIDs)) - return w.mysqlSink.WriteBlockEvent(ddl) + return nil case <-ticker.C: - log.Warn("DML events cannot be flushed in time", - zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + log.Warn("DML events cannot be flushed in time", append(fields, + zap.Int("total", len(events)), zap.Int64("flushed", flushed.Load()))...) } } } @@ -265,68 +348,48 @@ func (w *writer) globalWatermark() uint64 { } func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { - var ( - done = make(chan struct{}, 1) - - flushed atomic.Int64 - ) - watermark := w.globalWatermark() - resolvedEvents := make([]*event.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - messages := group.ResolveInto(watermark, nil) - events := make([]*event.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } - resolvedEvents = append(resolvedEvents, events...) - } - } - total := len(resolvedEvents) - if total == 0 { - return nil - } - for _, e := range resolvedEvents { - e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { - close(done) - } - }) - w.mysqlSink.AddDMLEvent(e) - log.Debug("flush DML event", zap.Int64("tableID", e.GetTableID()), - zap.Uint64("commitTs", e.GetCommitTs()), zap.Any("startTs", e.GetStartTs())) - } - - log.Info("flush DML events by watermark", zap.Uint64("watermark", watermark), zap.Int("total", total)) - start := time.Now() - ticker := time.NewTicker(time.Minute) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return context.Cause(ctx) - case <-done: - log.Info("flush DML events done", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Duration("duration", time.Since(start))) - return nil - case <-ticker.C: - log.Warn("DML events cannot be flushed in time", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + groups = append(groups, group) } } + total, err := w.flushEventsFromGroups(ctx, groups, watermark, zap.Uint64("watermark", watermark)) + if err != nil { + return err + } + if total != 0 { + stats := w.getSpillStore().Stats() + log.Info("flush DML events done", zap.Uint64("watermark", watermark), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Int64("spillPayloadWriteBytes", stats.PayloadWriteBytes), + zap.Int64("spillPayloadReadBytes", stats.PayloadReadBytes), + zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), + zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), + zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), + zap.Int64("spillIndexWriteCount", stats.IndexWriteCount), + zap.Int64("spillIndexReadCount", stats.IndexReadCount), + zap.Int64("spillAppliedEventCount", stats.AppliedEventCount), + zap.Int64("spillPendingBytes", stats.PendingBytes), + zap.Int("spillLivePayloads", stats.LivePayloads), + zap.Int("spillLiveSegments", stats.LiveSegments)) + } + return nil } // WriteMessage is to decode kafka message to event. // return true if the message is flushed to the downstream. // return error if flush messages failed. -func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool { +func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) (bool, error) { var ( partition = message.TopicPartition.Partition offset = message.TopicPartition.Offset ) progress := w.progresses[partition] + progress.decoder.SetSourcePosition(int64(offset)) progress.decoder.AddKeyValue(message.Key, message.Value) messageType, hasNext := progress.decoder.HasNext() @@ -335,6 +398,7 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool } needFlush := false + wasDraining := w.getSpillStore().ShouldDrain() switch messageType { case common.MessageTypeResolved: newWatermark := progress.decoder.NextResolvedEvent() @@ -349,25 +413,28 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool // but all DDL event messages should be consumed. ddl := progress.decoder.NextDDLEvent() - if dec, ok := progress.decoder.(*simple.Decoder); ok { + if dec, ok := progress.decoder.Unwrap().(*simple.Decoder); ok { cachedMessages := dec.GetCachedMessages() for _, dmlMessage := range cachedMessages { log.Info("simple protocol cached event resolved, append to the group", zap.Int64("tableID", dmlMessage.TableID), zap.Uint64("commitTs", dmlMessage.GetCommitTs()), zap.Int32("partition", partition), zap.Any("offset", offset)) - w.appendMessage2Group(dmlMessage, progress, offset) + progress.decoder.AttachCachedDMLMessage(dmlMessage) + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + return false, err + } } } w.onDDL(ddl) // DDL is broadcast to all partitions, but only handle the DDL from partition-0. if partition != 0 { - return false + return false, nil } // the Query maybe empty if using simple protocol, it's comes from `bootstrap` event, no need to handle it. if ddl.Query == "" { - return false + return false, nil } w.appendDDL(ddl) log.Info("DDL event received", @@ -389,7 +456,9 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool break } - w.appendMessage2Group(dmlMessage, progress, offset) + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + return false, err + } counter++ for { _, hasNext = progress.decoder.HasNext() @@ -405,7 +474,9 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool log.Debug("DML message is nil, it's cached", zap.Int32("partition", partition), zap.Any("offset", offset)) break } - w.appendMessage2Group(dmlMessage, progress, offset) + if err := w.appendMessage2Group(dmlMessage, progress, offset); err != nil { + return false, err + } counter++ } // If the message containing only one event exceeds the length limit, CDC will allow it and issue a warning. @@ -427,11 +498,16 @@ func (w *writer) WriteMessage(ctx context.Context, message *kafka.Message) bool if needFlush { return w.Write(ctx, messageType) } - return false + if !wasDraining && w.getSpillStore().ShouldDrain() { + if err := w.flushDMLEventsByWatermark(ctx); err != nil { + return false, err + } + } + return false, nil } // Write will synchronously write data downstream -func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool { +func (w *writer) Write(ctx context.Context, messageType common.MessageType) (bool, error) { // DDL events can be received out of commit-ts order (e.g. due to protocol-level broadcasting and // buffering differences between DDL kinds). We must execute DDLs in commit-ts order; otherwise a // "future" DDL that is not yet eligible (commitTs > watermark) can block executing earlier DDLs @@ -479,8 +555,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool break } if err := w.flushDDLEvent(ctx, todoDDL); err != nil { - log.Panic("write DDL event failed", zap.Error(err), - zap.String("DDL", todoDDL.Query), zap.Uint64("commitTs", todoDDL.GetCommitTs())) + return false, err } } @@ -488,7 +563,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool // since watermark is broadcast to all partitions, so that each partition can flush events individually. err := w.flushDMLEventsByWatermark(ctx) if err != nil { - log.Panic("flush dml events by the watermark failed", zap.Error(err)) + return false, err } } @@ -498,9 +573,9 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool log.Info("some DDL events will be flushed in the future", zap.Uint64("watermark", watermark), zap.Int("length", len(w.ddlList))) - return false + return false, nil } - return true + return true, nil } func (w *writer) onDDL(ddl *event.DDLEvent) { @@ -600,7 +675,11 @@ func (w *writer) messageWithPartitionCheck(message *common.DMLMessage, partition }) } -func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress, offset kafka.Offset) { +func (w *writer) appendMessage2Group( + message *common.DMLMessage, + progress *partitionProgress, + offset kafka.Offset, +) error { // if the kafka cluster is normal, this should not hit. // else if the cluster is abnormal, the consumer may consume old message, then cause the watermark fallback. var ( @@ -620,16 +699,23 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } group := progress.eventsGroup[tableID] if group == nil { - group = util.NewEventsGroup(progress.partition, tableID) + group = util.NewEventsGroup(progress.partition, tableID, w.getSpillStore()) + group.SetPostRestore(func(message *common.DMLMessage, sourcePosition int64) *common.DMLMessage { + return w.messageWithPartitionCheck(message, progress.partition, kafka.Offset(sourcePosition)) + }) progress.eventsGroup[tableID] = group } - message = w.messageWithPartitionCheck(message, progress.partition, offset) - group.AppendMessage(message) + if messageData, _ := message.SpillData(); messageData != nil { + messageData.SourcePosition = int64(offset) + } + if err := group.AppendMessage(message); err != nil { + return err + } if commitTs < progress.watermark { log.Warn("DML event fallback row, since less than the partition watermark, append it and sort before flush", zap.Int64("tableID", tableID), zap.Int32("partition", group.Partition), @@ -639,7 +725,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } if commitTs >= group.HighWatermark { log.Debug("DML event append to the group", @@ -647,7 +733,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.Uint64("commitTs", commitTs), zap.Uint64("HighWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return + return nil } log.Warn("DML event commit ts fallback, append it and sort before flush", zap.Int32("partition", progress.partition), zap.Any("offset", offset), @@ -656,6 +742,7 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) + return nil } func openDB(ctx context.Context, dsn string) (*sql.DB, error) { diff --git a/cmd/kafka-consumer/writer_test.go b/cmd/kafka-consumer/writer_test.go index 5cb904abc4..f06fb120c6 100644 --- a/cmd/kafka-consumer/writer_test.go +++ b/cmd/kafka-consumer/writer_test.go @@ -273,8 +273,10 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { ctrl := gomock.NewController(t) s := sinkmock.NewMockSink(ctrl) flushedCommitTs := make([]uint64, 0) + flushedRowTypeCounts := make([]int, 0) s.EXPECT().AddDMLEvent(gomock.Any()).Do(func(event *commonEvent.DMLEvent) { flushedCommitTs = append(flushedCommitTs, event.GetCommitTs()) + flushedRowTypeCounts = append(flushedRowTypeCounts, len(event.RowTypes)) event.PostFlush() }).Times(2) @@ -294,13 +296,23 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { protocol: config.ProtocolOpen, } - w.appendMessage2Group(newDMLMessageForWriterTest(20), p, kafka.Offset(1)) - w.appendMessage2Group(newDMLMessageForWriterTest(10), p, kafka.Offset(2)) - w.appendMessage2Group(newDMLMessageForWriterTest(20), p, kafka.Offset(3)) + for _, item := range []struct { + message *codeccommon.DMLMessage + offset kafka.Offset + }{ + {newDMLMessageForWriterTest(20), kafka.Offset(1)}, + {newDMLMessageForWriterTest(10), kafka.Offset(2)}, + {newDMLMessageForWriterTest(20), kafka.Offset(3)}, + } { + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(item.message), p, item.offset)) + } p.watermark = 20 - require.True(t, w.Write(ctx, codeccommon.MessageTypeResolved)) + needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) + require.NoError(t, err) + require.True(t, needCommit) require.Equal(t, []uint64{10, 20}, flushedCommitTs) + require.Equal(t, []int{1, 2}, flushedRowTypeCounts) } func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { @@ -313,7 +325,7 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { partition: 0, eventsGroup: make(map[int64]*util.EventsGroup), watermark: 20, - decoder: &singleDMLDecoder{message: newDMLMessageForWriterTest(10)}, + decoder: util.NewDMLMessageDecoder(&singleDMLDecoder{message: newDMLMessageForWriterTest(10)}), } w := &writer{ progresses: []*partitionProgress{progress}, @@ -323,9 +335,10 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { maxMessageBytes: 1, } - needCommit := w.WriteMessage(ctx, &kafka.Message{ + needCommit, err := w.WriteMessage(ctx, &kafka.Message{ TopicPartition: kafka.TopicPartition{Partition: 0, Offset: kafka.Offset(10)}, }) + require.NoError(t, err) require.False(t, needCommit) require.Nil(t, progress.eventsGroup[1]) @@ -350,10 +363,12 @@ func TestAppendMessageKeepsFallbackDMLAboveGlobalWatermark(t *testing.T) { protocol: config.ProtocolOpen, } - w.appendMessage2Group(newDMLMessageForWriterTest(10), progress, kafka.Offset(10)) + message := newDMLMessageForWriterTest(10) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(message), progress, kafka.Offset(10))) require.NotNil(t, progress.eventsGroup[1]) - resolved := progress.eventsGroup[1].ResolveInto(20, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(20, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(10), resolved[0].GetCommitTs()) } @@ -401,10 +416,13 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTableForAvro(t *testing.T) { } progress := w.progresses[0] - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) + first := codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)) + second := codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(first), progress, kafka.Offset(10))) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(second), progress, kafka.Offset(11))) - resolved := progress.eventsGroup[1].ResolveInto(150, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(100), resolved[0].GetCommitTs()) } @@ -449,10 +467,13 @@ func TestAppendRow2GroupKeepsDebeziumPartitionTableFallback(t *testing.T) { } progress := w.progresses[0] - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)), progress, kafka.Offset(10)) - w.appendMessage2Group(codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)), progress, kafka.Offset(11)) + first := codeccommon.NewDMLMessageFromEvent(newDMLEvent(200)) + second := codeccommon.NewDMLMessageFromEvent(newDMLEvent(100)) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(first), progress, kafka.Offset(10))) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(second), progress, kafka.Offset(11))) - resolved := progress.eventsGroup[1].ResolveInto(150, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(100), resolved[0].GetCommitTs()) }) @@ -463,6 +484,7 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { return codeccommon.NewDMLMessage(1, "test", "t", commitTs, common.RowTypeUpdate, func() *commonEvent.DMLEvent { return &commonEvent.DMLEvent{ PhysicalTableID: 1, + StartTs: commitTs - 1, CommitTs: commitTs, RowTypes: []common.RowType{common.RowTypeUpdate}, Rows: chunk.NewChunkWithCapacity(nil, 0), @@ -473,6 +495,16 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { }) } +func attachDMLMessageDataForWriterTest(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + type singleDMLDecoder struct { message *codeccommon.DMLMessage consumed bool diff --git a/cmd/pulsar-consumer/consumer.go b/cmd/pulsar-consumer/consumer.go index 8c75a8843b..e9284cbb7f 100644 --- a/cmd/pulsar-consumer/consumer.go +++ b/cmd/pulsar-consumer/consumer.go @@ -110,7 +110,10 @@ func (c *consumer) readMessage(ctx context.Context) error { return errors.Trace(ctx.Err()) case consumerMsg := <-msgChan: log.Debug("Received message", zap.Stringer("msgId", consumerMsg.ID()), zap.ByteString("content", consumerMsg.Payload())) - needCommit := c.writer.WriteMessage(ctx, consumerMsg) + needCommit, writeErr := c.writer.WriteMessage(ctx, consumerMsg) + if writeErr != nil { + return writeErr + } if !needCommit { continue } @@ -123,7 +126,13 @@ func (c *consumer) readMessage(ctx context.Context) error { } // Run the consumer, read data and write to the downstream target. -func (c *consumer) Run(ctx context.Context) error { +func (c *consumer) Run(ctx context.Context) (err error) { + defer func() { + if cleanupErr := c.writer.cleanupEventsGroups(); err == nil && cleanupErr != nil { + err = cleanupErr + } + }() + g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return c.writer.run(ctx) diff --git a/cmd/pulsar-consumer/main.go b/cmd/pulsar-consumer/main.go index c1a7ed385d..25dff1ea32 100644 --- a/cmd/pulsar-consumer/main.go +++ b/cmd/pulsar-consumer/main.go @@ -39,7 +39,7 @@ var ( ) func main() { - debug.SetMemoryLimit(14 * 1024 * 1024 * 1024) + debug.SetMemoryLimit(8 * 1024 * 1024 * 1024) cmd := &cobra.Command{ Use: "pulsar consumer", Run: run, diff --git a/cmd/pulsar-consumer/writer.go b/cmd/pulsar-consumer/writer.go index e0fb914e1e..a520feefa2 100644 --- a/cmd/pulsar-consumer/writer.go +++ b/cmd/pulsar-consumer/writer.go @@ -42,14 +42,14 @@ type partitionProgress struct { partition int32 watermark uint64 eventsGroup map[int64]*util.EventsGroup - decoder common.Decoder + decoder *util.DMLMessageDecoder } func newPartitionProgress(partition int32, decoder common.Decoder) *partitionProgress { return &partitionProgress{ partition: partition, eventsGroup: make(map[int64]*util.EventsGroup), - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } } @@ -77,6 +77,7 @@ type writer struct { protocol config.Protocol mysqlSink sink.Sink enableTableAcrossNodes bool + spillStore *util.SpillStore } func newWriter(ctx context.Context, o *option) *writer { @@ -87,6 +88,7 @@ func newWriter(ctx context.Context, o *option) *writer { ddlList: make([]*commonEvent.DDLEvent, 0), ddlWithMaxCommitTs: make(map[int64]uint64), enableTableAcrossNodes: putil.GetOrZero(o.replicaConfig.Scheduler.EnableTableAcrossNodes), + spillStore: util.NewSpillStore(), } var ( db *sql.DB @@ -139,46 +141,133 @@ func (w *writer) run(ctx context.Context) error { return w.mysqlSink.Run(ctx) } -func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) error { - var ( - done = make(chan struct{}, 1) +func (w *writer) getSpillStore() *util.SpillStore { + if w.spillStore == nil { + w.spillStore = util.NewSpillStore() + } + return w.spillStore +} - flushed atomic.Int64 - ) +func (w *writer) cleanupEventsGroups() error { + var cleanupErr error + for _, progress := range w.progresses { + for _, group := range progress.eventsGroup { + _ = group.Cleanup() + } + } + if err := w.getSpillStore().Cleanup(); err != nil { + cleanupErr = err + log.Warn("cleanup spill store failed", zap.Error(err)) + } + return cleanupErr +} +func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) error { tableIDs := w.getBlockTableIDs(ddl) commitTs := ddl.GetCommitTs() - resolvedEvents := make([]*commonEvent.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for tableID := range tableIDs { for _, progress := range w.progresses { g, ok := progress.eventsGroup[tableID] if !ok { continue } - messages := g.ResolveInto(commitTs, nil) - events := make([]*commonEvent.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + groups = append(groups, g) + } + } + total, err := w.flushEventsFromGroups(ctx, groups, commitTs, + zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query)) + if err != nil { + return err + } + + if total != 0 { + log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Any("tables", tableIDs)) + } + return w.mysqlSink.WriteBlockEvent(ddl) +} + +func (w *writer) flushEventsFromGroups( + ctx context.Context, groups []*util.EventsGroup, resolveTs uint64, fields ...zap.Field, +) (int, error) { + limit := w.getSpillStore().ResolveLimit() + batchEvents := make([]*commonEvent.DMLEvent, 0, limit.MaxMessages) + batchMessages := 0 + var batchBytes int64 + total := 0 + prepared := make([]*util.ResolveBatch, 0, len(groups)) + flush := func() error { + if err := w.flushDMLBatch(ctx, batchEvents, fields...); err != nil { + return err + } + for _, batch := range prepared { + if err := batch.Ack(); err != nil { + return err + } + } + total += len(batchEvents) + batchEvents = nil + prepared = prepared[:0] + batchMessages = 0 + batchBytes = 0 + return nil + } + + for { + hasMoreGroups := false + preparedAny := false + for _, group := range groups { + if batchMessages >= limit.MaxMessages || batchBytes >= limit.MaxBytes { + if err := flush(); err != nil { + return 0, err + } + } + remaining := util.ResolveLimit{ + MaxBytes: limit.MaxBytes - batchBytes, + MaxMessages: limit.MaxMessages - batchMessages, + } + batch, hasMore, err := group.PrepareResolve(resolveTs, remaining) + if err != nil { + return 0, err } - resolvedEvents = append(resolvedEvents, events...) + hasMoreGroups = hasMoreGroups || hasMore + if batch != nil { + preparedAny = true + prepared = append(prepared, batch) + batchEvents = append(batchEvents, util.DMLMessagesToEvents(batch.Messages)...) + batchMessages += len(batch.Messages) + batchBytes += batch.ResolvedBytes + } + } + if err := flush(); err != nil { + return 0, err + } + if !hasMoreGroups || !preparedAny { + break } } + return total, nil +} - total := len(resolvedEvents) - if total == 0 { - return w.mysqlSink.WriteBlockEvent(ddl) +func (w *writer) flushDMLBatch( + ctx context.Context, events []*commonEvent.DMLEvent, fields ...zap.Field, +) error { + if len(events) == 0 { + return nil } - for _, e := range resolvedEvents { + done := make(chan struct{}) + var flushed atomic.Int64 + for _, e := range events { e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { + if flushed.Inc() == int64(len(events)) { close(done) } }) w.mysqlSink.AddDMLEvent(e) } - - log.Info("flush DML events before DDL", zap.Uint64("DDLCommitTs", commitTs), zap.Int("total", total)) - start := time.Now() ticker := time.NewTicker(time.Minute) defer ticker.Stop() for { @@ -186,14 +275,10 @@ func (w *writer) flushDDLEvent(ctx context.Context, ddl *commonEvent.DDLEvent) e case <-ctx.Done(): return context.Cause(ctx) case <-done: - log.Info("flush DML events before DDL done", zap.Uint64("DDLCommitTs", commitTs), - zap.Int("total", total), zap.Duration("duration", time.Since(start)), - zap.Any("tables", tableIDs)) - return w.mysqlSink.WriteBlockEvent(ddl) + return nil case <-ticker.C: - log.Warn("DML events cannot be flushed in time", - zap.Uint64("DDLCommitTs", commitTs), zap.String("query", ddl.Query), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + log.Warn("DML events cannot be flushed in time", append(fields, + zap.Int("total", len(events)), zap.Int64("flushed", flushed.Load()))...) } } } @@ -257,60 +342,41 @@ func (w *writer) globalWatermark() uint64 { } func (w *writer) flushDMLEventsByWatermark(ctx context.Context) error { - var ( - done = make(chan struct{}, 1) - - flushed atomic.Int64 - ) - watermark := w.globalWatermark() - resolvedEvents := make([]*commonEvent.DMLEvent, 0) + start := time.Now() + groups := make([]*util.EventsGroup, 0) for _, p := range w.progresses { for _, group := range p.eventsGroup { - messages := group.ResolveInto(watermark, nil) - events := make([]*commonEvent.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) - } - resolvedEvents = append(resolvedEvents, events...) - } - } - total := len(resolvedEvents) - if total == 0 { - return nil - } - for _, e := range resolvedEvents { - e.AddPostFlushFunc(func() { - if flushed.Inc() == int64(total) { - close(done) - } - }) - w.mysqlSink.AddDMLEvent(e) - } - - log.Info("flush DML events by watermark", zap.Uint64("watermark", watermark), zap.Int("total", total)) - start := time.Now() - ticker := time.NewTicker(time.Minute) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return context.Cause(ctx) - case <-done: - log.Info("flush DML events done", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Duration("duration", time.Since(start))) - return nil - case <-ticker.C: - log.Warn("DML events cannot be flushed in time", zap.Uint64("watermark", watermark), - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + groups = append(groups, group) } } + total, err := w.flushEventsFromGroups(ctx, groups, watermark, zap.Uint64("watermark", watermark)) + if err != nil { + return err + } + if total != 0 { + stats := w.getSpillStore().Stats() + log.Info("flush DML events done", zap.Uint64("watermark", watermark), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Int64("spillPayloadWriteBytes", stats.PayloadWriteBytes), + zap.Int64("spillPayloadReadBytes", stats.PayloadReadBytes), + zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), + zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), + zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), + zap.Int64("spillIndexWriteCount", stats.IndexWriteCount), + zap.Int64("spillIndexReadCount", stats.IndexReadCount), + zap.Int64("spillAppliedEventCount", stats.AppliedEventCount), + zap.Int64("spillPendingBytes", stats.PendingBytes), + zap.Int("spillLivePayloads", stats.LivePayloads), + zap.Int("spillLiveSegments", stats.LiveSegments)) + } + return nil } // WriteMessage is to decode pulsar message to event. // return true if the message is flushed to the downstream. // return error if flush messages failed. -func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool { +func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) (bool, error) { progress := w.progresses[0] progress.decoder.AddKeyValue([]byte(message.Key()), message.Payload()) @@ -320,6 +386,7 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool } needFlush := false + wasDraining := w.getSpillStore().ShouldDrain() switch messageType { case common.MessageTypeResolved: newWatermark := progress.decoder.NextResolvedEvent() @@ -338,7 +405,7 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool // the Query maybe empty if using simple protocol, it's comes from `bootstrap` event, no need to handle it. if ddl.Query == "" { - return false + return false, nil } w.appendDDL(ddl) log.Info("DDL event received", @@ -351,18 +418,25 @@ func (w *writer) WriteMessage(ctx context.Context, message pulsar.Message) bool if dmlMessage == nil { log.Panic("DML message is nil, it's not expected") } - w.appendMessage2Group(dmlMessage, progress) + if err := w.appendMessage2Group(dmlMessage, progress); err != nil { + return false, err + } default: log.Panic("unknown message type", zap.Any("messageType", messageType)) } if needFlush { return w.Write(ctx, messageType) } - return false + if !wasDraining && w.getSpillStore().ShouldDrain() { + if err := w.flushDMLEventsByWatermark(ctx); err != nil { + return false, err + } + } + return false, nil } // Write will synchronously write data downstream -func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool { +func (w *writer) Write(ctx context.Context, messageType common.MessageType) (bool, error) { // DDL events can be received out of commit-ts order (e.g. due to protocol-level broadcasting and // buffering differences between DDL kinds). We must execute DDLs in commit-ts order; otherwise a // "future" DDL that is not yet eligible (commitTs > watermark) can block executing earlier DDLs @@ -409,8 +483,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool break } if err := w.flushDDLEvent(ctx, todoDDL); err != nil { - log.Panic("write DDL event failed", zap.Error(err), - zap.String("DDL", todoDDL.Query), zap.Uint64("commitTs", todoDDL.GetCommitTs())) + return false, err } } @@ -418,7 +491,7 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool // since watermark is broadcast to all partitions, so that each partition can flush events individually. err := w.flushDMLEventsByWatermark(ctx) if err != nil { - log.Panic("flush dml events by the watermark failed", zap.Error(err)) + return false, err } } @@ -428,9 +501,9 @@ func (w *writer) Write(ctx context.Context, messageType common.MessageType) bool log.Info("some DDL events will be flushed in the future", zap.Uint64("watermark", watermark), zap.Int("length", len(w.ddlList))) - return false + return false, nil } - return true + return true, nil } func (w *writer) onDDL(ddl *commonEvent.DDLEvent) { @@ -491,7 +564,10 @@ func (w *writer) addPartitionTable(schema, table string) { w.partitionTableAccessor.Add(schema, table) } -func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *partitionProgress) { +func (w *writer) appendMessage2Group( + message *common.DMLMessage, + progress *partitionProgress, +) error { var ( tableID = message.TableID schema = message.Schema @@ -508,15 +584,17 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } group := progress.eventsGroup[tableID] if group == nil { - group = util.NewEventsGroup(progress.partition, tableID) + group = util.NewEventsGroup(progress.partition, tableID, w.getSpillStore()) progress.eventsGroup[tableID] = group } - group.AppendMessage(message) + if err := group.AppendMessage(message); err != nil { + return err + } if commitTs < progress.watermark { log.Warn("DML event fallback row, since less than the partition watermark, append it and sort before flush", zap.Int64("tableID", tableID), zap.Int32("partition", group.Partition), @@ -525,14 +603,14 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) - return + return nil } if commitTs >= group.HighWatermark { log.Debug("DML event append to the group", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return + return nil } log.Warn("DML event commit ts fallback, append it and sort before flush", zap.Int32("partition", progress.partition), @@ -541,4 +619,5 @@ func (w *writer) appendMessage2Group(message *common.DMLMessage, progress *parti zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType), zap.Any("protocol", w.protocol), zap.Bool("enableTableAcrossNodes", w.enableTableAcrossNodes)) + return nil } diff --git a/cmd/pulsar-consumer/writer_test.go b/cmd/pulsar-consumer/writer_test.go index 35c9c037e3..0148efa4d8 100644 --- a/cmd/pulsar-consumer/writer_test.go +++ b/cmd/pulsar-consumer/writer_test.go @@ -273,8 +273,10 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { ctrl := gomock.NewController(t) s := sinkmock.NewMockSink(ctrl) flushedCommitTs := make([]uint64, 0) + flushedRowTypeCounts := make([]int, 0) s.EXPECT().AddDMLEvent(gomock.Any()).Do(func(event *commonEvent.DMLEvent) { flushedCommitTs = append(flushedCommitTs, event.GetCommitTs()) + flushedRowTypeCounts = append(flushedRowTypeCounts, len(event.RowTypes)) event.PostFlush() }).Times(2) @@ -289,13 +291,20 @@ func TestWriterWrite_sortsOutOfOrderDMLByWatermark(t *testing.T) { protocol: config.ProtocolCanalJSON, } - w.appendMessage2Group(newDMLMessageForWriterTest(20), p) - w.appendMessage2Group(newDMLMessageForWriterTest(10), p) - w.appendMessage2Group(newDMLMessageForWriterTest(20), p) + for _, message := range []*codeccommon.DMLMessage{ + newDMLMessageForWriterTest(20), + newDMLMessageForWriterTest(10), + newDMLMessageForWriterTest(20), + } { + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(message), p)) + } p.watermark = 20 - require.True(t, w.Write(ctx, codeccommon.MessageTypeResolved)) + needCommit, err := w.Write(ctx, codeccommon.MessageTypeResolved) + require.NoError(t, err) + require.True(t, needCommit) require.Equal(t, []uint64{10, 20}, flushedCommitTs) + require.Equal(t, []int{1, 2}, flushedRowTypeCounts) } func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { @@ -318,7 +327,7 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { partition: 0, eventsGroup: make(map[int64]*util.EventsGroup), watermark: 20, - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } w := &writer{ progresses: []*partitionProgress{progress}, @@ -326,7 +335,8 @@ func TestWriteMessageIgnoresFallbackDMLBelowGlobalWatermark(t *testing.T) { protocol: config.ProtocolCanalJSON, } - needCommit := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + needCommit, err := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + require.NoError(t, err) require.False(t, needCommit) require.Nil(t, progress.eventsGroup[1]) @@ -346,10 +356,12 @@ func TestAppendMessageKeepsFallbackDMLAboveGlobalWatermark(t *testing.T) { protocol: config.ProtocolCanalJSON, } - w.appendMessage2Group(newDMLMessageForWriterTest(10), progress) + message := newDMLMessageForWriterTest(10) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(message), progress)) require.NotNil(t, progress.eventsGroup[1]) - resolved := progress.eventsGroup[1].ResolveInto(20, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(20, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(10), resolved[0].GetCommitTs()) } @@ -381,20 +393,19 @@ func TestOnDDLMarksRoutedCreateTableLikePartitionTable(t *testing.T) { w.onDDL(ddl) require.True(t, w.partitionTableAccessor.IsPartitionTable("target", "dst")) - newDMLMessage := func(commitTs uint64) *codeccommon.DMLMessage { - return codeccommon.NewDMLMessage(1, "target", "dst", commitTs, common.RowTypeUpdate, nil) - } - progress := w.progresses[0] - w.appendMessage2Group(newDMLMessage(200), progress) - w.appendMessage2Group(newDMLMessage(100), progress) + first := newDMLMessageForWriterTest(200) + second := newDMLMessageForWriterTest(100) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(first), progress)) + require.NoError(t, w.appendMessage2Group(attachDMLMessageDataForWriterTest(second), progress)) - resolved := progress.eventsGroup[1].ResolveInto(150, nil) + resolved, err := progress.eventsGroup[1].ResolveInto(150, nil) + require.NoError(t, err) require.Len(t, resolved, 1) require.Equal(t, uint64(100), resolved[0].GetCommitTs()) } -func TestWriteMessageDefersDMLAssemblyUntilFlush(t *testing.T) { +func TestWriteMessageSpillsDMLImmediately(t *testing.T) { ctx := context.Background() ctrl := gomock.NewController(t) s := sinkmock.NewMockSink(ctrl) @@ -415,7 +426,7 @@ func TestWriteMessageDefersDMLAssemblyUntilFlush(t *testing.T) { progress := &partitionProgress{ partition: 0, eventsGroup: make(map[int64]*util.EventsGroup), - decoder: decoder, + decoder: util.NewDMLMessageDecoder(decoder), } w := &writer{ progresses: []*partitionProgress{progress}, @@ -423,21 +434,28 @@ func TestWriteMessageDefersDMLAssemblyUntilFlush(t *testing.T) { protocol: config.ProtocolCanalJSON, } - needCommit := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + needCommit, err := w.WriteMessage(ctx, fakePulsarMessage{key: "k", payload: []byte(`{"fake":"row"}`)}) + require.NoError(t, err) require.False(t, needCommit) require.Equal(t, 1, decoder.addKeyValueCount) require.Equal(t, 1, decoder.hasNextCount) require.Equal(t, 1, decoder.nextDMLMessageCount) require.Zero(t, decoder.toDMLEventCount) - require.Len(t, progress.eventsGroup[1].ResolveInto(99, nil), 0) + resolved, err := progress.eventsGroup[1].ResolveInto(99, nil) + require.NoError(t, err) + require.Len(t, resolved, 0) progress.watermark = 100 - require.True(t, w.Write(ctx, codeccommon.MessageTypeResolved)) - require.Equal(t, 1, decoder.addKeyValueCount) - require.Equal(t, 1, decoder.hasNextCount) - require.Equal(t, 1, decoder.nextDMLMessageCount) + needCommit, err = w.Write(ctx, codeccommon.MessageTypeResolved) + require.NoError(t, err) + require.True(t, needCommit) + require.Equal(t, 2, decoder.addKeyValueCount) + require.Equal(t, 3, decoder.hasNextCount) + require.Equal(t, 2, decoder.nextDMLMessageCount) require.Equal(t, 1, decoder.toDMLEventCount) - require.Empty(t, progress.eventsGroup[1].ResolveInto(100, nil)) + resolved, err = progress.eventsGroup[1].ResolveInto(100, nil) + require.NoError(t, err) + require.Empty(t, resolved) require.Equal(t, []byte(`{"fake":"row"}`), decoder.lastValue) } @@ -449,16 +467,18 @@ type deferredDMLDecoder struct { nextDMLMessageCount int toDMLEventCount int lastValue []byte + pending bool } func (d *deferredDMLDecoder) AddKeyValue(_, value []byte) { d.addKeyValueCount++ d.lastValue = append(d.lastValue[:0], value...) + d.pending = true } func (d *deferredDMLDecoder) HasNext() (codeccommon.MessageType, bool) { d.hasNextCount++ - return codeccommon.MessageTypeRow, true + return codeccommon.MessageTypeRow, d.pending } func (d *deferredDMLDecoder) NextResolvedEvent() uint64 { @@ -467,6 +487,7 @@ func (d *deferredDMLDecoder) NextResolvedEvent() uint64 { func (d *deferredDMLDecoder) NextDMLMessage() *codeccommon.DMLMessage { d.nextDMLMessageCount++ + d.pending = false return codeccommon.NewDMLMessage(1, "test", "t", d.row.CommitTs, common.RowTypeInsert, func() *commonEvent.DMLEvent { d.toDMLEventCount++ return d.row @@ -481,6 +502,7 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { return codeccommon.NewDMLMessage(1, "test", "t", commitTs, common.RowTypeUpdate, func() *commonEvent.DMLEvent { return &commonEvent.DMLEvent{ PhysicalTableID: 1, + StartTs: commitTs - 1, CommitTs: commitTs, RowTypes: []common.RowType{common.RowTypeUpdate}, Rows: chunk.NewChunkWithCapacity(nil, 0), @@ -491,6 +513,16 @@ func newDMLMessageForWriterTest(commitTs uint64) *codeccommon.DMLMessage { }) } +func attachDMLMessageDataForWriterTest(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + type fakePulsarMessage struct { key string payload []byte diff --git a/cmd/storage-consumer/consumer.go b/cmd/storage-consumer/consumer.go index d89de46cfe..b57c08c043 100644 --- a/cmd/storage-consumer/consumer.go +++ b/cmd/storage-consumer/consumer.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "math" "sort" "strconv" "strings" @@ -76,6 +77,7 @@ type consumer struct { // tableDMLIdxMap maintains a map of tableDMLIdxMap map[cloudstorage.DMLPathKey]fileIndexKeyMap eventsGroup map[int64]*util.EventsGroup + spillStore *util.SpillStore // tableDDLWatermark maintains a map of <`schema`.`table`, max executed DDL table version>. // DML files with smaller table versions are considered stale replays and should be ignored. tableDDLWatermark map[string]uint64 @@ -175,6 +177,7 @@ func newConsumer(ctx context.Context) (*consumer, error) { errCh: errCh, tableDMLIdxMap: make(map[cloudstorage.DMLPathKey]fileIndexKeyMap), eventsGroup: make(map[int64]*util.EventsGroup), + spillStore: util.NewSpillStore(), tableDDLWatermark: make(map[string]uint64), schemaFileMap: make(map[string]map[uint64]*cloudstorage.SchemaFile), tableIDGenerator: &fakeTableIDGenerator{ @@ -183,6 +186,13 @@ func newConsumer(ctx context.Context) (*consumer, error) { }, nil } +func (c *consumer) getSpillStore() *util.SpillStore { + if c.spillStore == nil { + c.spillStore = util.NewSpillStore() + } + return c.spillStore +} + // map1 - map2 func diffDMLMaps( map1, map2 map[cloudstorage.DMLPathKey]fileIndexKeyMap, @@ -282,33 +292,37 @@ func (c *consumer) getNewFiles( return tableDMLMap, err } -func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAcrossNodes bool) { +func (c *consumer) appendMessage2Group( + message *common.DMLMessage, + tableID int64, + enableTableAcrossNodes bool, +) error { var ( - tableID = message.TableID schema = message.Schema table = message.Table commitTs = message.GetCommitTs() ) group := c.eventsGroup[tableID] if group == nil { - group = util.NewEventsGroup(0, tableID) + group = util.NewEventsGroup(0, tableID, c.getSpillStore()) c.eventsGroup[tableID] = group } if commitTs >= group.HighWatermark { - group.AppendMessage(message) + if err := group.AppendMessage(message); err != nil { + return err + } log.Debug("DML event append to the group", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - return + return nil } if enableTableAcrossNodes { log.Warn("DML events fallback, but enableTableAcrossNodes is true, still append it", zap.Uint64("commitTs", commitTs), zap.Uint64("highWatermark", group.HighWatermark), zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), zap.Stringer("eventType", message.RowType)) - group.AppendMessage(message) - return + return group.AppendMessage(message) } log.Warn("dml event commit ts fallback, ignore", zap.Uint64("commitTs", commitTs), @@ -316,6 +330,7 @@ func (c *consumer) appendMessage2Group(message *common.DMLMessage, enableTableAc zap.String("schema", schema), zap.String("table", table), ) + return nil } // appendDMLEvents decodes RowChangedEvents from file content and append them to event group. @@ -355,10 +370,15 @@ func (c *consumer) appendDMLEvents( decoder.AddKeyValue(nil, content) } + spillDecoder := util.NewDMLMessageDecoderWithDataFactory(decoder, + func(_ common.Decoder, _, value []byte) *common.DMLMessageData { + return c.newDMLMessageData(ctx, schemaFile, value, tableID) + }) + spillDecoder.SetRawMessage(nil, content) cnt := 0 filteredCnt := 0 for { - tp, hasNext := decoder.HasNext() + tp, hasNext := spillDecoder.HasNext() if err != nil { log.Error("failed to decode message", zap.Error(err)) return err @@ -371,8 +391,10 @@ func (c *consumer) appendDMLEvents( if tp == common.MessageTypeRow { c.dmlCount.Add(1) - message := decoder.NextDMLMessage() - c.appendMessage2Group(messageWithPhysicalTableID(message, tableID), fileIdx.EnableTableAcrossNodes) + message := spillDecoder.NextDMLMessage() + if err := c.appendMessage2Group(message, tableID, fileIdx.EnableTableAcrossNodes); err != nil { + return err + } filteredCnt++ } } @@ -385,6 +407,40 @@ func (c *consumer) appendDMLEvents( return err } +func (c *consumer) newDMLMessageData( + ctx context.Context, + schemaFile cloudstorage.SchemaFile, + content []byte, + tableID int64, +) *common.DMLMessageData { + tableInfo := schemaFile.TableInfo() + selector := c.columnSelectors.GetForTableInfo(tableInfo) + messageData := util.NewDMLMessageDataWithDecoderFactory(nil, content, + func(_ []byte, value []byte) (common.Decoder, error) { + switch c.codecCfg.Protocol { + case config.ProtocolCsv: + return csv.NewDecoderWithColumnSelector(ctx, c.codecCfg, tableInfo, value, selector) + case config.ProtocolCanalJSON: + decoder := canal.NewTxnDecoder(c.codecCfg) + return decoder, nil + default: + return nil, errors.ErrSpillFileOp.FastGenByArgs("unsupported storage DML spill protocol") + } + }) + decode := messageData.Restorer.Decode + messageData.Restorer = common.NewDMLMessageRestorer(func(data []byte) ([]*common.DMLMessage, error) { + messages, err := decode(data) + if err != nil { + return nil, err + } + for i, message := range messages { + messages[i] = messageWithPhysicalTableID(message, tableID) + } + return messages, nil + }) + return messageData +} + func messageWithPhysicalTableID(message *common.DMLMessage, tableID int64) *common.DMLMessage { return common.NewDMLMessage(tableID, message.Schema, message.Table, message.GetCommitTs(), message.RowType, func() *event.DMLEvent { row := message.ToDMLEvent() @@ -398,28 +454,67 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { if group == nil { return nil } - messages := group.GetAllMessages() - if len(messages) == 0 { - return nil + start := time.Now() + total := 0 + for { + batch, hasMore, err := group.PrepareResolve( + math.MaxUint64, c.getSpillStore().ResolveLimit()) + if err != nil { + return err + } + if batch == nil { + break + } + events := util.DMLMessagesToEvents(batch.Messages) + if len(events) != 0 { + fields := []zap.Field{zap.Int64("tableID", tableID)} + if events[0].TableInfo != nil { + fields = append(fields, + zap.String("schema", events[0].TableInfo.GetSchemaName()), + zap.String("table", events[0].TableInfo.GetTableName())) + } + if err := c.flushDMLBatch(ctx, events, fields...); err != nil { + return err + } + total += len(events) + } + if err := batch.Ack(); err != nil { + return err + } + if !hasMore { + break + } } - events := make([]*event.DMLEvent, 0, len(messages)) - for _, message := range messages { - events = util.AppendOrMergeDMLEvent(events, message.ToDMLEvent()) + if total != 0 { + stats := c.getSpillStore().Stats() + log.Info("flush DML events done", zap.Int64("tableID", tableID), + zap.Int("total", total), zap.Duration("duration", time.Since(start)), + zap.Int64("spillPayloadWriteBytes", stats.PayloadWriteBytes), + zap.Int64("spillPayloadReadBytes", stats.PayloadReadBytes), + zap.Int64("spillPayloadWriteCount", stats.PayloadWriteCount), + zap.Int64("spillPayloadReadCount", stats.PayloadReadCount), + zap.Int64("spillPayloadDecodeCount", stats.PayloadDecodeCount), + zap.Int64("spillIndexWriteCount", stats.IndexWriteCount), + zap.Int64("spillIndexReadCount", stats.IndexReadCount), + zap.Int64("spillAppliedEventCount", stats.AppliedEventCount), + zap.Int64("spillPendingBytes", stats.PendingBytes), + zap.Int("spillLivePayloads", stats.LivePayloads), + zap.Int("spillLiveSegments", stats.LiveSegments)) } - total := len(events) - if total == 0 { + return nil +} + +func (c *consumer) flushDMLBatch( + ctx context.Context, events []*event.DMLEvent, fields ...zap.Field, +) error { + if len(events) == 0 { return nil } + total := len(events) var ( - schema string - table string + flushed atomic.Int64 + done = make(chan struct{}) ) - if events[0].TableInfo != nil { - schema = events[0].TableInfo.GetSchemaName() - table = events[0].TableInfo.GetTableName() - } - var flushed atomic.Int64 - done := make(chan struct{}) for _, e := range events { e.AddPostFlushFunc(func() { if flushed.Inc() == int64(total) { @@ -429,8 +524,6 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { c.sink.AddDMLEvent(e) } - // Make sure all events are flushed to downstream. - start := time.Now() ticker := time.NewTicker(defaultLogInterval) defer ticker.Stop() for { @@ -438,17 +531,26 @@ func (c *consumer) flushDMLEvents(ctx context.Context, tableID int64) error { case <-ctx.Done(): return context.Cause(ctx) case <-done: - log.Info("flush DML events done", - zap.String("schema", schema), zap.String("table", table), zap.Int64("tableID", tableID), - zap.Int("total", total), zap.Duration("duration", time.Since(start))) return nil case <-ticker.C: - log.Warn("DML events cannot be flushed in time", - zap.Int("total", total), zap.Int64("flushed", flushed.Load())) + log.Warn("DML events cannot be flushed in time", append(fields, + zap.Int("total", total), zap.Int64("flushed", flushed.Load()))...) } } } +func (c *consumer) cleanupEventsGroups() error { + var cleanupErr error + for _, group := range c.eventsGroup { + _ = group.Cleanup() + } + if err := c.getSpillStore().Cleanup(); err != nil { + cleanupErr = err + log.Warn("cleanup spill store failed", zap.Error(err)) + } + return cleanupErr +} + func (c *consumer) parseDMLIndexFile(ctx context.Context, path string, dmlkey cloudstorage.DMLPathKey) { if c.globalCheckpointTs > 0 && dmlkey.TableVersion > c.globalCheckpointTs { log.Debug("skip dml index file by checkpoint", @@ -707,6 +809,11 @@ func (c *consumer) handleNewFiles( if err := c.appendDMLEvents(ctx, tableID, schemaFile, key, fileIndex); err != nil { return err } + if c.getSpillStore().ShouldDrain() { + if err := c.flushDMLEvents(ctx, tableID); err != nil { + return err + } + } } } if err := c.flushDMLEvents(ctx, tableID); err != nil { @@ -766,7 +873,13 @@ func (c *consumer) handle(ctx context.Context) error { } } -func (c *consumer) run(ctx context.Context) error { +func (c *consumer) run(ctx context.Context) (err error) { + defer func() { + if cleanupErr := c.cleanupEventsGroups(); err == nil && cleanupErr != nil { + err = cleanupErr + } + }() + g, ctx := errgroup.WithContext(ctx) g.Go(func() error { return c.sink.Run(ctx) diff --git a/cmd/storage-consumer/main.go b/cmd/storage-consumer/main.go index 70256977f4..36a7c51ef0 100644 --- a/cmd/storage-consumer/main.go +++ b/cmd/storage-consumer/main.go @@ -20,6 +20,7 @@ import ( "net/url" "os" "os/signal" + "runtime/debug" "strings" "syscall" "time" @@ -82,6 +83,7 @@ func init() { } func main() { + debug.SetMemoryLimit(8 * 1024 * 1024 * 1024) var consumer *consumer var err error diff --git a/cmd/util/dml_message_decoder.go b/cmd/util/dml_message_decoder.go new file mode 100644 index 0000000000..5b8c4ea76d --- /dev/null +++ b/cmd/util/dml_message_decoder.go @@ -0,0 +1,112 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + +// DMLMessageDataFactory creates data shared by DML messages decoded from one +// input. It is called lazily when the decoder first returns a DML message. +type DMLMessageDataFactory func(codeccommon.Decoder, []byte, []byte) *codeccommon.DMLMessageData + +// DMLMessageDecoder attaches spill data to DML messages as they are decoded. +// It keeps raw input only until EventsGroup has written it to the spill file. +type DMLMessageDecoder struct { + codeccommon.Decoder + + key, value []byte + data *codeccommon.DMLMessageData + factory DMLMessageDataFactory + restorer *codeccommon.DMLMessageRestorer + share bool + position int64 +} + +// NewDMLMessageDecoder wraps a decoder with the standard raw-message restorer. +func NewDMLMessageDecoder(decoder codeccommon.Decoder) *DMLMessageDecoder { + d := NewDMLMessageDecoderWithDataFactory(decoder, + func(decoder codeccommon.Decoder, key, value []byte) *codeccommon.DMLMessageData { + return NewDMLMessageData(decoder, key, value) + }) + d.share = true + return d +} + +// NewDMLMessageDecoderWithDataFactory is for decoders such as CSV whose +// restore decoder must be constructed from the input value. +func NewDMLMessageDecoderWithDataFactory( + decoder codeccommon.Decoder, factory DMLMessageDataFactory, +) *DMLMessageDecoder { + return &DMLMessageDecoder{Decoder: decoder, factory: factory} +} + +// SetSourcePosition records broker-specific source metadata, such as a Kafka +// offset. The position is persisted in every event descriptor and can be used +// by an EventsGroup post-restore hook without retaining a closure per input. +func (d *DMLMessageDecoder) SetSourcePosition(position int64) { + d.position = position +} + +// AddKeyValue implements codeccommon.Decoder. +func (d *DMLMessageDecoder) AddKeyValue(key, value []byte) { + d.Decoder.AddKeyValue(key, value) + d.SetRawMessage(key, value) +} + +// SetRawMessage records input that was supplied while constructing a decoder, +// such as a CSV decoder. It does not pass the input to the wrapped decoder. +func (d *DMLMessageDecoder) SetRawMessage(key, value []byte) { + d.key = key + d.value = value + d.data = nil +} + +// NextDMLMessage implements codeccommon.Decoder. +func (d *DMLMessageDecoder) NextDMLMessage() *codeccommon.DMLMessage { + message := d.Decoder.NextDMLMessage() + if message != nil { + d.attachDMLMessage(message) + } + return message +} + +func (d *DMLMessageDecoder) attachDMLMessage(message *codeccommon.DMLMessage) { + if d.data == nil { + d.data = d.factory(d.Decoder, d.key, d.value) + if d.share { + if d.restorer == nil { + d.restorer = d.data.Restorer + } else { + d.data.Restorer = d.restorer + } + } + d.data.SourcePosition = d.position + } + d.data.AttachDMLMessage(message) +} + +// AttachCachedDMLMessage attaches data to a materialized DML message from +// Simple's DDL cache. It has no raw row payload to restore. +func (d *DMLMessageDecoder) AttachCachedDMLMessage(message *codeccommon.DMLMessage) { + data := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }) + data.SourcePosition = d.position + data.AttachDMLMessage(message) +} + +// Unwrap returns the decoder that produces protocol messages. +func (d *DMLMessageDecoder) Unwrap() codeccommon.Decoder { + return d.Decoder +} diff --git a/cmd/util/dml_message_decoder_test.go b/cmd/util/dml_message_decoder_test.go new file mode 100644 index 0000000000..fe756fafe9 --- /dev/null +++ b/cmd/util/dml_message_decoder_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +import ( + "testing" + + commonEvent "github.com/pingcap/ticdc/pkg/common/event" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" +) + +func TestDMLMessageDecoderAttachesSharedData(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(11) + decoder := &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{first, second}} + wrapped := NewDMLMessageDecoder(decoder) + + wrapped.AddKeyValue([]byte("key"), []byte("value")) + decodedFirst := wrapped.NextDMLMessage() + firstData, firstIndex := decodedFirst.SpillData() + require.Equal(t, []byte("key"), firstData.Key) + require.Equal(t, []byte("value"), firstData.Value) + require.Zero(t, firstIndex) + + decodedSecond := wrapped.NextDMLMessage() + secondData, secondIndex := decodedSecond.SpillData() + require.Same(t, firstData, secondData) + require.Equal(t, uint64(1), secondIndex) +} + +func TestDMLMessageDecoderSharesRestorerAcrossInputs(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(11) + decoder := &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{first, second}} + wrapped := NewDMLMessageDecoder(decoder) + + wrapped.SetSourcePosition(100) + wrapped.AddKeyValue([]byte("first-key"), []byte("first-value")) + firstData, _ := wrapped.NextDMLMessage().SpillData() + + wrapped.SetSourcePosition(101) + wrapped.AddKeyValue([]byte("second-key"), []byte("second-value")) + secondData, _ := wrapped.NextDMLMessage().SpillData() + + require.NotSame(t, firstData, secondData) + require.Same(t, firstData.Restorer, secondData.Restorer) + require.Equal(t, int64(100), firstData.SourcePosition) + require.Equal(t, int64(101), secondData.SourcePosition) +} + +func TestDMLMessageDecoderKeepsCustomRestorersPerInput(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(11) + decoder := &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{first, second}} + wrapped := NewDMLMessageDecoderWithDataFactory(decoder, + func(_ codeccommon.Decoder, key, value []byte) *codeccommon.DMLMessageData { + return codeccommon.NewDMLMessageData(key, value, + func([]byte) ([]*codeccommon.DMLMessage, error) { return nil, nil }) + }) + + wrapped.AddKeyValue([]byte("first-key"), []byte("first-value")) + firstData, _ := wrapped.NextDMLMessage().SpillData() + wrapped.AddKeyValue([]byte("second-key"), []byte("second-value")) + secondData, _ := wrapped.NextDMLMessage().SpillData() + + require.NotSame(t, firstData.Restorer, secondData.Restorer) +} + +func TestSharedRestorerDecodesMultipleInputs(t *testing.T) { + decoder := &resettableDMLDecoder{} + wrapped := NewDMLMessageDecoder(decoder) + group := NewEventsGroup(0, 1) + + for _, commitTs := range []byte{20, 10} { + wrapped.AddKeyValue(nil, []byte{commitTs}) + message := wrapped.NextDMLMessage() + require.NotNil(t, message) + require.NoError(t, group.AppendMessage(message)) + } + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Equal(t, []uint64{10, 20}, []uint64{ + messages[0].GetCommitTs(), messages[1].GetCommitTs(), + }) +} + +type dmlMessageDecoderStub struct { + messages []*codeccommon.DMLMessage +} + +func (d *dmlMessageDecoderStub) AddKeyValue(_, _ []byte) {} + +func (d *dmlMessageDecoderStub) HasNext() (codeccommon.MessageType, bool) { + return codeccommon.MessageTypeRow, len(d.messages) > 0 +} + +func (d *dmlMessageDecoderStub) NextResolvedEvent() uint64 { return 0 } + +func (d *dmlMessageDecoderStub) NextDMLMessage() *codeccommon.DMLMessage { + if len(d.messages) == 0 { + return nil + } + message := d.messages[0] + d.messages = d.messages[1:] + return message +} + +func (d *dmlMessageDecoderStub) NextDDLEvent() *commonEvent.DDLEvent { return nil } + +type resettableDMLDecoder struct { + message *codeccommon.DMLMessage +} + +func (d *resettableDMLDecoder) AddKeyValue(_, value []byte) { + if len(value) == 0 { + d.message = nil + return + } + d.message = newTestDMLMessage(uint64(value[0])) +} + +func (d *resettableDMLDecoder) HasNext() (codeccommon.MessageType, bool) { + return codeccommon.MessageTypeRow, d.message != nil +} + +func (d *resettableDMLDecoder) NextResolvedEvent() uint64 { return 0 } + +func (d *resettableDMLDecoder) NextDMLMessage() *codeccommon.DMLMessage { + message := d.message + d.message = nil + return message +} + +func (d *resettableDMLDecoder) NextDDLEvent() *commonEvent.DDLEvent { return nil } diff --git a/cmd/util/event_group.go b/cmd/util/event_group.go index d4d324282a..50ac0d2191 100644 --- a/cmd/util/event_group.go +++ b/cmd/util/event_group.go @@ -14,117 +14,1289 @@ package util import ( + "bytes" + "container/list" + "encoding/binary" + "fmt" "math" - "sort" + "os" + "path/filepath" + "sync/atomic" + "github.com/cockroachdb/pebble" "github.com/pingcap/log" commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/errors" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/spill" "go.uber.org/zap" + "golang.org/x/sys/unix" ) +const ( + eventsGroupSpillDirPattern = "ticdc-events-group-*" + payloadSpillPattern = "payload-*.spill" + spillIndexDir = "index" +) + +const ( + defaultSpillSegmentBytes = 128 * 1024 * 1024 + defaultSpillSegmentMessages = 100000 + defaultPendingHighBytes = 1024 * 1024 * 1024 + defaultPendingLowBytes = 512 * 1024 * 1024 + defaultResolveBatchBytes = 64 * 1024 * 1024 + defaultResolveBatchMessages = 10000 + defaultMessageMetadataBytes = 128 + defaultIndexBatchMessages = 10000 + defaultIndexCleanupMessages = 100000 + defaultDiskCheckBytes = 16 * 1024 * 1024 + defaultDiskUsageLimit = 0.90 + defaultIndexCacheBytes = 32 * 1024 * 1024 + defaultIndexMemTableBytes = 16 * 1024 * 1024 + spillRecordLengthBytes = 8 + eventIndexKeyBytes = 3 * 8 + eventIndexValueBytes = 6 * 8 +) + +type spillConfig struct { + segmentBytes int64 + segmentMessages int + pendingHighBytes int64 + pendingLowBytes int64 + resolveBatchBytes int64 + resolveBatchMessages int + messageMetadataBytes int64 + indexBatchMessages int + indexCleanupMessages int64 + diskCheckBytes int64 + diskUsageLimit float64 +} + +func defaultSpillConfig() spillConfig { + return spillConfig{ + segmentBytes: defaultSpillSegmentBytes, + segmentMessages: defaultSpillSegmentMessages, + pendingHighBytes: defaultPendingHighBytes, + pendingLowBytes: defaultPendingLowBytes, + resolveBatchBytes: defaultResolveBatchBytes, + resolveBatchMessages: defaultResolveBatchMessages, + messageMetadataBytes: defaultMessageMetadataBytes, + indexBatchMessages: defaultIndexBatchMessages, + indexCleanupMessages: defaultIndexCleanupMessages, + diskCheckBytes: defaultDiskCheckBytes, + diskUsageLimit: defaultDiskUsageLimit, + } +} + +// ResolveLimit bounds one batch restored from spill. +type ResolveLimit struct { + MaxBytes int64 + MaxMessages int +} + +// SpillStats reports process-wide payload I/O and live spill state. +type SpillStats struct { + PendingBytes int64 + PayloadWriteBytes int64 + PayloadReadBytes int64 + PayloadWriteCount int64 + PayloadReadCount int64 + PayloadDecodeCount int64 + LivePayloads int + LiveSegments int + IndexWriteCount int64 + IndexReadCount int64 + AppliedEventCount int64 +} + +type filesystemUsage struct { + usedBytes uint64 + totalBytes uint64 +} + +type payloadLocation struct { + segmentID uint64 + handle spill.Handle +} + +type spillSegment struct { + id uint64 + file *spill.RecordFile + bytes int64 + payloadCount int + pendingEvents int64 +} + +type registeredRestorer struct { + decode func([]byte) ([]*codeccommon.DMLMessage, error) + refs int64 +} + +type payloadCacheKey struct { + segmentID uint64 + offset int64 +} + +type payloadCacheEntry struct { + key payloadCacheKey + messages []*codeccommon.DMLMessage + bytes int64 + pins int + element *list.Element +} + +type spilledMessage struct { + key []byte + commitTs uint64 + location payloadLocation + dmlIndex uint64 + restorerID uint64 + sourcePosition int64 +} + +// SpillStore owns a disk-backed ordered event index and append-only payload +// segments shared by all event groups in one consumer. Pending DML descriptors +// live in Pebble rather than one Go object per row. Only a bounded restore cache +// and segment-level reference counts remain in memory. Filesystem usage is +// checked periodically while appending and before every applied-range delete; +// crossing the hard limit latches an error so the consumer terminates. +type SpillStore struct { + id uint64 + config spillConfig + rootDir string + index *pebble.DB + indexCache *pebble.Cache + indexBatch *pebble.Batch + indexBatchCount int + segments map[uint64]*spillSegment + activeSegment *spillSegment + restorers map[uint64]*registeredRestorer + cache map[payloadCacheKey]*payloadCacheEntry + cacheLRU list.List + cacheBytes int64 + pendingBytes int64 + livePayloads int + draining bool + nextSegmentID uint64 + nextGroupID uint64 + nextSequence uint64 + bytesSinceDiskCheck int64 + terminalErr error + + readRecord func(*spill.RecordFile, spill.Handle) ([]byte, error) + diskUsage func(string) (filesystemUsage, error) + stats SpillStats + + indexDeleteRangeCount int64 +} + +var nextSpillStoreID atomic.Uint64 + +// NewSpillStore creates a process-wide store with the default spill limits. +func NewSpillStore() *SpillStore { + return newSpillStore(defaultSpillConfig()) +} + +func newSpillStore(config spillConfig) *SpillStore { + return &SpillStore{ + id: nextSpillStoreID.Add(1), + config: config, + segments: make(map[uint64]*spillSegment), + restorers: make(map[uint64]*registeredRestorer), + cache: make(map[payloadCacheKey]*payloadCacheEntry), + readRecord: func(file *spill.RecordFile, handle spill.Handle) ([]byte, error) { + return file.Read(handle) + }, + diskUsage: getFilesystemUsage, + } +} + +// ResolveLimit returns the configured per-batch restore limit. +func (s *SpillStore) ResolveLimit() ResolveLimit { + return ResolveLimit{ + MaxBytes: s.config.resolveBatchBytes, + MaxMessages: s.config.resolveBatchMessages, + } +} + +// PendingBytes returns conservatively accounted payload, decoded cache, and +// logical index bytes. Physical index usage is covered by the filesystem guard. +func (s *SpillStore) PendingBytes() int64 { + return s.pendingBytes +} + +// ShouldDrain reports whether pending spill data has crossed the high watermark and not yet fallen below the low watermark. +func (s *SpillStore) ShouldDrain() bool { + return s.draining +} + +// Stats returns a snapshot of process-wide spill activity. +func (s *SpillStore) Stats() SpillStats { + stats := s.stats + stats.PendingBytes = s.pendingBytes + stats.LivePayloads = s.livePayloads + stats.LiveSegments = len(s.segments) + return stats +} + +func (s *SpillStore) addPending(bytes int64) { + if bytes <= 0 { + return + } + s.pendingBytes += bytes + if !s.draining && s.pendingBytes >= s.config.pendingHighBytes { + s.draining = true + log.Info("spill pending bytes reached high watermark", + zap.Int64("pendingBytes", s.pendingBytes), + zap.Int64("highWatermarkBytes", s.config.pendingHighBytes), + zap.Int64("lowWatermarkBytes", s.config.pendingLowBytes)) + } +} + +func (s *SpillStore) releasePending(bytes int64) { + if bytes <= 0 { + return + } + s.pendingBytes -= bytes + if s.pendingBytes < 0 { + log.Panic("spill pending bytes underflow", + zap.Int64("releasedBytes", bytes), zap.Int64("pendingBytes", s.pendingBytes)) + } + if s.draining && s.pendingBytes <= s.config.pendingLowBytes { + s.draining = false + log.Info("spill pending bytes fell below low watermark", + zap.Int64("pendingBytes", s.pendingBytes), + zap.Int64("highWatermarkBytes", s.config.pendingHighBytes), + zap.Int64("lowWatermarkBytes", s.config.pendingLowBytes)) + } +} + // EventsGroup stores change event messages. type EventsGroup struct { Partition int32 tableID int64 - messages []*codeccommon.DMLMessage - outOfOrder bool - HighWatermark uint64 + id uint64 + store *SpillStore + ownsStore bool + pendingCount int64 + segmentRefs map[uint64]int64 + restorerRefs map[uint64]int64 + batchPending bool + outOfOrder bool + lastAppendedTs uint64 + resolvedTs uint64 + appliedTs uint64 + postRestore func(*codeccommon.DMLMessage, int64) *codeccommon.DMLMessage + HighWatermark uint64 + + // indexCursor is the first logical key not acknowledged by the downstream. + indexCursor []byte + // indexCleanupStart begins the acknowledged prefix not yet deleted from Pebble. + indexCleanupStart []byte + indexCleanupCount int64 } // NewEventsGroup will create new event group. -func NewEventsGroup(partition int32, tableID int64) *EventsGroup { +func NewEventsGroup(partition int32, tableID int64, stores ...*SpillStore) *EventsGroup { + store := NewSpillStore() + ownsStore := true + if len(stores) != 0 && stores[0] != nil { + store = stores[0] + ownsStore = false + } + store.nextGroupID++ + lower, _ := eventIndexBounds(store.nextGroupID) return &EventsGroup{ - Partition: partition, - tableID: tableID, - messages: make([]*codeccommon.DMLMessage, 0, 1024), + Partition: partition, + tableID: tableID, + id: store.nextGroupID, + store: store, + ownsStore: ownsStore, + segmentRefs: make(map[uint64]int64), + restorerRefs: make(map[uint64]int64), + indexCursor: lower, + indexCleanupStart: append([]byte(nil), lower...), + } +} + +// SetPostRestore installs a bounded group-level hook for source metadata that +// must be applied after lazy decoding. Unlike the old per-input closure, this +// hook does not grow with the backlog. +func (g *EventsGroup) SetPostRestore( + restore func(*codeccommon.DMLMessage, int64) *codeccommon.DMLMessage, +) { + g.postRestore = restore +} + +func (s *SpillStore) ensureOpen() error { + if s.terminalErr != nil { + return s.terminalErr + } + if s.index != nil { + return nil + } + + rootDir, err := os.MkdirTemp(os.TempDir(), eventsGroupSpillDirPattern) + if err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "create spill store directory") + } + s.rootDir = rootDir + if err := s.checkDiskUsage(0, true); err != nil { + return err + } + + cache := pebble.NewCache(defaultIndexCacheBytes) + options := &pebble.Options{ + Cache: cache, + DisableWAL: true, + MaxOpenFiles: 128, + MemTableSize: defaultIndexMemTableBytes, + MemTableStopWritesThreshold: 2, + MaxConcurrentCompactions: func() int { return 1 }, + } + index, err := pebble.Open(filepath.Join(rootDir, spillIndexDir), options) + if err != nil { + cache.Unref() + return errors.WrapError(errors.ErrSpillFileOp, err, "open spill event index") + } + s.indexCache = cache + s.index = index + s.indexBatch = index.NewBatch() + return nil +} + +func (s *SpillStore) newSegment() error { + if err := s.ensureOpen(); err != nil { + return err + } + if err := s.checkDiskUsage(0, true); err != nil { + return err + } + file, err := spill.NewRecordFile(s.rootDir, payloadSpillPattern) + if err != nil { + return err + } + s.nextSegmentID++ + segment := &spillSegment{id: s.nextSegmentID, file: file} + s.segments[segment.id] = segment + s.activeSegment = segment + return nil +} + +func (s *SpillStore) prepareSegment(recordBytes int64) error { + if err := s.ensureOpen(); err != nil { + return err + } + segment := s.activeSegment + if segment != nil && (segment.bytes+recordBytes > s.config.segmentBytes || + segment.payloadCount >= s.config.segmentMessages) { + s.activeSegment = nil + } + if s.activeSegment == nil { + return s.newSegment() + } + return nil +} + +func (s *SpillStore) sealFullSegment() { + segment := s.activeSegment + if segment == nil { + return + } + if segment.bytes >= s.config.segmentBytes || + segment.payloadCount >= s.config.segmentMessages { + s.activeSegment = nil + } +} + +func spillMessageDataSize(key, value []byte) int64 { + if len(key) == 0 && len(value) == 0 { + return spillRecordLengthBytes + 1 + } + return spillRecordLengthBytes + 2*8 + int64(len(key)) + int64(len(value)) +} + +func appendMessageData(file *spill.RecordFile, key, value []byte) (spill.Handle, error) { + if len(key) == 0 && len(value) == 0 { + return file.AppendChunks([]byte{0}) + } + var keyLen, valueLen [8]byte + binary.BigEndian.PutUint64(keyLen[:], uint64(len(key))) + binary.BigEndian.PutUint64(valueLen[:], uint64(len(value))) + return file.AppendChunks(keyLen[:], key, valueLen[:], value) +} + +func (s *SpillStore) acquirePayload(data *codeccommon.DMLMessageData) (payloadLocation, error) { + if segmentID, offset, length, ok := data.SpillLocation(s.id); ok { + if _, exists := s.segments[segmentID]; exists { + return payloadLocation{ + segmentID: segmentID, + handle: spill.Handle{Offset: offset, Length: length}, + }, nil + } + } + recordBytes := spillMessageDataSize(data.Key, data.Value) + if err := s.checkDiskUsage(recordBytes, false); err != nil { + return payloadLocation{}, err + } + if err := s.prepareSegment(recordBytes); err != nil { + return payloadLocation{}, err + } + segment := s.activeSegment + handle, err := appendMessageData(segment.file, data.Key, data.Value) + if err != nil { + return payloadLocation{}, err + } + data.SetSpillLocation(s.id, segment.id, handle.Offset, handle.Length) + segment.bytes += recordBytes + segment.payloadCount++ + s.livePayloads++ + s.addPending(recordBytes) + s.stats.PayloadWriteBytes += int64(handle.Length) + s.stats.PayloadWriteCount++ + s.sealFullSegment() + return payloadLocation{segmentID: segment.id, handle: handle}, nil +} + +func (s *SpillStore) cleanupSegment(segment *spillSegment) { + if s.activeSegment == segment { + s.activeSegment = nil + } + s.evictSegmentCache(segment.id) + if err := segment.file.Cleanup(); err != nil { + log.Warn("cleanup spill segment failed", zap.String("path", segment.file.Path()), zap.Error(err)) + return + } + s.releasePending(segment.bytes) + s.livePayloads -= segment.payloadCount + delete(s.segments, segment.id) +} + +func (s *SpillStore) appendEventIndex(key, value []byte) error { + if err := s.ensureOpen(); err != nil { + return err + } + if err := s.checkDiskUsage(int64(len(key)+len(value)), false); err != nil { + return err + } + if err := s.indexBatch.Set(key, value, nil); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "append spill event index") + } + s.indexBatchCount++ + if s.indexBatchCount >= s.config.indexBatchMessages { + return s.flushEventIndex() + } + return nil +} + +func (s *SpillStore) flushEventIndex() error { + if s.indexBatchCount == 0 { + return nil + } + if err := s.indexBatch.Commit(pebble.NoSync); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "commit spill event index") + } + if err := s.indexBatch.Close(); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "close committed spill event index batch") + } + s.indexBatch = s.index.NewBatch() + s.indexBatchCount = 0 + return nil +} + +func encodeEventIndexKey(groupID, commitTs, sequence uint64) []byte { + key := make([]byte, eventIndexKeyBytes) + binary.BigEndian.PutUint64(key[0:8], groupID) + binary.BigEndian.PutUint64(key[8:16], commitTs) + binary.BigEndian.PutUint64(key[16:24], sequence) + return key +} + +func eventIndexBounds(groupID uint64) ([]byte, []byte) { + lower := make([]byte, 8) + upper := make([]byte, 8) + binary.BigEndian.PutUint64(lower, groupID) + binary.BigEndian.PutUint64(upper, groupID+1) + return lower, upper +} + +func nextEventIndexKey(key []byte) []byte { + return append(append([]byte(nil), key...), 0) +} + +func encodeEventIndexValue( + location payloadLocation, dmlIndex, restorerID uint64, sourcePosition int64, +) []byte { + value := make([]byte, eventIndexValueBytes) + binary.BigEndian.PutUint64(value[0:8], location.segmentID) + binary.BigEndian.PutUint64(value[8:16], uint64(location.handle.Offset)) + binary.BigEndian.PutUint64(value[16:24], location.handle.Length) + binary.BigEndian.PutUint64(value[24:32], dmlIndex) + binary.BigEndian.PutUint64(value[32:40], restorerID) + binary.BigEndian.PutUint64(value[40:48], uint64(sourcePosition)) + return value +} + +func decodeEventIndexCommitTs(key []byte) (uint64, error) { + if len(key) != eventIndexKeyBytes { + return 0, errors.ErrSpillFileOp.FastGenByArgs("invalid spill event index key") + } + return binary.BigEndian.Uint64(key[8:16]), nil +} + +func decodeSpilledMessage(key, value []byte) (spilledMessage, error) { + commitTs, err := decodeEventIndexCommitTs(key) + if err != nil { + return spilledMessage{}, err + } + if len(value) != eventIndexValueBytes { + return spilledMessage{}, errors.ErrSpillFileOp.FastGenByArgs("invalid spill event index value") + } + return spilledMessage{ + key: append([]byte(nil), key...), + commitTs: commitTs, + location: payloadLocation{ + segmentID: binary.BigEndian.Uint64(value[0:8]), + handle: spill.Handle{ + Offset: int64(binary.BigEndian.Uint64(value[8:16])), + Length: binary.BigEndian.Uint64(value[16:24]), + }, + }, + dmlIndex: binary.BigEndian.Uint64(value[24:32]), + restorerID: binary.BigEndian.Uint64(value[32:40]), + sourcePosition: int64(binary.BigEndian.Uint64(value[40:48])), + }, nil +} + +func (s *SpillStore) trimPayloadCache() { + byteLimit := s.config.resolveBatchBytes + if byteLimit <= 0 { + byteLimit = defaultResolveBatchBytes + } + entryLimit := s.config.resolveBatchMessages + if entryLimit <= 0 { + entryLimit = defaultResolveBatchMessages + } + for s.cacheBytes > byteLimit || len(s.cache) > entryLimit { + var victim *payloadCacheEntry + for element := s.cacheLRU.Back(); element != nil; element = element.Prev() { + entry := element.Value.(*payloadCacheEntry) + if entry.pins == 0 { + victim = entry + break + } + } + if victim == nil { + return + } + s.removePayloadCacheEntry(victim) + } +} + +func (s *SpillStore) removePayloadCacheEntry(entry *payloadCacheEntry) { + if entry == nil || entry.element == nil { + return } + delete(s.cache, entry.key) + s.cacheLRU.Remove(entry.element) + entry.element = nil + entry.messages = nil + s.cacheBytes -= entry.bytes + s.releasePending(entry.bytes) } -// AppendMessage appends a message to event groups. -func (g *EventsGroup) AppendMessage(message *codeccommon.DMLMessage) { +func (s *SpillStore) unpinPayloads(payloads []*payloadCacheEntry) { + for _, payload := range payloads { + payload.pins-- + if payload.pins < 0 { + log.Panic("spill payload cache pin underflow", + zap.Uint64("segmentID", payload.key.segmentID), zap.Int64("offset", payload.key.offset)) + } + } + s.trimPayloadCache() +} + +func (s *SpillStore) evictSegmentCache(segmentID uint64) { + for _, entry := range s.cache { + if entry.key.segmentID == segmentID { + s.removePayloadCacheEntry(entry) + } + } +} + +func (g *EventsGroup) releaseEvent(segmentID, restorerID uint64, count int64) { + g.releaseSegmentRefs(segmentID, count) + g.releaseRestorerRefs(restorerID, count) +} + +func (g *EventsGroup) releaseSegmentRefs(segmentID uint64, count int64) { + segment := g.store.segments[segmentID] + if segment == nil || segment.pendingEvents < count || g.segmentRefs[segmentID] < count { + log.Panic("spill segment reference underflow", + zap.Uint64("segmentID", segmentID), zap.Int64("released", count)) + } + segment.pendingEvents -= count + g.segmentRefs[segmentID] -= count + if g.segmentRefs[segmentID] == 0 { + delete(g.segmentRefs, segmentID) + } + if segment.pendingEvents == 0 { + g.store.cleanupSegment(segment) + } +} + +func (g *EventsGroup) releaseRestorerRefs(restorerID uint64, count int64) { + restorer := g.store.restorers[restorerID] + if restorer == nil || restorer.refs < count || g.restorerRefs[restorerID] < count { + log.Panic("spill restorer reference underflow", + zap.Uint64("restorerID", restorerID), zap.Int64("released", count)) + } + restorer.refs -= count + g.restorerRefs[restorerID] -= count + if g.restorerRefs[restorerID] == 0 { + delete(g.restorerRefs, restorerID) + } + if restorer.refs == 0 { + delete(g.store.restorers, restorerID) + } +} + +func getFilesystemUsage(path string) (filesystemUsage, error) { + var stat unix.Statfs_t + if err := unix.Statfs(path, &stat); err != nil { + return filesystemUsage{}, errors.WrapError(errors.ErrSpillFileOp, err, "read spill filesystem usage") + } + if stat.Bsize <= 0 { + return filesystemUsage{}, errors.ErrSpillFileOp.FastGenByArgs("spill filesystem has invalid block size") + } + blockSize := uint64(stat.Bsize) + totalBytes := uint64(stat.Blocks) * blockSize + availableBytes := uint64(stat.Bavail) * blockSize + if availableBytes > totalBytes { + return filesystemUsage{}, errors.ErrSpillFileOp.FastGenByArgs("spill filesystem has invalid available blocks") + } + return filesystemUsage{usedBytes: totalBytes - availableBytes, totalBytes: totalBytes}, nil +} + +func (s *SpillStore) checkDiskUsage(nextWriteBytes int64, force bool) error { + if s.terminalErr != nil { + return s.terminalErr + } + s.bytesSinceDiskCheck += nextWriteBytes + if !force && s.config.diskCheckBytes > 0 && s.bytesSinceDiskCheck < s.config.diskCheckBytes { + return nil + } + path := s.rootDir + if path == "" { + path = os.TempDir() + } + usage, err := s.diskUsage(path) + if err != nil { + return err + } + s.bytesSinceDiskCheck = 0 + if usage.totalBytes == 0 { + return errors.ErrSpillFileOp.FastGenByArgs("spill filesystem has zero capacity") + } + projectedUsed := usage.usedBytes + if nextWriteBytes > 0 && uint64(nextWriteBytes) <= ^uint64(0)-projectedUsed { + projectedUsed += uint64(nextWriteBytes) + } + ratio := float64(projectedUsed) / float64(usage.totalBytes) + if ratio <= s.config.diskUsageLimit { + return nil + } + message := fmt.Sprintf( + "spill filesystem usage %.2f%% exceeds %.2f%% limit: path=%s usedBytes=%d totalBytes=%d", + ratio*100, s.config.diskUsageLimit*100, path, projectedUsed, usage.totalBytes) + s.terminalErr = errors.ErrSpillFileOp.FastGenByArgs(message) + log.Error("spill filesystem usage limit exceeded; stop consumer", + zap.String("path", path), + zap.Uint64("usedBytes", projectedUsed), + zap.Uint64("totalBytes", usage.totalBytes), + zap.Float64("usagePercent", ratio*100), + zap.Float64("limitPercent", s.config.diskUsageLimit*100)) + return s.terminalErr +} + +func (g *EventsGroup) deleteAppliedIndex(end []byte) error { + if bytes.Compare(g.indexCleanupStart, end) >= 0 { + return nil + } + if err := g.store.checkDiskUsage(eventIndexKeyBytes+eventIndexValueBytes, true); err != nil { + return err + } + if err := g.store.index.DeleteRange(g.indexCleanupStart, end, pebble.NoSync); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "delete applied spill event range") + } + g.store.indexDeleteRangeCount++ + return nil +} + +// prepareIndexAppend preserves ordering if a caller appends an event older +// than the already applied cursor. Consumers normally reject such events by +// their global watermark. Keeping the fallback here prevents the generic +// EventsGroup from skipping a late event or deleting it with a future +// coalesced range tombstone. +func (g *EventsGroup) prepareIndexAppend(key []byte) error { + if bytes.Compare(key, g.indexCursor) >= 0 { + return nil + } + if g.indexCleanupCount != 0 { + if err := g.store.flushEventIndex(); err != nil { + return err + } + if err := g.deleteAppliedIndex(g.indexCursor); err != nil { + return err + } + } + g.indexCursor = append(g.indexCursor[:0], key...) + g.indexCleanupStart = append(g.indexCleanupStart[:0], key...) + g.indexCleanupCount = 0 + return nil +} + +// Cleanup removes all temporary index and payload state when a consumer stops. +func (s *SpillStore) Cleanup() error { + var cleanupErr error + if s.indexBatch != nil { + if err := s.indexBatch.Close(); err != nil && cleanupErr == nil { + cleanupErr = errors.WrapError(errors.ErrSpillFileOp, err, "close spill event index batch") + } + s.indexBatch = nil + } + if s.index != nil { + if err := s.index.Close(); err != nil && cleanupErr == nil { + cleanupErr = errors.WrapError(errors.ErrSpillFileOp, err, "close spill event index") + } + s.index = nil + } + if s.indexCache != nil { + s.indexCache.Unref() + s.indexCache = nil + } + for _, segment := range s.segments { + if err := segment.file.Cleanup(); err != nil && cleanupErr == nil { + cleanupErr = err + } + } + if s.rootDir != "" { + if err := os.RemoveAll(s.rootDir); err != nil && cleanupErr == nil { + cleanupErr = errors.WrapError(errors.ErrSpillFileOp, err, "remove spill store directory") + } + } + s.rootDir = "" + s.activeSegment = nil + clear(s.segments) + clear(s.restorers) + clear(s.cache) + s.cacheLRU.Init() + s.cacheBytes = 0 + s.pendingBytes = 0 + s.livePayloads = 0 + s.draining = false + s.indexBatchCount = 0 + return cleanupErr +} + +// AppendMessage appends an opaque codec payload to the spill file. It does +// not call DMLMessage.ToDMLEvent; the restored message remains lazy until the +// consumer flushes it against a watermark or DDL barrier. +func (g *EventsGroup) AppendMessage( + message *codeccommon.DMLMessage, +) error { + if message == nil { + return errors.ErrSpillFileOp.FastGenByArgs("cannot spill nil DML message") + } + messageData, dmlIndex := message.SpillData() + if messageData == nil || messageData.Restorer == nil || messageData.Restorer.Decode == nil { + return errors.ErrSpillFileOp.FastGenByArgs("cannot spill DML message without decode function") + } commitTs := message.GetCommitTs() - if len(g.messages) > 0 && commitTs < g.messages[len(g.messages)-1].GetCommitTs() { + g.store.nextSequence++ + key := encodeEventIndexKey(g.id, commitTs, g.store.nextSequence) + if err := g.prepareIndexAppend(key); err != nil { + return err + } + location, err := g.store.acquirePayload(messageData) + if err != nil { + return err + } + if g.pendingCount > 0 && commitTs < g.lastAppendedTs { g.outOfOrder = true } if commitTs > g.HighWatermark { g.HighWatermark = commitTs } - g.messages = append(g.messages, message) + g.lastAppendedTs = commitTs + value := encodeEventIndexValue(location, dmlIndex, messageData.Restorer.ID, messageData.SourcePosition) + if err := g.store.appendEventIndex(key, value); err != nil { + return err + } + + segment := g.store.segments[location.segmentID] + segment.pendingEvents++ + restorer := g.store.restorers[messageData.Restorer.ID] + if restorer == nil { + restorer = ®isteredRestorer{decode: messageData.Restorer.Decode} + g.store.restorers[messageData.Restorer.ID] = restorer + } + restorer.refs++ + g.pendingCount++ + g.segmentRefs[location.segmentID]++ + g.restorerRefs[messageData.Restorer.ID]++ + g.store.addPending(g.store.config.messageMetadataBytes) + g.store.stats.IndexWriteCount++ + return nil +} + +// ResolveBatch owns a prepared group prefix until the downstream confirms it. +type ResolveBatch struct { + Messages []*codeccommon.DMLMessage + ResolvedBytes int64 + group *EventsGroup + entries []spilledMessage + payloads []*payloadCacheEntry + acked bool +} + +// Ack releases a prepared group prefix after downstream flush callbacks complete. +func (b *ResolveBatch) Ack() error { + if b == nil || b.acked { + return nil + } + if err := b.group.ack(b); err != nil { + return err + } + b.acked = true + return nil } -// ResolveInto appends all messages with CommitTs <= resolve into dst in commit-ts order and removes -// them from the group. ResolveInto copies pointers into dst first, then clears the resolved messages -// so Go GC can reclaim them once downstream is done with them. -func (g *EventsGroup) ResolveInto(resolve uint64, dst []*codeccommon.DMLMessage) []*codeccommon.DMLMessage { - if len(g.messages) == 0 { - return dst +// PrepareResolve restores one bounded group prefix without removing it. +func (g *EventsGroup) PrepareResolve( + resolve uint64, limit ResolveLimit, +) (*ResolveBatch, bool, error) { + if g.batchPending { + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("events group already has a pending resolve batch") + } + if resolve > g.resolvedTs { + g.resolvedTs = resolve + } + if g.pendingCount == 0 { + return nil, false, nil + } + if err := g.store.flushEventIndex(); err != nil { + return nil, false, err } - if g.outOfOrder { - sort.SliceStable(g.messages, func(i, j int) bool { - return g.messages[i].GetCommitTs() < g.messages[j].GetCommitTs() - }) + _, upper := eventIndexBounds(g.id) + lower := g.indexCursor + iterator, err := g.store.index.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) + if err != nil { + return nil, false, errors.WrapError(errors.ErrSpillFileOp, err, "create spill event iterator") } + defer iterator.Close() - resolvedCount := sort.Search(len(g.messages), func(i int) bool { - return g.messages[i].GetCommitTs() > resolve - }) + // Most groups have no event below a given global watermark. Allocate the + // batch lazily so polling those groups does not create a full-sized slice. + var entries []spilledMessage + seenPayloads := make(map[payloadCacheKey]struct{}) + var plannedBytes int64 + var lastCommitTs uint64 + valid := iterator.First() + for valid { + commitTs, err := decodeEventIndexCommitTs(iterator.Key()) + if err != nil { + return nil, false, err + } + if commitTs > g.resolvedTs { + break + } + entry, err := decodeSpilledMessage(iterator.Key(), iterator.Value()) + if err != nil { + return nil, false, err + } + payloadKey := payloadCacheKey{segmentID: entry.location.segmentID, offset: entry.location.handle.Offset} + additionalBytes := int64(0) + if _, ok := seenPayloads[payloadKey]; !ok { + if _, cached := g.store.cache[payloadKey]; !cached { + additionalBytes = int64(entry.location.handle.Length) + } + } + if len(entries) > 0 && exceedsResolveLimit(len(entries), plannedBytes, additionalBytes, limit) && + commitTs != lastCommitTs { + break + } + if entries == nil { + capacity := boundedMessageCapacity(limit) + if int64(capacity) > g.pendingCount { + capacity = int(g.pendingCount) + } + entries = make([]spilledMessage, 0, capacity) + } + entries = append(entries, entry) + plannedBytes += additionalBytes + seenPayloads[payloadKey] = struct{}{} + lastCommitTs = commitTs + valid = iterator.Next() + } + if err := iterator.Error(); err != nil { + return nil, false, errors.WrapError(errors.ErrSpillFileOp, err, "iterate spill event index") + } + if len(entries) == 0 { + return nil, false, nil + } + hasMore := false + if valid { + nextCommitTs, err := decodeEventIndexCommitTs(iterator.Key()) + if err != nil { + return nil, false, err + } + hasMore = nextCommitTs <= g.resolvedTs + } if g.outOfOrder { - log.Warn("DML events are out of order before flush, sort them", + log.Warn("DML events were appended out of order; restore from ordered spill index", zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), - zap.Uint64("resolveTs", resolve), - zap.Int("resolved", resolvedCount)) + zap.Uint64("resolveTs", g.resolvedTs), + zap.Int("resolved", len(entries))) g.outOfOrder = false } - if resolvedCount == 0 { - return dst + batch := &ResolveBatch{ + Messages: make([]*codeccommon.DMLMessage, 0, len(entries)), + group: g, + entries: entries, } + loaded := make(map[payloadCacheKey]*payloadCacheEntry) + for _, message := range entries { + payloadKey := payloadCacheKey{segmentID: message.location.segmentID, offset: message.location.handle.Offset} + payload := loaded[payloadKey] + readBytes := int64(0) + if payload == nil { + payload, readBytes, err = g.store.loadAndPinPayload(message) + if err != nil { + g.store.unpinPayloads(batch.payloads) + return nil, false, err + } + loaded[payloadKey] = payload + batch.payloads = append(batch.payloads, payload) + } + if message.dmlIndex >= uint64(len(payload.messages)) { + g.store.unpinPayloads(batch.payloads) + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("DML spill message index is out of range") + } + restored := payload.messages[message.dmlIndex] + if restored == nil { + g.store.unpinPayloads(batch.payloads) + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("DML spill message is nil") + } + if g.postRestore != nil { + restored = g.postRestore(restored, message.sourcePosition) + } + if restored == nil { + g.store.unpinPayloads(batch.payloads) + return nil, false, errors.ErrSpillFileOp.FastGenByArgs("post-restore returned nil DML message") + } + batch.Messages = append(batch.Messages, restored) + batch.ResolvedBytes += readBytes + } + g.store.stats.IndexReadCount += int64(len(entries)) + g.batchPending = true + return batch, hasMore, nil +} - dst = append(dst, g.messages[:resolvedCount]...) - remainingCount := len(g.messages) - resolvedCount - copy(g.messages, g.messages[resolvedCount:]) - clear(g.messages[remainingCount:]) - g.messages = g.messages[:remainingCount] - if len(g.messages) != 0 { - firstCommitTs := g.messages[0].GetCommitTs() - log.Debug("not all events resolved", - zap.Int32("partition", g.Partition), zap.Int64("tableID", g.tableID), - zap.Int("resolved", resolvedCount), zap.Int("remained", len(g.messages)), - zap.Uint64("resolveTs", resolve), zap.Uint64("firstCommitTs", firstCommitTs)) +func boundedMessageCapacity(limit ResolveLimit) int { + if limit.MaxMessages <= 0 || limit.MaxMessages > defaultResolveBatchMessages { + return defaultResolveBatchMessages + } + return limit.MaxMessages +} + +func exceedsResolveLimit(count int, bytes, additionalBytes int64, limit ResolveLimit) bool { + maxBytes := limit.MaxBytes + if maxBytes <= 0 { + maxBytes = math.MaxInt64 + } + maxMessages := limit.MaxMessages + if maxMessages <= 0 { + maxMessages = int(^uint(0) >> 1) + } + return count >= maxMessages || bytes+additionalBytes > maxBytes +} + +func (s *SpillStore) loadAndPinPayload(message spilledMessage) (*payloadCacheEntry, int64, error) { + key := payloadCacheKey{segmentID: message.location.segmentID, offset: message.location.handle.Offset} + if cached := s.cache[key]; cached != nil { + cached.pins++ + s.cacheLRU.MoveToFront(cached.element) + return cached, 0, nil + } + segment := s.segments[message.location.segmentID] + if segment == nil || segment.file == nil { + return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("spill payload segment is missing") + } + restorer := s.restorers[message.restorerID] + if restorer == nil || restorer.decode == nil { + return nil, 0, errors.ErrSpillFileOp.FastGenByArgs("DML spill restorer is missing") + } + data, err := s.readRecord(segment.file, message.location.handle) + if err != nil { + return nil, 0, err + } + messages, err := restorer.decode(data) + if err != nil { + return nil, 0, err + } + entry := &payloadCacheEntry{ + key: key, + messages: messages, + bytes: int64(message.location.handle.Length) + + int64(len(messages))*s.config.messageMetadataBytes, + pins: 1, } - return dst + entry.element = s.cacheLRU.PushFront(entry) + s.cache[key] = entry + s.cacheBytes += entry.bytes + s.addPending(entry.bytes) + s.stats.PayloadReadBytes += int64(message.location.handle.Length) + s.stats.PayloadReadCount++ + s.stats.PayloadDecodeCount++ + s.trimPayloadCache() + return entry, int64(message.location.handle.Length), nil +} + +// ResolveInto appends all messages with CommitTs <= resolve into dst in commit-ts order. +func (g *EventsGroup) ResolveInto( + resolve uint64, dst []*codeccommon.DMLMessage, +) ([]*codeccommon.DMLMessage, error) { + dst, _, _, err := g.ResolveIntoBatch(resolve, dst, ResolveLimit{}) + return dst, err +} + +// ResolveIntoBatch appends one bounded batch of messages with CommitTs <= resolve into dst in commit-ts order. +// A single commit-ts group can exceed the limits so that one transaction is never split. +func (g *EventsGroup) ResolveIntoBatch( + resolve uint64, dst []*codeccommon.DMLMessage, limit ResolveLimit, +) ([]*codeccommon.DMLMessage, bool, int64, error) { + batch, hasMore, err := g.PrepareResolve(resolve, limit) + if err != nil || batch == nil { + return dst, hasMore, 0, err + } + dst = append(dst, batch.Messages...) + resolvedBytes := batch.ResolvedBytes + if err := batch.Ack(); err != nil { + return dst, hasMore, resolvedBytes, err + } + return dst, hasMore, resolvedBytes, nil +} + +func (g *EventsGroup) ack(batch *ResolveBatch) error { + if len(batch.entries) == 0 { + g.batchPending = false + g.store.unpinPayloads(batch.payloads) + return nil + } + end := nextEventIndexKey(batch.entries[len(batch.entries)-1].key) + indexCleanupCount := g.indexCleanupCount + int64(len(batch.entries)) + cleanupMessages := g.store.config.indexCleanupMessages + if cleanupMessages <= 0 { + cleanupMessages = defaultIndexCleanupMessages + } + indexDeleted := indexCleanupCount >= cleanupMessages + if indexDeleted { + if err := g.deleteAppliedIndex(end); err != nil { + return err + } + } + for _, message := range batch.entries { + g.releaseEvent(message.location.segmentID, message.restorerID, 1) + } + g.pendingCount -= int64(len(batch.entries)) + g.store.releasePending(int64(len(batch.entries)) * g.store.config.messageMetadataBytes) + g.store.stats.AppliedEventCount += int64(len(batch.entries)) + lastCommitTs := batch.entries[len(batch.entries)-1].commitTs + if lastCommitTs > g.appliedTs { + g.appliedTs = lastCommitTs + } + g.indexCursor = end + if indexDeleted { + g.indexCleanupStart = append(g.indexCleanupStart[:0], end...) + g.indexCleanupCount = 0 + } else { + g.indexCleanupCount = indexCleanupCount + } + g.batchPending = false + g.store.unpinPayloads(batch.payloads) + if g.pendingCount == 0 && g.ownsStore { + return g.store.Cleanup() + } + return nil } // GetAllMessages gets all messages. -func (g *EventsGroup) GetAllMessages() []*codeccommon.DMLMessage { +func (g *EventsGroup) GetAllMessages() ([]*codeccommon.DMLMessage, error) { return g.ResolveInto(math.MaxUint64, nil) } -// AppendOrMergeDMLEvent appends a DML event, or merges it into the previous event -// when both events belong to the same table group and have the same commit-ts. -func AppendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { - var lastDMLEvent *commonEvent.DMLEvent - if len(events) > 0 { - lastDMLEvent = events[len(events)-1] +// Cleanup removes pending spill records when the consumer is stopping. +func (g *EventsGroup) Cleanup() error { + if g.batchPending { + return errors.ErrSpillFileOp.FastGenByArgs("cannot clean events group with pending resolve batch") + } + if g.store.index != nil { + if err := g.store.flushEventIndex(); err != nil { + return err + } + lower, upper := eventIndexBounds(g.id) + if err := g.store.index.DeleteRange(lower, upper, pebble.NoSync); err != nil { + return errors.WrapError(errors.ErrSpillFileOp, err, "delete spill event group") + } + g.store.indexDeleteRangeCount++ + } + for segmentID, count := range g.segmentRefs { + g.releaseSegmentRefs(segmentID, count) + } + for restorerID, count := range g.restorerRefs { + g.releaseRestorerRefs(restorerID, count) + } + g.store.releasePending(g.pendingCount * g.store.config.messageMetadataBytes) + g.pendingCount = 0 + clear(g.segmentRefs) + clear(g.restorerRefs) + lower, _ := eventIndexBounds(g.id) + g.indexCursor = lower + g.indexCleanupStart = append(g.indexCleanupStart[:0], lower...) + g.indexCleanupCount = 0 + g.batchPending = false + if g.ownsStore { + return g.store.Cleanup() + } + return nil +} + +// DMLMessagesToEvents materializes messages and merges compatible adjacent +// messages before they are handed to the downstream sink. +func DMLMessagesToEvents(messages []*codeccommon.DMLMessage) []*commonEvent.DMLEvent { + events := make([]*commonEvent.DMLEvent, 0, len(messages)) + for _, message := range messages { + events = appendOrMergeDMLEvent(events, message.ToDMLEvent()) } + return events +} - if lastDMLEvent == nil || lastDMLEvent.GetCommitTs() < row.GetCommitTs() { +func appendOrMergeDMLEvent(events []*commonEvent.DMLEvent, row *commonEvent.DMLEvent) []*commonEvent.DMLEvent { + if len(events) == 0 || !sameDMLTransaction(events[len(events)-1], row) { return append(events, row) } - if lastDMLEvent.GetCommitTs() == row.GetCommitTs() { - lastDMLEvent.Rows.Append(row.Rows, 0, row.Rows.NumRows()) - lastDMLEvent.RowTypes = append(lastDMLEvent.RowTypes, row.RowTypes...) - lastDMLEvent.Length += row.Length - lastDMLEvent.PostTxnFlushed = append(lastDMLEvent.PostTxnFlushed, row.PostTxnFlushed...) - return events + last := events[len(events)-1] + lastRowTypeCount := len(last.RowTypes) + rowRowTypeCount := len(row.RowTypes) + last.Rows.Append(row.Rows, 0, row.Rows.NumRows()) + last.RowTypes = append(last.RowTypes, row.RowTypes...) + last.RowKeys = appendOptionalDMLValues(last.RowKeys, row.RowKeys, lastRowTypeCount, rowRowTypeCount) + last.Checksum = appendOptionalDMLValues(last.Checksum, row.Checksum, lastRowTypeCount, rowRowTypeCount) + last.Length += row.Length + last.ApproximateSize += row.ApproximateSize + last.PostTxnEnqueued = append(last.PostTxnEnqueued, row.PostTxnEnqueued...) + last.PostTxnFlushed = append(last.PostTxnFlushed, row.PostTxnFlushed...) + return events +} + +func sameDMLTransaction(last, row *commonEvent.DMLEvent) bool { + return last != nil && row != nil && last.CommitTs == row.CommitTs +} + +func appendOptionalDMLValues[T any](last, row []T, lastRowTypeCount, rowRowTypeCount int) []T { + if len(last) == 0 && len(row) != 0 { + last = make([]T, lastRowTypeCount) + } else if len(last) != 0 && len(row) == 0 { + row = make([]T, rowRowTypeCount) } + return append(last, row...) +} - log.Panic("append event with smaller commit ts", - zap.Int64("tableID", row.GetTableID()), - zap.Uint64("lastCommitTs", lastDMLEvent.GetCommitTs()), zap.Uint64("commitTs", row.GetCommitTs())) - return events +// NewDMLMessageData preserves one original codec input. The decoder is used +// only during ResolveInto, after the DDL or watermark barrier selects schema. +func NewDMLMessageData(decoder codeccommon.Decoder, key, value []byte) *codeccommon.DMLMessageData { + return NewDMLMessageDataWithDecoderFactory(key, value, + func([]byte, []byte) (codeccommon.Decoder, error) { return decoder, nil }) +} + +// NewDMLMessageDataWithDecoderFactory is for decoders such as CSV whose +// input is supplied during construction rather than through AddKeyValue. +func NewDMLMessageDataWithDecoderFactory( + key, value []byte, + decoderFactory func([]byte, []byte) (codeccommon.Decoder, error), +) *codeccommon.DMLMessageData { + return codeccommon.NewDMLMessageData(key, value, + func(data []byte) ([]*codeccommon.DMLMessage, error) { + key, value, err := unmarshalDMLMessageData(data) + if err != nil { + return nil, err + } + decoder, err := decoderFactory(key, value) + if err != nil { + return nil, errors.WrapError(errors.ErrSpillFileOp, err, "create DML spill decoder") + } + return restoreDMLMessages(decoder, key, value) + }) +} + +func restoreDMLMessages( + decoder codeccommon.Decoder, key, value []byte, +) ([]*codeccommon.DMLMessage, error) { + decoder.AddKeyValue(key, value) + messages := make([]*codeccommon.DMLMessage, 0, 1) + for { + messageType, hasNext := decoder.HasNext() + if !hasNext { + if len(messages) == 0 { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload has no message") + } + return messages, nil + } + if messageType != codeccommon.MessageTypeRow { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload contains a non-DML message") + } + message := decoder.NextDMLMessage() + if message == nil { + return nil, errors.ErrSpillFileOp.FastGenByArgs("DML spill payload cannot be restored") + } + messages = append(messages, message) + } +} + +func unmarshalDMLMessageData(data []byte) ([]byte, []byte, error) { + key, data, err := readSpillBytes(data) + if err != nil { + return nil, nil, err + } + value, data, err := readSpillBytes(data) + if err != nil { + return nil, nil, err + } + if len(data) != 0 { + return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("unexpected trailing DML spill data") + } + return key, value, nil +} + +func readSpillBytes(data []byte) ([]byte, []byte, error) { + length, data, err := readSpillUint64(data) + if err != nil { + return nil, nil, err + } + if length > uint64(len(data)) { + return nil, nil, errors.ErrSpillFileOp.FastGenByArgs("invalid DML spill payload length") + } + return data[:length], data[length:], nil +} + +func readSpillUint64(data []byte) (uint64, []byte, error) { + if len(data) < 8 { + return 0, nil, errors.ErrSpillFileOp.FastGenByArgs("truncated DML spill data") + } + return binary.BigEndian.Uint64(data[:8]), data[8:], nil } diff --git a/cmd/util/event_group_test.go b/cmd/util/event_group_test.go index a756f297bd..1b94d7ca2c 100644 --- a/cmd/util/event_group_test.go +++ b/cmd/util/event_group_test.go @@ -14,19 +14,160 @@ package util import ( + "errors" + "fmt" + "math" + "os" "testing" + "github.com/cockroachdb/pebble" "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" + "github.com/pingcap/ticdc/pkg/integrity" codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/spill" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tidb/pkg/util/chunk" "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" ) func newTestDMLMessage(commitTs uint64) *codeccommon.DMLMessage { - return codeccommon.NewDMLMessage(1, "test", "t", commitTs, common.RowTypeInsert, nil) + return codeccommon.NewDMLMessageFromEvent(newTestDMLEvent(commitTs, common.RowTypeInsert)) +} + +func attachTestDMLMessageData(message *codeccommon.DMLMessage) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + +func attachTestDMLMessageDataWithPayload( + message *codeccommon.DMLMessage, key, value []byte, +) *codeccommon.DMLMessage { + messageData := codeccommon.NewDMLMessageData(key, value, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }, + ) + messageData.AttachDMLMessage(message) + return message +} + +func readGroupIndex(t *testing.T, group *EventsGroup) []spilledMessage { + return readGroupIndexFrom(t, group, group.indexCursor) +} + +func readGroupIndexFrom(t *testing.T, group *EventsGroup, lower []byte) []spilledMessage { + t.Helper() + require.NoError(t, group.store.flushEventIndex()) + _, upper := eventIndexBounds(group.id) + iterator, err := group.store.index.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) + require.NoError(t, err) + defer func() { require.NoError(t, iterator.Close()) }() + + entries := make([]spilledMessage, 0) + for valid := iterator.First(); valid; valid = iterator.Next() { + entry, err := decodeSpilledMessage(iterator.Key(), iterator.Value()) + require.NoError(t, err) + entries = append(entries, entry) + } + require.NoError(t, iterator.Error()) + return entries +} + +func TestEventsGroupResolvesFromAppliedIndexCursor(t *testing.T) { + config := defaultSpillConfig() + config.resolveBatchMessages = 1 + config.indexCleanupMessages = 1000 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + const messageCount = 129 + for commitTs := uint64(1); commitTs <= messageCount; commitTs++ { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + for resolveTs := uint64(1); resolveTs < messageCount; resolveTs++ { + batch, _, err := group.PrepareResolve(resolveTs, store.ResolveLimit()) + require.NoError(t, err) + require.NoError(t, batch.Ack()) + } + + require.Zero(t, store.indexDeleteRangeCount) + lower, _ := eventIndexBounds(group.id) + require.Len(t, readGroupIndexFrom(t, group, lower), messageCount) + entries := readGroupIndex(t, group) + require.Len(t, entries, 1) + require.Equal(t, uint64(messageCount), entries[0].commitTs) +} + +func TestEventsGroupCoalescesAppliedIndexCleanup(t *testing.T) { + config := defaultSpillConfig() + config.resolveBatchMessages = 1 + config.indexCleanupMessages = 64 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + const messageCount = 257 + for commitTs := uint64(1); commitTs <= messageCount; commitTs++ { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + + for resolveTs := uint64(1); resolveTs < messageCount; resolveTs++ { + batch, _, err := group.PrepareResolve(resolveTs, store.ResolveLimit()) + require.NoError(t, err) + require.NotNil(t, batch) + require.Len(t, batch.Messages, 1) + require.Equal(t, resolveTs, batch.Messages[0].GetCommitTs()) + require.NoError(t, batch.Ack()) + } + + require.Equal(t, int64(4), store.indexDeleteRangeCount) + require.Zero(t, group.indexCleanupCount) + require.Equal(t, int64(1), group.pendingCount) + entries := readGroupIndex(t, group) + require.Len(t, entries, 1) + require.Equal(t, uint64(messageCount), entries[0].commitTs) +} + +func TestEventsGroupRestoresLateEventBehindAppliedCursor(t *testing.T) { + config := defaultSpillConfig() + config.indexCleanupMessages = 100 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + for _, commitTs := range []uint64{10, 20, 30} { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + resolved, err := group.ResolveInto(20, nil) + require.NoError(t, err) + require.Equal(t, []uint64{10, 20}, []uint64{ + resolved[0].GetCommitTs(), resolved[1].GetCommitTs(), + }) + require.Zero(t, store.indexDeleteRangeCount) + + require.NoError(t, group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(15)))) + require.Equal(t, int64(1), store.indexDeleteRangeCount) + resolved, err = group.GetAllMessages() + require.NoError(t, err) + require.Equal(t, []uint64{15, 30}, []uint64{ + resolved[0].GetCommitTs(), resolved[1].GetCommitTs(), + }) } func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.DMLEvent { @@ -39,43 +180,625 @@ func newTestDMLEvent(commitTs uint64, rowTypes ...common.RowType) *commonEvent.D } } -func TestEventsGroupResolveIntoAppendsAndClearsResolvedMessages(t *testing.T) { +func newMergeTestTableInfo(tableID int64, updateTS uint64, columnCount int) *common.TableInfo { + columns := make([]*model.ColumnInfo, columnCount) + for i := range columns { + columns[i] = &model.ColumnInfo{ + ID: int64(i + 1), + Offset: i, + Name: ast.NewCIStr(fmt.Sprintf("c%d", i)), + FieldType: *types.NewFieldType(mysql.TypeLonglong), + } + } + return common.WrapTableInfo("test", &model.TableInfo{ + ID: tableID, + Name: ast.NewCIStr("t"), + UpdateTS: updateTS, + Columns: columns, + }) +} + +func newMergeTestDMLEvent(commitTs uint64, tableInfo *common.TableInfo, value int64) *commonEvent.DMLEvent { + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 1) + for column := range tableInfo.GetFieldSlice() { + rows.AppendInt64(column, value) + } + return &commonEvent.DMLEvent{ + DispatcherID: common.DispatcherID{Low: 1}, + PhysicalTableID: tableInfo.TableName.TableID, + StartTs: commitTs - 1, + CommitTs: commitTs, + Length: 1, + RowTypes: []common.RowType{common.RowTypeInsert}, + Rows: rows, + TableInfo: tableInfo, + TableInfoVersion: tableInfo.GetUpdateTS(), + } +} + +func newMergeTestDMLMessage(event *commonEvent.DMLEvent) *codeccommon.DMLMessage { + return codeccommon.NewDMLMessage(event.GetTableID(), event.TableInfo.GetSchemaName(), event.TableInfo.GetTableName(), + event.GetCommitTs(), event.RowTypes[0], func() *commonEvent.DMLEvent { return event }) +} + +func TestAppendOrMergeDMLEvent(t *testing.T) { + t.Run("merge compatible events", func(t *testing.T) { + tableInfo := newMergeTestTableInfo(1, 10, 1) + first := newMergeTestDMLEvent(100, tableInfo, 1) + second := newMergeTestDMLEvent(100, tableInfo, 2) + first.RowKeys = [][]byte{[]byte("first")} + second.RowKeys = [][]byte{[]byte("second")} + first.Checksum = []*integrity.Checksum{{Current: 1}} + second.Checksum = []*integrity.Checksum{{Current: 2}} + var flushed []int + first.AddPostFlushFunc(func() { flushed = append(flushed, 1) }) + second.AddPostFlushFunc(func() { flushed = append(flushed, 2) }) + + events := DMLMessagesToEvents([]*codeccommon.DMLMessage{ + newMergeTestDMLMessage(first), + newMergeTestDMLMessage(second), + }) + + require.Len(t, events, 1) + require.Same(t, first, events[0]) + require.Equal(t, int32(2), first.Length) + require.Equal(t, 2, first.Rows.NumRows()) + require.Equal(t, []byte("second"), first.RowKeys[1]) + require.Equal(t, uint32(2), first.Checksum[1].Current) + first.PostFlush() + require.Equal(t, []int{1, 2}, flushed) + }) + + t.Run("keep different commit timestamps separate", func(t *testing.T) { + tableInfo := newMergeTestTableInfo(1, 10, 1) + first := newMergeTestDMLEvent(100, tableInfo, 1) + second := newMergeTestDMLEvent(101, tableInfo, 2) + + events := DMLMessagesToEvents([]*codeccommon.DMLMessage{ + newMergeTestDMLMessage(first), + newMergeTestDMLMessage(second), + }) + + require.Len(t, events, 2) + }) + + t.Run("merge compatible events restored from spill", func(t *testing.T) { + tableInfo := newMergeTestTableInfo(1, 10, 1) + group := NewEventsGroup(0, 1) + first := newMergeTestDMLEvent(100, tableInfo, 1) + second := newMergeTestDMLEvent(100, tableInfo, 2) + firstMessage := newMergeTestDMLMessage(first) + secondMessage := newMergeTestDMLMessage(second) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(firstMessage))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(secondMessage))) + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 2) + events := DMLMessagesToEvents(messages) + + require.Len(t, events, 1) + require.Equal(t, 2, events[0].Rows.NumRows()) + }) +} + +func TestEventsGroupSharesRawMessageData(t *testing.T) { + first := newTestDMLMessage(10) + second := newTestDMLMessage(10) + messageData := codeccommon.NewDMLMessageData(nil, []byte("raw message"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{first, second}, nil + }, + ) + + group := NewEventsGroup(0, 1) + messageData.AttachDMLMessage(first) + require.NoError(t, group.AppendMessage(first)) + messageData.AttachDMLMessage(second) + require.NoError(t, group.AppendMessage(second)) + entries := readGroupIndex(t, group) + require.Len(t, entries, 2) + require.Equal(t, entries[0].location, entries[1].location) + require.Equal(t, uint64(0), entries[0].dmlIndex) + require.Equal(t, uint64(1), entries[1].dmlIndex) + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Same(t, first, messages[0]) + require.Same(t, second, messages[1]) +} + +func TestEventsGroupRestoresSharedSpillInputOnce(t *testing.T) { + // A single canal-json input can contain thousands of DML messages. Restoring + // each ordinal must not re-decode the complete input. + inputMessages := []*codeccommon.DMLMessage{ + newTestDMLMessage(30), + newTestDMLMessage(10), + newTestDMLMessage(20), + } + var decoderCount int + key := []byte("raw key") + value := []byte("raw message") + messageData := NewDMLMessageDataWithDecoderFactory(key, value, + func(restoredKey, restoredValue []byte) (codeccommon.Decoder, error) { + decoderCount++ + require.Equal(t, key, restoredKey) + require.Equal(t, value, restoredValue) + return &dmlMessageDecoderStub{messages: []*codeccommon.DMLMessage{ + newTestDMLMessage(30), + newTestDMLMessage(10), + newTestDMLMessage(20), + }}, nil + }) + + group := NewEventsGroup(0, 1) + for _, message := range inputMessages { + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) + } + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Equal(t, 1, decoderCount) + require.Equal(t, []uint64{10, 20, 30}, []uint64{ + messages[0].GetCommitTs(), messages[1].GetCommitTs(), messages[2].GetCommitTs(), + }) +} + +func TestEventsGroupReadsLargeSharedPayloadOnceAcrossBatches(t *testing.T) { + const messageCount = 27020 + + config := defaultSpillConfig() + config.resolveBatchMessages = 10000 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + originalMessages := make([]*codeccommon.DMLMessage, messageCount) + restoredMessages := make([]*codeccommon.DMLMessage, messageCount) + for i := range messageCount { + commitTs := uint64(i + 1) + originalMessages[i] = newTestDMLMessage(commitTs) + restoredMessages[i] = newTestDMLMessage(commitTs) + } + decodeCount := 0 + messageData := codeccommon.NewDMLMessageData(nil, []byte("one large object payload"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + decodeCount++ + return restoredMessages, nil + }) + for _, message := range originalMessages { + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) + } + + readRecord := store.readRecord + readCount := 0 + store.readRecord = func(file *spill.RecordFile, handle spill.Handle) ([]byte, error) { + readCount++ + return readRecord(file, handle) + } + + resolved := 0 + batchCount := 0 + for { + batch, hasMore, err := group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + require.NotNil(t, batch) + batchCount++ + resolved += len(batch.Messages) + if batchCount == 1 { + require.Positive(t, batch.ResolvedBytes) + } else { + require.Zero(t, batch.ResolvedBytes) + } + require.NoError(t, batch.Ack()) + if !hasMore { + break + } + } + + require.Equal(t, messageCount, resolved) + require.Equal(t, 3, batchCount) + require.Equal(t, 1, readCount) + require.Equal(t, 1, decodeCount) + require.Equal(t, int64(1), store.Stats().PayloadWriteCount) + require.Equal(t, int64(1), store.Stats().PayloadReadCount) + require.Equal(t, int64(1), store.Stats().PayloadDecodeCount) + require.Equal(t, store.Stats().PayloadWriteBytes, store.Stats().PayloadReadBytes) + require.Zero(t, store.PendingBytes()) + require.Empty(t, store.restorers) + require.Empty(t, store.segments) +} + +func TestEventsGroupsSharePayloadUntilEveryGroupAcks(t *testing.T) { + store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + firstGroup := NewEventsGroup(0, 1, store) + secondGroup := NewEventsGroup(0, 2, store) + first := newTestDMLMessage(1) + second := newTestDMLMessage(2) + decodeCount := 0 + messageData := codeccommon.NewDMLMessageData(nil, []byte("shared across groups"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + decodeCount++ + return []*codeccommon.DMLMessage{first, second}, nil + }) + messageData.AttachDMLMessage(first) + require.NoError(t, firstGroup.AppendMessage(first)) + messageData.AttachDMLMessage(second) + require.NoError(t, secondGroup.AppendMessage(second)) + + readRecord := store.readRecord + readCount := 0 + store.readRecord = func(file *spill.RecordFile, handle spill.Handle) ([]byte, error) { + readCount++ + return readRecord(file, handle) + } + + firstBatch, _, err := firstGroup.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + secondBatch, _, err := secondGroup.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + require.Equal(t, 1, readCount) + require.Equal(t, 1, decodeCount) + require.Equal(t, 1, store.Stats().LivePayloads) + require.Len(t, store.segments, 1) + entry := readGroupIndex(t, firstGroup)[0] + spillPath := store.segments[entry.location.segmentID].file.Path() + + require.NoError(t, firstBatch.Ack()) + require.Equal(t, 1, store.Stats().LivePayloads) + require.FileExists(t, spillPath) + require.NoError(t, secondBatch.Ack()) + require.Zero(t, store.Stats().LivePayloads) + require.Empty(t, store.segments) + _, err = os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) +} + +func TestEventsGroupPrepareDoesNotReleaseBeforeAck(t *testing.T) { + store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + message := attachTestDMLMessageDataWithPayload(newTestDMLMessage(1), nil, []byte("payload")) + require.NoError(t, group.AppendMessage(message)) + entry := readGroupIndex(t, group)[0] + spillPath := store.segments[entry.location.segmentID].file.Path() + + batch, _, err := group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.NoError(t, err) + require.Len(t, batch.Messages, 1) + require.Equal(t, int64(1), group.pendingCount) + require.FileExists(t, spillPath) + _, _, err = group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + require.Error(t, err) + + require.NoError(t, batch.Ack()) + require.Zero(t, group.pendingCount) + _, err = os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) +} + +func TestEventsGroupSegmentsAndPendingWatermarks(t *testing.T) { + config := defaultSpillConfig() + config.segmentBytes = 64 + config.pendingHighBytes = 82 + config.pendingLowBytes = 41 + config.messageMetadataBytes = 1 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + first := attachTestDMLMessageDataWithPayload(newTestDMLMessage(1), nil, make([]byte, 16)) + second := attachTestDMLMessageDataWithPayload(newTestDMLMessage(2), nil, make([]byte, 16)) + require.NoError(t, group.AppendMessage(first)) + firstEntry := readGroupIndex(t, group)[0] + firstSegment := store.segments[firstEntry.location.segmentID] + firstPath := firstSegment.file.Path() + require.NoError(t, group.AppendMessage(second)) + entries := readGroupIndex(t, group) + secondPath := store.segments[entries[1].location.segmentID].file.Path() + + require.Len(t, store.segments, 2) + require.Equal(t, int64(82), store.PendingBytes()) + require.True(t, store.ShouldDrain()) + + messages, hasMore, _, err := group.ResolveIntoBatch(1, nil, ResolveLimit{MaxMessages: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, messages, 1) + require.Equal(t, int64(41), store.PendingBytes()) + require.False(t, store.ShouldDrain()) + require.Len(t, store.segments, 1) + _, err = os.Stat(firstPath) + require.True(t, os.IsNotExist(err)) + require.FileExists(t, secondPath) + + require.NoError(t, group.Cleanup()) + require.Zero(t, store.PendingBytes()) + _, err = os.Stat(secondPath) + require.True(t, os.IsNotExist(err)) +} + +func TestEventsGroupKeepsSharedPayloadInOneSegment(t *testing.T) { + config := defaultSpillConfig() + config.segmentMessages = 1 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + first := newTestDMLMessage(1) + second := newTestDMLMessage(2) + messageData := codeccommon.NewDMLMessageData(nil, []byte("shared payload"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{first, second}, nil + }) + + messageData.AttachDMLMessage(first) + require.NoError(t, group.AppendMessage(first)) + messageData.AttachDMLMessage(second) + require.NoError(t, group.AppendMessage(second)) + + entries := readGroupIndex(t, group) + require.Len(t, store.segments, 1) + require.Equal(t, entries[0].location, entries[1].location) + require.NoError(t, group.Cleanup()) +} + +func TestEventsGroupAllowsOversizeSegment(t *testing.T) { + config := defaultSpillConfig() + config.segmentBytes = 16 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + + message := attachTestDMLMessageDataWithPayload( + newTestDMLMessage(1), []byte("oversize-key"), []byte("oversize-value")) + require.NoError(t, group.AppendMessage(message)) + require.Len(t, store.segments, 1) + require.Nil(t, store.activeSegment) + require.Greater(t, store.PendingBytes(), config.segmentBytes) + + messages, hasMore, _, err := group.ResolveIntoBatch( + math.MaxUint64, nil, ResolveLimit{MaxBytes: 1, MaxMessages: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, messages, 1) + require.Zero(t, store.PendingBytes()) +} + +func TestEventsGroupRestoreErrorDoesNotReleasePendingData(t *testing.T) { + config := defaultSpillConfig() + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + wantErr := errors.New("restore failed") + messageData := codeccommon.NewDMLMessageData([]byte("key"), []byte("value"), + func([]byte) ([]*codeccommon.DMLMessage, error) { + return nil, wantErr + }) + message := newTestDMLMessage(1) + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) + pendingBytes := store.PendingBytes() + + _, _, _, err := group.ResolveIntoBatch(math.MaxUint64, nil, store.ResolveLimit()) + require.ErrorIs(t, err, wantErr) + require.Equal(t, int64(1), group.pendingCount) + require.Equal(t, pendingBytes, store.PendingBytes()) + require.NoError(t, group.Cleanup()) + require.Zero(t, store.PendingBytes()) +} + +func TestEventsGroupResolveIntoBatchBounds(t *testing.T) { + group := NewEventsGroup(0, 1) + for _, commitTs := range []uint64{1, 2, 2, 3} { + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(commitTs)))) + } + + messages, hasMore, _, err := group.ResolveIntoBatch( + math.MaxUint64, nil, ResolveLimit{MaxMessages: 2}) + require.NoError(t, err) + require.True(t, hasMore) + require.Len(t, messages, 3) + require.Equal(t, []uint64{1, 2, 2}, []uint64{ + messages[0].GetCommitTs(), messages[1].GetCommitTs(), messages[2].GetCommitTs(), + }) + + messages, hasMore, _, err = group.ResolveIntoBatch( + math.MaxUint64, nil, ResolveLimit{MaxBytes: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, messages, 1) + require.Equal(t, uint64(3), messages[0].GetCommitTs()) +} + +func TestSpillStoreAllowsPendingAboveHighWatermark(t *testing.T) { + config := defaultSpillConfig() + config.pendingHighBytes = 10 + config.pendingLowBytes = 5 + store := newSpillStore(config) + + store.addPending(11) + require.True(t, store.ShouldDrain()) + store.addPending(100) + require.Equal(t, int64(111), store.PendingBytes()) + require.True(t, store.ShouldDrain()) + + store.releasePending(106) + require.Equal(t, int64(5), store.PendingBytes()) + require.False(t, store.ShouldDrain()) +} + +func TestSpillStoreDefaults(t *testing.T) { + store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + require.Equal(t, int64(128*1024*1024), store.config.segmentBytes) + require.Equal(t, int64(1024*1024*1024), store.config.pendingHighBytes) + require.Equal(t, int64(512*1024*1024), store.config.pendingLowBytes) + require.Equal(t, ResolveLimit{MaxBytes: 64 * 1024 * 1024, MaxMessages: 10000}, store.ResolveLimit()) + require.Equal(t, int64(100000), store.config.indexCleanupMessages) + require.Equal(t, 0.90, store.config.diskUsageLimit) +} + +func TestSpillStoreRejectsWriteAboveDiskUsageLimit(t *testing.T) { + config := defaultSpillConfig() + config.diskCheckBytes = 1 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + store.diskUsage = func(string) (filesystemUsage, error) { + return filesystemUsage{usedBytes: 91, totalBytes: 100}, nil + } + group := NewEventsGroup(0, 1, store) + + err := group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(1))) + require.ErrorContains(t, err, "spill filesystem usage") + require.ErrorContains(t, err, "90.00% limit") + require.Same(t, store.terminalErr, err) + require.Zero(t, group.pendingCount) + require.Empty(t, store.segments) + require.NoError(t, store.Cleanup()) +} + +func TestEventsGroupTracksResolvedAndAppliedFrontiers(t *testing.T) { + store := NewSpillStore() + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(1)))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(newTestDMLMessage(2)))) + + batch, hasMore, err := group.PrepareResolve(1, store.ResolveLimit()) + require.NoError(t, err) + require.False(t, hasMore) + require.Equal(t, uint64(2), group.HighWatermark) + require.Equal(t, uint64(1), group.resolvedTs) + require.Zero(t, group.appliedTs) + require.Equal(t, int64(2), group.pendingCount) + + require.NoError(t, batch.Ack()) + require.Equal(t, uint64(1), group.appliedTs) + require.Equal(t, int64(1), group.pendingCount) + require.Equal(t, int64(1), store.Stats().AppliedEventCount) + require.NoError(t, group.Cleanup()) + require.NoError(t, store.Cleanup()) +} + +func TestEventsGroupRestoresPersistedSourcePosition(t *testing.T) { + group := NewEventsGroup(3, 1) + message := newTestDMLMessage(10) + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { + return []*codeccommon.DMLMessage{message}, nil + }) + messageData.SourcePosition = 42 + messageData.AttachDMLMessage(message) + var restoredPosition int64 + group.SetPostRestore(func(message *codeccommon.DMLMessage, position int64) *codeccommon.DMLMessage { + restoredPosition = position + return message + }) + + require.NoError(t, group.AppendMessage(message)) + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 1) + require.Equal(t, int64(42), restoredPosition) +} + +func TestSpillStoreRestoreCacheHasByteAndEntryBounds(t *testing.T) { + config := defaultSpillConfig() + config.resolveBatchBytes = 1 << 20 + config.resolveBatchMessages = 2 + store := newSpillStore(config) + t.Cleanup(func() { require.NoError(t, store.Cleanup()) }) + group := NewEventsGroup(0, 1, store) + for commitTs := uint64(1); commitTs <= 4; commitTs++ { + require.NoError(t, group.AppendMessage( + attachTestDMLMessageDataWithPayload(newTestDMLMessage(commitTs), nil, []byte{byte(commitTs)}))) + } + + for commitTs := uint64(1); commitTs <= 3; commitTs++ { + batch, hasMore, err := group.PrepareResolve(4, ResolveLimit{MaxMessages: 1}) + require.NoError(t, err) + require.True(t, hasMore) + require.Equal(t, commitTs, batch.Messages[0].GetCommitTs()) + require.NoError(t, batch.Ack()) + require.LessOrEqual(t, len(store.cache), 2) + require.LessOrEqual(t, store.cacheBytes, config.resolveBatchBytes) + } + + batch, hasMore, err := group.PrepareResolve(4, ResolveLimit{MaxMessages: 1}) + require.NoError(t, err) + require.False(t, hasMore) + require.NoError(t, batch.Ack()) + require.Empty(t, store.cache) +} + +func TestEventsGroupKeepsPerEventMetadataOnDisk(t *testing.T) { + group := NewEventsGroup(0, 1) + const messageCount = 2048 + messages := make([]*codeccommon.DMLMessage, 0, messageCount) + for i := 1; i <= messageCount; i++ { + messages = append(messages, newTestDMLMessage(uint64(i))) + } + messageData := codeccommon.NewDMLMessageData(nil, nil, + func([]byte) ([]*codeccommon.DMLMessage, error) { return messages, nil }) + for _, message := range messages { + messageData.AttachDMLMessage(message) + require.NoError(t, group.AppendMessage(message)) + } + require.Equal(t, int64(messageCount), group.pendingCount) + require.Len(t, group.segmentRefs, 1) + require.Len(t, group.restorerRefs, 1) + + resolved, hasMore, _, err := group.ResolveIntoBatch(1536, nil, ResolveLimit{}) + require.NoError(t, err) + require.False(t, hasMore) + require.Len(t, resolved, 1536) + require.Equal(t, int64(512), group.pendingCount) + require.Len(t, readGroupIndex(t, group), 512) + require.NoError(t, group.Cleanup()) +} + +func TestEventsGroupResolveIntoAppendsAndCleansResolvedSpillRecords(t *testing.T) { // Scenario: A consumer resolves events by watermark/commit-ts and appends them into a downstream - // batch slice. We must clear resolved messages in the group's backing array to avoid retaining - // already-flushed events and causing unbounded memory growth. + // batch slice. Buffered messages are held only by a spill file, and the file must be cleaned once + // all of its records have been resolved. // // Steps: // 1. Append 3 events with increasing CommitTs. // 2. Call ResolveInto with resolve=2 and a nil dst. // 3. Verify (a) returned events are correct, (b) group keeps only the remaining event, - // (c) resolved messages in the original backing slice are cleared (nil'd). + // (c) the file survives the partial resolve. group := NewEventsGroup(0, 1) m1 := newTestDMLMessage(1) m2 := newTestDMLMessage(2) m3 := newTestDMLMessage(3) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) - // Keep a reference to the original slice header so we can validate that ResolveInto clears - // resolved messages in-place (this is what prevents GC retention of flushed events). - original := group.messages + spillPath := group.store.activeSegment.file.Path() var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(2, dst) + dst, err := group.ResolveInto(2, dst) + require.NoError(t, err) require.Len(t, dst, 2) - require.Same(t, m1, dst[0]) - require.Same(t, m2, dst[1]) + require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m2.GetCommitTs(), dst[1].GetCommitTs()) - require.Len(t, group.messages, 1) - require.Same(t, m3, group.messages[0]) + require.Equal(t, int64(1), group.pendingCount) + entries := readGroupIndex(t, group) + require.Len(t, entries, 1) + require.Equal(t, m3.GetCommitTs(), entries[0].commitTs) + require.FileExists(t, spillPath) - // The unresolved event is compacted to the front, and the tail is cleared so the group - // doesn't keep flushed events alive via its backing array. - require.Same(t, m3, original[0]) - require.Nil(t, original[1]) - require.Nil(t, original[2]) + _, err = group.GetAllMessages() + require.NoError(t, err) + require.Nil(t, group.store.activeSegment) + require.Empty(t, group.store.segments) + _, err = os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) } func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { @@ -84,21 +807,18 @@ func TestEventsGroupResolveIntoNoopWhenNothingResolved(t *testing.T) { group := NewEventsGroup(0, 1) m1 := newTestDMLMessage(10) m2 := newTestDMLMessage(20) - group.AppendMessage(m1) - group.AppendMessage(m2) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) - original := group.messages dst := make([]*codeccommon.DMLMessage, 0, 1) - dst = group.ResolveInto(5, dst) + dst, err := group.ResolveInto(5, dst) + require.NoError(t, err) require.Len(t, dst, 0) - require.Len(t, group.messages, 2) - require.Same(t, m1, group.messages[0]) - require.Same(t, m2, group.messages[1]) - - // No prefix should be cleared because nothing was resolved. - require.Same(t, m1, original[0]) - require.Same(t, m2, original[1]) + require.Equal(t, int64(2), group.pendingCount) + entries := readGroupIndex(t, group) + require.Equal(t, m1.GetCommitTs(), entries[0].commitTs) + require.Equal(t, m2.GetCommitTs(), entries[1].commitTs) } func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { @@ -107,20 +827,23 @@ func TestEventsGroupResolveIntoClearsAllWhenFullyResolved(t *testing.T) { group := NewEventsGroup(0, 1) m1 := newTestDMLMessage(1) m2 := newTestDMLMessage(2) - group.AppendMessage(m1) - group.AppendMessage(m2) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) - original := group.messages + spillPath := group.store.activeSegment.file.Path() var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(100, dst) + dst, err := group.ResolveInto(100, dst) + require.NoError(t, err) require.Len(t, dst, 2) - require.Same(t, m1, dst[0]) - require.Same(t, m2, dst[1]) + require.Equal(t, m1.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m2.GetCommitTs(), dst[1].GetCommitTs()) - require.Len(t, group.messages, 0) - require.Nil(t, original[0]) - require.Nil(t, original[1]) + require.Zero(t, group.pendingCount) + require.Nil(t, group.store.activeSegment) + require.Empty(t, group.store.segments) + _, err = os.Stat(spillPath) + require.True(t, os.IsNotExist(err)) } func TestEventsGroupResolveIntoSortsOutOfOrderResolvedMessages(t *testing.T) { @@ -128,23 +851,21 @@ func TestEventsGroupResolveIntoSortsOutOfOrderResolvedMessages(t *testing.T) { m1 := newTestDMLMessage(20) m2 := newTestDMLMessage(10) m3 := newTestDMLMessage(30) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) - original := group.messages var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(25, dst) + dst, err := group.ResolveInto(25, dst) + require.NoError(t, err) require.Len(t, dst, 2) - require.Same(t, m2, dst[0]) - require.Same(t, m1, dst[1]) + require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m1.GetCommitTs(), dst[1].GetCommitTs()) - require.Len(t, group.messages, 1) - require.Same(t, m3, group.messages[0]) - require.Same(t, m3, original[0]) - require.Nil(t, original[1]) - require.Nil(t, original[2]) + require.Equal(t, int64(1), group.pendingCount) + entries := readGroupIndex(t, group) + require.Equal(t, m3.GetCommitTs(), entries[0].commitTs) } func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { @@ -152,18 +873,19 @@ func TestEventsGroupResolveIntoKeepsSameCommitTsStable(t *testing.T) { m1 := newTestDMLMessage(20) m2 := newTestDMLMessage(10) m3 := newTestDMLMessage(20) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) var dst []*codeccommon.DMLMessage - dst = group.ResolveInto(20, dst) + dst, err := group.ResolveInto(20, dst) + require.NoError(t, err) require.Len(t, dst, 3) - require.Same(t, m2, dst[0]) - require.Same(t, m1, dst[1]) - require.Same(t, m3, dst[2]) - require.Empty(t, group.messages) + require.Equal(t, m2.GetCommitTs(), dst[0].GetCommitTs()) + require.Equal(t, m1.GetCommitTs(), dst[1].GetCommitTs()) + require.Equal(t, m3.GetCommitTs(), dst[2].GetCommitTs()) + require.Zero(t, group.pendingCount) } func TestEventsGroupGetAllMessagesSortsOutOfOrderMessages(t *testing.T) { @@ -171,17 +893,151 @@ func TestEventsGroupGetAllMessagesSortsOutOfOrderMessages(t *testing.T) { m1 := newTestDMLMessage(20) m2 := newTestDMLMessage(10) m3 := newTestDMLMessage(30) - group.AppendMessage(m1) - group.AppendMessage(m2) - group.AppendMessage(m3) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m1))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m2))) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(m3))) - messages := group.GetAllMessages() + messages, err := group.GetAllMessages() + require.NoError(t, err) require.Len(t, messages, 3) - require.Same(t, m2, messages[0]) - require.Same(t, m1, messages[1]) - require.Same(t, m3, messages[2]) - require.Empty(t, group.messages) + require.Equal(t, m2.GetCommitTs(), messages[0].GetCommitTs()) + require.Equal(t, m1.GetCommitTs(), messages[1].GetCommitTs()) + require.Equal(t, m3.GetCommitTs(), messages[2].GetCommitTs()) + require.Zero(t, group.pendingCount) +} + +func TestEventsGroupRestoresSpilledEventRowsAndTableInfo(t *testing.T) { + tableInfo := common.WrapTableInfo("test", &model.TableInfo{ + ID: 1, + Name: ast.NewCIStr("t"), + Columns: []*model.ColumnInfo{ + { + ID: 1, + Name: ast.NewCIStr("id"), + FieldType: *types.NewFieldType(mysql.TypeLonglong), + }, + }, + }) + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 1) + rows.AppendInt64(0, 42) + event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) + event.Rows = rows + event.RowTypes = []common.RowType{common.RowTypeInsert} + event.Length = 1 + event.TableInfoVersion = 88 + event.ReplicatingTs = 99 + event.Checksum = []*integrity.Checksum{{Current: 1, Previous: 2, Corrupted: true, Version: 3}} + + group := NewEventsGroup(0, 1) + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 1) + restored := messages[0].ToDMLEvent() + require.Equal(t, uint64(100), restored.CommitTs) + require.Equal(t, uint64(88), restored.TableInfoVersion) + require.Equal(t, uint64(99), restored.ReplicatingTs) + require.Equal(t, event.Checksum, restored.Checksum) + require.Equal(t, "test", restored.TableInfo.GetSchemaName()) + require.Equal(t, "t", restored.TableInfo.GetTableName()) + require.Equal(t, int64(42), restored.Rows.GetRow(0).GetInt64(0)) +} + +func TestEventsGroupRestoresRowsFromSharedChunk(t *testing.T) { + tableInfo := common.WrapTableInfo("test", &model.TableInfo{ + ID: 1, + Name: ast.NewCIStr("t"), + Columns: []*model.ColumnInfo{ + {ID: 1, Name: ast.NewCIStr("id"), FieldType: *types.NewFieldType(mysql.TypeLonglong)}, + }, + }) + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 4) + for i := range int64(4) { + rows.AppendInt64(0, i) + } + + group := NewEventsGroup(0, 1) + for _, offset := range []int{0, 2} { + event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) + event.Rows = rows + // A decoded update occupies two RowTypes entries, matching its before + // and after rows in the shared chunk. + event.RowTypes = []common.RowType{common.RowTypeUpdate, common.RowTypeUpdate} + event.Length = 1 + event.PreviousTotalOffset = offset + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) + } + + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 2) + require.Zero(t, messages[0].ToDMLEvent().PreviousTotalOffset) + require.Equal(t, 2, messages[1].ToDMLEvent().PreviousTotalOffset) + for _, message := range messages { + require.Equal(t, 4, message.ToDMLEvent().Rows.NumRows()) + } + + second := messages[1].ToDMLEvent() + row, ok := second.GetNextRow() + require.True(t, ok) + require.Equal(t, int64(2), row.PreRow.GetInt64(0)) + require.Equal(t, int64(3), row.Row.GetInt64(0)) + _, ok = second.GetNextRow() + require.False(t, ok) +} + +func TestEventsGroupRestoresCompactUpdateRows(t *testing.T) { + tableInfo := common.WrapTableInfo("test", &model.TableInfo{ + ID: 1, + Name: ast.NewCIStr("t"), + Columns: []*model.ColumnInfo{ + {ID: 1, Name: ast.NewCIStr("id"), FieldType: *types.NewFieldType(mysql.TypeLonglong)}, + }, + }) + rows := chunk.NewChunkWithCapacity(tableInfo.GetFieldSlice(), 2) + rows.AppendInt64(0, 1) + rows.AppendInt64(0, 2) + + // The Avro decoder represents an update with one RowType even though the + // chunk still contains both before and after rows. + event := commonEvent.NewDMLEvent(common.NewDispatcherID(), 1, 90, 100, tableInfo) + event.Rows = rows + event.RowTypes = []common.RowType{common.RowTypeUpdate} + event.Length = 1 + + group := NewEventsGroup(0, 1) + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) + messages, err := group.GetAllMessages() + require.NoError(t, err) + require.Len(t, messages, 1) + + restored := messages[0].ToDMLEvent() + require.Equal(t, 2, restored.Rows.NumRows()) + row, ok := restored.GetNextRow() + require.True(t, ok) + require.Equal(t, int64(1), row.PreRow.GetInt64(0)) + require.Equal(t, int64(2), row.Row.GetInt64(0)) + _, ok = restored.GetNextRow() + require.False(t, ok) +} + +func TestEventsGroupSpillDoesNotSignalDownstreamCallbacks(t *testing.T) { + event := newTestDMLEvent(100, common.RowTypeInsert) + var enqueued, flushed int + event.AddPostEnqueueFunc(func() { enqueued++ }) + event.AddPostFlushFunc(func() { flushed++ }) + + group := NewEventsGroup(0, 1) + defer func() { require.NoError(t, group.Cleanup()) }() + message := codeccommon.NewDMLMessageFromEvent(event) + require.NoError(t, group.AppendMessage(attachTestDMLMessageData(message))) + require.Zero(t, enqueued) + require.Zero(t, flushed) } func BenchmarkEventsGroupResolveInto(b *testing.B) { @@ -215,50 +1071,73 @@ func BenchmarkEventsGroupResolveInto(b *testing.B) { lastIndex := len(source) - 1 source[lastIndex-1], source[lastIndex] = source[lastIndex], source[lastIndex-1] } - group := NewEventsGroup(0, 1) - group.messages = make([]*codeccommon.DMLMessage, 0, messageCount) dst := make([]*codeccommon.DMLMessage, 0, messageCount) b.ReportAllocs() b.ResetTimer() - for b.Loop() { - if len(group.messages) != messageCount { - group.messages = append(group.messages[:0], source...) - group.outOfOrder = benchmark.outOfOrder + b.StopTimer() + for range b.N { + group := NewEventsGroup(0, 1) + for _, message := range source { + if err := group.AppendMessage(attachTestDMLMessageData(message)); err != nil { + b.Fatal(err) + } + } + b.StartTimer() + var err error + dst, err = group.ResolveInto(benchmark.resolveTs, dst[:0]) + b.StopTimer() + if err != nil { + b.Fatal(err) + } + if err := group.Cleanup(); err != nil { + b.Fatal(err) } - dst = group.ResolveInto(benchmark.resolveTs, dst[:0]) } }) } } -func TestAppendOrMergeDMLEventMergesSameCommitTs(t *testing.T) { - var flushed []int - e1 := newTestDMLEvent(10, common.RowTypeInsert) - e1.AddPostFlushFunc(func() { flushed = append(flushed, 1) }) - e2 := newTestDMLEvent(10, common.RowTypeDelete) - e2.AddPostFlushFunc(func() { flushed = append(flushed, 2) }) - - events := AppendOrMergeDMLEvent(nil, e1) - events = AppendOrMergeDMLEvent(events, e2) +func BenchmarkEventsGroupResolveIncrementally(b *testing.B) { + const ( + messageCount = 32 * 1024 + batchMessages = 16 + ) - require.Len(t, events, 1) - require.Same(t, e1, events[0]) - require.Equal(t, int32(2), events[0].Length) - require.Equal(t, []common.RowType{common.RowTypeInsert, common.RowTypeDelete}, events[0].RowTypes) - - events[0].PostFlush() - require.Equal(t, []int{1, 2}, flushed) -} - -func TestAppendOrMergeDMLEventAppendsDifferentCommitTs(t *testing.T) { - e1 := newTestDMLEvent(10, common.RowTypeInsert) - e2 := newTestDMLEvent(20, common.RowTypeDelete) + oldLogLevel := log.GetLevel() + log.SetLevel(zapcore.FatalLevel) + b.Cleanup(func() { log.SetLevel(oldLogLevel) }) - events := AppendOrMergeDMLEvent(nil, e1) - events = AppendOrMergeDMLEvent(events, e2) + b.ReportAllocs() + for range b.N { + b.StopTimer() + config := defaultSpillConfig() + config.resolveBatchMessages = batchMessages + store := newSpillStore(config) + group := NewEventsGroup(0, 1, store) + for i := 1; i <= messageCount; i++ { + if err := group.AppendMessage( + attachTestDMLMessageData(newTestDMLMessage(uint64(i)))); err != nil { + b.Fatal(err) + } + } - require.Len(t, events, 2) - require.Same(t, e1, events[0]) - require.Same(t, e2, events[1]) + b.StartTimer() + for { + batch, hasMore, err := group.PrepareResolve(math.MaxUint64, store.ResolveLimit()) + if err != nil { + b.Fatal(err) + } + if err := batch.Ack(); err != nil { + b.Fatal(err) + } + if !hasMore { + break + } + } + b.StopTimer() + if err := store.Cleanup(); err != nil { + b.Fatal(err) + } + } } diff --git a/pkg/sink/codec/common/decoder.go b/pkg/sink/codec/common/decoder.go index 86e5a8f5fa..9553daea9f 100644 --- a/pkg/sink/codec/common/decoder.go +++ b/pkg/sink/codec/common/decoder.go @@ -14,10 +14,95 @@ package common import ( + "sync/atomic" + commonType "github.com/pingcap/ticdc/pkg/common" commonEvent "github.com/pingcap/ticdc/pkg/common/event" ) +// DMLMessageData keeps the original encoded input needed to restore a DML +// message after it has been spilled. One input can be attached to multiple +// DMLMessages; Attach assigns each message its ordinal in that input. +type DMLMessageData struct { + ID uint64 + Key []byte + Value []byte + Restorer *DMLMessageRestorer + SourcePosition int64 + + nextDMLIndex uint64 + + spillStoreID uint64 + spillSegmentID uint64 + spillOffset int64 + spillLength uint64 +} + +var nextDMLMessageDataID atomic.Uint64 + +// DMLMessageRestorer restores every DML message from one encoded input. A +// restorer can be shared by all inputs decoded by the same decoder context, so +// a spill queue does not need to retain one closure per input message. +type DMLMessageRestorer struct { + ID uint64 + Decode func([]byte) ([]*DMLMessage, error) +} + +var nextDMLMessageRestorerID atomic.Uint64 + +// NewDMLMessageRestorer creates a decoder context that can be shared by many +// DMLMessageData values. +func NewDMLMessageRestorer(decode func([]byte) ([]*DMLMessage, error)) *DMLMessageRestorer { + return &DMLMessageRestorer{ + ID: nextDMLMessageRestorerID.Add(1), + Decode: decode, + } +} + +// NewDMLMessageData creates data shared by DMLMessages decoded from one input. +func NewDMLMessageData( + key, value []byte, + decode func([]byte) ([]*DMLMessage, error), +) *DMLMessageData { + return NewDMLMessageDataWithRestorer(key, value, NewDMLMessageRestorer(decode)) +} + +// NewDMLMessageDataWithRestorer creates input data using a reusable decoder +// context. +func NewDMLMessageDataWithRestorer( + key, value []byte, + restorer *DMLMessageRestorer, +) *DMLMessageData { + return &DMLMessageData{ + ID: nextDMLMessageDataID.Add(1), + Key: key, + Value: value, + Restorer: restorer, + } +} + +// SpillLocation returns the payload record already written by storeID. +func (d *DMLMessageData) SpillLocation(storeID uint64) ( + segmentID uint64, offset int64, length uint64, ok bool, +) { + if d == nil || d.spillStoreID != storeID || d.spillSegmentID == 0 { + return 0, 0, 0, false + } + return d.spillSegmentID, d.spillOffset, d.spillLength, true +} + +// SetSpillLocation records the payload location while this input is being +// appended. The input object is short-lived; the location is copied into each +// disk-backed event descriptor. +func (d *DMLMessageData) SetSpillLocation( + storeID, segmentID uint64, offset int64, length uint64, +) { + d.spillStoreID = storeID + d.spillSegmentID = segmentID + d.spillOffset = offset + d.spillLength = length +} + type DMLMessage struct { TableID int64 Schema string @@ -28,6 +113,8 @@ type DMLMessage struct { // toDMLEvent may be called after the decoder has consumed later messages. // It must only use data captured by this DMLMessage and must not depend on decoder cursor state. toDMLEvent func() *commonEvent.DMLEvent + spillData *DMLMessageData + dmlIndex uint64 } func NewDMLMessage( @@ -74,6 +161,21 @@ func (m *DMLMessage) ToDMLEvent() *commonEvent.DMLEvent { return m.toDMLEvent() } +// AttachDMLMessageData attaches the original input required to restore this +// message after spill. It must be called once for every decoded DML, including +// DMLs the consumer later discards. +func (d *DMLMessageData) AttachDMLMessage(message *DMLMessage) { + message.spillData = d + message.dmlIndex = d.nextDMLIndex + d.nextDMLIndex++ +} + +// SpillData returns the data and ordinal attached while this message was +// decoded. They are used by the consumer's in-memory events group only. +func (m *DMLMessage) SpillData() (*DMLMessageData, uint64) { + return m.spillData, m.dmlIndex +} + // Decoder is an abstraction for events decoder // this interface is only for testing now type Decoder interface {