diff --git a/downstreamadapter/sink/kafka/helper.go b/downstreamadapter/sink/kafka/helper.go index fbbdd4cd50..6f555bdb61 100644 --- a/downstreamadapter/sink/kafka/helper.go +++ b/downstreamadapter/sink/kafka/helper.go @@ -51,6 +51,9 @@ func (c components) close() { if c.claimCheck != nil { c.claimCheck.Close() } + if c.factory != nil { + c.factory.Close() + } } func newKafkaSinkComponent( @@ -85,7 +88,7 @@ func newKafkaSinkComponent( } options.Topic = topic - comp.factory, err = kafka.NewSaramaFactory(ctx, options, changefeedID) + comp.factory, err = kafka.NewFactory(ctx, options, changefeedID) if err != nil { return comp, protocol, err } @@ -109,6 +112,9 @@ func newKafkaSinkComponent( if err != nil { return comp, protocol, err } + if options.Client == kafka.KafkaClientFranz { + encoderConfig.WithKafkaRecordBatchSize() + } comp.claimCheck, err = claimcheck.New(ctx, encoderConfig.LargeMessageHandle, changefeedID) if err != nil { diff --git a/downstreamadapter/sink/kafka/sink.go b/downstreamadapter/sink/kafka/sink.go index c960f0cca1..4f6302ccc3 100644 --- a/downstreamadapter/sink/kafka/sink.go +++ b/downstreamadapter/sink/kafka/sink.go @@ -72,10 +72,6 @@ func (s *sink) SinkType() common.SinkType { return common.KafkaSinkType } -var createKafkaFactory = func(createSaramaFactory func() (kafka.Factory, error)) (kafka.Factory, error) { - return createSaramaFactory() -} - func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, sinkConfig *config.SinkConfig) error { protocol, err := helper.GetProtocol(util.GetOrZero(sinkConfig.Protocol)) if err != nil { @@ -100,6 +96,9 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, if err != nil { return err } + if options.Client == kafka.KafkaClientFranz { + encoderConfig.WithKafkaRecordBatchSize() + } claimCheck, err := claimcheck.New(ctx, encoderConfig.LargeMessageHandle, changefeedID) if err != nil { @@ -116,12 +115,11 @@ func Verify(ctx context.Context, changefeedID common.ChangeFeedID, uri *url.URL, return err } - factory, err := createKafkaFactory(func() (kafka.Factory, error) { - return kafka.NewSaramaFactory(ctx, options, changefeedID) - }) + factory, err := kafka.NewFactory(ctx, options, changefeedID) if err != nil { return err } + defer factory.Close() adminClient, err := factory.AdminClient(ctx) if err != nil { @@ -427,7 +425,6 @@ func (s *sink) sendMessages(ctx context.Context) error { for _, message := range future.Messages { start := time.Now() if err = s.statistics.RecordBatchExecution(func() (int, int64, error) { - message.SetPartitionKey(future.Key.PartitionKey) if err = s.dmlProducer.AsyncSend( ctx, future.Key.Topic, @@ -471,11 +468,11 @@ func (s *sink) sendDDLEvent(event *commonEvent.DDLEvent) error { ddlType := e.GetDDLType().String() if s.partitionRule == helper.PartitionAll { err = s.statistics.RecordDDLExecution(func() (string, error) { - return ddlType, s.ddlProducer.SendMessages(topic, partitionNum, message) + return ddlType, s.ddlProducer.SendMessages(s.ctx, topic, partitionNum, message) }) } else { err = s.statistics.RecordDDLExecution(func() (string, error) { - return ddlType, s.ddlProducer.SendMessage(topic, 0, message) + return ddlType, s.ddlProducer.SendMessage(s.ctx, topic, 0, message) }) } if err != nil { @@ -541,7 +538,7 @@ func (s *sink) sendCheckpoint(ctx context.Context) error { if err != nil { return err } - err = s.ddlProducer.SendMessages(topic, partitionNum, msg) + err = s.ddlProducer.SendMessages(ctx, topic, partitionNum, msg) if err != nil { return err } @@ -552,7 +549,7 @@ func (s *sink) sendCheckpoint(ctx context.Context) error { if err != nil { return err } - err = s.ddlProducer.SendMessages(topic, partitionNum, msg) + err = s.ddlProducer.SendMessages(ctx, topic, partitionNum, msg) if err != nil { return err } diff --git a/downstreamadapter/sink/kafka/sink_test.go b/downstreamadapter/sink/kafka/sink_test.go index 47133a2723..0b3ecfe725 100644 --- a/downstreamadapter/sink/kafka/sink_test.go +++ b/downstreamadapter/sink/kafka/sink_test.go @@ -35,6 +35,7 @@ import ( "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/pingcap/tidb/pkg/meta/model" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" "go.uber.org/atomic" ) @@ -91,6 +92,9 @@ func TestSinkWorkersReturnContextError(t *testing.T) { } func TestVerifyInvalidConfig(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, kafkaSinkTestTopic)) + defer cluster.Close() + schemaRegistry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "invalid response", http.StatusInternalServerError) })) @@ -101,28 +105,10 @@ func TestVerifyInvalidConfig(t *testing.T) { Protocol: &avroProtocol, SchemaRegistry: &schemaRegistry.URL, } - sinkURI, err := url.Parse("kafka://127.0.0.1:9092/" + kafkaSinkTestTopic + + sinkURI, err := url.Parse("kafka://" + cluster.ListenAddrs()[0] + "/" + kafkaSinkTestTopic + "?required-acks=1&kafka-version=2.4.0") require.NoError(t, err) - ctrl := gomock.NewController(t) - adminClient := kafka.NewMockAdminClient(ctrl) - factory := kafka.NewMockFactory(ctrl) - gomock.InOrder( - factory.EXPECT().AdminClient(gomock.Any()).Return(adminClient, nil), - adminClient.EXPECT().GetTopicsMeta([]string{kafkaSinkTestTopic}, false).Return( - map[string]kafka.TopicDetail{kafkaSinkTestTopic: {Name: kafkaSinkTestTopic}}, nil), - adminClient.EXPECT().Close(), - ) - - originalCreateKafkaFactory := createKafkaFactory - createKafkaFactory = func(_ func() (kafka.Factory, error)) (kafka.Factory, error) { - return factory, nil - } - t.Cleanup(func() { - createKafkaFactory = originalCreateKafkaFactory - }) - changefeedID := common.NewChangefeedID4Test("test", "verify-invalid-config") err = Verify(context.Background(), changefeedID, sinkURI, sinkConfig) require.ErrorContains(t, err, "ErrAvroSchemaAPIError") @@ -208,7 +194,7 @@ func TestKafkaSinkBasicFunctionality(t *testing.T) { } return nil }).Times(2) - syncProducer.EXPECT().SendMessages(gomock.Any(), int32(1), gomock.Any()).Return(nil) + syncProducer.EXPECT().SendMessages(gomock.Any(), gomock.Any(), int32(1), gomock.Any()).Return(nil) defer cancel() go kafkaSink.Run(ctx) @@ -248,6 +234,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { gomock.InOrder( adminClient.EXPECT().Close(), topicManager.EXPECT().Close(), + factory.EXPECT().Close(), ) kafkaSink, err := newWithComponents( @@ -276,6 +263,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { asyncProducer.EXPECT().Close(), adminClient.EXPECT().Close(), topicManager.EXPECT().Close(), + factory.EXPECT().Close(), ) kafkaSink, err := newWithComponents( @@ -307,6 +295,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { asyncProducer.EXPECT().Close().Do(func() { closeCount.Add(1) }), adminClient.EXPECT().Close().Do(func() { closeCount.Add(1) }), topicManager.EXPECT().Close().Do(func() { closeCount.Add(1) }), + factory.EXPECT().Close().Do(func() { closeCount.Add(1) }), ) kafkaSink, err := newWithComponents( @@ -322,7 +311,7 @@ func TestKafkaSinkConstructionAndCleanup(t *testing.T) { require.True(t, kafkaSink.IsNormal()) kafkaSink.Close() - require.Equal(t, int64(4), closeCount.Load()) + require.Equal(t, int64(5), closeCount.Load()) require.False(t, kafkaSink.IsNormal()) kafkaSink.AddDMLEvent(&commonEvent.DMLEvent{}) require.Zero(t, kafkaSink.eventChan.Len()) @@ -443,8 +432,8 @@ func TestKafkaSinkDDL(t *testing.T) { kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(4), nil) - syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(4), gomock.Any()). - DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { + syncProducer.EXPECT().SendMessages(gomock.Any(), kafkaSinkTestTopic, int32(4), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ int32, message *codecCommon.Message) error { require.NotEmpty(t, message.Key) require.NotEmpty(t, message.Value) return nil @@ -457,8 +446,8 @@ func TestKafkaSinkDDL(t *testing.T) { kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( t, t.Context(), config.ProtocolCanalJSON, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(4), nil) - syncProducer.EXPECT().SendMessage(kafkaSinkTestTopic, int32(0), gomock.Any()). - DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { + syncProducer.EXPECT().SendMessage(gomock.Any(), kafkaSinkTestTopic, int32(0), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ int32, message *codecCommon.Message) error { require.NotEmpty(t, message.Value) return nil }) @@ -480,7 +469,7 @@ func TestKafkaSinkDDL(t *testing.T) { t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) cause := errors.ErrKafkaSendMessage.GenWithStackByArgs() topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(2), nil) - syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(2), gomock.Any()).Return(cause) + syncProducer.EXPECT().SendMessages(gomock.Any(), kafkaSinkTestTopic, int32(2), gomock.Any()).Return(cause) require.Equal(t, cause, kafkaSink.WriteBlockEvent(ddlEvent)) require.False(t, kafkaSink.IsNormal()) @@ -508,8 +497,8 @@ func TestKafkaSinkCheckpoint(t *testing.T) { kafkaSink, topicManager, _, syncProducer := newKafkaSinkForTest( t, t.Context(), config.ProtocolOpen, &config.SinkConfig{}) topicManager.EXPECT().GetPartitionNum(gomock.Any(), kafkaSinkTestTopic).Return(int32(3), nil) - syncProducer.EXPECT().SendMessages(kafkaSinkTestTopic, int32(3), gomock.Any()). - DoAndReturn(func(_ string, _ int32, message *codecCommon.Message) error { + syncProducer.EXPECT().SendMessages(gomock.Any(), kafkaSinkTestTopic, int32(3), gomock.Any()). + DoAndReturn(func(_ context.Context, _ string, _ int32, message *codecCommon.Message) error { require.NotEmpty(t, message.Key) return nil }) @@ -535,7 +524,7 @@ func TestKafkaSinkCheckpoint(t *testing.T) { partitionCounts := map[string]int32{"topic-a": 2, "topic-b": 3, kafkaSinkTestTopic: 4} for topic, partitionCount := range partitionCounts { topicManager.EXPECT().GetPartitionNum(gomock.Any(), topic).Return(partitionCount, nil) - syncProducer.EXPECT().SendMessages(topic, partitionCount, gomock.Any()).Return(nil) + syncProducer.EXPECT().SendMessages(gomock.Any(), topic, partitionCount, gomock.Any()).Return(nil) } kafkaSink.checkpointChan <- 100 close(kafkaSink.checkpointChan) @@ -567,7 +556,7 @@ func TestKafkaSinkCheckpoint(t *testing.T) { // return the error and stop, so exactly one GetPartitionNum and one // SendMessages call are expected regardless of the topic order. topicManager.EXPECT().GetPartitionNum(gomock.Any(), gomock.Any()).Return(int32(2), nil) - syncProducer.EXPECT().SendMessages(gomock.Any(), int32(2), gomock.Any()).Return(cause) + syncProducer.EXPECT().SendMessages(gomock.Any(), gomock.Any(), int32(2), gomock.Any()).Return(cause) kafkaSink.checkpointChan <- 100 require.Equal(t, cause, kafkaSink.sendCheckpoint(t.Context())) @@ -623,6 +612,7 @@ func newKafkaSinkForTest( factory.EXPECT().AsyncProducer(gomock.Any()).Return(asyncProducer, nil) factory.EXPECT().SyncProducer(gomock.Any()).Return(syncProducer, nil) factory.EXPECT().MetricsCollector(nil).Return(noopMetricsCollector{}) + factory.EXPECT().Close().AnyTimes() kafkaSink, err := newWithComponents(ctx, changefeedID, common.DefaultKeyspaceID, protocol, components{ encoderGroup: encoderGroup, diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go index 7dbfe31ac5..29d3d0d42c 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager.go @@ -125,7 +125,7 @@ func (m *kafkaTopicManager) backgroundRefreshMeta(ctx context.Context) { case <-ticker.C: // We ignore the error here, because the error may be caused by the // network problem, and we can try to get the metadata next time. - topicPartitionNums, _ := m.fetchAllTopicsPartitionsNum() + topicPartitionNums, _ := m.fetchAllTopicsPartitionsNum(ctx) for topic, partitionNum := range topicPartitionNums { m.tryUpdatePartitionsAndLogging(topic, partitionNum) } @@ -157,7 +157,7 @@ func (m *kafkaTopicManager) tryUpdatePartitionsAndLogging(topic string, partitio // The error returned by this method could be a transient error that is fixable by the underlying logic. // When handling this error, please be cautious. // If you simply throw the error to the caller, it may impact the robustness of your program. -func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, error) { +func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum(ctx context.Context) (map[string]int32, error) { var topics []string m.topics.Range(func(key, _ any) bool { topics = append(topics, key.(string)) @@ -165,7 +165,7 @@ func (m *kafkaTopicManager) fetchAllTopicsPartitionsNum() (map[string]int32, err }) start := time.Now() - numPartitions, err := m.admin.GetTopicsPartitionsNum(topics) + numPartitions, err := m.admin.GetTopicsPartitionsNum(ctx, topics) if err != nil { log.Warn( "kafka topic metadata refresh failed", @@ -202,7 +202,7 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( err := retry.Do(ctx, func() error { // ignoreTopicError is set to false since we just create the topic, // make sure the topic is visible. - meta, err := m.admin.GetTopicsMeta(topics, false) + meta, err := m.admin.GetTopicsMeta(ctx, topics, false) if err != nil { return err } @@ -231,20 +231,17 @@ func (m *kafkaTopicManager) waitUntilTopicVisible( // createTopic creates a topic with the given name // and returns the number of partitions. -func (m *kafkaTopicManager) createTopic( - _ context.Context, - topicName string, -) (int32, error) { +func (m *kafkaTopicManager) createTopic(ctx context.Context, topicName string) (int32, error) { if !m.cfg.AutoCreate { return 0, errors.ErrKafkaInvalidConfig.GenWithStack("`auto-create-topic` is false, and %s not found", topicName) } - if err := m.cfg.ValidateReplicationFactor(m.admin); err != nil { + if err := m.cfg.ValidateReplicationFactor(ctx, m.admin); err != nil { return 0, err } start := time.Now() - err := m.admin.CreateTopic(&kafka.TopicDetail{ + err := m.admin.CreateTopic(ctx, &kafka.TopicDetail{ Name: topicName, NumPartitions: m.cfg.PartitionNum, ReplicationFactor: m.cfg.ReplicationFactor, @@ -274,7 +271,7 @@ func (m *kafkaTopicManager) CreateTopicAndWaitUntilVisible( ctx context.Context, topicName string, ) (int32, error) { // If the topic is not in the cache, try to get its metadata. - topicDetails, err := m.admin.GetTopicsMeta([]string{topicName}, false) + topicDetails, err := m.admin.GetTopicsMeta(ctx, []string{topicName}, false) if err == nil { if numPartition, ok := m.tryStoreTopicMeta(topicName, topicDetails); ok { return numPartition, nil diff --git a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go index 9ba464da7c..a992d2d6a9 100644 --- a/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go +++ b/downstreamadapter/sink/topicmanager/kafka_topic_manager_test.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/sink/kafka" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kerr" ) const kafkaTopicManagerTestTopic = "mock_topic" @@ -38,7 +39,7 @@ func TestCreateTopic(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{kafkaTopicManagerTestTopic}, false).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{kafkaTopicManagerTestTopic}, false).Return( map[string]kafka.TopicDetail{ kafkaTopicManagerTestTopic: {Name: kafkaTopicManagerTestTopic, NumPartitions: 2}, }, nil) @@ -63,8 +64,8 @@ func TestCreateTopic(t *testing.T) { var createdTopic *kafka.TopicDetail postCreateDescribeCount := 0 var manager *kafkaTopicManager - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).DoAndReturn( - func([]string, bool) (map[string]kafka.TopicDetail, error) { + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).DoAndReturn( + func(context.Context, []string, bool) (map[string]kafka.TopicDetail, error) { if createdTopic == nil { return nil, errors.WrapError(errors.ErrKafkaAdminAPI, sarama.ErrUnknownTopicOrPartition, "describe-topic", "new-topic") } @@ -78,8 +79,8 @@ func TestCreateTopic(t *testing.T) { createdTopic.Name: {Name: createdTopic.Name, NumPartitions: createdTopic.NumPartitions}, }, nil }).Times(3) - adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail *kafka.TopicDetail) error { + adminClient.EXPECT().CreateTopic(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, detail *kafka.TopicDetail) error { copy := *detail createdTopic = © return nil @@ -116,7 +117,7 @@ func TestCreateTopic(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) manager := newKafkaTopicManager( "new-topic", changefeedID, @@ -139,10 +140,10 @@ func TestCreateTopic(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) var createdTopic *kafka.TopicDetail - adminClient.EXPECT().CreateTopic(gomock.Any()).DoAndReturn( - func(detail *kafka.TopicDetail) error { + adminClient.EXPECT().CreateTopic(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, detail *kafka.TopicDetail) error { copy := *detail createdTopic = © return errors.ErrKafkaAdminAPI.GenWithStackByArgs("create-topic", detail.Name) @@ -170,8 +171,8 @@ func TestCreateTopicValidatesReplicationFactor(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) - adminClient.EXPECT().GetBrokerConfig(kafka.MinInsyncReplicasConfigName).Return("2", true, nil) + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"new-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), kafka.MinInsyncReplicasConfigName).Return("2", true, nil) manager := newKafkaTopicManager( "new-topic", common.NewChangefeedID4Test("test", "test"), @@ -192,23 +193,35 @@ func TestCreateTopicValidatesReplicationFactor(t *testing.T) { func TestWaitUntilTopicVisibleUnretryableError(t *testing.T) { t.Parallel() - ctrl := gomock.NewController(t) - adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"invalid-topic"}, false).Return( - nil, - errors.WrapError(errors.ErrKafkaAdminAPI, sarama.ErrInvalidTopic, "describe-topic", "invalid-topic"), - ).Times(1) - manager := newKafkaTopicManager( - "invalid-topic", - common.NewChangefeedID4Test("test", "test"), - adminClient, - &kafka.AutoCreateTopicConfig{PartitionNum: 2}, - ) - - err := manager.waitUntilTopicVisible(context.Background(), "invalid-topic") - - require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) - require.ErrorIs(t, err, sarama.ErrInvalidTopic) + for _, test := range []struct { + name string + cause error + }{ + {name: "sarama", cause: sarama.ErrInvalidTopic}, + {name: "franz-go", cause: kerr.InvalidTopicException}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + adminClient := kafka.NewMockAdminClient(ctrl) + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"invalid-topic"}, false).Return( + nil, + errors.WrapError(errors.ErrKafkaAdminAPI, test.cause, "describe-topic", "invalid-topic"), + ).Times(1) + manager := newKafkaTopicManager( + "invalid-topic", + common.NewChangefeedID4Test("test", "test"), + adminClient, + &kafka.AutoCreateTopicConfig{PartitionNum: 2}, + ) + + err := manager.waitUntilTopicVisible(context.Background(), "invalid-topic") + + require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) + require.ErrorIs(t, err, test.cause) + }) + } } func TestGetTopicManagerStartsBackgroundRefreshAfterTopicReady(t *testing.T) { @@ -216,7 +229,7 @@ func TestGetTopicManagerStartsBackgroundRefreshAfterTopicReady(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"existing-topic"}, false).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"existing-topic"}, false).Return( map[string]kafka.TopicDetail{ "existing-topic": {Name: "existing-topic", NumPartitions: 2}, }, nil) @@ -239,7 +252,7 @@ func TestCreateTopicWithTopicDescribeDenied(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"default-topic"}, false).Return( nil, errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs("describe-topic", "default-topic")) manager := newKafkaTopicManager( "default-topic", @@ -266,8 +279,8 @@ func TestCreateTopicWithCreateDenied(t *testing.T) { ctrl := gomock.NewController(t) adminClient := kafka.NewMockAdminClient(ctrl) - adminClient.EXPECT().GetTopicsMeta([]string{"default-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) - adminClient.EXPECT().CreateTopic(&kafka.TopicDetail{ + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{"default-topic"}, false).Return(map[string]kafka.TopicDetail{}, nil) + adminClient.EXPECT().CreateTopic(gomock.Any(), &kafka.TopicDetail{ Name: "default-topic", NumPartitions: 2, ReplicationFactor: 1, diff --git a/go.mod b/go.mod index 44c8c0cf19..8744adfd19 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/imdario/mergo v0.3.16 github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc github.com/jarcoal/httpmock v1.2.0 + github.com/jcmturner/gokrb5/v8 v8.4.4 github.com/json-iterator/go v1.1.12 github.com/klauspost/compress v1.19.0 github.com/linkedin/goavro/v2 v2.14.0 @@ -74,6 +75,11 @@ require ( github.com/tikv/pd v1.1.0-beta.0.20260604125942-9f1c47b1e851 github.com/tikv/pd/client v0.0.0-20260805103528-afa43111d149 github.com/tinylib/msgp v1.5.0 + github.com/twmb/franz-go v1.21.6 + github.com/twmb/franz-go/pkg/kadm v1.18.0 + github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 + github.com/twmb/franz-go/pkg/kmsg v1.13.1 + github.com/twmb/franz-go/pkg/sasl/kerberos v1.1.0 github.com/uber-go/atomic v1.4.0 github.com/xdg/scram v1.0.5 github.com/zeebo/assert v1.3.0 @@ -243,7 +249,6 @@ require ( github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jedib0t/go-pretty/v6 v6.2.2 // indirect github.com/jellydator/ttlcache/v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 513defe13c..54946c2407 100644 --- a/go.sum +++ b/go.sum @@ -549,6 +549,7 @@ github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVET github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= @@ -962,6 +963,18 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 h1:6fotK7 github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75/go.mod h1:KO6IkyS8Y3j8OdNO85qEYBsRPuteD+YciPomcXdrMnk= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/twmb/franz-go v1.7.0/go.mod h1:PMze0jNfNghhih2XHbkmTFykbMF5sJqmNJB31DOOzro= +github.com/twmb/franz-go v1.21.6 h1:+v0dQJVIIuw9uPmPWmPrkoUHs1pPeV8MSwA4eU/Y2kY= +github.com/twmb/franz-go v1.21.6/go.mod h1:wMepkgCatAdV9vCsuwM+wr+C1fl7KV/41+uHGAjt/wc= +github.com/twmb/franz-go/pkg/kadm v1.18.0 h1:WRf/LZmDdcDXwX7WMbtDU++v+b3NzYh2bCGoPMmzirw= +github.com/twmb/franz-go/pkg/kadm v1.18.0/go.mod h1:XeLhGoLXLFzK8/ryv5FfpxPxGwj4oFEGpPJMB/x6KDE= +github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7 h1:OLL/h1pOsMuCnJEp3IW0L2W3jHATeTUMAmE6PxxbhtU= +github.com/twmb/franz-go/pkg/kfake v0.0.0-20260727183601-4176fc0fcaf7/go.mod h1:9j4VxU2ng6tHgD4lIkNJ5OJ3D6vgPhhIp3tBa7dJgLA= +github.com/twmb/franz-go/pkg/kmsg v1.2.0/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY= +github.com/twmb/franz-go/pkg/kmsg v1.13.1 h1:fG5kItwysTk5UXqVwb64EpQEy3TydF3vYYK21nUQ+bI= +github.com/twmb/franz-go/pkg/kmsg v1.13.1/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY= +github.com/twmb/franz-go/pkg/sasl/kerberos v1.1.0 h1:alKdbddkPw3rDh+AwmUEwh6HNYgTvDSFIe/GWYRR9RM= +github.com/twmb/franz-go/pkg/sasl/kerberos v1.1.0/go.mod h1:k8BoBjyUbFj34f0rRbn+Ky12sZFAPbmShrg0karAIMo= github.com/twmb/murmur3 v1.1.6 h1:mqrRot1BRxm+Yct+vavLMou2/iJt0tNVTTC0QoIjaZg= github.com/twmb/murmur3 v1.1.6/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ= github.com/uber-go/atomic v1.4.0 h1:yOuPqEq4ovnhEjpHmfFwsqBXDYbQeT6Nb0bwD6XnD5o= @@ -1111,6 +1124,8 @@ golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= @@ -1166,8 +1181,11 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -1228,6 +1246,7 @@ golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220909162455-aba9fc2a8ff2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/metrics/grafana/ticdc_new_arch.json b/metrics/grafana/ticdc_new_arch.json index aa3463392f..b02af79518 100644 --- a/metrics/grafana/ticdc_new_arch.json +++ b/metrics/grafana/ticdc_new_arch.json @@ -28344,6 +28344,878 @@ ], "title": "Pulsar Sink", "type": "row" + }, + { + "collapsed": true, + "datasource": "${DS_TEST-CLUSTER}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 62100, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Bytes written to brokers per second.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 62101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests currently awaiting a broker response.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 62102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Inflight Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 request latency for each broker.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 62103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "1000 * sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "1000 * histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 62104, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 records per topic-partition batch.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 62105, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Per Batch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute ratio of uncompressed bytes to compressed bytes.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 62106, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compression Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum one-minute average and p99 broker throttle time.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 62107, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "1000 * max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker)) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "1000 * max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, namespace,changefeed, instance, broker))) by (namespace,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 62109, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", namespace=~\"$namespace\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (namespace,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{namespace}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Kafka Sink V2", + "type": "row" } ], "refresh": "10s", diff --git a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json index 538dc0a33e..1f13b87a4e 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_next_gen.json +++ b/metrics/nextgengrafana/ticdc_new_arch_next_gen.json @@ -28344,6 +28344,878 @@ ], "title": "Pulsar Sink", "type": "row" + }, + { + "collapsed": true, + "datasource": "${DS_TEST-CLUSTER}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 62100, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Bytes written to brokers per second.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 62101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests currently awaiting a broker response.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 62102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Inflight Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 request latency for each broker.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 62103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "1000 * sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "1000 * histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 62104, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 records per topic-partition batch.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 62105, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Per Batch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute ratio of uncompressed bytes to compressed bytes.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 62106, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compression Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum one-minute average and p99 broker throttle time.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 62107, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "1000 * max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "1000 * max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 62109, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", sharedpool_id=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Kafka Sink V2", + "type": "row" } ], "refresh": "10s", diff --git a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json index d17672f4b1..2511a575b5 100644 --- a/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json +++ b/metrics/nextgengrafana/ticdc_new_arch_with_keyspace_name.json @@ -11536,6 +11536,878 @@ ], "title": "DDL", "type": "row" + }, + { + "collapsed": true, + "datasource": "${DS_TEST-CLUSTER}", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 62100, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Bytes written to brokers per second.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 62101, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_outgoing_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Outgoing Bytes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests currently awaiting a broker response.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 62102, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(ticdc_sink_kafka_franz_producer_in_flight_requests{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Inflight Requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 request latency for each broker.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "id": 62103, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "1000 * sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "1000 * histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Latency", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Requests per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "id": 62104, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_requests_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute average and p99 records per topic-partition batch.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "hiddenSeries": false, + "id": 62105, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_records_per_batch_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance))", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Records Per Batch", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "One-minute ratio of uncompressed bytes to compressed bytes.", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "hiddenSeries": false, + "id": 62106, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "100 * sum(rate(ticdc_sink_kafka_franz_producer_uncompressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance) / sum(rate(ticdc_sink_kafka_franz_producer_compressed_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Compression Ratio", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Maximum one-minute average and p99 broker throttle time.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "hiddenSeries": false, + "id": 62107, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "1000 * max(sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker) / sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker)) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-avg", + "refId": "A" + }, + { + "exemplar": true, + "expr": "1000 * max(histogram_quantile(0.99, sum(rate(ticdc_sink_kafka_franz_producer_throttle_time_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (le, keyspace_name,changefeed, instance, broker))) by (keyspace_name,changefeed, instance)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-p99", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throttle Time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_TEST-CLUSTER}", + "description": "Responses per second, grouped by broker and result.", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "hiddenSeries": false, + "id": 62109, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "percentage": false, + "pluginVersion": "7.5.17", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum(rate(ticdc_sink_kafka_franz_producer_responses_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", keyspace_name=~\"$keyspace_name\",changefeed=~\"$changefeed\", instance=~\"$ticdc_instance\"}[1m])) by (keyspace_name,changefeed, instance, broker, result)", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{keyspace_name}}-{{changefeed}}-{{instance}}-{{broker}}-{{result}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Response Rate", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 1, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Kafka Sink V2", + "type": "row" } ], "refresh": "10s", diff --git a/pkg/sink/codec/avro/encoder.go b/pkg/sink/codec/avro/encoder.go index fdad6c7dbf..7d518b2c2b 100644 --- a/pkg/sink/codec/avro/encoder.go +++ b/pkg/sink/codec/avro/encoder.go @@ -88,12 +88,13 @@ func (a *BatchEncoder) AppendRowChangedEvent( message.Callback = e.Callback message.IncRowsCount() - if message.Length() > a.config.MaxMessageBytes { + length := a.config.MessageLength(message) + if length > a.config.MaxMessageBytes { log.Warn("Single message is too large for avro", zap.Int("maxMessageBytes", a.config.MaxMessageBytes), - zap.Int("length", message.Length()), + zap.Int("length", length), zap.Any("table", e.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), message.Length(), a.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs(e.TableInfo.GetTargetTableName(), length, a.config.MaxMessageBytes) } a.result = append(a.result, message) diff --git a/pkg/sink/codec/canal/canal_json_encoder.go b/pkg/sink/codec/canal/canal_json_encoder.go index dc076f3ee0..0c914e2f91 100644 --- a/pkg/sink/codec/canal/canal_json_encoder.go +++ b/pkg/sink/codec/canal/canal_json_encoder.go @@ -463,8 +463,8 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( m.IncRowsCount() targetTable := e.TableInfo.GetTargetTableName() - originLength := m.Length() - if m.Length() > c.config.MaxMessageBytes { + originLength := c.config.MessageLength(m) + if originLength > c.config.MaxMessageBytes { // for single message that is longer than max-message-bytes, do not send it. if c.config.LargeMessageHandle.Disabled() { log.Error("Single message is too large for canal-json", @@ -487,7 +487,7 @@ func (c *JSONRowEventEncoder) AppendRowChangedEvent( } m.Value = value - length := m.Length() + length := c.config.MessageLength(m) if length > c.config.MaxMessageBytes { log.Error("Single message is still too large for canal-json only encode handle-key columns", zap.Int("maxMessageBytes", c.config.MaxMessageBytes), @@ -540,7 +540,7 @@ func (c *JSONRowEventEncoder) newClaimCheckLocationMessage( result.Callback = event.Callback result.IncRowsCount() - length := result.Length() + length := c.config.MessageLength(result) if length > c.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json, when create the claim check location message", zap.Int("maxMessageBytes", c.config.MaxMessageBytes), diff --git a/pkg/sink/codec/canal/canal_json_txn_encoder.go b/pkg/sink/codec/canal/canal_json_txn_encoder.go index 61f20ab173..6c789b10c3 100644 --- a/pkg/sink/codec/canal/canal_json_txn_encoder.go +++ b/pkg/sink/codec/canal/canal_json_txn_encoder.go @@ -50,7 +50,7 @@ func (j *JSONTxnEventEncoder) AppendTxnEvent(rowEvents []*commonEvent.RowEvent) if err != nil { return err } - length := len(value) + common.MaxRecordOverhead + length := j.config.MessageLengthForKeyValue(0, len(value)) if length > j.config.MaxMessageBytes { log.Warn("Single message is too large for canal-json", zap.Int("maxMessageBytes", j.config.MaxMessageBytes), diff --git a/pkg/sink/codec/common/config.go b/pkg/sink/codec/common/config.go index 58c34aaab9..10687af7db 100644 --- a/pkg/sink/codec/common/config.go +++ b/pkg/sink/codec/common/config.go @@ -48,6 +48,8 @@ type Config struct { // MaxBatchedBytes controls open-protocol encoder's maximum number of events for a batched message. MaxBatchSize int + useKafkaRecordBatchSize bool + // DeleteOnlyHandleKeyColumns is true, for the delete event only output the handle key columns. DeleteOnlyHandleKeyColumns bool @@ -382,6 +384,30 @@ func (c *Config) WithMaxBatchedBytes(bytes int) *Config { return c } +// WithKafkaRecordBatchSize applies encoder byte limits to the complete +// uncompressed Kafka record batch rather than a client-specific estimate. +func (c *Config) WithKafkaRecordBatchSize() *Config { + c.useKafkaRecordBatchSize = true + return c +} + +// MessageLength returns the size used by encoder byte-limit checks. +func (c *Config) MessageLength(message *Message) int { + if c.useKafkaRecordBatchSize { + return message.KafkaRecordBatchLength() + } + return message.Length() +} + +// MessageLengthForKeyValue returns the size used by encoder byte-limit checks +// before a Message has been constructed. +func (c *Config) MessageLengthForKeyValue(keyLength, valueLength int) int { + if c.useKafkaRecordBatchSize { + return keyLength + valueLength + kafkaRecordBatchOverhead + } + return keyLength + valueLength + MaxRecordOverhead +} + // WithChangefeedID set the `changefeedID` func (c *Config) WithChangefeedID(id common.ChangeFeedID) *Config { c.ChangefeedID = id diff --git a/pkg/sink/codec/common/message.go b/pkg/sink/codec/common/message.go index 5cfd56b010..869942ddd1 100644 --- a/pkg/sink/codec/common/message.go +++ b/pkg/sink/codec/common/message.go @@ -25,6 +25,13 @@ import ( // which will be treated as `version = 2` by sarama producer. const MaxRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 +// kafkaRecordBatchOverhead is the maximum framing added to one headerless +// record in a Kafka record batch. franz-go counts 65 bytes of fixed batch +// framing. The record adds at most 19 bytes: 5 for its length, 1 each for +// attributes, timestamp delta and offset delta, 5 each for key and value +// lengths, and 1 for the zero header count. +const kafkaRecordBatchOverhead = 65 + 19 + // MessageType is the type of message, which is used by MqSink and RedoLog. type MessageType int @@ -94,6 +101,12 @@ func (m *Message) Length() int { return len(m.Key) + len(m.Value) + MaxRecordOverhead } +// KafkaRecordBatchLength returns a conservative uncompressed size for a record +// batch containing only this message and no record headers. +func (m *Message) KafkaRecordBatchLength() int { + return len(m.Key) + len(m.Value) + kafkaRecordBatchOverhead +} + // GetRowsCount returns the number of rows batched in one Message func (m *Message) GetRowsCount() int { return m.rowsCount diff --git a/pkg/sink/codec/common/message_test.go b/pkg/sink/codec/common/message_test.go new file mode 100644 index 0000000000..886c5b76d0 --- /dev/null +++ b/pkg/sink/codec/common/message_test.go @@ -0,0 +1,35 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/config" + "github.com/stretchr/testify/require" +) + +func TestKafkaRecordBatchLength(t *testing.T) { + message := NewMsg([]byte("k"), []byte("v")) + require.Equal(t, len(message.Key)+len(message.Value)+84, message.KafkaRecordBatchLength()) +} + +func TestConfigMessageLength(t *testing.T) { + message := NewMsg([]byte("k"), []byte("v")) + codecConfig := NewConfig(config.ProtocolOpen) + require.Equal(t, message.Length(), codecConfig.MessageLength(message)) + + codecConfig.WithKafkaRecordBatchSize() + require.Equal(t, message.KafkaRecordBatchLength(), codecConfig.MessageLength(message)) +} diff --git a/pkg/sink/codec/open/codec.go b/pkg/sink/codec/open/codec.go index 454cb0d372..a9e3e80970 100644 --- a/pkg/sink/codec/open/codec.go +++ b/pkg/sink/codec/open/codec.go @@ -112,9 +112,9 @@ func encodeRowChangedEvent( return nil, nil, 0, err } - // for single message that is longer than max-message-bytes - // 16 is the length of `keyLenByte` and `valueLenByte`, 8 is the length of `versionHead` - length := len(key) + len(valueCompressed) + common.MaxRecordOverhead + 16 + 8 + // Open Protocol adds an 8-byte version, key length, and value length to + // the encoded row before it becomes one Kafka record. + length := config.MessageLengthForKeyValue(len(key)+16, len(valueCompressed)+8) return key, valueCompressed, length, nil } diff --git a/pkg/sink/codec/open/encoder.go b/pkg/sink/codec/open/encoder.go index b8e608a298..df12ad7664 100644 --- a/pkg/sink/codec/open/encoder.go +++ b/pkg/sink/codec/open/encoder.go @@ -155,8 +155,6 @@ func (d *batchEncoder) Build() (messages []*common.Message) { } func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { - length := len(key) + len(value) + 16 - var ( keyLenByte [8]byte valueLenByte [8]byte @@ -164,7 +162,15 @@ func (d *batchEncoder) pushMessage(key, value []byte, callback func()) { binary.BigEndian.PutUint64(keyLenByte[:], uint64(len(key))) binary.BigEndian.PutUint64(valueLenByte[:], uint64(len(value))) - if len(d.messages) == 0 || d.messages[len(d.messages)-1].Length()+length > d.config.MaxBatchedBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { + var batchedMessageLength int + if len(d.messages) > 0 { + latestMessage := d.messages[len(d.messages)-1] + batchedMessageLength = d.config.MessageLengthForKeyValue( + len(latestMessage.Key)+len(keyLenByte)+len(key), + len(latestMessage.Value)+len(valueLenByte)+len(value), + ) + } + if len(d.messages) == 0 || batchedMessageLength > d.config.MaxBatchedBytes || d.messages[len(d.messages)-1].GetRowsCount() >= d.config.MaxBatchSize { d.finalizeCallback() // create a new message versionHead := make([]byte, 8) diff --git a/pkg/sink/codec/open/encoder_test.go b/pkg/sink/codec/open/encoder_test.go index de88347061..0b1fc9909f 100644 --- a/pkg/sink/codec/open/encoder_test.go +++ b/pkg/sink/codec/open/encoder_test.go @@ -857,6 +857,23 @@ func TestEncoderMultipleMessage(t *testing.T) { common.CompareRow(t, insertEvents[2].Event, insertEvents[2].TableInfo, change, decoded.TableInfo) } +func TestEncoderUsesKafkaRecordBatchSize(t *testing.T) { + codecConfig := common.NewConfig(config.ProtocolOpen). + WithMaxMessageBytes(200). + WithMaxBatchedBytes(120). + WithKafkaRecordBatchSize() + encoder := &batchEncoder{config: codecConfig} + + encoder.pushMessage([]byte("k"), []byte("v"), nil) + require.Len(t, encoder.messages, 1) + require.LessOrEqual(t, encoder.messages[0].KafkaRecordBatchLength(), codecConfig.MaxBatchedBytes) + + encoder.pushMessage([]byte("k"), []byte("v"), nil) + require.Len(t, encoder.messages, 2) + require.Equal(t, 1, encoder.messages[0].GetRowsCount()) + require.Equal(t, 1, encoder.messages[1].GetRowsCount()) +} + func TestMessageTooLarge(t *testing.T) { ctx := context.Background() codecConfig := common.NewConfig(config.ProtocolOpen).WithMaxMessageBytes(100) diff --git a/pkg/sink/codec/simple/encoder.go b/pkg/sink/codec/simple/encoder.go index c8a4208f59..3b18aa44dd 100644 --- a/pkg/sink/codec/simple/encoder.go +++ b/pkg/sink/codec/simple/encoder.go @@ -62,7 +62,8 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co } result.IncRowsCount() - length := result.Length() + length := e.config.MessageLength(result) + originLength := length if length <= e.config.MaxMessageBytes { e.messages = append(e.messages, result) return nil @@ -95,11 +96,12 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co } result.Value = value - if result.Length() <= e.config.MaxMessageBytes { + length = e.config.MessageLength(result) + if length <= e.config.MaxMessageBytes { log.Warn("Single message is too large for simple, only encode handle key columns", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("originLength", length), - zap.Int("length", result.Length()), + zap.Int("originLength", originLength), + zap.Int("length", length), zap.Any("table", event.TableInfo.TableName)) e.messages = append(e.messages, result) return nil @@ -107,9 +109,9 @@ func (e *Encoder) AppendRowChangedEvent(ctx context.Context, _ string, event *co log.Error("Single message is still too large for simple after only encode handle key columns", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("length", result.Length()), + zap.Int("length", length), zap.Any("table", event.TableInfo.TableName)) - return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return errors.ErrMessageTooLarge.GenWithStackByArgs(event.TableInfo.GetTargetTableName(), length, e.config.MaxMessageBytes) } // Build implement the RowEventEncoder interface @@ -148,12 +150,13 @@ func (e *Encoder) EncodeDDLEvent(event *commonEvent.DDLEvent) (*common.Message, } result := common.NewMsg(nil, value) - if result.Length() > e.config.MaxMessageBytes { + length := e.config.MessageLength(result) + if length > e.config.MaxMessageBytes { log.Error("DDL message is too large for simple", zap.Int("maxMessageBytes", e.config.MaxMessageBytes), - zap.Int("length", result.Length()), + zap.Int("length", length), zap.String("table", event.GetTargetTableName())) - return nil, errors.ErrMessageTooLarge.GenWithStackByArgs(event.GetTargetTableName(), result.Length(), e.config.MaxMessageBytes) + return nil, errors.ErrMessageTooLarge.GenWithStackByArgs(event.GetTargetTableName(), length, e.config.MaxMessageBytes) } return result, nil } diff --git a/pkg/sink/kafka/admin.go b/pkg/sink/kafka/admin.go index 8c1a54f2ec..df47e5c70a 100644 --- a/pkg/sink/kafka/admin.go +++ b/pkg/sink/kafka/admin.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "strconv" "strings" @@ -21,6 +22,7 @@ import ( "github.com/pingcap/log" "github.com/pingcap/ticdc/pkg/common" "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kerr" "go.uber.org/zap" ) @@ -47,18 +49,16 @@ type saramaClusterAdmin interface { Close() error } -func (a *saramaAdminClient) GetAllBrokers() []Broker { +func (a *saramaAdminClient) GetAllBrokers(_ context.Context) []Broker { brokers := a.client.Brokers() result := make([]Broker, 0, len(brokers)) for _, broker := range brokers { - result = append(result, Broker{ - ID: broker.ID(), - }) + result = append(result, Broker{ID: broker.ID()}) } return result } -func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, bool, error) { +func (a *saramaAdminClient) GetBrokerConfig(_ context.Context, configName string) (string, bool, error) { _, controller, err := a.admin.DescribeCluster() if err != nil { if IsAuthorizationFailed(err) { @@ -90,7 +90,7 @@ func (a *saramaAdminClient) GetBrokerConfig(configName string) (string, bool, er return "", false, nil } -func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) (string, bool, error) { +func (a *saramaAdminClient) GetTopicConfig(_ context.Context, topicName string, configName string) (string, bool, error) { configEntries, err := a.admin.DescribeConfig(sarama.ConfigResource{ Type: sarama.TopicResource, Name: topicName, @@ -114,7 +114,7 @@ func (a *saramaAdminClient) GetTopicConfig(topicName string, configName string) return "", false, nil } -func (a *saramaAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { +func (a *saramaAdminClient) GetTopicsMeta(_ context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { result := make(map[string]TopicDetail, len(topics)) metaList, err := a.admin.DescribeTopics(topics) @@ -156,10 +156,10 @@ func IsAuthorizationFailed(err error) bool { errors.Is(err, sarama.ErrClusterAuthorizationFailed) } -// IsUnretryableKafkaError reports whether err is not retryable. +// IsUnretryableSaramaError reports whether a Sarama error is not retryable. // See Apache Kafka protocol error definitions: // https://kafka.apache.org/38/generated/protocol_errors.html -func IsUnretryableKafkaError(err error) bool { +func IsUnretryableSaramaError(err error) bool { if IsAuthorizationFailed(err) || errors.Is(err, errors.ErrKafkaInvalidConfig) || errors.Is(err, sarama.ErrInvalidTopic) || @@ -176,7 +176,22 @@ func IsUnretryableKafkaError(err error) bool { return errors.As(err, &configErr) } -func (a *saramaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { +// IsUnretryableKafkaError reports whether a Kafka error is not retryable. +func IsUnretryableKafkaError(err error) bool { + if errors.Is(err, errors.ErrKafkaAuthorizationFailed) || + errors.Is(err, errors.ErrKafkaInvalidConfig) { + return true + } + + var kafkaErr *kerr.Error + if errors.As(err, &kafkaErr) { + return !kafkaErr.Retriable + } + + return IsUnretryableSaramaError(err) +} + +func (a *saramaAdminClient) GetTopicsPartitionsNum(_ context.Context, topics []string) (map[string]int32, error) { result := make(map[string]int32, len(topics)) for _, topic := range topics { partition, err := a.client.Partitions(topic) @@ -192,7 +207,7 @@ func (a *saramaAdminClient) GetTopicsPartitionsNum(topics []string) (map[string] return result, nil } -func (a *saramaAdminClient) CreateTopic(detail *TopicDetail) error { +func (a *saramaAdminClient) CreateTopic(_ context.Context, detail *TopicDetail) error { request := &sarama.TopicDetail{ NumPartitions: detail.NumPartitions, ReplicationFactor: detail.ReplicationFactor, diff --git a/pkg/sink/kafka/admin_client.go b/pkg/sink/kafka/admin_client.go index cea348c2a2..bb7680e73c 100644 --- a/pkg/sink/kafka/admin_client.go +++ b/pkg/sink/kafka/admin_client.go @@ -13,6 +13,8 @@ package kafka +import "context" + // TopicDetail represent a topic's detail information. type TopicDetail struct { Name string @@ -29,23 +31,23 @@ type Broker struct { // which supports managing and inspecting topics, brokers, configurations and ACLs. type AdminClient interface { // GetAllBrokers return all brokers among the cluster - GetAllBrokers() []Broker + GetAllBrokers(ctx context.Context) []Broker // GetBrokerConfig returns the broker-level configuration and whether it exists. - GetBrokerConfig(configName string) (value string, found bool, err error) + GetBrokerConfig(ctx context.Context, configName string) (value string, found bool, err error) // GetTopicConfig returns the topic-level configuration and whether it exists. - GetTopicConfig(topicName string, configName string) (value string, found bool, err error) + GetTopicConfig(ctx context.Context, topicName string, configName string) (value string, found bool, err error) // GetTopicsMeta return all target topics' metadata // if `ignoreTopicError` is true, ignore the topic error and return the metadata of valid topics - GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) + GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) // GetTopicsPartitionsNum return the number of partitions of each topic. - GetTopicsPartitionsNum(topics []string) (map[string]int32, error) + GetTopicsPartitionsNum(ctx context.Context, topics []string) (map[string]int32, error) // CreateTopic creates a new topic. - CreateTopic(detail *TopicDetail) error + CreateTopic(ctx context.Context, detail *TopicDetail) error // Close shuts down the admin client. Close() diff --git a/pkg/sink/kafka/admin_client_mock.go b/pkg/sink/kafka/admin_client_mock.go index 2d2778ea3b..50e80191c9 100644 --- a/pkg/sink/kafka/admin_client_mock.go +++ b/pkg/sink/kafka/admin_client_mock.go @@ -5,6 +5,7 @@ package kafka import ( + context "context" reflect "reflect" gomock "github.com/golang/mock/gomock" @@ -46,37 +47,37 @@ func (mr *MockAdminClientMockRecorder) Close() *gomock.Call { } // CreateTopic mocks base method. -func (m *MockAdminClient) CreateTopic(detail *TopicDetail) error { +func (m *MockAdminClient) CreateTopic(ctx context.Context, detail *TopicDetail) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTopic", detail) + ret := m.ctrl.Call(m, "CreateTopic", ctx, detail) ret0, _ := ret[0].(error) return ret0 } // CreateTopic indicates an expected call of CreateTopic. -func (mr *MockAdminClientMockRecorder) CreateTopic(detail interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) CreateTopic(ctx, detail interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockAdminClient)(nil).CreateTopic), detail) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTopic", reflect.TypeOf((*MockAdminClient)(nil).CreateTopic), ctx, detail) } // GetAllBrokers mocks base method. -func (m *MockAdminClient) GetAllBrokers() []Broker { +func (m *MockAdminClient) GetAllBrokers(ctx context.Context) []Broker { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAllBrokers") + ret := m.ctrl.Call(m, "GetAllBrokers", ctx) ret0, _ := ret[0].([]Broker) return ret0 } // GetAllBrokers indicates an expected call of GetAllBrokers. -func (mr *MockAdminClientMockRecorder) GetAllBrokers() *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetAllBrokers(ctx interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrokers", reflect.TypeOf((*MockAdminClient)(nil).GetAllBrokers)) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAllBrokers", reflect.TypeOf((*MockAdminClient)(nil).GetAllBrokers), ctx) } // GetBrokerConfig mocks base method. -func (m *MockAdminClient) GetBrokerConfig(configName string) (string, bool, error) { +func (m *MockAdminClient) GetBrokerConfig(ctx context.Context, configName string) (string, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBrokerConfig", configName) + ret := m.ctrl.Call(m, "GetBrokerConfig", ctx, configName) ret0, _ := ret[0].(string) ret1, _ := ret[1].(bool) ret2, _ := ret[2].(error) @@ -84,15 +85,15 @@ func (m *MockAdminClient) GetBrokerConfig(configName string) (string, bool, erro } // GetBrokerConfig indicates an expected call of GetBrokerConfig. -func (mr *MockAdminClientMockRecorder) GetBrokerConfig(configName interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetBrokerConfig(ctx, configName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockAdminClient)(nil).GetBrokerConfig), configName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBrokerConfig", reflect.TypeOf((*MockAdminClient)(nil).GetBrokerConfig), ctx, configName) } // GetTopicConfig mocks base method. -func (m *MockAdminClient) GetTopicConfig(topicName, configName string) (string, bool, error) { +func (m *MockAdminClient) GetTopicConfig(ctx context.Context, topicName, configName string) (string, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicConfig", topicName, configName) + ret := m.ctrl.Call(m, "GetTopicConfig", ctx, topicName, configName) ret0, _ := ret[0].(string) ret1, _ := ret[1].(bool) ret2, _ := ret[2].(error) @@ -100,37 +101,37 @@ func (m *MockAdminClient) GetTopicConfig(topicName, configName string) (string, } // GetTopicConfig indicates an expected call of GetTopicConfig. -func (mr *MockAdminClientMockRecorder) GetTopicConfig(topicName, configName interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetTopicConfig(ctx, topicName, configName interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockAdminClient)(nil).GetTopicConfig), topicName, configName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicConfig", reflect.TypeOf((*MockAdminClient)(nil).GetTopicConfig), ctx, topicName, configName) } // GetTopicsMeta mocks base method. -func (m *MockAdminClient) GetTopicsMeta(topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { +func (m *MockAdminClient) GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsMeta", topics, ignoreTopicError) + ret := m.ctrl.Call(m, "GetTopicsMeta", ctx, topics, ignoreTopicError) ret0, _ := ret[0].(map[string]TopicDetail) ret1, _ := ret[1].(error) return ret0, ret1 } // GetTopicsMeta indicates an expected call of GetTopicsMeta. -func (mr *MockAdminClientMockRecorder) GetTopicsMeta(topics, ignoreTopicError interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetTopicsMeta(ctx, topics, ignoreTopicError interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsMeta), topics, ignoreTopicError) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsMeta", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsMeta), ctx, topics, ignoreTopicError) } // GetTopicsPartitionsNum mocks base method. -func (m *MockAdminClient) GetTopicsPartitionsNum(topics []string) (map[string]int32, error) { +func (m *MockAdminClient) GetTopicsPartitionsNum(ctx context.Context, topics []string) (map[string]int32, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", topics) + ret := m.ctrl.Call(m, "GetTopicsPartitionsNum", ctx, topics) ret0, _ := ret[0].(map[string]int32) ret1, _ := ret[1].(error) return ret0, ret1 } // GetTopicsPartitionsNum indicates an expected call of GetTopicsPartitionsNum. -func (mr *MockAdminClientMockRecorder) GetTopicsPartitionsNum(topics interface{}) *gomock.Call { +func (mr *MockAdminClientMockRecorder) GetTopicsPartitionsNum(ctx, topics interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsPartitionsNum), topics) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTopicsPartitionsNum", reflect.TypeOf((*MockAdminClient)(nil).GetTopicsPartitionsNum), ctx, topics) } diff --git a/pkg/sink/kafka/factory.go b/pkg/sink/kafka/factory.go index c19089de4c..4ca1912569 100644 --- a/pkg/sink/kafka/factory.go +++ b/pkg/sink/kafka/factory.go @@ -16,11 +16,22 @@ package kafka import ( "context" - "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/pingcap/ticdc/pkg/common" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" ) -// Factory is used to produce all kafka components. +// NewFactory selects the Kafka client for one changefeed without automatic fallback. +func NewFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { + if o.Client == KafkaClientSarama { + return newSaramaFactory(ctx, o, changefeedID) + } + return newFranzFactory(ctx, o, changefeedID) +} + +// Factory creates Kafka components for one changefeed. type Factory interface { + // Close releases factory-owned resources after all components are closed. + Close() // AdminClient return a kafka cluster admin client AdminClient(ctx context.Context) (AdminClient, error) // SyncProducer creates a sync producer to writer message to kafka @@ -36,35 +47,29 @@ type SyncProducer interface { // SendMessage produces a given message, and returns only when it either has // succeeded or failed to produce. It will return the partition and the offset // of the produced message, or an error if the message failed to produce. - SendMessage(topic string, partitionNum int32, message *common.Message) error + SendMessage(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error // SendMessages produces a given set of messages, and returns only when all // messages in the set have either succeeded or failed. Note that messages // can succeed and fail individually; if some succeed and some fail, // SendMessages will return an error. - SendMessages(topic string, partitionNum int32, message *common.Message) error + SendMessages(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error - // Close shuts down the producer; you must call this function before a producer - // object passes out of scope, as it may otherwise leak memory. - // You must call this before calling Close on the underlying client. + // Close stops the producer. It must be called before Factory.Close. Close() } // AsyncProducer is the kafka async producer type AsyncProducer interface { - // Close shuts down the producer and waits for any buffered messages to be - // flushed. You must call this function before a producer object passes out of - // scope, as it may otherwise leak memory. You must call this before process - // shutting down, or you may lose messages. You must call this before calling - // Close on the underlying client. + // Close stops the producer. It must be called before Factory.Close. Close() // AsyncSend is the input channel for the user to write messages to that they // wish to send. - AsyncSend(ctx context.Context, topic string, partition int32, message *common.Message) error + AsyncSend(ctx context.Context, topic string, partition int32, message *codecCommon.Message) error - // AsyncRunCallback process the messages that has sent to kafka, - // and run tha attached callback. the caller should call this - // method in a background goroutine + // AsyncRunCallback invokes callbacks for successfully delivered messages and + // returns the first terminal delivery error. The caller should run it in a + // background goroutine. AsyncRunCallback(ctx context.Context) error } diff --git a/pkg/sink/kafka/factory_mock.go b/pkg/sink/kafka/factory_mock.go index ecc8fe131c..a1f76e6f5a 100644 --- a/pkg/sink/kafka/factory_mock.go +++ b/pkg/sink/kafka/factory_mock.go @@ -65,6 +65,18 @@ func (mr *MockFactoryMockRecorder) AsyncProducer(ctx interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AsyncProducer", reflect.TypeOf((*MockFactory)(nil).AsyncProducer), ctx) } +// Close mocks base method. +func (m *MockFactory) Close() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Close") +} + +// Close indicates an expected call of Close. +func (mr *MockFactoryMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockFactory)(nil).Close)) +} + // MetricsCollector mocks base method. func (m *MockFactory) MetricsCollector(adminClient AdminClient) MetricsCollector { m.ctrl.T.Helper() @@ -130,31 +142,31 @@ func (mr *MockSyncProducerMockRecorder) Close() *gomock.Call { } // SendMessage mocks base method. -func (m *MockSyncProducer) SendMessage(topic string, partitionNum int32, message *common.Message) error { +func (m *MockSyncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *common.Message) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessage", topic, partitionNum, message) + ret := m.ctrl.Call(m, "SendMessage", ctx, topic, partitionNum, message) ret0, _ := ret[0].(error) return ret0 } // SendMessage indicates an expected call of SendMessage. -func (mr *MockSyncProducerMockRecorder) SendMessage(topic, partitionNum, message interface{}) *gomock.Call { +func (mr *MockSyncProducerMockRecorder) SendMessage(ctx, topic, partitionNum, message interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), topic, partitionNum, message) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessage", reflect.TypeOf((*MockSyncProducer)(nil).SendMessage), ctx, topic, partitionNum, message) } // SendMessages mocks base method. -func (m *MockSyncProducer) SendMessages(topic string, partitionNum int32, message *common.Message) error { +func (m *MockSyncProducer) SendMessages(ctx context.Context, topic string, partitionNum int32, message *common.Message) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMessages", topic, partitionNum, message) + ret := m.ctrl.Call(m, "SendMessages", ctx, topic, partitionNum, message) ret0, _ := ret[0].(error) return ret0 } // SendMessages indicates an expected call of SendMessages. -func (mr *MockSyncProducerMockRecorder) SendMessages(topic, partitionNum, message interface{}) *gomock.Call { +func (mr *MockSyncProducerMockRecorder) SendMessages(ctx, topic, partitionNum, message interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), topic, partitionNum, message) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMessages", reflect.TypeOf((*MockSyncProducer)(nil).SendMessages), ctx, topic, partitionNum, message) } // MockAsyncProducer is a mock of AsyncProducer interface. diff --git a/pkg/sink/kafka/franz_admin.go b/pkg/sink/kafka/franz_admin.go new file mode 100644 index 0000000000..c37b07b85f --- /dev/null +++ b/pkg/sink/kafka/franz_admin.go @@ -0,0 +1,259 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strconv" + "strings" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" +) + +type admin struct { + changefeed common.ChangeFeedID + admin *kadm.Client + closeClient func() +} + +const adminMetadataMinAge = 100 * time.Millisecond + +func newAdmin(ctx context.Context, changefeedID common.ChangeFeedID, clientOpts []kgo.Opt) (*admin, error) { + opts := make([]kgo.Opt, 0, len(clientOpts)+3) + opts = append(opts, clientOpts...) + opts = append(opts, kgo.WithContext(ctx), kgo.WithLogger(newClientLogger(changefeedID, "admin"))) + // It must stay below the visibility retry interval to avoid retrying a cached topic-not-found result. + opts = append(opts, kgo.MetadataMinAge(adminMetadataMinAge)) + + client, err := kgo.NewClient(opts...) + if err != nil { + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + + return &admin{ + changefeed: changefeedID, + admin: kadm.NewClient(client), + closeClient: client.Close, + }, nil +} + +func (a *admin) GetAllBrokers(ctx context.Context) []Broker { + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + return nil + } + + brokers := make([]Broker, 0, len(meta.Brokers)) + for id := range meta.Brokers { + brokers = append(brokers, Broker{ID: int32(id)}) + } + return brokers +} + +func (a *admin) GetBrokerConfig(ctx context.Context, configName string) (string, bool, error) { + meta, err := a.admin.BrokerMetadata(ctx) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-cluster", "cluster") + } + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-cluster", "cluster") + } + + if meta.Controller < 0 { + return "", false, errors.ErrKafkaAdminAPI.GenWithStackByArgs("describe-cluster", "cluster") + } + + configs, err := a.admin.DescribeBrokerConfigs(ctx, meta.Controller) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) + } + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) + } + + controllerName := strconv.Itoa(int(meta.Controller)) + resource, err := configs.On(controllerName, nil) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", configName) + } + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", configName) + } + + if resource.Err != nil { + if isAuthorizationFailed(resource.Err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", configName) + } + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", configName) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + return entry.MaybeValue(), true, nil + } + } + return "", false, nil +} + +func (a *admin) GetTopicConfig(ctx context.Context, topicName string, configName string) (string, bool, error) { + configs, err := a.admin.DescribeTopicConfigs(ctx, topicName) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) + } + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) + } + + resource, err := configs.On(topicName, nil) + if err != nil { + if isAuthorizationFailed(err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-config", topicName) + } + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-config", topicName) + } + + if resource.Err != nil { + if isAuthorizationFailed(resource.Err) { + return "", false, errors.WrapError(errors.ErrKafkaAuthorizationFailed, resource.Err, "describe-config", topicName) + } + return "", false, errors.WrapError(errors.ErrKafkaAdminAPI, resource.Err, "describe-config", topicName) + } + + for _, entry := range resource.Configs { + if entry.Key == configName { + return entry.MaybeValue(), true, nil + } + } + return "", false, nil +} + +func (a *admin) GetTopicsMeta(ctx context.Context, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + if len(topics) == 0 { + return make(map[string]TopicDetail), nil + } + + meta, err := a.admin.Metadata(ctx, topics...) + if err != nil { + resource := strings.Join(topics, ",") + if ignoreTopicError && errors.Is(err, kerr.TopicAuthorizationFailed) { + log.Warn("kafka topic metadata refresh failed", + zap.String("keyspace", a.changefeed.Keyspace()), + zap.String("changefeed", a.changefeed.Name()), + zap.String("topic", resource), + zap.Error(err)) + return make(map[string]TopicDetail), nil + } + if isAuthorizationFailed(err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "describe-topics", resource) + } + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, err, "describe-topics", resource) + } + + return topicDetailsFromMetadata(meta, topics, ignoreTopicError) +} + +func topicDetailsFromMetadata(meta kadm.Metadata, topics []string, ignoreTopicError bool) (map[string]TopicDetail, error) { + result := make(map[string]TopicDetail, len(topics)) + for _, topic := range topics { + detail, ok := meta.Topics[topic] + if !ok { + if ignoreTopicError { + continue + } + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, kerr.UnknownTopicOrPartition, "describe-topic", topic) + } + + if detail.Err == nil { + result[topic] = TopicDetail{ + Name: topic, + NumPartitions: int32(len(detail.Partitions)), + } + continue + } + + if ignoreTopicError { + continue + } + + if isAuthorizationFailed(detail.Err) { + return nil, errors.WrapError(errors.ErrKafkaAuthorizationFailed, detail.Err, "describe-topic", topic) + } + return nil, errors.WrapError(errors.ErrKafkaAdminAPI, detail.Err, "describe-topic", topic) + } + return result, nil +} + +func isAuthorizationFailed(err error) bool { + return errors.Is(err, errors.ErrKafkaAuthorizationFailed) || + errors.Is(err, kerr.TopicAuthorizationFailed) || + errors.Is(err, kerr.ClusterAuthorizationFailed) +} + +func (a *admin) GetTopicsPartitionsNum(ctx context.Context, topics []string) (map[string]int32, error) { + details, err := a.GetTopicsMeta(ctx, topics, false) + if err != nil { + return nil, err + } + + partitions := make(map[string]int32, len(details)) + for topic, detail := range details { + partitions[topic] = detail.NumPartitions + } + return partitions, nil +} + +func (a *admin) CreateTopic(ctx context.Context, detail *TopicDetail) error { + responses, err := a.admin.CreateTopics(ctx, detail.NumPartitions, detail.ReplicationFactor, nil, detail.Name) + if err != nil { + if isAuthorizationFailed(err) { + return errors.WrapError(errors.ErrKafkaAuthorizationFailed, err, "create-topic", detail.Name) + } + return errors.WrapError(errors.ErrKafkaAdminAPI, err, "create-topic", detail.Name) + } + + resp, ok := responses[detail.Name] + if !ok { + return errors.ErrKafkaAdminAPI.GenWithStackByArgs("create-topic", detail.Name) + } + + if resp.Err == nil { + return nil + } + + if errors.Is(resp.Err, kerr.TopicAlreadyExists) { + return nil + } + + if errors.Is(resp.Err, kerr.InvalidReplicationFactor) { + return errors.WrapError(errors.ErrKafkaInvalidConfig, resp.Err) + } + + if isAuthorizationFailed(resp.Err) { + return errors.WrapError(errors.ErrKafkaAuthorizationFailed, resp.Err, "create-topic", detail.Name) + } + return errors.WrapError(errors.ErrKafkaAdminAPI, resp.Err, "create-topic", detail.Name) +} + +func (a *admin) Close() { + if a.closeClient != nil { + a.closeClient() + } +} diff --git a/pkg/sink/kafka/franz_admin_test.go b/pkg/sink/kafka/franz_admin_test.go new file mode 100644 index 0000000000..a9860ad221 --- /dev/null +++ b/pkg/sink/kafka/franz_admin_test.go @@ -0,0 +1,389 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "io" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func TestIsUnretryableClientError(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + unretryable bool + }{ + {name: "unknown topic", err: kerr.UnknownTopicOrPartition}, + {name: "leader unavailable", err: kerr.LeaderNotAvailable}, + {name: "request timeout", err: kerr.RequestTimedOut}, + {name: "network exception", err: kerr.NetworkException}, + {name: "controller changed", err: kerr.NotController}, + {name: "EOF", err: io.EOF}, + {name: "invalid topic", err: kerr.InvalidTopicException, unretryable: true}, + {name: "invalid config", err: kerr.InvalidConfig, unretryable: true}, + {name: "SASL authentication failure", err: kerr.SaslAuthenticationFailed, unretryable: true}, + {name: "unsupported SASL mechanism", err: kerr.UnsupportedSaslMechanism, unretryable: true}, + {name: "illegal SASL state", err: kerr.IllegalSaslState, unretryable: true}, + {name: "unsupported version", err: kerr.UnsupportedVersion, unretryable: true}, + {name: "invalid request", err: kerr.InvalidRequest, unretryable: true}, + { + name: "wrapped invalid topic", + err: errors.WrapError(errors.ErrKafkaAdminAPI, kerr.InvalidTopicException, "describe-topic", "test-topic"), + unretryable: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, test.unretryable, IsUnretryableKafkaError(test.err)) + }) + } +} + +func TestFranzTopicDetailsFromMetadata(t *testing.T) { + t.Parallel() + + const topic = "topic" + testCases := []struct { + name string + metadata kadm.Metadata + ignoreTopicError bool + expected map[string]TopicDetail + expectedError error + expectedCause error + }{ + { + name: "success", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Partitions: kadm.PartitionDetails{0: {}, 1: {}}}, + }}, + expected: map[string]TopicDetail{ + topic: {Name: topic, NumPartitions: 2}, + }, + }, + { + name: "ignore unknown topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, + }}, + ignoreTopicError: true, + expected: map[string]TopicDetail{}, + }, + { + name: "strict unknown topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.UnknownTopicOrPartition}, + }}, + expectedError: errors.ErrKafkaAdminAPI, + expectedCause: kerr.UnknownTopicOrPartition, + }, + { + name: "strict missing topic", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{}}, + expectedError: errors.ErrKafkaAdminAPI, + expectedCause: kerr.UnknownTopicOrPartition, + }, + { + name: "return authorization failure", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.TopicAuthorizationFailed}, + }}, + expectedError: errors.ErrKafkaAuthorizationFailed, + expectedCause: kerr.TopicAuthorizationFailed, + }, + { + name: "ignore authorization failure", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.TopicAuthorizationFailed}, + }}, + ignoreTopicError: true, + expected: map[string]TopicDetail{}, + }, + { + name: "ignore general failure", + metadata: kadm.Metadata{Topics: kadm.TopicDetails{ + topic: {Topic: topic, Err: kerr.InvalidTopicException}, + }}, + ignoreTopicError: true, + expected: map[string]TopicDetail{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + actual, err := topicDetailsFromMetadata(tc.metadata, []string{topic}, tc.ignoreTopicError) + if tc.expectedError != nil { + require.ErrorIs(t, err, tc.expectedError) + if tc.expectedCause != nil { + require.ErrorIs(t, err, tc.expectedCause) + } + return + } + require.NoError(t, err) + require.Equal(t, tc.expected, actual) + }) + } +} + +func TestFranzGetTopicsMetaIgnoresTopicAuthorizationFailure(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1)) + defer cluster.Close() + + cluster.ControlKey(int16(kmsg.Metadata), func(req kmsg.Request) (kmsg.Response, error, bool) { + request := req.(*kmsg.MetadataRequest) + response := request.ResponseKind().(*kmsg.MetadataResponse) + for _, requestTopic := range request.Topics { + responseTopic := kmsg.NewMetadataResponseTopic() + responseTopic.Topic = requestTopic.Topic + responseTopic.ErrorCode = kerr.TopicAuthorizationFailed.Code + response.Topics = append(response.Topics, responseTopic) + } + return response, nil, true + }) + + o := testOptions(cluster.ListenAddrs()) + admin, err := newAdmin( + t.Context(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "ignore-topic-authorization"), + testClientOptions(t, o), + ) + require.NoError(t, err) + defer admin.Close() + + topics, err := admin.GetTopicsMeta(t.Context(), []string{"topic"}, true) + require.NoError(t, err) + require.Empty(t, topics) + + _, err = admin.GetTopicsMeta(t.Context(), []string{"topic"}, false) + require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) + require.ErrorIs(t, err, kerr.TopicAuthorizationFailed) +} + +func TestFranzIsAuthorizationFailed(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + expected bool + }{ + { + name: "TiCDC authorization error", + err: errors.ErrKafkaAuthorizationFailed.GenWithStackByArgs("describe-topic", "test-topic"), + expected: true, + }, + {name: "topic authorization error", err: kerr.TopicAuthorizationFailed, expected: true}, + {name: "cluster authorization error", err: kerr.ClusterAuthorizationFailed, expected: true}, + {name: "general error", err: kerr.InvalidTopicException}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { require.Equal(t, test.expected, isAuthorizationFailed(test.err)) }) + } +} + +func TestAdminHonorsCallContext(t *testing.T) { + o := testOptions([]string{"127.0.0.1:1"}) + admin, err := newAdmin( + t.Context(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "context"), + testClientOptions(t, o), + ) + require.NoError(t, err) + t.Cleanup(admin.Close) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + _, err = admin.GetTopicsMeta(ctx, []string{"topic"}, false) + require.ErrorIs(t, err, context.Canceled) +} + +func TestAdminOperations(t *testing.T) { + const existingTopic = "existing-topic" + ctx := t.Context() + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, existingTopic)) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + + admin, err := newAdmin( + ctx, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test"), + testClientOptions(t, o), + ) + require.NoError(t, err) + defer admin.Close() + + require.Len(t, admin.GetAllBrokers(ctx), 1) + + value, found, err := admin.GetBrokerConfig(ctx, "message.max.bytes") + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "1048588", value) + + _, found, err = admin.GetBrokerConfig(ctx, "missing") + require.NoError(t, err) + require.False(t, found) + + value, found, err = admin.GetTopicConfig(ctx, existingTopic, "max.message.bytes") + require.NoError(t, err) + require.True(t, found) + require.Equal(t, "1048588", value) + + _, found, err = admin.GetTopicConfig(ctx, existingTopic, "missing") + require.NoError(t, err) + require.False(t, found) + + partitions, err := admin.GetTopicsPartitionsNum(ctx, []string{existingTopic}) + require.NoError(t, err) + require.Equal(t, map[string]int32{existingTopic: 3}, partitions) + + const topic = "test-topic" + topics, err := admin.GetTopicsMeta(ctx, []string{topic}, true) + require.NoError(t, err) + require.Empty(t, topics) + + err = admin.CreateTopic(ctx, &TopicDetail{ + Name: topic, + NumPartitions: 3, + ReplicationFactor: 1, + }) + require.NoError(t, err) + + require.Eventually(t, func() bool { + topics, err = admin.GetTopicsMeta(ctx, []string{topic}, false) + return err == nil && topics[topic].NumPartitions == 3 + }, time.Second, 20*time.Millisecond) + + require.NoError(t, admin.CreateTopic(ctx, &TopicDetail{Name: topic, NumPartitions: 3, ReplicationFactor: 1})) +} + +func TestAdminConfigErrors(t *testing.T) { + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, "topic")) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + admin, err := newAdmin( + t.Context(), + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "config-errors"), + testClientOptions(t, o), + ) + require.NoError(t, err) + defer admin.Close() + + for _, test := range []struct { + name string + broker bool + responseError *kerr.Error + expectedError error + }{ + {name: "broker authorization", broker: true, responseError: kerr.ClusterAuthorizationFailed, expectedError: errors.ErrKafkaAuthorizationFailed}, + {name: "broker error", broker: true, responseError: kerr.InvalidRequest, expectedError: errors.ErrKafkaAdminAPI}, + {name: "topic authorization", responseError: kerr.TopicAuthorizationFailed, expectedError: errors.ErrKafkaAuthorizationFailed}, + {name: "topic error", responseError: kerr.InvalidTopicException, expectedError: errors.ErrKafkaAdminAPI}, + } { + t.Run(test.name, func(t *testing.T) { + cluster.ControlKey(int16(kmsg.DescribeConfigs), func(req kmsg.Request) (kmsg.Response, error, bool) { + request := req.(*kmsg.DescribeConfigsRequest) + response := req.ResponseKind().(*kmsg.DescribeConfigsResponse) + for _, requested := range request.Resources { + resource := kmsg.NewDescribeConfigsResponseResource() + resource.ResourceType = requested.ResourceType + resource.ResourceName = requested.ResourceName + resource.ErrorCode = test.responseError.Code + response.Resources = append(response.Resources, resource) + } + return response, nil, true + }) + + if test.broker { + _, _, err = admin.GetBrokerConfig(t.Context(), "message.max.bytes") + } else { + _, _, err = admin.GetTopicConfig(t.Context(), "topic", "max.message.bytes") + } + require.ErrorIs(t, err, test.expectedError) + require.ErrorIs(t, err, test.responseError) + }) + } +} + +func TestCreateTopicErrors(t *testing.T) { + ctx := t.Context() + cluster := kfake.MustCluster(kfake.NumBrokers(1)) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + + admin, err := newAdmin( + ctx, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "create-errors"), + testClientOptions(t, o), + ) + require.NoError(t, err) + defer admin.Close() + + detail := &TopicDetail{Name: "topic", NumPartitions: 1, ReplicationFactor: 1} + + cluster.ControlKey(int16(kmsg.CreateTopics), func(req kmsg.Request) (kmsg.Response, error, bool) { + return req.ResponseKind(), nil, true + }) + require.ErrorIs(t, admin.CreateTopic(ctx, detail), errors.ErrKafkaAdminAPI) + + for _, test := range []struct { + name string + code int16 + expected error + }{ + { + name: "invalid replication factor", + code: kerr.InvalidReplicationFactor.Code, + expected: errors.ErrKafkaInvalidConfig, + }, + { + name: "authorization", + code: kerr.TopicAuthorizationFailed.Code, + expected: errors.ErrKafkaAuthorizationFailed, + }, + { + name: "admin API", + code: kerr.InvalidTopicException.Code, + expected: errors.ErrKafkaAdminAPI, + }, + } { + t.Run(test.name, func(t *testing.T) { + cluster.ControlKey(int16(kmsg.CreateTopics), func(req kmsg.Request) (kmsg.Response, error, bool) { + response := req.ResponseKind().(*kmsg.CreateTopicsResponse) + topic := kmsg.NewCreateTopicsResponseTopic() + topic.Topic, topic.ErrorCode = detail.Name, test.code + response.Topics = append(response.Topics, topic) + return response, nil, true + }) + + require.ErrorIs(t, admin.CreateTopic(ctx, detail), test.expected) + }) + } +} diff --git a/pkg/sink/kafka/franz_async_producer.go b/pkg/sink/kafka/franz_async_producer.go new file mode 100644 index 0000000000..2966c10568 --- /dev/null +++ b/pkg/sink/kafka/franz_async_producer.go @@ -0,0 +1,115 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "sync/atomic" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" +) + +type asyncProducer struct { + client *kgo.Client + changefeedID common.ChangeFeedID + + closed atomic.Bool + resultCh chan asyncProduceResult +} + +type asyncProduceResult struct { + callback func() + logInfo *codeccommon.MessageLogInfo + err error +} + +func (p *asyncProducer) Close() { + if !p.closed.CompareAndSwap(false, true) { + return + } + + start := time.Now() + log.Info("kafka async producer closed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.Duration("duration", time.Since(start))) +} + +func (p *asyncProducer) AsyncSend(ctx context.Context, topic string, partition int32, message *codeccommon.Message) error { + if p.closed.Load() { + return errors.ErrKafkaSinkClosed.GenWithStackByArgs() + } + + record := &kgo.Record{ + Topic: topic, + Partition: partition, + Key: message.Key, + Value: message.Value, + } + + callback := message.Callback + logInfo := message.LogInfo + promise := func(_ *kgo.Record, err error) { + result := asyncProduceResult{ + callback: callback, + logInfo: logInfo, + err: err, + } + select { + case p.resultCh <- result: + case <-ctx.Done(): + case <-p.client.Context().Done(): + } + } + + // Produce can buffer a record without checking ctx, so reject prior cancellation. + // If it waits for buffer space, canceling ctx makes it return; propagate the cause below. + select { + case <-ctx.Done(): + return context.Cause(ctx) + default: + } + p.client.Produce(ctx, record, promise) + return context.Cause(ctx) +} + +func (p *asyncProducer) AsyncRunCallback(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case <-p.client.Context().Done(): + return context.Cause(p.client.Context()) + case result := <-p.resultCh: + if result.err != nil { + log.Error("kafka message send failed", + zap.String("keyspace", p.changefeedID.Keyspace()), + zap.String("changefeed", p.changefeedID.Name()), + zap.String("eventContext", BuildEventLogContext( + p.changefeedID.Keyspace(), p.changefeedID.Name(), result.logInfo)), + zap.Error(result.err)) + return errors.WrapError(errors.ErrKafkaSendMessage, result.err) + } + if result.callback != nil { + result.callback() + } + } + } +} diff --git a/pkg/sink/kafka/franz_async_producer_test.go b/pkg/sink/kafka/franz_async_producer_test.go new file mode 100644 index 0000000000..067f0241fd --- /dev/null +++ b/pkg/sink/kafka/franz_async_producer_test.go @@ -0,0 +1,284 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func TestAsyncSendClosed(t *testing.T) { + producer := &asyncProducer{} + producer.closed.Store(true) + + err := producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{}) + + require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) +} + +func TestAsyncSendCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + producer := &asyncProducer{} + require.ErrorIs(t, producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{}), context.Canceled) +} + +func TestAsyncPartition(t *testing.T) { + const topic = "async-partition" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) + defer cluster.Close() + partition := make(chan int32, 1) + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + partition <- req.(*kmsg.ProduceRequest).Topics[0].Partitions[0].Partition + return produceResponseWithError(req, -1, 0) + }) + o := testOptions(cluster.ListenAddrs()) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-partition"), + resultCh: make(chan asyncProduceResult, 1), + } + + require.NoError(t, producer.AsyncSend(t.Context(), topic, 2, &codeccommon.Message{Value: []byte("value")})) + require.Eventually(t, func() bool { return client.BufferedProduceRecords() == 0 }, time.Second, time.Millisecond) + require.Equal(t, int32(2), <-partition) +} + +func TestAsyncCallbackStopsWithClient(t *testing.T) { + client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) + require.NoError(t, err) + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback-close"), + resultCh: make(chan asyncProduceResult, 1), + } + + done := make(chan error, 1) + go func() { done <- producer.AsyncRunCallback(context.Background()) }() + client.Close() + + select { + case err = <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("callback runner did not stop with the client") + } +} + +func TestFactoryCloseUnblocksPromise(t *testing.T) { + client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1")) + require.NoError(t, err) + factory := &franzFactory{client: client} + defer factory.Close() + + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-close-full-result-channel"), + resultCh: make(chan asyncProduceResult, 1), + } + producer.resultCh <- asyncProduceResult{} + require.NoError(t, producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{})) + require.Equal(t, int64(1), client.BufferedProduceRecords()) + + factory.Close() + + require.Eventually(t, func() bool { + return client.BufferedProduceRecords() == 0 + }, time.Second, time.Millisecond) +} + +func TestAsyncCallbackOnce(t *testing.T) { + const topic = "async-topic" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-success"), + resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), + } + defer producer.Close() + + var calls atomic.Int32 + called := make(chan struct{}, 10) + message := &codeccommon.Message{ + Value: []byte("value"), + Callback: func() { + calls.Add(1) + called <- struct{}{} + }, + } + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, message)) + require.Eventually(t, func() bool { return producer.client.BufferedProduceRecords() == 0 }, time.Second, time.Millisecond) + require.Zero(t, calls.Load()) + + callbackCtx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- producer.AsyncRunCallback(callbackCtx) }() + + select { + case <-called: + case <-time.After(time.Second): + t.Fatal("produce callback was not called") + } + + time.Sleep(20 * time.Millisecond) + require.Equal(t, int32(1), calls.Load()) + cancel() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestAsyncCallbackIsolation(t *testing.T) { + const topic = "async-callback-isolation" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-callback-isolation"), + resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), + } + defer producer.Close() + + callbackStarted := make(chan struct{}) + releaseCallback := make(chan struct{}) + secondCallback := make(chan struct{}) + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, &codeccommon.Message{ + Value: []byte("first"), + Callback: func() { + close(callbackStarted) + <-releaseCallback + }, + })) + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, &codeccommon.Message{ + Value: []byte("second"), + Callback: func() { close(secondCallback) }, + })) + + callbackCtx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- producer.AsyncRunCallback(callbackCtx) }() + select { + case <-callbackStarted: + case <-time.After(time.Second): + t.Fatal("produce callback was not called") + } + require.Eventually(t, func() bool { return producer.client.BufferedProduceRecords() == 0 }, time.Second, time.Millisecond) + select { + case <-secondCallback: + t.Fatal("second callback ran before the first callback completed") + case <-time.After(20 * time.Millisecond): + } + + close(releaseCallback) + select { + case <-secondCallback: + case <-time.After(time.Second): + t.Fatal("second callback was not called") + } + cancel() + require.ErrorIs(t, <-done, context.Canceled) +} + +func TestAsyncProduceFailure(t *testing.T) { + const topic = "async-error" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + return produceResponseWithError(req, 0, kerr.InvalidTopicException.Code) + }) + o := testOptions(cluster.ListenAddrs()) + + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "async-error"), + resultCh: make(chan asyncProduceResult, producerMaxBufferedRecords), + } + defer producer.Close() + + var callbackCalled atomic.Bool + message := &codeccommon.Message{ + Value: []byte("value"), + Callback: func() { callbackCalled.Store(true) }, + } + require.NoError(t, producer.AsyncSend(context.Background(), topic, 0, message)) + + callbackCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + err = producer.AsyncRunCallback(callbackCtx) + requireKafkaSendError(t, err, kerr.InvalidTopicException) + require.False(t, callbackCalled.Load()) +} + +func TestBufferBackpressure(t *testing.T) { + client, err := kgo.NewClient(kgo.SeedBrokers("127.0.0.1:1"), kgo.MaxBufferedBytes(10), kgo.RecordRetries(100)) + require.NoError(t, err) + + producer := &asyncProducer{ + client: client, + changefeedID: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "backpressure"), + resultCh: make(chan asyncProduceResult, 2), + } + t.Cleanup(client.Close) + t.Cleanup(producer.Close) + + require.NoError(t, producer.AsyncSend(context.Background(), "topic", 0, &codeccommon.Message{Value: make([]byte, 10)})) + require.Equal(t, int64(1), client.BufferedProduceRecords()) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- producer.AsyncSend(ctx, "topic", 0, &codeccommon.Message{Value: make([]byte, 10)}) }() + + select { + case <-done: + t.Fatal("second send did not wait for buffer space") + case <-time.After(50 * time.Millisecond): + } + + cancel() + + select { + case err = <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("canceled send remained blocked") + } +} diff --git a/pkg/sink/kafka/franz_config.go b/pkg/sink/kafka/franz_config.go new file mode 100644 index 0000000000..14c02faed3 --- /dev/null +++ b/pkg/sink/kafka/franz_config.go @@ -0,0 +1,190 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "crypto/tls" + "strings" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/sasl" + "github.com/twmb/franz-go/pkg/sasl/oauth" + "github.com/twmb/franz-go/pkg/sasl/plain" + "github.com/twmb/franz-go/pkg/sasl/scram" + "go.uber.org/zap" +) + +// franz-go counts a record from Produce acceptance until its delivery callback +// returns, including metadata lookup, batching, sending, Broker response, and +// retries. A record larger than the byte limit fails immediately; otherwise, +// either limit blocks later Produce calls. Their ratio is 2 KiB per record, so +// bytes govern larger records while count bounds smaller record objects. +const ( + producerMaxBufferedBytes = 128 << 20 + producerMaxBufferedRecords = 1 << 16 +) + +// producerMaxRequestBytes matches franz-go's default BrokerMaxWriteBytes and Kafka's default socket.request.max.bytes. +const producerMaxRequestBytes = 100 << 20 + +// The shared client uses these options for all Kafka requests. Producer options +// below apply only to Produce requests and buffered records. +func clientOptions(ctx context.Context, o *options) ([]kgo.Opt, error) { + opts := []kgo.Opt{ + kgo.SeedBrokers(o.BrokerEndpoints...), + kgo.ClientID(o.ClientID), + kgo.DialTimeout(o.DialTimeout), + // franz-go does not expose an independent socket read timeout. This value + // sets the socket write deadline and is added to each request-specific + // Broker processing timeout to form the socket read deadline. + kgo.RequestTimeoutOverhead(o.WriteTimeout), + } + + if o.EnableTLS { + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + } + if o.Credential != nil && o.Credential.IsTLSEnabled() { + var err error + tlsConfig, err = o.Credential.ToTLSConfig() + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + } + tlsConfig.InsecureSkipVerify = o.InsecureSkipVerify + opts = append(opts, kgo.DialTLSConfig(tlsConfig)) + } + + if o.sasl != nil && o.sasl.mechanism != "" { + mechanism, err := buildSASLMechanism(ctx, o.sasl) + if err != nil { + return nil, err + } + opts = append(opts, kgo.SASL(mechanism)) + } + return opts, nil +} + +func producerOptions(o *options) []kgo.Opt { + return []kgo.Opt{ + kgo.RecordPartitioner(kgo.ManualPartitioner()), + kgo.RequiredAcks(requiredAcks(o.RequiredAcks)), + // Retried requests may create duplicates because broker-side producer ID deduplication is disabled. + kgo.DisableIdempotentWrite(), + // More than one in-flight request can reorder records when an earlier request is retried. + kgo.MaxProduceRequestsInflightPerBroker(1), + // The default of five retries allows six Produce attempts. franz-go's + // default jittered backoff adds about 6.2s to 9.3s across five retries. + kgo.RecordRetries(o.MaxRetry), + kgo.UnknownTopicRetries(o.MaxRetry), + // Limit each client to 128 MiB of buffered payload. The in-flight limit + // applies per broker and does not bound records queued for other brokers, + // metadata, or retries, so the producer needs a separate byte limit. + // 128 MiB exceeds the 100 MiB record batch limit, so a valid single + // record is not rejected by the buffer limit. + kgo.MaxBufferedBytes(producerMaxBufferedBytes), + kgo.MaxBufferedRecords(producerMaxBufferedRecords), + // A record batch must fit in the 100 MiB Produce request limit. + kgo.ProducerBatchMaxBytes(int32(min(o.MaxMessageBytes, producerMaxRequestBytes))), + // This value limits how long the Broker may process a Produce request; + // read-timeout is therefore not an exact socket read deadline. Together + // with RequestTimeoutOverhead above, the socket write deadline is + // write-timeout, the Broker processing timeout is read-timeout, and the + // socket read deadline is read-timeout plus write-timeout. + // With the default 10s timeout, the Broker may process a Produce request + // for 10s, the socket write deadline is 10s, and the socket read deadline + // is 20s. Across six attempts, consecutive timeouts take about 66s-69s + // when the Broker returns on its processing deadline, 126s-129s when it + // never replies, or 186s-189s if every write and read reaches its deadline. + // Buffering, metadata lookup, connection setup, and Broker throttling are + // not included; the caller context is the end-to-end bound. + // A Broker processing timeout returns REQUEST_TIMED_OUT, which franz-go + // retries. The original record may already be stored, so retries may create + // duplicates while idempotent writes are disabled. Exhausting the retry + // budget fails the record and reports the error through its callback. + kgo.ProduceRequestTimeout(o.ReadTimeout), + kgo.ProducerLinger(0), + compressionOption(o.Compression), + } +} + +func buildSASLMechanism(ctx context.Context, cfg *saslConfig) (sasl.Mechanism, error) { + switch cfg.mechanism { + case plainMechanism: + return plain.Auth{User: cfg.user, Pass: cfg.password}.AsMechanism(), nil + case scram256Mechanism: + return scram.Auth{User: cfg.user, Pass: cfg.password}.AsSha256Mechanism(), nil + case scram512Mechanism: + return scram.Auth{User: cfg.user, Pass: cfg.password}.AsSha512Mechanism(), nil + case oauthMechanism: + return buildOAuthMechanism(ctx, cfg.oauth2) + case gssapiMechanism: + return buildGSSAPIMechanism(cfg.gssapi) + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl mechanism %s", cfg.mechanism) + } +} + +func buildOAuthMechanism(ctx context.Context, cfg oauth2Config) (sasl.Mechanism, error) { + tokenSource, err := newOAuthTokenSource(ctx, cfg) + if err != nil { + return nil, err + } + // One token source shares cached credentials across broker connections and refreshes them on expiry. + return oauth.Oauth(func(context.Context) (oauth.Auth, error) { + token, err := tokenSource.Token() + if err != nil { + return oauth.Auth{}, errors.WrapError(errors.ErrNewKafkaSink, err) + } + return oauth.Auth{Token: token.AccessToken}, nil + }), nil +} + +func requiredAcks(required RequiredAcks) kgo.Acks { + switch required { + case WaitForAll: + return kgo.AllISRAcks() + case WaitForLocal: + return kgo.LeaderAck() + case NoResponse: + return kgo.NoAck() + default: + log.Warn("unsupported required acks", zap.Int16("requiredAcks", int16(required))) + return kgo.AllISRAcks() + } +} + +func compressionOption(compression string) kgo.Opt { + var codec kgo.CompressionCodec + switch strings.ToLower(strings.TrimSpace(compression)) { + case "", "none": + codec = kgo.NoCompression() + case "gzip": + codec = kgo.GzipCompression() + case "snappy": + codec = kgo.SnappyCompression() + case "lz4": + codec = kgo.Lz4Compression() + case "zstd": + codec = kgo.ZstdCompression() + default: + log.Warn("unsupported kafka compression algorithm", zap.String("compression", compression)) + codec = kgo.NoCompression() + } + return kgo.ProducerBatchCompression(codec) +} diff --git a/pkg/sink/kafka/franz_config_test.go b/pkg/sink/kafka/franz_config_test.go new file mode 100644 index 0000000000..4d451fdefa --- /dev/null +++ b/pkg/sink/kafka/franz_config_test.go @@ -0,0 +1,323 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "crypto/tls" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/jcmturner/gokrb5/v8/iana/etypeID" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/sasl" + "golang.org/x/oauth2" +) + +func testOptions(brokers []string) *options { + o := NewOptions() + o.BrokerEndpoints = brokers + o.MaxMessageBytes = 1 << 20 + o.MaxRetry = 1 + o.DialTimeout = time.Second + o.ReadTimeout = time.Second + o.WriteTimeout = time.Second + return o +} + +func testClientOptions(t *testing.T, o *options) []kgo.Opt { + t.Helper() + opts, err := clientOptions(t.Context(), o) + require.NoError(t, err) + return opts +} + +func TestFranzRequiredAcks(t *testing.T) { + for _, test := range []struct { + required RequiredAcks + expected kgo.Acks + }{ + {required: WaitForAll, expected: kgo.AllISRAcks()}, + {required: WaitForLocal, expected: kgo.LeaderAck()}, + {required: NoResponse, expected: kgo.NoAck()}, + {required: RequiredAcks(2), expected: kgo.AllISRAcks()}, + } { + require.Equal(t, test.expected, requiredAcks(test.required)) + } +} + +func TestClientOptions(t *testing.T) { + t.Run("timeouts", func(t *testing.T) { + o := testOptions([]string{"127.0.0.1:9092"}) + o.ReadTimeout = 3 * time.Second + o.WriteTimeout = 2 * time.Second + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, 2*time.Second, client.OptValue(kgo.RequestTimeoutOverhead)) + require.Equal(t, 3*time.Second, client.OptValue(kgo.ProduceRequestTimeout)) + }) + + t.Run("TLS", func(t *testing.T) { + ca, err := security.NewCA() + require.NoError(t, err) + certPEM, keyPEM, err := ca.GenerateCerts("localhost") + require.NoError(t, err) + certificate, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.TLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{certificate}, + })) + defer cluster.Close() + + dir := t.TempDir() + caPath := filepath.Join(dir, "ca.pem") + certPath := filepath.Join(dir, "cert.pem") + keyPath := filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(caPath, ca.CAPEM, 0o600)) + require.NoError(t, os.WriteFile(certPath, certPEM, 0o600)) + require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600)) + o := testOptions(cluster.ListenAddrs()) + o.EnableTLS = true + o.Credential = &security.Credential{CAPath: caPath, CertPath: certPath, KeyPath: keyPath} + client, err := kgo.NewClient(testClientOptions(t, o)...) + require.NoError(t, err) + defer client.Close() + + metadata, err := kadm.NewClient(client).Metadata(t.Context()) + require.NoError(t, err) + require.Len(t, metadata.Brokers, 1) + }) + + t.Run("SASL", func(t *testing.T) { + cluster := kfake.MustCluster( + kfake.NumBrokers(1), + kfake.EnableSASL(), + kfake.Superuser("PLAIN", "alice", "secret"), + ) + defer cluster.Close() + o := testOptions(cluster.ListenAddrs()) + o.sasl = &saslConfig{mechanism: plainMechanism, user: "alice", password: "secret"} + client, err := kgo.NewClient(testClientOptions(t, o)...) + require.NoError(t, err) + defer client.Close() + + metadata, err := kadm.NewClient(client).Metadata(t.Context()) + require.NoError(t, err) + require.Len(t, metadata.Brokers, 1) + }) +} + +func TestProducerLimits(t *testing.T) { + for _, test := range []struct { + name string + maxMessageBytes int + expectedBatch int32 + }{ + {name: "configured", maxMessageBytes: 1048588, expectedBatch: 1048588}, + {name: "at request limit", maxMessageBytes: producerMaxRequestBytes, expectedBatch: producerMaxRequestBytes}, + {name: "above request limit", maxMessageBytes: 128 << 20, expectedBatch: producerMaxRequestBytes}, + } { + t.Run(test.name, func(t *testing.T) { + o := testOptions([]string{"127.0.0.1:9092"}) + o.MaxMessageBytes = test.maxMessageBytes + + client, err := kgo.NewClient(producerOptions(o)...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, test.expectedBatch, client.OptValue(kgo.ProducerBatchMaxBytes)) + require.Equal(t, int64(producerMaxBufferedBytes), client.OptValue(kgo.MaxBufferedBytes)) + require.Equal(t, int64(producerMaxBufferedRecords), client.OptValue(kgo.MaxBufferedRecords)) + require.Equal(t, int64(o.MaxRetry), client.OptValue(kgo.RecordRetries)) + require.Equal(t, int64(o.MaxRetry), client.OptValue(kgo.UnknownTopicRetries)) + require.Equal(t, int32(producerMaxRequestBytes), client.OptValue(kgo.BrokerMaxWriteBytes)) + require.Equal(t, time.Duration(0), client.OptValue(kgo.ProducerLinger)) + }) + } +} + +func TestCompressionOptions(t *testing.T) { + for _, test := range []struct { + name string + compression string + expected kgo.CompressionCodec + }{ + {name: "none", compression: "none", expected: kgo.NoCompression()}, + {name: "gzip", compression: "gzip", expected: kgo.GzipCompression()}, + {name: "snappy", compression: "snappy", expected: kgo.SnappyCompression()}, + {name: "lz4", compression: "lz4", expected: kgo.Lz4Compression()}, + {name: "zstd", compression: "zstd", expected: kgo.ZstdCompression()}, + {name: "unknown falls back to none", compression: "unknown", expected: kgo.NoCompression()}, + } { + t.Run(test.name, func(t *testing.T) { + cfg := testOptions([]string{"127.0.0.1:9092"}) + cfg.Compression = test.compression + + producerOpts := producerOptions(cfg) + + client, err := kgo.NewClient(producerOpts...) + require.NoError(t, err) + defer client.Close() + + require.Equal(t, []kgo.CompressionCodec{test.expected}, client.OptValue(kgo.ProducerBatchCompression)) + }) + } +} + +func TestGSSAPIMechanism(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "krb5.conf") + require.NoError(t, os.WriteFile(configPath, []byte(`[libdefaults] + default_realm = EXAMPLE.COM +[realms] + EXAMPLE.COM = { + kdc = localhost:88 + } +`), 0o600)) + keytabPath := filepath.Join(dir, "client.keytab") + kt := keytab.New() + require.NoError(t, kt.AddEntry("alice", "EXAMPLE.COM", "pwd", time.Now(), 1, etypeID.AES256_CTS_HMAC_SHA1_96)) + keytabBytes, err := kt.Marshal() + require.NoError(t, err) + require.NoError(t, os.WriteFile(keytabPath, keytabBytes, 0o600)) + + for _, cfg := range []gssapiConfig{ + {authType: userAuth, password: "pwd"}, + {authType: keyTabAuth, keyTabPath: keytabPath}, + } { + cfg.kerberosConfigPath = configPath + cfg.serviceName = "kafka" + cfg.username = "alice" + cfg.realm = "EXAMPLE.COM" + + mechanism, err := buildSASLMechanism(t.Context(), &saslConfig{ + mechanism: gssapiMechanism, + gssapi: cfg, + }) + require.NoError(t, err) + require.Equal(t, "GSSAPI", mechanism.Name()) + closing, ok := mechanism.(sasl.ClosingMechanism) + require.True(t, ok) + closing.Close() + + next, err := buildGSSAPIMechanism(cfg) + require.NoError(t, err) + require.NotSame(t, mechanism, next) + nextClosing, ok := next.(sasl.ClosingMechanism) + require.True(t, ok) + nextClosing.Close() + } +} + +func TestBuildFranzSASLMechanisms(t *testing.T) { + for _, mechanism := range []saslMechanism{plainMechanism, scram256Mechanism, scram512Mechanism} { + actual, err := buildSASLMechanism(t.Context(), &saslConfig{ + mechanism: mechanism, + user: "alice", + password: "secret", + }) + require.NoError(t, err) + require.Equal(t, string(mechanism), actual.Name()) + } + + _, err := buildSASLMechanism(t.Context(), &saslConfig{mechanism: "unknown"}) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) +} + +func TestFranzOAuth(t *testing.T) { + t.Run("token reuse", func(t *testing.T) { + var tokenRequests atomic.Int32 + request := make(chan url.Values, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + t.Errorf("parse token request: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + tokenRequests.Add(1) + select { + case request <- r.PostForm: + default: + } + w.Header().Set("Content-Type", "application/json") + if _, err := io.WriteString(w, `{"access_token":"token","token_type":"bearer"}`); err != nil { + t.Errorf("write token response: %v", err) + } + })) + defer server.Close() + + mechanism, err := buildSASLMechanism(t.Context(), &saslConfig{ + mechanism: oauthMechanism, + oauth2: oauth2Config{ + clientID: "client", + clientSecret: "secret", + tokenURL: server.URL, + scopes: []string{"scope-a", "scope-b"}, + grantType: "custom", + audience: "audience", + }, + }) + require.NoError(t, err) + _, _, err = mechanism.Authenticate(context.Background(), "") + require.NoError(t, err) + _, _, err = mechanism.Authenticate(context.Background(), "") + require.NoError(t, err) + + form := <-request + require.Equal(t, int32(1), tokenRequests.Load()) + require.Equal(t, "custom", form.Get("grant_type")) + require.Equal(t, "audience", form.Get("audience")) + require.Equal(t, "scope-a scope-b", form.Get("scope")) + }) + + t.Run("endpoint error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + if _, err := io.WriteString(w, `{"error":"invalid_client"}`); err != nil { + t.Errorf("write token error response: %v", err) + } + })) + defer server.Close() + mechanism, err := buildOAuthMechanism(t.Context(), oauth2Config{tokenURL: server.URL}) + require.NoError(t, err) + + _, _, err = mechanism.Authenticate(context.Background(), "") + require.ErrorIs(t, err, errors.ErrNewKafkaSink) + var retrieveErr *oauth2.RetrieveError + require.ErrorAs(t, err, &retrieveErr) + }) + + t.Run("invalid URL", func(t *testing.T) { + _, err := buildOAuthMechanism(t.Context(), oauth2Config{tokenURL: "http://example.com/%%"}) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + }) +} diff --git a/pkg/sink/kafka/franz_factory.go b/pkg/sink/kafka/franz_factory.go new file mode 100644 index 0000000000..bdd0dea2b6 --- /dev/null +++ b/pkg/sink/kafka/franz_factory.go @@ -0,0 +1,134 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strings" + "sync" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/kadm" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" +) + +type franzFactory struct { + changefeedID common.ChangeFeedID + client *kgo.Client + closeOnce sync.Once +} + +func newFranzFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { + adminOpts, err := clientOptions(ctx, o) + if err != nil { + return nil, err + } + admin, err := newAdmin(ctx, changefeedID, adminOpts) + if err != nil { + return nil, err + } + err = adjustOptions(ctx, changefeedID, admin, o, o.Topic) + admin.Close() + if err != nil { + return nil, err + } + // The temporary admin and shared client must not own the same SASL authentication state. + clientOpts, err := clientOptions(ctx, o) + if err != nil { + return nil, err + } + producerOpts := producerOptions(o) + metricsHook := newMetricsHook(changefeedID) + opts := make([]kgo.Opt, 0, len(clientOpts)+len(producerOpts)+4) + opts = append(opts, clientOpts...) + opts = append(opts, + kgo.WithContext(ctx), + kgo.WithLogger(newClientLogger(changefeedID, "shared")), + kgo.WithHooks(metricsHook), + kgo.MetadataMinAge(adminMetadataMinAge)) + opts = append(opts, producerOpts...) + + client, err := kgo.NewClient(opts...) + if err != nil { + cleanupMetrics(changefeedID) + return nil, errors.WrapError(errors.ErrNewKafkaSink, err) + } + + compression := strings.ToLower(strings.TrimSpace(o.Compression)) + if compression == "" { + compression = "none" + } + + log.Info("kafka sink configuration resolved", + zap.String("namespace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.String("client", KafkaClientFranz), + zap.String("topic", o.Topic), + zap.Int32("partitionNum", o.PartitionNum), + zap.Int("maxMessageBytes", o.MaxMessageBytes), + zap.Int("maxBatchedBytes", o.MaxBatchedBytes), + zap.String("compression", compression), + zap.Int16("requiredAcks", int16(o.RequiredAcks)), + zap.Int("maxRetry", o.MaxRetry), + zap.Duration("dialTimeout", o.DialTimeout), + zap.Duration("readTimeout", o.ReadTimeout), + zap.Duration("writeTimeout", o.WriteTimeout)) + return &franzFactory{ + changefeedID: changefeedID, + client: client, + }, nil +} + +func (f *franzFactory) AdminClient(context.Context) (AdminClient, error) { + return &admin{ + changefeed: f.changefeedID, + admin: kadm.NewClient(f.client), + }, nil +} + +func (f *franzFactory) SyncProducer(context.Context) (SyncProducer, error) { + return &syncProducer{ + id: f.changefeedID, + client: f.client, + }, nil +} + +func (f *franzFactory) AsyncProducer(context.Context) (AsyncProducer, error) { + return &asyncProducer{ + client: f.client, + changefeedID: f.changefeedID, + resultCh: make(chan asyncProduceResult, 1), + }, nil +} + +func (f *franzFactory) Close() { + f.closeOnce.Do(func() { + if f.client != nil { + f.client.Close() + } + cleanupMetrics(f.changefeedID) + }) +} + +func (f *franzFactory) MetricsCollector(AdminClient) MetricsCollector { + return noopMetricsCollector{} +} + +type noopMetricsCollector struct{} + +func (noopMetricsCollector) Run(ctx context.Context) { <-ctx.Done() } diff --git a/pkg/sink/kafka/franz_factory_test.go b/pkg/sink/kafka/franz_factory_test.go new file mode 100644 index 0000000000..64c6364a60 --- /dev/null +++ b/pkg/sink/kafka/franz_factory_test.go @@ -0,0 +1,75 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" +) + +func TestSharedClientLifecycle(t *testing.T) { + const topic = "shared-client" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + o := testOptions(cluster.ListenAddrs()) + o.Topic = topic + created, err := newFranzFactory( + t.Context(), + o, + common.NewChangefeedID4Test(common.DefaultKeyspaceName, "shared-client"), + ) + require.NoError(t, err) + factory := created.(*franzFactory) + defer factory.Close() + + adminClient, err := factory.AdminClient(t.Context()) + require.NoError(t, err) + + asyncClient, err := factory.AsyncProducer(t.Context()) + require.NoError(t, err) + asyncProducer := asyncClient.(*asyncProducer) + + syncClient, err := factory.SyncProducer(t.Context()) + require.NoError(t, err) + syncProducer := syncClient.(*syncProducer) + + require.Same(t, factory.client, asyncProducer.client) + require.Same(t, factory.client, syncProducer.client) + require.NoError(t, syncProducer.SendMessage(t.Context(), topic, 0, &codeccommon.Message{Value: []byte("sync")})) + + syncProducer.Close() + require.NoError(t, asyncProducer.AsyncSend(t.Context(), topic, 0, &codeccommon.Message{Value: []byte("value")})) + require.Eventually(t, func() bool { + return asyncProducer.client.BufferedProduceRecords() == 0 + }, time.Second, time.Millisecond) + + asyncProducer.Close() + topics, err := adminClient.GetTopicsMeta(context.Background(), []string{topic}, false) + require.NoError(t, err) + require.Contains(t, topics, topic) + + adminClient.Close() + require.NoError(t, factory.client.Context().Err()) + factory.Close() + require.ErrorIs(t, asyncProducer.client.Context().Err(), context.Canceled) + factory.Close() +} diff --git a/pkg/sink/kafka/franz_gssapi.go b/pkg/sink/kafka/franz_gssapi.go new file mode 100644 index 0000000000..a053230e79 --- /dev/null +++ b/pkg/sink/kafka/franz_gssapi.go @@ -0,0 +1,85 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "github.com/jcmturner/gokrb5/v8/client" + "github.com/jcmturner/gokrb5/v8/config" + "github.com/jcmturner/gokrb5/v8/keytab" + "github.com/pingcap/ticdc/pkg/errors" + "github.com/twmb/franz-go/pkg/sasl" + "github.com/twmb/franz-go/pkg/sasl/kerberos" +) + +func buildGSSAPIMechanism(g gssapiConfig) (sasl.Mechanism, error) { + if err := validateGSSAPIConfig(g); err != nil { + return nil, err + } + + krbClient, err := newKerberosClient(g) + if err != nil { + return nil, err + } + return kerberos.Auth{Client: krbClient, Service: g.serviceName}.AsMechanismWithClose(), nil +} + +func validateGSSAPIConfig(g gssapiConfig) error { + if g.serviceName == "" { + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-service-name must not be empty when sasl mechanism is GSSAPI") + } + if g.kerberosConfigPath == "" { + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-kerberos-config-path must not be empty when sasl mechanism is GSSAPI") + } + if g.username == "" { + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-user must not be empty when sasl mechanism is GSSAPI") + } + if g.realm == "" { + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-realm must not be empty when sasl mechanism is GSSAPI") + } + + switch g.authType { + case userAuth: + if g.password == "" { + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-password must not be empty when sasl-gssapi-auth-type is USER") + } + case keyTabAuth: + if g.keyTabPath == "" { + return errors.ErrKafkaInvalidConfig.GenWithStack("sasl-gssapi-keytab-path must not be empty when sasl-gssapi-auth-type is KEYTAB") + } + default: + return errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl-gssapi-auth-type %d", g.authType) + } + return nil +} + +func newKerberosClient(g gssapiConfig) (*client.Client, error) { + cfg, err := config.Load(g.kerberosConfigPath) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + + switch g.authType { + case userAuth: + return client.NewWithPassword(g.username, g.realm, g.password, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil + case keyTabAuth: + kt, err := keytab.Load(g.keyTabPath) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + return client.NewWithKeytab(g.username, g.realm, kt, cfg, client.DisablePAFXFAST(g.disablePAFXFAST)), nil + default: + return nil, errors.ErrKafkaInvalidConfig.GenWithStack("unsupported sasl-gssapi-auth-type %d", g.authType) + } +} diff --git a/pkg/sink/kafka/franz_gssapi_test.go b/pkg/sink/kafka/franz_gssapi_test.go new file mode 100644 index 0000000000..0326d3cd2b --- /dev/null +++ b/pkg/sink/kafka/franz_gssapi_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func TestGSSAPIConfigValidation(t *testing.T) { + valid := gssapiConfig{ + authType: userAuth, + kerberosConfigPath: "/etc/krb5.conf", + serviceName: "kafka", + username: "alice", + password: "secret", + realm: "EXAMPLE.COM", + } + + for _, mutate := range []func(*gssapiConfig){ + func(cfg *gssapiConfig) { cfg.serviceName = "" }, + func(cfg *gssapiConfig) { cfg.kerberosConfigPath = "" }, + func(cfg *gssapiConfig) { cfg.username = "" }, + func(cfg *gssapiConfig) { cfg.realm = "" }, + func(cfg *gssapiConfig) { cfg.password = "" }, + func(cfg *gssapiConfig) { cfg.authType = 0 }, + func(cfg *gssapiConfig) { cfg.authType, cfg.keyTabPath = keyTabAuth, "" }, + } { + cfg := valid + mutate(&cfg) + + _, err := buildGSSAPIMechanism(cfg) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + } +} + +func TestGSSAPIMissingConfig(t *testing.T) { + mechanism, err := buildGSSAPIMechanism(gssapiConfig{ + authType: userAuth, + kerberosConfigPath: "/path/that/does/not/exist", + serviceName: "kafka", + username: "alice", + password: "secret", + realm: "EXAMPLE.COM", + }) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.Nil(t, mechanism) +} diff --git a/pkg/sink/kafka/franz_logger.go b/pkg/sink/kafka/franz_logger.go new file mode 100644 index 0000000000..54e5854683 --- /dev/null +++ b/pkg/sink/kafka/franz_logger.go @@ -0,0 +1,99 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "fmt" + "strings" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// logValueLimit bounds individual string fields emitted by the franz-go logger. +const logValueLimit = 1024 + +type clientLogger struct{ logger *zap.Logger } + +func newClientLogger(changefeedID common.ChangeFeedID, role string) kgo.Logger { + logger := log.L().With( + zap.String("component", "kafka-client"), + zap.String("keyspace", changefeedID.Keyspace()), + zap.String("changefeed", changefeedID.Name()), + zap.String("role", role), + ).WithOptions(zap.WrapCore(func(core zapcore.Core) zapcore.Core { + return zapcore.NewSamplerWithOptions(core, time.Minute, 5, 100) + })) + return &clientLogger{logger: logger} +} + +func (l *clientLogger) Level() kgo.LogLevel { + if log.GetLevel() <= zapcore.DebugLevel { + return kgo.LogLevelInfo + } + return kgo.LogLevelWarn +} + +func (l *clientLogger) Log(level kgo.LogLevel, msg string, keyvals ...any) { + fields := make([]zap.Field, 0, (len(keyvals)+1)/2) + for i := 0; i < len(keyvals); i += 2 { + key := fmt.Sprint(keyvals[i]) + value := any("") + if i+1 < len(keyvals) { + value = keyvals[i+1] + } + + if isSensitiveLogKey(key) { + value = "[redacted]" + } else if text, ok := value.(string); ok && len(text) > logValueLimit { + value = text[:logValueLimit] + } + fields = append(fields, zap.Any(key, value)) + } + + switch level { + case kgo.LogLevelError: + l.logger.Error(msg, fields...) + case kgo.LogLevelWarn: + l.logger.Warn(msg, fields...) + default: + l.logger.Debug(msg, fields...) + } +} + +func isSensitiveLogKey(key string) bool { + key = strings.ToLower(key) + if key == "key" || key == "value" { + return true + } + + for _, fragment := range []string{ + "password", + "passwd", + "secret", + "token", + "authorization", + "credential", + "sasl", + } { + if strings.Contains(key, fragment) { + return true + } + } + return false +} diff --git a/pkg/sink/kafka/franz_logger_test.go b/pkg/sink/kafka/franz_logger_test.go new file mode 100644 index 0000000000..2507911ae9 --- /dev/null +++ b/pkg/sink/kafka/franz_logger_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "strings" + "testing" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestLoggerLevelAndFiltering(t *testing.T) { + oldLevel := log.GetLevel() + defer log.SetLevel(oldLevel) + + clientLogger := newClientLogger(common.NewChangefeedID4Test(common.DefaultKeyspaceName, "logger"), "producer").(*clientLogger) + + log.SetLevel(zapcore.InfoLevel) + require.Equal(t, kgo.LogLevelWarn, clientLogger.Level()) + + log.SetLevel(zapcore.DebugLevel) + require.Equal(t, kgo.LogLevelInfo, clientLogger.Level()) + + for _, key := range []string{"password", "access_token", "key", "value", "sasl-user"} { + require.True(t, isSensitiveLogKey(key)) + } + + require.NotPanics(t, func() { clientLogger.Log(kgo.LogLevelWarn, "odd key value", "key-only") }) +} + +func TestLoggerPreservesContextAndRedactsValues(t *testing.T) { + core, logs := observer.New(zapcore.DebugLevel) + restore := log.ReplaceGlobals(zap.New(core), nil) + defer restore() + + clientLogger := newClientLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") + clientLogger.Log( + kgo.LogLevelWarn, + "connection failed", + "password", "secret", + "payload", strings.Repeat("x", logValueLimit+10), + "odd", + ) + + entries := logs.FilterMessage("connection failed").AllUntimed() + require.Len(t, entries, 1) + + fields := entries[0].ContextMap() + require.Equal(t, "kafka-client", fields["component"]) + require.Equal(t, "keyspace", fields["keyspace"]) + require.Equal(t, "changefeed", fields["changefeed"]) + require.Equal(t, "producer", fields["role"]) + require.Equal(t, "[redacted]", fields["password"]) + require.Equal(t, strings.Repeat("x", logValueLimit), fields["payload"]) + require.Equal(t, "", fields["odd"]) +} + +func TestLoggerSamplesRepeatedMessages(t *testing.T) { + core, logs := observer.New(zapcore.DebugLevel) + restore := log.ReplaceGlobals(zap.New(core), nil) + defer restore() + + clientLogger := newClientLogger(common.NewChangefeedID4Test("keyspace", "changefeed"), "producer") + for range 105 { + clientLogger.Log(kgo.LogLevelWarn, "repeated") + } + + require.Len(t, logs.FilterMessage("repeated").AllUntimed(), 6) +} diff --git a/pkg/sink/kafka/franz_metrics.go b/pkg/sink/kafka/franz_metrics.go new file mode 100644 index 0000000000..5489960389 --- /dev/null +++ b/pkg/sink/kafka/franz_metrics.go @@ -0,0 +1,84 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import "github.com/prometheus/client_golang/prometheus" + +var ( + requestsInFlight = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_in_flight_requests", + Help: "Current franz-go requests awaiting a response.", + }, []string{"namespace", "changefeed", "broker"}) + + outgoingBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_outgoing_bytes_total", + Help: "Total bytes written by franz-go, excluding TLS overhead.", + }, []string{"namespace", "changefeed", "broker"}) + + requestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_requests_total", + Help: "Total franz-go requests by broker and write result.", + }, []string{"namespace", "changefeed", "broker", "result"}) + + responsesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_responses_total", + Help: "Total franz-go responses by broker and read result.", + }, []string{"namespace", "changefeed", "broker", "result"}) + + requestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_request_duration_seconds", + Help: "Franz-go request end-to-end duration in seconds.", + Buckets: prometheus.DefBuckets, + }, []string{"namespace", "changefeed", "broker"}) + + throttleTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_throttle_time_seconds", + Help: "Kafka broker throttle time reported to the producer in seconds.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 20), + }, []string{"namespace", "changefeed", "broker"}) + + recordsPerBatch = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_records_per_batch", + Help: "Records in each successfully written franz-go topic-partition batch.", + Buckets: prometheus.ExponentialBuckets(1, 2, 15), + }, []string{"namespace", "changefeed"}) + + uncompressedBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_uncompressed_bytes_total", + Help: "Total record bytes before compression in successfully written franz-go batches.", + }, []string{"namespace", "changefeed"}) + + compressedBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "ticdc", + Subsystem: "sink", + Name: "kafka_franz_producer_compressed_bytes_total", + Help: "Total record bytes after compression in successfully written franz-go batches.", + }, []string{"namespace", "changefeed"}) +) diff --git a/pkg/sink/kafka/franz_metrics_hook.go b/pkg/sink/kafka/franz_metrics_hook.go new file mode 100644 index 0000000000..daf19ddc12 --- /dev/null +++ b/pkg/sink/kafka/franz_metrics_hook.go @@ -0,0 +1,172 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "strconv" + "sync" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/prometheus/client_golang/prometheus" + "github.com/twmb/franz-go/pkg/kgo" +) + +// metricsHook adapts client callbacks to TiCDC's Kafka sink metrics. +// franz-go calls these hook methods while writing requests, receiving responses, +// and flushing produce batches. The hook does not poll Kafka; it only records +// raw callback values for Prometheus. +type metricsHook struct { + keyspace string + changefeed string + + brokers sync.Map + + recordsPerBatch prometheus.Observer + uncompressedBytesTotal prometheus.Counter + compressedBytesTotal prometheus.Counter +} + +type brokerMetrics struct { + outgoingBytesTotal prometheus.Counter + requestsSuccess prometheus.Counter + requestsWriteError prometheus.Counter + responsesSuccess prometheus.Counter + responsesReadError prometheus.Counter + requestsInFlight prometheus.Gauge + requestDuration prometheus.Observer + throttleTime prometheus.Observer +} + +const ( + // Result values are fixed to keep the broker-level metric label cardinality bounded. + metricResultSuccess = "success" + metricResultWriteError = "write_error" + metricResultReadError = "read_error" +) + +func newMetricsHook(changefeedID common.ChangeFeedID) *metricsHook { + keyspace := changefeedID.Keyspace() + changefeed := changefeedID.Name() + return &metricsHook{ + keyspace: keyspace, + changefeed: changefeed, + recordsPerBatch: recordsPerBatch.WithLabelValues(keyspace, changefeed), + uncompressedBytesTotal: uncompressedBytesTotal.WithLabelValues(keyspace, changefeed), + compressedBytesTotal: compressedBytesTotal.WithLabelValues(keyspace, changefeed), + } +} + +func (h *metricsHook) OnBrokerThrottle(meta kgo.BrokerMetadata, throttleInterval time.Duration, _ bool) { + if meta.NodeID < 0 { + return + } + + h.broker(meta.NodeID).throttleTime.Observe(throttleInterval.Seconds()) +} + +func (h *metricsHook) broker(nodeID int32) *brokerMetrics { + if cached, ok := h.brokers.Load(nodeID); ok { + return cached.(*brokerMetrics) + } + + brokerID := strconv.Itoa(int(nodeID)) + metrics := &brokerMetrics{ + outgoingBytesTotal: outgoingBytesTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID), + requestsSuccess: requestsTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultSuccess), + requestsWriteError: requestsTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultWriteError), + responsesSuccess: responsesTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultSuccess), + responsesReadError: responsesTotal.WithLabelValues(h.keyspace, h.changefeed, brokerID, metricResultReadError), + requestsInFlight: requestsInFlight.WithLabelValues(h.keyspace, h.changefeed, brokerID), + requestDuration: requestDuration.WithLabelValues(h.keyspace, h.changefeed, brokerID), + throttleTime: throttleTime.WithLabelValues(h.keyspace, h.changefeed, brokerID), + } + + actual, _ := h.brokers.LoadOrStore(nodeID, metrics) + return actual.(*brokerMetrics) +} + +// cleanupMetrics removes producer series after all clients are closed. +func cleanupMetrics(changefeedID common.ChangeFeedID) { + labels := prometheus.Labels{ + "namespace": changefeedID.Keyspace(), + "changefeed": changefeedID.Name(), + } + + outgoingBytesTotal.DeletePartialMatch(labels) + requestsTotal.DeletePartialMatch(labels) + responsesTotal.DeletePartialMatch(labels) + requestsInFlight.DeletePartialMatch(labels) + requestDuration.DeletePartialMatch(labels) + throttleTime.DeletePartialMatch(labels) + recordsPerBatch.DeletePartialMatch(labels) + uncompressedBytesTotal.DeletePartialMatch(labels) + compressedBytesTotal.DeletePartialMatch(labels) +} + +func (h *metricsHook) OnBrokerWrite(meta kgo.BrokerMetadata, _ int16, bytesWritten int, _ time.Duration, _ time.Duration, err error) { + if meta.NodeID < 0 { + return + } + + metrics := h.broker(meta.NodeID) + + if bytesWritten > 0 { + metrics.outgoingBytesTotal.Add(float64(bytesWritten)) + } + + if err != nil { + metrics.requestsWriteError.Inc() + } else { + metrics.requestsSuccess.Inc() + metrics.requestsInFlight.Inc() + } +} + +func (h *metricsHook) OnBrokerE2E(meta kgo.BrokerMetadata, _ int16, e2e kgo.BrokerE2E) { + if meta.NodeID < 0 { + return + } + + metrics := h.broker(meta.NodeID) + + if e2e.WriteErr == nil { + metrics.requestsInFlight.Dec() + if e2e.BytesRead > 0 || e2e.ReadErr != nil { + if e2e.ReadErr != nil { + metrics.responsesReadError.Inc() + } else { + metrics.responsesSuccess.Inc() + } + } + } + + if e2e.Err() == nil { + metrics.requestDuration.Observe(e2e.DurationE2E().Seconds()) + } +} + +func (h *metricsHook) OnProduceBatchWritten(_ kgo.BrokerMetadata, _ string, _ int32, m kgo.ProduceBatchMetrics) { + if m.NumRecords > 0 { + h.recordsPerBatch.Observe(float64(m.NumRecords)) + } + + if m.UncompressedBytes > 0 { + h.uncompressedBytesTotal.Add(float64(m.UncompressedBytes)) + } + + if m.CompressedBytes > 0 { + h.compressedBytesTotal.Add(float64(m.CompressedBytes)) + } +} diff --git a/pkg/sink/kafka/franz_metrics_hook_test.go b/pkg/sink/kafka/franz_metrics_hook_test.go new file mode 100644 index 0000000000..1f9c2d51f0 --- /dev/null +++ b/pkg/sink/kafka/franz_metrics_hook_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kgo" +) + +func TestMetricsHook(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "metrics-hook") + cleanupMetrics(changefeedID) + t.Cleanup(func() { cleanupMetrics(changefeedID) }) + hook := newMetricsHook(changefeedID) + meta := kgo.BrokerMetadata{NodeID: 1} + + hook.OnBrokerWrite(meta, 0, 12, 0, 0, nil) + hook.OnBrokerE2E(meta, 0, kgo.BrokerE2E{ + BytesRead: 8, + TimeToWrite: time.Millisecond, + TimeToRead: time.Millisecond, + }) + hook.OnProduceBatchWritten(meta, "topic", 0, kgo.ProduceBatchMetrics{ + NumRecords: 3, + UncompressedBytes: 10, + CompressedBytes: 5, + }) + + metrics := hook.broker(1) + require.Same(t, metrics, hook.broker(1)) + require.Equal(t, float64(12), testutil.ToFloat64(metrics.outgoingBytesTotal)) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.requestsSuccess)) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.responsesSuccess)) + require.Equal(t, float64(0), testutil.ToFloat64(metrics.requestsInFlight)) + require.Equal(t, float64(10), testutil.ToFloat64( + uncompressedBytesTotal.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name()), + )) + require.Equal(t, float64(5), testutil.ToFloat64( + compressedBytesTotal.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name()), + )) + batchMetric, ok := hook.recordsPerBatch.(prometheus.Metric) + require.True(t, ok) + batchHistogram := &dto.Metric{} + require.NoError(t, batchMetric.Write(batchHistogram)) + require.Equal(t, uint64(1), batchHistogram.GetHistogram().GetSampleCount()) + require.Equal(t, float64(3), batchHistogram.GetHistogram().GetSampleSum()) + + registry := prometheus.NewRegistry() + InitMetrics(registry) + metricFamilies, err := registry.Gather() + require.NoError(t, err) + names := make([]string, 0, len(metricFamilies)) + for _, family := range metricFamilies { + if strings.HasPrefix(family.GetName(), "ticdc_sink_kafka_franz_producer_") { + names = append(names, family.GetName()) + } + } + require.ElementsMatch(t, []string{ + "ticdc_sink_kafka_franz_producer_compressed_bytes_total", + "ticdc_sink_kafka_franz_producer_in_flight_requests", + "ticdc_sink_kafka_franz_producer_outgoing_bytes_total", + "ticdc_sink_kafka_franz_producer_records_per_batch", + "ticdc_sink_kafka_franz_producer_request_duration_seconds", + "ticdc_sink_kafka_franz_producer_requests_total", + "ticdc_sink_kafka_franz_producer_responses_total", + "ticdc_sink_kafka_franz_producer_throttle_time_seconds", + "ticdc_sink_kafka_franz_producer_uncompressed_bytes_total", + }, names) + + hook.OnBrokerWrite(meta, 0, 0, 0, 0, nil) + hook.OnBrokerE2E(meta, 0, kgo.BrokerE2E{ + BytesRead: 8, + TimeToWrite: 3 * time.Millisecond, + ReadWait: time.Millisecond, + TimeToRead: 4 * time.Millisecond, + }) + + metric, ok := metrics.requestDuration.(prometheus.Metric) + require.True(t, ok) + + histogram := &dto.Metric{} + require.NoError(t, metric.Write(histogram)) + require.Equal(t, uint64(2), histogram.GetHistogram().GetSampleCount()) + require.InDelta(t, 0.010, histogram.GetHistogram().GetSampleSum(), 0.000001) + + hook.OnBrokerWrite(meta, 0, 0, 0, 0, context.DeadlineExceeded) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.requestsWriteError)) + + hook.OnBrokerWrite(meta, 0, 0, 0, 0, nil) + hook.OnBrokerE2E(meta, 0, kgo.BrokerE2E{ReadErr: context.Canceled}) + require.Equal(t, float64(1), testutil.ToFloat64(metrics.responsesReadError)) + require.Equal(t, float64(0), testutil.ToFloat64(metrics.requestsInFlight)) + + hook.OnBrokerWrite(kgo.BrokerMetadata{NodeID: -1}, 0, 1, 0, 0, nil) + hook.OnBrokerE2E(kgo.BrokerMetadata{NodeID: -1}, 0, kgo.BrokerE2E{}) + + hook.OnBrokerThrottle(meta, 10*time.Millisecond, true) + hook.OnBrokerThrottle(meta, 50*time.Millisecond, false) + hook.OnBrokerThrottle(kgo.BrokerMetadata{NodeID: 2}, 30*time.Millisecond, true) + hook.OnBrokerThrottle(kgo.BrokerMetadata{NodeID: -1}, time.Second, true) + + throttleMetric, ok := metrics.throttleTime.(prometheus.Metric) + require.True(t, ok) + throttleHistogram := &dto.Metric{} + require.NoError(t, throttleMetric.Write(throttleHistogram)) + require.Equal(t, uint64(2), throttleHistogram.GetHistogram().GetSampleCount()) + require.InDelta(t, 0.06, throttleHistogram.GetHistogram().GetSampleSum(), 0.000001) + keyspace, changefeed, broker := changefeedID.Keyspace(), changefeedID.Name(), "1" + cleanupMetrics(changefeedID) + require.False(t, outgoingBytesTotal.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, requestsTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultSuccess)) + require.False(t, requestsTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultWriteError)) + require.False(t, responsesTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultSuccess)) + require.False(t, responsesTotal.DeleteLabelValues(keyspace, changefeed, broker, metricResultReadError)) + require.False(t, requestsInFlight.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, requestDuration.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, throttleTime.DeleteLabelValues(keyspace, changefeed, broker)) + require.False(t, recordsPerBatch.DeleteLabelValues(keyspace, changefeed)) + require.False(t, uncompressedBytesTotal.DeleteLabelValues(keyspace, changefeed)) + require.False(t, compressedBytesTotal.DeleteLabelValues(keyspace, changefeed)) +} diff --git a/pkg/sink/kafka/franz_sync_producer.go b/pkg/sink/kafka/franz_sync_producer.go new file mode 100644 index 0000000000..cfc341ea14 --- /dev/null +++ b/pkg/sink/kafka/franz_sync_producer.go @@ -0,0 +1,88 @@ +// Copyright 2025 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "sync/atomic" + "time" + + "github.com/pingcap/log" + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codecCommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/twmb/franz-go/pkg/kgo" + "go.uber.org/zap" +) + +type syncProducer struct { + id common.ChangeFeedID + + client *kgo.Client + closed atomic.Bool +} + +func (p *syncProducer) SendMessage(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { + record := &kgo.Record{ + Topic: topic, + Partition: partitionNum, + Key: message.Key, + Value: message.Value, + } + return p.sendRecords(ctx, message, record) +} + +func (p *syncProducer) SendMessages(ctx context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { + records := make([]*kgo.Record, 0, partitionNum) + for i := 0; i < int(partitionNum); i++ { + records = append(records, &kgo.Record{ + Topic: topic, + Partition: int32(i), + Key: message.Key, + Value: message.Value, + }) + } + + return p.sendRecords(ctx, message, records...) +} + +func (p *syncProducer) sendRecords(ctx context.Context, message *codecCommon.Message, records ...*kgo.Record) error { + if p.closed.Load() { + return errors.ErrKafkaSinkClosed.GenWithStackByArgs() + } + + err := p.client.ProduceSync(ctx, records...).FirstErr() + if err == nil { + return nil + } + + log.Error("kafka message send failed", + zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), + zap.String("eventContext", BuildEventLogContext(p.id.Keyspace(), p.id.Name(), message.LogInfo)), + zap.Error(err)) + return errors.WrapError(errors.ErrKafkaSendMessage, err) +} + +func (p *syncProducer) Close() { + if !p.closed.CompareAndSwap(false, true) { + log.Warn("kafka ddl producer already closed", + zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name())) + return + } + + start := time.Now() + log.Info("kafka ddl producer closed", + zap.String("keyspace", p.id.Keyspace()), zap.String("changefeed", p.id.Name()), + zap.Duration("duration", time.Since(start))) +} diff --git a/pkg/sink/kafka/franz_sync_producer_test.go b/pkg/sink/kafka/franz_sync_producer_test.go new file mode 100644 index 0000000000..2bfd09399d --- /dev/null +++ b/pkg/sink/kafka/franz_sync_producer_test.go @@ -0,0 +1,155 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "slices" + "sync" + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/pingcap/ticdc/pkg/errors" + codeccommon "github.com/pingcap/ticdc/pkg/sink/codec/common" + "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kerr" + "github.com/twmb/franz-go/pkg/kfake" + "github.com/twmb/franz-go/pkg/kgo" + "github.com/twmb/franz-go/pkg/kmsg" +) + +func TestSyncProducerClosed(t *testing.T) { + producer := &syncProducer{} + producer.closed.Store(true) + + err := producer.SendMessage(t.Context(), "topic", 1, &codeccommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) + + err = producer.SendMessages(t.Context(), "topic", 1, &codeccommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaSinkClosed) +} + +func TestSyncProducerPartitions(t *testing.T) { + const topic = "sync-topic" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) + defer cluster.Close() + var mu sync.Mutex + var partitions []int32 + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + cluster.KeepControl() + request := req.(*kmsg.ProduceRequest) + mu.Lock() + defer mu.Unlock() + for _, requestTopic := range request.Topics { + for _, partition := range requestTopic.Partitions { + partitions = append(partitions, partition.Partition) + } + } + return produceResponseWithError(req, -1, 0) + }) + o := testOptions(cluster.ListenAddrs()) + + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + producer := &syncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "sync"), + client: client, + } + defer producer.Close() + + require.NoError(t, producer.SendMessage(t.Context(), topic, 2, &codeccommon.Message{Key: []byte("key"), Value: []byte("value")})) + mu.Lock() + require.Equal(t, []int32{2}, partitions) + partitions = nil + mu.Unlock() + require.NoError(t, producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("all")})) + mu.Lock() + slices.Sort(partitions) + require.Equal(t, []int32{0, 1, 2}, partitions) + mu.Unlock() +} + +func TestSyncProducerPartialFailure(t *testing.T) { + const topic = "partial-failure" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(3, topic)) + defer cluster.Close() + + cluster.ControlKey(int16(kmsg.Produce), func(req kmsg.Request) (kmsg.Response, error, bool) { + return produceResponseWithError(req, 1, kerr.InvalidTopicException.Code) + }) + o := testOptions(cluster.ListenAddrs()) + + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + producer := &syncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "partial"), + client: client, + } + defer producer.Close() + + err = producer.SendMessages(t.Context(), topic, 3, &codeccommon.Message{Value: []byte("value")}) + requireKafkaSendError(t, err, kerr.InvalidTopicException) +} + +func TestSyncProducerContext(t *testing.T) { + o := testOptions([]string{"127.0.0.1:1"}) + client, err := kgo.NewClient(append(testClientOptions(t, o), producerOptions(o)...)...) + require.NoError(t, err) + defer client.Close() + producer := &syncProducer{ + id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "canceled"), + client: client, + } + defer producer.Close() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + err = producer.SendMessage(ctx, "topic", 0, &codeccommon.Message{}) + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) + require.ErrorIs(t, err, context.Canceled) +} + +func TestFranzSyncProducerClose(t *testing.T) { + producer := &syncProducer{id: common.NewChangefeedID4Test(common.DefaultKeyspaceName, "close")} + + producer.Close() + producer.Close() + require.True(t, producer.closed.Load()) +} + +func produceResponseWithError(req kmsg.Request, failedPartition int32, errorCode int16) (kmsg.Response, error, bool) { + request := req.(*kmsg.ProduceRequest) + response := request.ResponseKind().(*kmsg.ProduceResponse) + + for _, requestTopic := range request.Topics { + responseTopic := kmsg.NewProduceResponseTopic() + responseTopic.Topic = requestTopic.Topic + responseTopic.TopicID = requestTopic.TopicID + + for _, requestPartition := range requestTopic.Partitions { + responsePartition := kmsg.NewProduceResponseTopicPartition() + responsePartition.Partition = requestPartition.Partition + if requestPartition.Partition == failedPartition { + responsePartition.ErrorCode = errorCode + } + responseTopic.Partitions = append(responseTopic.Partitions, responsePartition) + } + + response.Topics = append(response.Topics, responseTopic) + } + return response, nil, true +} diff --git a/pkg/sink/kafka/metrics.go b/pkg/sink/kafka/metrics.go index fc1ebf7594..f75f700cc0 100644 --- a/pkg/sink/kafka/metrics.go +++ b/pkg/sink/kafka/metrics.go @@ -89,6 +89,17 @@ var ( // InitMetrics registers all metrics in this file. func InitMetrics(registry *prometheus.Registry) { + registry.MustRegister( + requestsInFlight, + outgoingBytesTotal, + requestsTotal, + responsesTotal, + requestDuration, + throttleTime, + recordsPerBatch, + uncompressedBytesTotal, + compressedBytesTotal, + ) registry.MustRegister(compressionRatioGauge) registry.MustRegister(recordsPerRequestGauge) registry.MustRegister(throttleTimeGauge) diff --git a/pkg/sink/kafka/metrics_collector.go b/pkg/sink/kafka/metrics_collector.go index 674afa7a7f..60b48ac628 100644 --- a/pkg/sink/kafka/metrics_collector.go +++ b/pkg/sink/kafka/metrics_collector.go @@ -93,7 +93,7 @@ func (m *saramaMetricsCollector) Run(ctx context.Context) { } func (m *saramaMetricsCollector) updateBrokers(ctx context.Context) { - brokers := m.adminClient.GetAllBrokers() + brokers := m.adminClient.GetAllBrokers(ctx) for _, b := range brokers { m.brokers[b.ID] = struct{}{} } @@ -125,6 +125,7 @@ func (m *saramaMetricsCollector) collectProducerMetrics() { func (m *saramaMetricsCollector) collectBrokerMetrics() { keyspace := m.changefeedID.Keyspace() changefeedID := m.changefeedID.Name() + for id := range m.brokers { brokerID := strconv.Itoa(int(id)) outgoingByteRateMetric := m.registry.Get( diff --git a/pkg/sink/kafka/metrics_collector_test.go b/pkg/sink/kafka/metrics_collector_test.go new file mode 100644 index 0000000000..3f2237e331 --- /dev/null +++ b/pkg/sink/kafka/metrics_collector_test.go @@ -0,0 +1,56 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "testing" + + "github.com/pingcap/ticdc/pkg/common" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/rcrowley/go-metrics" + "github.com/stretchr/testify/require" +) + +func TestCollectBrokerThrottleTime(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "throttle-time") + + registry := metrics.NewRegistry() + firstBroker := metrics.NewHistogram(metrics.NewUniformSample(10)) + firstBroker.Update(10) + firstBroker.Update(50) + require.NoError(t, registry.Register(getBrokerMetricName(throttleTimeMetricNamePrefix, "1"), firstBroker)) + + secondBroker := metrics.NewHistogram(metrics.NewUniformSample(10)) + secondBroker.Update(40) + require.NoError(t, registry.Register(getBrokerMetricName(throttleTimeMetricNamePrefix, "2"), secondBroker)) + + collector := saramaMetricsCollector{ + changefeedID: changefeedID, + brokers: map[int32]struct{}{1: {}, 2: {}}, + registry: registry, + } + collector.collectBrokerMetrics() + + require.Equal(t, 0.03, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", avg))) + require.Equal(t, 0.05, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", p99))) + require.Equal(t, 0.04, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", avg))) + require.Equal(t, 0.04, testutil.ToFloat64(throttleTimeGauge.WithLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", p99))) + + collector.cleanupMetrics() + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", avg)) + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "1", p99)) + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", avg)) + require.False(t, throttleTimeGauge.DeleteLabelValues(changefeedID.Keyspace(), changefeedID.Name(), "2", p99)) +} diff --git a/pkg/sink/kafka/oauth2.go b/pkg/sink/kafka/oauth2.go new file mode 100644 index 0000000000..ae53e01290 --- /dev/null +++ b/pkg/sink/kafka/oauth2.go @@ -0,0 +1,88 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net/http" + "net/url" + "os" + + "github.com/pingcap/ticdc/pkg/errors" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +func newOAuthTokenSource(ctx context.Context, cfg oauth2Config) (oauth2.TokenSource, error) { + endpointParams := url.Values{} + if cfg.grantType != "" { + endpointParams.Set("grant_type", cfg.grantType) + } + if cfg.audience != "" { + endpointParams.Set("audience", cfg.audience) + } + + tokenURL, err := url.Parse(cfg.tokenURL) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if cfg.caPath != "" { + httpClient, err := oauthHTTPClient(cfg.caPath) + if err != nil { + return nil, err + } + ctx = context.WithValue(ctx, oauth2.HTTPClient, httpClient) + } + + config := clientcredentials.Config{ + ClientID: cfg.clientID, + ClientSecret: cfg.clientSecret, + TokenURL: tokenURL.String(), + EndpointParams: endpointParams, + Scopes: cfg.scopes, + } + return config.TokenSource(ctx), nil +} + +func oauthHTTPClient(caPath string) (*http.Client, error) { + caPEM, err := os.ReadFile(caPath) + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + + rootCAs, err := x509.SystemCertPool() + if err != nil { + return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) + } + if !rootCAs.AppendCertsFromPEM(caPEM) { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "OAuth2 CA file %q does not contain a valid certificate", caPath) + } + + defaultTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, errors.ErrKafkaInvalidConfig.GenWithStack( + "cannot configure OAuth2 CA file %q with HTTP transport type %T", + caPath, http.DefaultTransport) + } + transport := defaultTransport.Clone() + if transport.TLSClientConfig == nil { + transport.TLSClientConfig = &tls.Config{} + } + transport.TLSClientConfig.RootCAs = rootCAs + return &http.Client{Transport: transport}, nil +} diff --git a/pkg/sink/kafka/oauth2_test.go b/pkg/sink/kafka/oauth2_test.go new file mode 100644 index 0000000000..814ca89eff --- /dev/null +++ b/pkg/sink/kafka/oauth2_test.go @@ -0,0 +1,109 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "context" + "encoding/pem" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/pingcap/ticdc/pkg/security" + "github.com/stretchr/testify/require" + "golang.org/x/oauth2" +) + +func TestOAuthCA(t *testing.T) { + server := newTLSTokenServer(t) + tokenSource, err := newOAuthTokenSource(t.Context(), newOAuthConfig(server.URL, writeServerCA(t, server))) + require.NoError(t, err) + token, err := tokenSource.Token() + require.NoError(t, err) + require.Equal(t, "access-token", token.AccessToken) +} + +func TestOAuthCAErrors(t *testing.T) { + t.Run("missing file", func(t *testing.T) { + caPath := filepath.Join(t.TempDir(), "missing-ca.pem") + _, err := newOAuthTokenSource(t.Context(), newOAuthConfig("https://example.com/token", caPath)) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.ErrorContains(t, err, caPath) + }) + + t.Run("invalid PEM", func(t *testing.T) { + caPath := filepath.Join(t.TempDir(), "invalid-ca.pem") + require.NoError(t, os.WriteFile(caPath, []byte("not a certificate"), 0o600)) + _, err := newOAuthTokenSource(t.Context(), newOAuthConfig("https://example.com/token", caPath)) + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + require.ErrorContains(t, err, "does not contain a valid certificate") + }) + + t.Run("mismatched certificate", func(t *testing.T) { + server := newTLSTokenServer(t) + unrelatedCA, err := security.NewCA() + require.NoError(t, err) + caPath := filepath.Join(t.TempDir(), "unrelated-ca.pem") + require.NoError(t, os.WriteFile(caPath, unrelatedCA.CAPEM, 0o600)) + tokenSource, err := newOAuthTokenSource(t.Context(), newOAuthConfig(server.URL, caPath)) + require.NoError(t, err) + _, err = tokenSource.Token() + require.ErrorContains(t, err, "certificate signed by unknown authority") + }) +} + +func TestOAuthContextClient(t *testing.T) { + server := newTLSTokenServer(t) + ctx := context.WithValue(t.Context(), oauth2.HTTPClient, server.Client()) + tokenSource, err := newOAuthTokenSource(ctx, newOAuthConfig(server.URL, "")) + require.NoError(t, err) + token, err := tokenSource.Token() + require.NoError(t, err) + require.Equal(t, "access-token", token.AccessToken) +} + +func newTLSTokenServer(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if _, err := io.WriteString(w, `{"access_token":"access-token","token_type":"bearer"}`); err != nil { + t.Errorf("write token response: %v", err) + } + })) + t.Cleanup(server.Close) + return server +} + +func writeServerCA(t *testing.T, server *httptest.Server) string { + t.Helper() + caPath := filepath.Join(t.TempDir(), "ca.pem") + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) + require.NotNil(t, caPEM) + require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) + return caPath +} + +func newOAuthConfig(tokenURL, caPath string) oauth2Config { + return oauth2Config{ + clientID: "client-id", + clientSecret: "client-secret", + tokenURL: tokenURL, + caPath: caPath, + } +} diff --git a/pkg/sink/kafka/options.go b/pkg/sink/kafka/options.go index 07c44901ac..2f98dfee48 100644 --- a/pkg/sink/kafka/options.go +++ b/pkg/sink/kafka/options.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "encoding/base64" "fmt" "net/http" @@ -40,6 +41,10 @@ const ( defaultMaxRetry = 5 // defaultTimeout is the default timeout for Kafka connections. defaultTimeout = 10 * time.Second + // KafkaClientFranz is the default Kafka client implementation. + KafkaClientFranz = "franz" + // KafkaClientSarama keeps the master implementation available as a fallback. + KafkaClientSarama = "sarama" ) const ( @@ -108,6 +113,7 @@ func requireAcksFromString(acks int) (RequiredAcks, error) { } type urlConfig struct { + KafkaClient *string `form:"kafka-client"` PartitionNum *int32 `form:"partition-num"` ReplicationFactor *int16 `form:"replication-factor"` KafkaVersion *string `form:"kafka-version"` @@ -140,6 +146,7 @@ type urlConfig struct { // options stores Kafka sink configurations type options struct { + Client string Topic string BrokerEndpoints []string @@ -177,6 +184,7 @@ type options struct { // NewOptions returns a default Kafka configuration func NewOptions() *options { return &options{ + Client: KafkaClientFranz, Version: "2.4.0", MaxMessageBytes: config.DefaultMaxMessageBytes, MaxBatchedBytes: config.DefaultMaxMessageBytes, @@ -262,6 +270,12 @@ func (o *options) Apply(changefeedID common.ChangeFeedID, o.MaxMessageBytes = *urlParameter.MaxMessageBytes } o.MaxBatchedBytes = o.MaxMessageBytes + if urlParameter.KafkaClient != nil { + o.Client = strings.ToLower(strings.TrimSpace(*urlParameter.KafkaClient)) + } + if o.Client != KafkaClientFranz && o.Client != KafkaClientSarama { + return errors.ErrKafkaInvalidConfig.GenWithStack("invalid kafka-client %q, only support franz and sarama", o.Client) + } if urlParameter.MaxRetry != nil && *urlParameter.MaxRetry >= 0 { o.MaxRetry = *urlParameter.MaxRetry @@ -574,12 +588,12 @@ func (o *options) DeriveTopicConfig() *AutoCreateTopicConfig { // ValidateReplicationFactor checks whether a topic created with this config // can satisfy the configured acknowledgment requirement. -func (c *AutoCreateTopicConfig) ValidateReplicationFactor(admin AdminClient) error { +func (c *AutoCreateTopicConfig) ValidateReplicationFactor(ctx context.Context, admin AdminClient) error { if c.RequiredAcks != WaitForAll { return nil } - raw, found, err := admin.GetBrokerConfig(MinInsyncReplicasConfigName) + raw, found, err := admin.GetBrokerConfig(ctx, MinInsyncReplicasConfigName) if err != nil { log.Warn("kafka broker configuration lookup failed, skipping replication factor validation", zap.String("configName", MinInsyncReplicasConfigName), @@ -636,15 +650,10 @@ func NewKafkaClientID(captureAddr string, // adjustOptions adjusts options with Kafka runtime metadata. // It overwrites MaxMessageBytes with the final producer message limit derived // from the topic or broker configuration. -func adjustOptions( - changefeedID common.ChangeFeedID, - admin AdminClient, - options *options, - topic string, -) error { +func adjustOptions(ctx context.Context, changefeedID common.ChangeFeedID, admin AdminClient, options *options, topic string) error { // The topic may not exist yet and will be created later by the topic manager, // so ignore per-topic metadata errors here. - topics, err := admin.GetTopicsMeta([]string{topic}, true) + topics, err := admin.GetTopicsMeta(ctx, []string{topic}, true) if err != nil { return err } @@ -653,9 +662,9 @@ func adjustOptions( // once we have found the topic, no matter `auto-create-topic`, // make sure user input parameters are valid. if exists { - err = adjustExistingTopicOption(changefeedID, admin, options, info) + err = adjustExistingTopicOption(ctx, changefeedID, admin, options, info) } else { - adjustNewTopicOptions(admin, changefeedID, options) + adjustNewTopicOptions(ctx, admin, changefeedID, options) } if err != nil { return err @@ -665,13 +674,8 @@ func adjustOptions( return nil } -func adjustExistingTopicOption( - changefeedID common.ChangeFeedID, - admin AdminClient, - options *options, - info TopicDetail, -) error { - maxMessageBytes, found, err := getTopicMaxMessageBytes(admin, info.Name) +func adjustExistingTopicOption(ctx context.Context, changefeedID common.ChangeFeedID, admin AdminClient, options *options, info TopicDetail) error { + maxMessageBytes, found, err := getTopicMaxMessageBytes(ctx, admin, info.Name) if err != nil || !found { log.Warn("kafka topic `max.message.bytes` unavailable, using configured value", zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), @@ -686,14 +690,10 @@ func adjustExistingTopicOption( return nil } -func adjustNewTopicOptions( - admin AdminClient, - changefeedID common.ChangeFeedID, - options *options, -) { +func adjustNewTopicOptions(ctx context.Context, admin AdminClient, changefeedID common.ChangeFeedID, options *options) { // when create the topic, `max.message.bytes` is decided by the broker, // it would use broker's `message.max.bytes` to set topic's `max.message.bytes`. - messageMaxBytes, found, err := getBrokerMaxMessageBytes(admin) + messageMaxBytes, found, err := getBrokerMaxMessageBytes(ctx, admin) if err != nil || !found { log.Warn("kafka broker `message.max.bytes` unavailable, using configured value", zap.String("namespace", changefeedID.Keyspace()), zap.String("changefeed", changefeedID.Name()), @@ -708,15 +708,8 @@ func adjustNewTopicOptions( } } -func getTopicMaxMessageBytes( - admin AdminClient, - topic string, -) (int, bool, error) { - raw, found, err := getTopicConfig( - admin, topic, - TopicMaxMessageBytesConfigName, - BrokerMessageMaxBytesConfigName, - ) +func getTopicMaxMessageBytes(ctx context.Context, admin AdminClient, topic string) (int, bool, error) { + raw, found, err := getTopicConfig(ctx, admin, topic, TopicMaxMessageBytesConfigName, BrokerMessageMaxBytesConfigName) if err != nil { return 0, false, err } @@ -730,8 +723,8 @@ func getTopicMaxMessageBytes( return maxMessageBytes, true, nil } -func getBrokerMaxMessageBytes(admin AdminClient) (int, bool, error) { - raw, found, err := admin.GetBrokerConfig(BrokerMessageMaxBytesConfigName) +func getBrokerMaxMessageBytes(ctx context.Context, admin AdminClient) (int, bool, error) { + raw, found, err := admin.GetBrokerConfig(ctx, BrokerMessageMaxBytesConfigName) if err != nil { return 0, false, err } @@ -749,16 +742,11 @@ func getBrokerMaxMessageBytes(admin AdminClient) (int, bool, error) { // If the topic does not have this configuration, // we will try to get it from the broker's configuration. // NOTICE: The configuration names of topic and broker may be different for the same configuration. -func getTopicConfig( - admin AdminClient, - topicName string, - topicConfigName string, - brokerConfigName string, -) (string, bool, error) { - c, found, err := admin.GetTopicConfig(topicName, topicConfigName) +func getTopicConfig(ctx context.Context, admin AdminClient, topicName, topicConfigName, brokerConfigName string) (string, bool, error) { + c, found, err := admin.GetTopicConfig(ctx, topicName, topicConfigName) if err == nil && found { return c, true, nil } - return admin.GetBrokerConfig(brokerConfigName) + return admin.GetBrokerConfig(ctx, brokerConfigName) } diff --git a/pkg/sink/kafka/options_test.go b/pkg/sink/kafka/options_test.go index 32bb58436f..c742ce1b80 100644 --- a/pkg/sink/kafka/options_test.go +++ b/pkg/sink/kafka/options_test.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "fmt" "net/url" "strconv" @@ -28,6 +29,7 @@ import ( "github.com/pingcap/ticdc/pkg/errors" "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" + "github.com/twmb/franz-go/pkg/kfake" ) const ( @@ -38,6 +40,78 @@ const ( mockTopicMessageMaxBytes = "1048588" ) +func TestKafkaClientSelection(t *testing.T) { + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "client-selection") + require.Equal(t, KafkaClientFranz, NewOptions().Client) + + for _, test := range []struct { + name string + uri string + expected string + wantErr bool + }{ + { + name: "URI selects sarama", + uri: "kafka://127.0.0.1:9092/topic?kafka-client=sarama", + expected: KafkaClientSarama, + }, + { + name: "URI value is case insensitive", + uri: "kafka://127.0.0.1:9092/topic?kafka-client=FRANZ", + expected: KafkaClientFranz, + }, + { + name: "invalid client", + uri: "kafka://127.0.0.1:9092/topic?kafka-client=other", + wantErr: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + sinkURI, err := url.Parse(test.uri) + require.NoError(t, err) + + options := NewOptions() + err = options.Apply(changefeedID, sinkURI, &config.SinkConfig{}) + if test.wantErr { + require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) + return + } + + require.NoError(t, err) + require.Equal(t, test.expected, options.Client) + }) + } +} + +func TestFactorySelection(t *testing.T) { + const topic = "factory-selection" + cluster := kfake.MustCluster(kfake.NumBrokers(1), kfake.SeedTopics(1, topic)) + defer cluster.Close() + + changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "factory-selection") + for _, test := range []struct { + client string + expected Factory + }{ + {client: KafkaClientFranz, expected: &franzFactory{}}, + {client: KafkaClientSarama, expected: &saramaFactory{}}, + } { + t.Run(test.client, func(t *testing.T) { + o := NewOptions() + o.Client = test.client + o.ClientID = "ticdc-test" + o.BrokerEndpoints = cluster.ListenAddrs() + o.Topic = topic + + factory, err := NewFactory(context.Background(), o, changefeedID) + require.NoError(t, err) + require.IsType(t, test.expected, factory) + + factory.Close() + }) + } +} + func TestCompleteOptions(t *testing.T) { options := NewOptions() @@ -647,13 +721,13 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) gomock.InOrder( - adminClient.EXPECT().GetTopicsMeta([]string{topicName}, true).Return( + adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{topicName}, true).Return( map[string]TopicDetail{ topicName: {Name: topicName, NumPartitions: 3}, }, nil), - adminClient.EXPECT().GetTopicConfig(topicName, TopicMaxMessageBytesConfigName). + adminClient.EXPECT().GetTopicConfig(gomock.Any(), topicName, TopicMaxMessageBytesConfigName). Return("", false, nil), - adminClient.EXPECT().GetBrokerConfig(BrokerMessageMaxBytesConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), BrokerMessageMaxBytesConfigName). Return(mockBrokerMessageMaxBytes, true, nil), ) sinkURI, err := url.Parse(fmt.Sprintf( @@ -668,7 +742,7 @@ func TestAdjustConfigFallsBackToBrokerMessageMaxBytesWhenTopicConfigMissing(t *t require.Equal(t, test.configuredMaxMessageBytes, options.MaxMessageBytes) require.Equal(t, test.configuredMaxMessageBytes, options.MaxBatchedBytes) - err = adjustOptions(changefeedID, adminClient, options, topicName) + err = adjustOptions(t.Context(), changefeedID, adminClient, options, topicName) require.NoError(t, err) require.NotEqual(t, test.configuredMaxMessageBytes, options.MaxMessageBytes) @@ -686,9 +760,9 @@ func TestValidateReplicationFactor(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) gomock.InOrder( - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("2", true, nil), - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("", false, nil), ) @@ -697,7 +771,7 @@ func TestValidateReplicationFactor(t *testing.T) { ReplicationFactor: 1, RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.Regexp( t, ".*`replication-factor` 1 is smaller than the `min.insync.replicas` 2 of broker.*", @@ -709,7 +783,7 @@ func TestValidateReplicationFactor(t *testing.T) { ReplicationFactor: 1, RequiredAcks: WaitForLocal, } - err = localAcksConfig.ValidateReplicationFactor(adminClient) + err = localAcksConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) missingBrokerConfig := &AutoCreateTopicConfig{ @@ -717,13 +791,13 @@ func TestValidateReplicationFactor(t *testing.T) { ReplicationFactor: 1, RequiredAcks: WaitForAll, } - err = missingBrokerConfig.ValidateReplicationFactor(adminClient) + err = missingBrokerConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) t.Run("replication factor satisfies min insync replicas", func(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("2", true, nil) topicConfig := &AutoCreateTopicConfig{ @@ -731,14 +805,14 @@ func TestValidateReplicationFactor(t *testing.T) { RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) }) t.Run("invalid min insync replicas", func(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("invalid", true, nil) topicConfig := &AutoCreateTopicConfig{ @@ -746,7 +820,7 @@ func TestValidateReplicationFactor(t *testing.T) { RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) }) @@ -757,7 +831,7 @@ func TestValidateReplicationFactor(t *testing.T) { "describe-config", MinInsyncReplicasConfigName, ) - adminClient.EXPECT().GetBrokerConfig(MinInsyncReplicasConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), MinInsyncReplicasConfigName). Return("", false, lookupErr) topicConfig := &AutoCreateTopicConfig{ @@ -765,7 +839,7 @@ func TestValidateReplicationFactor(t *testing.T) { RequiredAcks: WaitForAll, } - err := topicConfig.ValidateReplicationFactor(adminClient) + err := topicConfig.ValidateReplicationFactor(t.Context(), adminClient) require.NoError(t, err) }) } @@ -930,7 +1004,7 @@ func TestConfigurationCombinations(t *testing.T) { ctrl := gomock.NewController(t) adminClient := NewMockAdminClient(ctrl) - metadataCall := adminClient.EXPECT().GetTopicsMeta([]string{topic}, true) + metadataCall := adminClient.EXPECT().GetTopicsMeta(gomock.Any(), []string{topic}, true) sourceMaxMessageBytes := a.brokerMessageMaxBytes if topic == defaultMockTopicName { metadataCall.Return(map[string]TopicDetail{ @@ -938,7 +1012,7 @@ func TestConfigurationCombinations(t *testing.T) { }, nil) gomock.InOrder( metadataCall, - adminClient.EXPECT().GetTopicConfig(topic, TopicMaxMessageBytesConfigName). + adminClient.EXPECT().GetTopicConfig(gomock.Any(), topic, TopicMaxMessageBytesConfigName). Return(a.topicMaxMessageBytes, true, nil), ) sourceMaxMessageBytes = a.topicMaxMessageBytes @@ -946,7 +1020,7 @@ func TestConfigurationCombinations(t *testing.T) { metadataCall.Return(map[string]TopicDetail{}, nil) gomock.InOrder( metadataCall, - adminClient.EXPECT().GetBrokerConfig(BrokerMessageMaxBytesConfigName). + adminClient.EXPECT().GetBrokerConfig(gomock.Any(), BrokerMessageMaxBytesConfigName). Return(a.brokerMessageMaxBytes, true, nil), ) } @@ -959,7 +1033,7 @@ func TestConfigurationCombinations(t *testing.T) { expectedMaxMessageBytes, err := strconv.Atoi(sourceMaxMessageBytes) require.NoError(t, err) changefeedID := common.NewChangefeedID4Test(common.DefaultKeyspaceName, "test") - err = adjustOptions(changefeedID, adminClient, options, topic) + err = adjustOptions(t.Context(), changefeedID, adminClient, options, topic) require.Nil(t, err) require.Equal(t, expectedMaxMessageBytes, options.MaxMessageBytes) require.Equal( diff --git a/pkg/sink/kafka/producer_test.go b/pkg/sink/kafka/producer_test.go new file mode 100644 index 0000000000..53259d3c60 --- /dev/null +++ b/pkg/sink/kafka/producer_test.go @@ -0,0 +1,31 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kafka + +import ( + "strings" + "testing" + + "github.com/pingcap/ticdc/pkg/errors" + "github.com/stretchr/testify/require" +) + +func requireKafkaSendError(t *testing.T, err, cause error) { + t.Helper() + require.ErrorIs(t, err, errors.ErrKafkaSendMessage) + require.ErrorIs(t, err, cause) + require.Equal(t, 1, strings.Count(err.Error(), string(errors.ErrKafkaSendMessage.RFCCode()))) + require.NotContains(t, err.Error(), "keyspace=test") +} diff --git a/pkg/sink/kafka/sarama_admin_test.go b/pkg/sink/kafka/sarama_admin_test.go index d0ce07fc1d..9afa80c407 100644 --- a/pkg/sink/kafka/sarama_admin_test.go +++ b/pkg/sink/kafka/sarama_admin_test.go @@ -45,7 +45,7 @@ func TestGetBrokerConfig(t *testing.T) { changefeed: common.NewChangeFeedIDWithName("test", "default"), admin: admin, } - value, found, err := client.GetBrokerConfig("message.max.bytes") + value, found, err := client.GetBrokerConfig(t.Context(), "message.max.bytes") require.NoError(t, err) require.True(t, found) @@ -62,7 +62,7 @@ func TestGetBrokerConfig(t *testing.T) { changefeed: common.NewChangeFeedIDWithName("test", "default"), admin: admin, } - value, found, err := client.GetBrokerConfig("missing") + value, found, err := client.GetBrokerConfig(t.Context(), "missing") require.NoError(t, err) require.False(t, found) @@ -79,7 +79,7 @@ func TestGetBrokerConfig(t *testing.T) { changefeed: common.NewChangeFeedIDWithName("test", "default"), admin: admin, } - _, _, err := client.GetBrokerConfig("missing") + _, _, err := client.GetBrokerConfig(t.Context(), "missing") require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) require.ErrorIs(t, err, cause) @@ -104,7 +104,7 @@ func TestGetTopicConfig(t *testing.T) { admin: admin, } - value, found, err := client.GetTopicConfig("test-topic", "max.message.bytes") + value, found, err := client.GetTopicConfig(t.Context(), "test-topic", "max.message.bytes") require.NoError(t, err) require.True(t, found) @@ -120,7 +120,7 @@ func TestGetTopicConfig(t *testing.T) { admin: admin, } - value, found, err := client.GetTopicConfig("test-topic", "missing") + value, found, err := client.GetTopicConfig(t.Context(), "test-topic", "missing") require.NoError(t, err) require.False(t, found) @@ -136,7 +136,7 @@ func TestGetTopicConfig(t *testing.T) { admin: admin, } - _, _, err := client.GetTopicConfig("test-topic", "missing") + _, _, err := client.GetTopicConfig(t.Context(), "test-topic", "missing") require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) require.ErrorIs(t, err, context.DeadlineExceeded) @@ -165,7 +165,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"valid-topic", "missing-topic"}, false) + topics, err := client.GetTopicsMeta(t.Context(), []string{"valid-topic", "missing-topic"}, false) require.Nil(t, topics) require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) @@ -190,7 +190,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"valid-topic", "missing-topic"}, true) + topics, err := client.GetTopicsMeta(t.Context(), []string{"valid-topic", "missing-topic"}, true) require.NoError(t, err) require.Equal(t, map[string]TopicDetail{ @@ -210,7 +210,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"missing-topic"}, false) + topics, err := client.GetTopicsMeta(t.Context(), []string{"missing-topic"}, false) require.NoError(t, err) require.Empty(t, topics) @@ -227,7 +227,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + _, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, false) require.ErrorIs(t, err, errors.ErrKafkaAdminAPI) require.ErrorIs(t, err, sarama.ErrInvalidTopic) @@ -245,7 +245,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + _, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, false) require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) require.NotErrorIs(t, err, errors.ErrKafkaAdminAPI) @@ -265,7 +265,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - _, err := client.GetTopicsMeta([]string{"test-topic"}, false) + _, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, false) require.ErrorIs(t, err, errors.ErrKafkaAuthorizationFailed) require.NotErrorIs(t, err, errors.ErrKafkaAdminAPI) @@ -284,7 +284,7 @@ func TestGetTopicsMeta(t *testing.T) { admin: admin, } - topics, err := client.GetTopicsMeta([]string{"test-topic"}, true) + topics, err := client.GetTopicsMeta(t.Context(), []string{"test-topic"}, true) require.NoError(t, err) require.Empty(t, topics) @@ -312,7 +312,7 @@ func TestIsAuthorizationFailed(t *testing.T) { } } -func TestIsUnretryableKafkaError(t *testing.T) { +func TestIsUnretryableSaramaError(t *testing.T) { t.Parallel() tests := []struct { @@ -354,7 +354,7 @@ func TestIsUnretryableKafkaError(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - require.Equal(t, test.unretryable, IsUnretryableKafkaError(test.err)) + require.Equal(t, test.unretryable, IsUnretryableSaramaError(test.err)) }) } } @@ -387,7 +387,7 @@ func TestCreateTopic(t *testing.T) { admin: admin, } - err := client.CreateTopic(&TopicDetail{ + err := client.CreateTopic(t.Context(), &TopicDetail{ Name: "test-topic", NumPartitions: 3, ReplicationFactor: 2, diff --git a/pkg/sink/kafka/sarama_factory.go b/pkg/sink/kafka/sarama_factory.go index e86cdc78a3..1fcf601793 100644 --- a/pkg/sink/kafka/sarama_factory.go +++ b/pkg/sink/kafka/sarama_factory.go @@ -32,12 +32,10 @@ type saramaFactory struct { metricRegistry metrics.Registry } -// NewSaramaFactory constructs a Factory with sarama implementation. -func NewSaramaFactory( - ctx context.Context, - o *options, - changefeedID common.ChangeFeedID, -) (Factory, error) { +func (*saramaFactory) Close() {} + +// newSaramaFactory constructs a Factory with sarama implementation. +func newSaramaFactory(ctx context.Context, o *options, changefeedID common.ChangeFeedID) (Factory, error) { start := time.Now() config, err := newSaramaConfig(ctx, o) duration := time.Since(start) @@ -59,7 +57,7 @@ func NewSaramaFactory( admin.Close() }() - if err = adjustOptions(changefeedID, admin, o, o.Topic); err != nil { + if err = adjustOptions(ctx, changefeedID, admin, o, o.Topic); err != nil { return nil, err } log.Info("kafka sink configuration resolved", @@ -182,9 +180,7 @@ func (f *saramaFactory) AsyncProducer(ctx context.Context) (AsyncProducer, error }, nil } -func (f *saramaFactory) MetricsCollector( - adminClient AdminClient, -) MetricsCollector { +func (f *saramaFactory) MetricsCollector(adminClient AdminClient) MetricsCollector { return &saramaMetricsCollector{ changefeedID: f.changefeedID, adminClient: adminClient, diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider.go b/pkg/sink/kafka/sarama_oauth2_token_provider.go index 6adbc56659..0c8d80eaf6 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider.go @@ -15,16 +15,9 @@ package kafka import ( "context" - "crypto/tls" - "crypto/x509" - "net/http" - "net/url" - "os" "github.com/IBM/sarama" - "github.com/pingcap/ticdc/pkg/errors" "golang.org/x/oauth2" - "golang.org/x/oauth2/clientcredentials" ) // tokenProvider is a user-defined callback for generating @@ -56,69 +49,9 @@ func (t *tokenProvider) Token() (*sarama.AccessToken, error) { } func newTokenProvider(ctx context.Context, o *options) (sarama.AccessTokenProvider, error) { - // grant_type is by default going to be set to 'client_credentials' by the - // client credentials library as defined by the spec, however non-compliant - // auth server implementations may want a custom type - endpointParams := url.Values{} - if o.sasl.oauth2.grantType != "" { - endpointParams.Set("grant_type", o.sasl.oauth2.grantType) - } - - // audience is an optional parameter that can be used to specify the - // intended audience of the token. - if o.sasl.oauth2.audience != "" { - endpointParams.Set("audience", o.sasl.oauth2.audience) - } - - tokenURL, err := url.Parse(o.sasl.oauth2.tokenURL) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - - if o.sasl.oauth2.caPath != "" { - ctx, err = contextWithOAuthCA(ctx, o.sasl.oauth2.caPath) - if err != nil { - return nil, err - } - } - - cfg := clientcredentials.Config{ - ClientID: o.sasl.oauth2.clientID, - ClientSecret: o.sasl.oauth2.clientSecret, - TokenURL: tokenURL.String(), - EndpointParams: endpointParams, - Scopes: o.sasl.oauth2.scopes, - } - return &tokenProvider{ - tokenSource: cfg.TokenSource(ctx), - }, nil -} - -func contextWithOAuthCA(ctx context.Context, caPath string) (context.Context, error) { - caPEM, err := os.ReadFile(caPath) - if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - - rootCAs, err := x509.SystemCertPool() + tokenSource, err := newOAuthTokenSource(ctx, o.sasl.oauth2) if err != nil { - return nil, errors.WrapError(errors.ErrKafkaInvalidConfig, err) - } - if !rootCAs.AppendCertsFromPEM(caPEM) { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "OAuth2 CA file %q does not contain a valid certificate", caPath) - } - - defaultTransport, ok := http.DefaultTransport.(*http.Transport) - if !ok { - return nil, errors.ErrKafkaInvalidConfig.GenWithStack( - "cannot configure OAuth2 CA file %q with HTTP transport type %T", - caPath, http.DefaultTransport) - } - transport := defaultTransport.Clone() - if transport.TLSClientConfig == nil { - transport.TLSClientConfig = &tls.Config{} + return nil, err } - transport.TLSClientConfig.RootCAs = rootCAs - return context.WithValue(ctx, oauth2.HTTPClient, &http.Client{Transport: transport}), nil + return &tokenProvider{tokenSource: tokenSource}, nil } diff --git a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go index fc2dffa6a2..d7d0473c32 100644 --- a/pkg/sink/kafka/sarama_oauth2_token_provider_test.go +++ b/pkg/sink/kafka/sarama_oauth2_token_provider_test.go @@ -14,18 +14,13 @@ package kafka import ( - "context" - "encoding/pem" "io" "net/http" "net/http/httptest" "net/url" - "os" - "path/filepath" "testing" "github.com/pingcap/ticdc/pkg/errors" - "github.com/pingcap/ticdc/pkg/security" "github.com/stretchr/testify/require" "golang.org/x/oauth2" ) @@ -137,106 +132,3 @@ func TestTokenProviderPropagatesEndpointError(t *testing.T) { require.Equal(t, "invalid_client", retrieveErr.ErrorCode) require.Equal(t, "bad credentials", retrieveErr.ErrorDescription) } - -func TestTokenProviderUsesOAuthCA(t *testing.T) { - t.Parallel() - - server := newTLSTokenServer(t) - caPath := writeServerCA(t, server) - options := newOAuthOptions(server.URL, caPath) - - provider, err := newTokenProvider(t.Context(), options) - require.NoError(t, err) - token, err := provider.Token() - require.NoError(t, err) - require.Equal(t, "access-token", token.Token) -} - -func TestTokenProviderRejectsInvalidOAuthCA(t *testing.T) { - t.Parallel() - - t.Run("missing file", func(t *testing.T) { - t.Parallel() - - caPath := filepath.Join(t.TempDir(), "missing-ca.pem") - _, err := newTokenProvider(t.Context(), newOAuthOptions("https://example.com/token", caPath)) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) - require.ErrorContains(t, err, caPath) - require.ErrorContains(t, err, "no such file") - }) - - t.Run("invalid PEM", func(t *testing.T) { - t.Parallel() - - caPath := filepath.Join(t.TempDir(), "invalid-ca.pem") - require.NoError(t, os.WriteFile(caPath, []byte("not a certificate"), 0o600)) - _, err := newTokenProvider(t.Context(), newOAuthOptions("https://example.com/token", caPath)) - require.ErrorIs(t, err, errors.ErrKafkaInvalidConfig) - require.ErrorContains(t, err, caPath) - require.ErrorContains(t, err, "does not contain a valid certificate") - }) -} - -func TestTokenProviderRejectsMismatchedOAuthCA(t *testing.T) { - t.Parallel() - - tokenServer := newTLSTokenServer(t) - unrelatedCA, err := security.NewCA() - require.NoError(t, err) - caPath := filepath.Join(t.TempDir(), "unrelated-ca.pem") - require.NoError(t, os.WriteFile(caPath, unrelatedCA.CAPEM, 0o600)) - options := newOAuthOptions(tokenServer.URL, caPath) - - provider, err := newTokenProvider(t.Context(), options) - require.NoError(t, err) - _, err = provider.Token() - require.ErrorContains(t, err, "certificate signed by unknown authority") -} - -func TestTokenProviderWithoutOAuthCAKeepsContextHTTPClient(t *testing.T) { - t.Parallel() - - server := newTLSTokenServer(t) - ctx := context.WithValue(t.Context(), oauth2.HTTPClient, server.Client()) - provider, err := newTokenProvider(ctx, newOAuthOptions(server.URL, "")) - require.NoError(t, err) - token, err := provider.Token() - require.NoError(t, err) - require.Equal(t, "access-token", token.Token) -} - -func newTLSTokenServer(t *testing.T) *httptest.Server { - t.Helper() - - server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.Header().Set("Content-Type", "application/json") - if _, err := io.WriteString(w, `{"access_token":"access-token","token_type":"bearer"}`); err != nil { - t.Errorf("write token response: %v", err) - } - })) - t.Cleanup(server.Close) - return server -} - -func writeServerCA(t *testing.T, server *httptest.Server) string { - t.Helper() - - caPath := filepath.Join(t.TempDir(), "ca.pem") - caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw}) - require.NotNil(t, caPEM) - require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) - return caPath -} - -func newOAuthOptions(tokenURL, caPath string) *options { - return &options{ - sasl: &saslConfig{ - oauth2: oauth2Config{ - clientID: "client-id", - clientSecret: "client-secret", - tokenURL: tokenURL, - caPath: caPath, - }, - }, - } -} diff --git a/pkg/sink/kafka/sarama_sync_producer.go b/pkg/sink/kafka/sarama_sync_producer.go index fcf1c9c258..628ea6b51e 100644 --- a/pkg/sink/kafka/sarama_sync_producer.go +++ b/pkg/sink/kafka/sarama_sync_producer.go @@ -14,6 +14,7 @@ package kafka import ( + "context" "time" "github.com/IBM/sarama" @@ -43,7 +44,7 @@ type saramaSyncProducer struct { closed *atomic.Bool } -func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, message *codecCommon.Message) error { +func (p *saramaSyncProducer) SendMessage(_ context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } @@ -66,7 +67,7 @@ func (p *saramaSyncProducer) SendMessage(topic string, partitionNum int32, messa return errors.WrapError(errors.ErrKafkaSendMessage, err) } -func (p *saramaSyncProducer) SendMessages(topic string, partitionNum int32, message *codecCommon.Message) error { +func (p *saramaSyncProducer) SendMessages(_ context.Context, topic string, partitionNum int32, message *codecCommon.Message) error { if p.closed.Load() { return errors.ErrKafkaSinkClosed.GenWithStackByArgs() } diff --git a/pkg/sink/kafka/sarama_sync_producer_test.go b/pkg/sink/kafka/sarama_sync_producer_test.go index e00e0dafa8..27f1888826 100644 --- a/pkg/sink/kafka/sarama_sync_producer_test.go +++ b/pkg/sink/kafka/sarama_sync_producer_test.go @@ -16,7 +16,6 @@ package kafka import ( "context" "io" - "strings" "testing" "github.com/golang/mock/gomock" @@ -32,8 +31,8 @@ func TestProducerRejectsSendAfterClose(t *testing.T) { message := &codecCommon.Message{} syncProducer := &saramaSyncProducer{closed: atomic.NewBool(true)} - require.ErrorIs(t, syncProducer.SendMessage("topic", 1, message), errors.ErrKafkaSinkClosed) - require.ErrorIs(t, syncProducer.SendMessages("topic", 1, message), errors.ErrKafkaSinkClosed) + require.ErrorIs(t, syncProducer.SendMessage(t.Context(), "topic", 1, message), errors.ErrKafkaSinkClosed) + require.ErrorIs(t, syncProducer.SendMessages(t.Context(), "topic", 1, message), errors.ErrKafkaSinkClosed) asyncProducer := &saramaAsyncProducer{closed: atomic.NewBool(true)} require.ErrorIs(t, asyncProducer.AsyncSend(context.Background(), "topic", 0, message), errors.ErrKafkaSinkClosed) @@ -88,7 +87,7 @@ func TestSyncProducerErrorWrappedOnce(t *testing.T) { producer.EXPECT().SendMessage(gomock.Any()).Return(int32(0), int64(0), cause) }, send: func(producer *saramaSyncProducer, message *codecCommon.Message) error { - return producer.SendMessage("topic", 0, message) + return producer.SendMessage(t.Context(), "topic", 0, message) }, }, { @@ -97,7 +96,7 @@ func TestSyncProducerErrorWrappedOnce(t *testing.T) { producer.EXPECT().SendMessages(gomock.Any()).Return(cause) }, send: func(producer *saramaSyncProducer, message *codecCommon.Message) error { - return producer.SendMessages("topic", 1, message) + return producer.SendMessages(t.Context(), "topic", 1, message) }, }, } @@ -131,11 +130,3 @@ func TestAsyncProducerErrorWrappedOnce(t *testing.T) { requireKafkaSendError(t, err, cause) } - -func requireKafkaSendError(t *testing.T, err, cause error) { - t.Helper() - require.ErrorIs(t, err, errors.ErrKafkaSendMessage) - require.ErrorIs(t, err, cause) - require.Equal(t, 1, strings.Count(err.Error(), string(errors.ErrKafkaSendMessage.RFCCode()))) - require.NotContains(t, err.Error(), "keyspace=test") -}